├── version.txt ├── settings.gradle ├── resources └── img │ └── RecallIcon.png ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── .github └── dependabot.yml ├── config ├── suppressions.xml ├── header.txt ├── intellij_code_style.xml └── checkstyle.xml ├── .gitignore ├── .circleci └── config.yml ├── recall-annotations └── src │ ├── main │ └── java │ │ └── com │ │ └── aitusoftware │ │ └── recall │ │ └── annotation │ │ ├── model │ │ ├── Field.java │ │ └── Type.java │ │ ├── LookupField.java │ │ └── Recall.java │ └── test │ └── java │ └── com │ └── aitusoftware │ └── recall │ └── annotation │ └── example │ └── ExampleObject.java ├── recall-store └── src │ ├── test │ └── java │ │ └── com │ │ └── aitusoftware │ │ └── recall │ │ ├── store │ │ ├── HeaderTest.java │ │ ├── UnsafeBufferOpsTest.java │ │ ├── ByteBufferOpsTest.java │ │ ├── UnsafeBufferStoreTest.java │ │ └── ByteBufferStoreTest.java │ │ ├── example │ │ ├── CharSequenceMapExample.java │ │ ├── OrderByteBufferTranscoder.java │ │ ├── OrderUnsafeBufferTranscoder.java │ │ └── Order.java │ │ └── map │ │ ├── ByteSequenceMapTest.java │ │ └── CharSequenceMapTest.java │ └── main │ └── java │ └── com │ └── aitusoftware │ └── recall │ ├── persistence │ ├── IdAccessor.java │ ├── Encoder.java │ ├── Decoder.java │ └── AsciiCharSequence.java │ ├── store │ ├── Version.java │ ├── Store.java │ ├── Header.java │ ├── ByteBufferOps.java │ ├── SingleTypeStore.java │ ├── BufferOps.java │ ├── UnsafeBufferOps.java │ └── BufferStore.java │ └── map │ ├── SequenceMap.java │ ├── ByteSequenceMap.java │ └── CharSequenceMap.java ├── recall-sbe └── src │ ├── main │ └── java │ │ └── com │ │ └── aitusoftware │ │ └── recall │ │ └── sbe │ │ ├── SbeMessageBufferDecoder.java │ │ ├── SbeMessageBufferEncoder.java │ │ └── SbeMessageStoreFactory.java │ └── test │ ├── java │ └── com │ │ └── aitusoftware │ │ └── recall │ │ └── sbe │ │ ├── SbeMessageBufferEncoderTest.java │ │ └── SbeObjectStoreTest.java │ └── resources │ └── sbe-schema.xml ├── gradlew.bat ├── gradlew ├── README.md └── LICENSE /version.txt: -------------------------------------------------------------------------------- 1 | 1.1.0 2 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include 'recall-store' 2 | include 'recall-sbe' 3 | // include 'recall-annotations' 4 | -------------------------------------------------------------------------------- /resources/img/RecallIcon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aitusoftware/recall/HEAD/resources/img/RecallIcon.png -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aitusoftware/recall/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gradle" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" -------------------------------------------------------------------------------- /config/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStorePath=wrapper/dists 5 | zipStoreBase=GRADLE_USER_HOME 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | */build/ 25 | *.iml 26 | .gradle/ 27 | .idea/ 28 | todo.txt 29 | recall-sbe/out 30 | recall-store/out 31 | build/ 32 | -------------------------------------------------------------------------------- /config/header.txt: -------------------------------------------------------------------------------- 1 | Copyright 2019 Aitu Software Limited. 2 | 3 | https://aitusoftware.com 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: circleci/openjdk:8-jdk 6 | working_directory: ~/repo 7 | environment: 8 | JVM_OPTS: -Xmx3200m 9 | TERM: dumb 10 | 11 | steps: 12 | - checkout 13 | - restore_cache: 14 | keys: 15 | - v1-dependencies-{{ checksum "build.gradle" }} 16 | # fallback to using the latest cache if no exact match is found 17 | - v1-dependencies- 18 | 19 | - run: gradle dependencies 20 | 21 | - save_cache: 22 | paths: 23 | - ~/.gradle 24 | key: v1-dependencies-{{ checksum "build.gradle" }} 25 | - run: gradle licenseFormat check 26 | -------------------------------------------------------------------------------- /config/intellij_code_style.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 17 | -------------------------------------------------------------------------------- /recall-annotations/src/main/java/com/aitusoftware/recall/annotation/model/Field.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.annotation.model; 19 | 20 | public final class Field 21 | { 22 | } 23 | -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/store/HeaderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import static org.junit.jupiter.api.Assertions.*; 21 | 22 | class HeaderTest 23 | { 24 | 25 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/persistence/IdAccessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.persistence; 19 | 20 | /** 21 | * Function to map an instance to an ID. 22 | * 23 | * @param the type of the instance 24 | */ 25 | @FunctionalInterface 26 | public interface IdAccessor 27 | { 28 | /** 29 | * Return the ID belonging to the value. 30 | * 31 | * @param value input value 32 | * @return the ID of the value 33 | */ 34 | long getId(T value); 35 | } 36 | -------------------------------------------------------------------------------- /recall-annotations/src/main/java/com/aitusoftware/recall/annotation/model/Type.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.annotation.model; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | public final class Type 24 | { 25 | private final List fieldList = new ArrayList<>(); 26 | 27 | public void addField(final Field field) 28 | { 29 | fieldList.add(field); 30 | } 31 | 32 | public List getFieldList() 33 | { 34 | return fieldList; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /recall-annotations/src/main/java/com/aitusoftware/recall/annotation/LookupField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.annotation; 19 | 20 | import java.lang.annotation.ElementType; 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.RetentionPolicy; 23 | import java.lang.annotation.Target; 24 | 25 | /** 26 | * Annotation to indicate that a mapping should be generated for the annotated field. 27 | */ 28 | @Retention(RetentionPolicy.RUNTIME) 29 | @Target(ElementType.METHOD) 30 | public @interface LookupField 31 | { 32 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/persistence/Encoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.persistence; 19 | 20 | /** 21 | * Serialiser to be used with a {@code Store}. 22 | * 23 | * @param type of the buffer 24 | * @param type of the instance 25 | */ 26 | @FunctionalInterface 27 | public interface Encoder 28 | { 29 | /** 30 | * Encodes the specified value into the supplied buffer. 31 | * 32 | * @param buffer target buffer 33 | * @param offset offset into the buffer 34 | * @param value value to encode 35 | */ 36 | void store(B buffer, int offset, T value); 37 | } 38 | -------------------------------------------------------------------------------- /recall-annotations/src/test/java/com/aitusoftware/recall/annotation/example/ExampleObject.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.annotation.example; 19 | 20 | import com.aitusoftware.recall.annotation.LookupField; 21 | import com.aitusoftware.recall.annotation.Recall; 22 | 23 | @Recall 24 | public interface ExampleObject 25 | { 26 | long longValue(); 27 | 28 | @LookupField 29 | int intValue(); 30 | 31 | char charValue(); 32 | 33 | byte byteValue(); 34 | 35 | @LookupField 36 | CharSequence textValue(); 37 | 38 | default String intToString(int input) 39 | { 40 | return Integer.toString(input); 41 | } 42 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/persistence/Decoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.persistence; 19 | 20 | /** 21 | * Deserialiser to be used with a {@code Store}. 22 | * 23 | * @param type of the buffer 24 | * @param type of the instance 25 | */ 26 | @FunctionalInterface 27 | public interface Decoder 28 | { 29 | /** 30 | * Decodes the value at the specified offset into the supplied container instance. 31 | * 32 | * @param buffer source buffer 33 | * @param offset offset into the buffer 34 | * @param container receiver for the data 35 | */ 36 | void load(B buffer, int offset, T container); 37 | } 38 | -------------------------------------------------------------------------------- /recall-sbe/src/main/java/com/aitusoftware/recall/sbe/SbeMessageBufferDecoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.sbe; 19 | 20 | import com.aitusoftware.recall.persistence.Decoder; 21 | import org.agrona.concurrent.UnsafeBuffer; 22 | import org.agrona.sbe.MessageDecoderFlyweight; 23 | 24 | /** 25 | * Decoder for SBE-encoded messages. 26 | * 27 | * @param the type of the message 28 | */ 29 | public final class SbeMessageBufferDecoder implements Decoder 30 | { 31 | /** 32 | * {@inheritDoc} 33 | */ 34 | @Override 35 | public void load(final UnsafeBuffer buffer, final int offset, final T container) 36 | { 37 | container.wrap(buffer, offset, container.sbeBlockLength(), 38 | container.sbeSchemaVersion()); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/store/Version.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | public enum Version 21 | { 22 | ONE(1); 23 | 24 | private final int versionNumber; 25 | 26 | Version(final int versionNumber) 27 | { 28 | this.versionNumber = versionNumber; 29 | } 30 | 31 | public static Version from(final int versionNumber) 32 | { 33 | for (final Version version : values()) 34 | { 35 | if (version.versionNumber == versionNumber) 36 | { 37 | return version; 38 | } 39 | } 40 | 41 | throw new IllegalArgumentException("Unknown version number: " + versionNumber); 42 | } 43 | 44 | public int getVersionNumber() 45 | { 46 | return versionNumber; 47 | } 48 | } -------------------------------------------------------------------------------- /recall-annotations/src/main/java/com/aitusoftware/recall/annotation/Recall.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.annotation; 19 | 20 | import java.lang.annotation.ElementType; 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.RetentionPolicy; 23 | import java.lang.annotation.Target; 24 | 25 | /** 26 | * Annotation that indicates that the annotated type should trigger code generation. 27 | */ 28 | @Retention(RetentionPolicy.RUNTIME) 29 | @Target(ElementType.TYPE) 30 | public @interface Recall 31 | { 32 | /** 33 | * Prefix that will be added to generated class names. 34 | * 35 | * @return the prefix 36 | */ 37 | String classPrefix() default ""; 38 | 39 | /** 40 | * Suffix that will be added to generated class names. 41 | * 42 | * @return the suffix 43 | */ 44 | String classSuffix() default "Impl"; 45 | } 46 | -------------------------------------------------------------------------------- /recall-sbe/src/test/java/com/aitusoftware/recall/sbe/SbeMessageBufferEncoderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.sbe; 19 | 20 | import com.aitusoftware.recall.sbe.example.CarDecoder; 21 | import org.agrona.concurrent.UnsafeBuffer; 22 | import org.junit.jupiter.api.Assertions; 23 | import org.junit.jupiter.api.Test; 24 | 25 | class SbeMessageBufferEncoderTest 26 | { 27 | private final SbeMessageBufferEncoder encoder = new SbeMessageBufferEncoder<>(10); 28 | 29 | @Test 30 | void shouldThrowExceptionIfEncodedLengthExceedMaxMessageLength() 31 | { 32 | final CarDecoder carDecoder = new CarDecoder(); 33 | carDecoder.wrap(new UnsafeBuffer(new byte[64]), 0, carDecoder.sbeBlockLength(), 0); 34 | Assertions.assertThrows(IllegalArgumentException.class, 35 | () -> encoder.store(new UnsafeBuffer(new byte[0]), 0, carDecoder), 36 | "Unable to encode message of length 49"); 37 | } 38 | } -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/store/UnsafeBufferOpsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import org.agrona.concurrent.UnsafeBuffer; 21 | import org.junit.jupiter.api.Test; 22 | 23 | import java.nio.charset.StandardCharsets; 24 | import java.util.Arrays; 25 | 26 | import static com.google.common.truth.Truth.assertThat; 27 | 28 | class UnsafeBufferOpsTest 29 | { 30 | @Test 31 | void shouldCopyBytes() 32 | { 33 | final byte[] data = "0123456789ABCDE".getBytes(StandardCharsets.UTF_8); 34 | final UnsafeBuffer source = new UnsafeBuffer(); 35 | source.wrap(data); 36 | final UnsafeBuffer target = new UnsafeBuffer(new byte[32]); 37 | final UnsafeBuffer expected = new UnsafeBuffer(new byte[32]); 38 | expected.putBytes(7, data); 39 | 40 | final UnsafeBufferOps ops = new UnsafeBufferOps(); 41 | ops.copyBytes(source, target, 0, 7, source.capacity()); 42 | 43 | assertThat(Arrays.equals(target.byteArray(), expected.byteArray())).isTrue(); 44 | } 45 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/map/SequenceMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.map; 19 | 20 | public interface SequenceMap 21 | { 22 | /** 23 | * Insert a value into the map. 24 | * 25 | * @param value value to use as a key 26 | * @param id id to store 27 | */ 28 | void put(T value, long id); 29 | 30 | /** 31 | * Searches the map for a given key. 32 | * 33 | * @param value the key to search for 34 | * @return the retrieved value, or {@code missingValue} if it was not present 35 | */ 36 | long get(T value); 37 | 38 | /** 39 | * Removes an entry for a given key. 40 | * 41 | * @param value the key to search for 42 | * @return the stored value, or {@code missingValue} if the key was not present 43 | */ 44 | long remove(T value); 45 | 46 | /** 47 | * Returns the number of entries in the map. 48 | * 49 | * @return the number of entries 50 | */ 51 | int size(); 52 | 53 | /** 54 | * Allocates a new buffer and copies existing entries to it. 55 | */ 56 | void rehash(); 57 | } 58 | -------------------------------------------------------------------------------- /recall-sbe/src/main/java/com/aitusoftware/recall/sbe/SbeMessageBufferEncoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.sbe; 19 | 20 | import com.aitusoftware.recall.persistence.Encoder; 21 | import org.agrona.concurrent.UnsafeBuffer; 22 | import org.agrona.sbe.MessageDecoderFlyweight; 23 | 24 | /** 25 | * Encoder for SBE-encoded messages. 26 | * 27 | * @param the type of the message 28 | */ 29 | public final class SbeMessageBufferEncoder implements Encoder 30 | { 31 | private final int maxMessageLength; 32 | 33 | /** 34 | * Maximum length of any message that will be stored. 35 | * 36 | * @param maxMessageLength max message length, in bytes 37 | */ 38 | public SbeMessageBufferEncoder(final int maxMessageLength) 39 | { 40 | this.maxMessageLength = maxMessageLength; 41 | } 42 | 43 | /** 44 | * {@inheritDoc} 45 | */ 46 | @Override 47 | public void store(final UnsafeBuffer buffer, final int offset, final T value) 48 | { 49 | final int encodedLength = value.encodedLength(); 50 | if (encodedLength > maxMessageLength) 51 | { 52 | throw new IllegalArgumentException("Unable to encode message of length " + encodedLength); 53 | } 54 | buffer.putBytes(offset, value.buffer(), value.offset(), encodedLength); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/store/ByteBufferOpsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | import java.nio.ByteBuffer; 23 | import java.nio.charset.StandardCharsets; 24 | import java.util.Arrays; 25 | 26 | import static com.google.common.truth.Truth.assertThat; 27 | 28 | class ByteBufferOpsTest 29 | { 30 | @Test 31 | void shouldCopyBytes() 32 | { 33 | final byte[] data = "0123456789ABCDE".getBytes(StandardCharsets.UTF_8); 34 | final ByteBuffer source = ByteBuffer.wrap(data); 35 | final ByteBuffer target = ByteBuffer.allocate(32); 36 | final ByteBuffer expected = ByteBuffer.allocate(32); 37 | expected.position(7); 38 | expected.put(data); 39 | 40 | final ByteBufferOps ops = new ByteBufferOps(); 41 | ops.copyBytes(source, target, 0, 7, source.capacity()); 42 | 43 | assertThat(Arrays.equals(target.array(), expected.array())).isTrue(); 44 | } 45 | 46 | @Test 47 | void shouldCopyWithinSameBuffer() 48 | { 49 | final byte[] data = "0123456789ABCDE".getBytes(StandardCharsets.UTF_8); 50 | final ByteBuffer source = ByteBuffer.allocate(64); 51 | source.put(data); 52 | 53 | final ByteBuffer expected = ByteBuffer.allocate(64); 54 | expected.put(data); 55 | expected.position(19); 56 | expected.put(data); 57 | 58 | final ByteBufferOps ops = new ByteBufferOps(); 59 | ops.copyBytes(source, source, 0, 19, data.length); 60 | 61 | assertThat(Arrays.equals(source.array(), expected.array())).isTrue(); 62 | } 63 | } -------------------------------------------------------------------------------- /recall-sbe/src/main/java/com/aitusoftware/recall/sbe/SbeMessageStoreFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.sbe; 19 | 20 | import com.aitusoftware.recall.persistence.IdAccessor; 21 | import com.aitusoftware.recall.store.BufferStore; 22 | import com.aitusoftware.recall.store.SingleTypeStore; 23 | import com.aitusoftware.recall.store.UnsafeBufferOps; 24 | import org.agrona.concurrent.UnsafeBuffer; 25 | import org.agrona.sbe.MessageDecoderFlyweight; 26 | 27 | import java.util.function.IntFunction; 28 | 29 | /** 30 | * Factory for providing a {@link SingleTypeStore} for an SBE-encoded message. 31 | */ 32 | public final class SbeMessageStoreFactory 33 | { 34 | private SbeMessageStoreFactory() 35 | { 36 | } 37 | 38 | /** 39 | * Creates a {@link SingleTypeStore} for the specified type. 40 | * 41 | * @param decoderFlyweight SBE decoder type 42 | * @param maxMessageLength max message length 43 | * @param maxRecords max number of records 44 | * @param bufferFactory factory for {@link UnsafeBuffer} instances 45 | * @param idAccessor function to retrieve the ID of the type 46 | * @param the type of the SBE message 47 | * @return the store 48 | */ 49 | public static SingleTypeStore forSbeMessage( 50 | final T decoderFlyweight, 51 | final int maxMessageLength, 52 | final int maxRecords, 53 | final IntFunction bufferFactory, 54 | final IdAccessor idAccessor) 55 | { 56 | final BufferStore store = new BufferStore<>( 57 | maxMessageLength, maxRecords, bufferFactory, new UnsafeBufferOps()); 58 | 59 | return new SingleTypeStore<>(store, new SbeMessageBufferDecoder<>(), 60 | new SbeMessageBufferEncoder<>(maxMessageLength), idAccessor); 61 | } 62 | } -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/persistence/AsciiCharSequence.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.persistence; 19 | 20 | /** 21 | * Mutable {@link CharSequence}. 22 | */ 23 | public final class AsciiCharSequence implements CharSequence 24 | { 25 | private final char[] content; 26 | private int length; 27 | 28 | /** 29 | * Construct a new instance. 30 | * 31 | * @param value the initial value 32 | */ 33 | public AsciiCharSequence(final String value) 34 | { 35 | content = new char[value.length() * 4]; 36 | System.arraycopy(value.toCharArray(), 0, content, 0, value.length()); 37 | length = value.length(); 38 | } 39 | 40 | /** 41 | * Construct a new instance. 42 | * 43 | * @param maxLength maximum length that will be encoded 44 | */ 45 | public AsciiCharSequence(final int maxLength) 46 | { 47 | this.content = new char[maxLength]; 48 | } 49 | 50 | /** 51 | * Reset the CharSequence. 52 | */ 53 | public void reset() 54 | { 55 | length = 0; 56 | } 57 | 58 | /** 59 | * Append a character. 60 | * 61 | * @param value value to append 62 | */ 63 | public void append(final char value) 64 | { 65 | content[length++] = value; 66 | } 67 | 68 | /** 69 | * {@inheritDoc} 70 | */ 71 | @Override 72 | public int length() 73 | { 74 | return length; 75 | } 76 | 77 | /** 78 | * {@inheritDoc} 79 | */ 80 | @Override 81 | public char charAt(final int i) 82 | { 83 | return content[i]; 84 | } 85 | 86 | /** 87 | * {@inheritDoc} 88 | */ 89 | @Override 90 | public CharSequence subSequence(final int i, final int i1) 91 | { 92 | throw new UnsupportedOperationException(); 93 | } 94 | 95 | /** 96 | * {@inheritDoc} 97 | */ 98 | @Override 99 | public String toString() 100 | { 101 | return new String(content, 0, length); 102 | } 103 | } -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/example/CharSequenceMapExample.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.example; 19 | 20 | import static com.google.common.truth.Truth.assertThat; 21 | 22 | import java.nio.ByteBuffer; 23 | import java.util.concurrent.atomic.AtomicInteger; 24 | 25 | import com.aitusoftware.recall.map.CharSequenceMap; 26 | import com.aitusoftware.recall.store.BufferStore; 27 | import com.aitusoftware.recall.store.ByteBufferOps; 28 | import com.aitusoftware.recall.store.SingleTypeStore; 29 | 30 | public class CharSequenceMapExample 31 | { 32 | private static final int MAX_RECORD_LENGTH = 128; 33 | private static final int MAX_KEY_LENGTH = 64; 34 | private static final int INITIAL_SIZE = 20; 35 | 36 | private final OrderByteBufferTranscoder transcoder = 37 | new OrderByteBufferTranscoder(); 38 | private final SingleTypeStore store = 39 | new SingleTypeStore<>( 40 | new BufferStore<>(MAX_RECORD_LENGTH, INITIAL_SIZE, 41 | ByteBuffer::allocateDirect, new ByteBufferOps()), 42 | transcoder, transcoder, Order::getId); 43 | private final CharSequenceMap orderBySymbol = 44 | new CharSequenceMap(MAX_KEY_LENGTH, INITIAL_SIZE, Long.MIN_VALUE); 45 | 46 | private void execute() 47 | { 48 | final String[] symbols = new String[INITIAL_SIZE]; 49 | for (int i = 0; i < INITIAL_SIZE; i++) 50 | { 51 | final Order order = Order.of(i); 52 | 53 | store.store(order); 54 | orderBySymbol.put(order.getSymbol(), order.getId()); 55 | symbols[i] = order.getSymbol().toString(); 56 | } 57 | 58 | final Order container = Order.of(-1L); 59 | final AtomicInteger matchCount = new AtomicInteger(); 60 | for (int i = 0; i < INITIAL_SIZE; i++) 61 | { 62 | final String searchTerm = symbols[i]; 63 | final long id = orderBySymbol.get(searchTerm); 64 | 65 | assertThat(store.load(id, container)).isTrue(); 66 | System.out.printf("Order with symbol %s has id %d%n", searchTerm, id); 67 | } 68 | } 69 | 70 | public static void main(final String[] args) 71 | { 72 | new CharSequenceMapExample().execute(); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/store/Store.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import com.aitusoftware.recall.persistence.Decoder; 21 | import com.aitusoftware.recall.persistence.Encoder; 22 | import com.aitusoftware.recall.persistence.IdAccessor; 23 | 24 | import java.nio.channels.FileChannel; 25 | 26 | /** 27 | * A storage medium for serialisable data. 28 | * 29 | * @param type of the backing buffer that data will be serialised to 30 | */ 31 | public interface Store 32 | { 33 | /** 34 | * Attempts to load the value belonging to the specified identifier. 35 | * 36 | * @param id the identifier of the value to retrieve 37 | * @param decoder the {@link Decoder} to use to deserialise the data 38 | * @param container the instance to deserialise data into 39 | * @param the type of the object being deserialised 40 | * @return indicates whether the identifier was found in the store 41 | */ 42 | boolean load(long id, Decoder decoder, T container); 43 | 44 | /** 45 | * Attempts to store the a value. 46 | * 47 | * @param encoder the {@link Encoder} to use to serialise the data 48 | * @param value the data to serialise 49 | * @param idAccessor the function to retrieve the identifier of the value 50 | * @param the type of the data 51 | */ 52 | void store(Encoder encoder, T value, IdAccessor idAccessor); 53 | 54 | /** 55 | * Attempts to remove the value belonging to the specified identifier. 56 | * 57 | * @param id the identifier of the value to remove 58 | * @return indicates whether the value was removed 59 | */ 60 | boolean remove(long id); 61 | 62 | /** 63 | * Perform a compaction operation on the underlying store. 64 | * This is implementation dependent. 65 | */ 66 | void compact(); 67 | 68 | /** 69 | * If storage is provided by an in-memory store, persist it to a more 70 | * reliable medium (e.g. sync to disk). 71 | */ 72 | void sync(); 73 | 74 | /** 75 | * Write contents of store to the supplied {@code FileChannel}. 76 | * 77 | * @param output the file to write to 78 | */ 79 | void writeTo(FileChannel output); 80 | 81 | /** 82 | * Return the current utilisation of the Store capacity. 83 | * 84 | * @return current utilisation 85 | */ 86 | float utilisation(); 87 | 88 | /** 89 | * Return the number of elements in the Store. 90 | * 91 | * @return number of elements 92 | */ 93 | int size(); 94 | 95 | /** 96 | * Clears all entries from the Store. 97 | */ 98 | void clear(); 99 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/store/Header.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import java.nio.ByteBuffer; 21 | import java.nio.ByteOrder; 22 | 23 | final class Header 24 | { 25 | static final int LENGTH = 4 * Integer.BYTES; 26 | private static final int VERSION_OFFSET = 0; 27 | private static final int STORE_LENGTH_OFFSET = Integer.BYTES; 28 | private static final int RECORD_LENGTH_OFFSET = 2 * Integer.BYTES; 29 | private static final int WRITE_OFFSET_OFFSET = 3 * Integer.BYTES; 30 | private static final ByteOrder STORAGE_ORDER = ByteOrder.LITTLE_ENDIAN; 31 | 32 | private Version version; 33 | private int storeLength; 34 | private int maxRecordLength; 35 | private int nextWriteOffset; 36 | 37 | void readFrom(final ByteBuffer headerBuffer) 38 | { 39 | version = Version.from(headerBuffer.order(STORAGE_ORDER).getInt(VERSION_OFFSET)); 40 | storeLength = headerBuffer.order(STORAGE_ORDER).getInt(STORE_LENGTH_OFFSET); 41 | maxRecordLength = headerBuffer.order(STORAGE_ORDER).getInt(RECORD_LENGTH_OFFSET); 42 | nextWriteOffset = headerBuffer.order(STORAGE_ORDER).getInt(WRITE_OFFSET_OFFSET); 43 | } 44 | 45 | void writeTo(final B input, final BufferOps bufferOps, final int offset) 46 | { 47 | final ByteOrder bufferOrder = bufferOps.byteOrder(); 48 | bufferOps.writeInt(input, offset + VERSION_OFFSET, littleEndian(version.getVersionNumber(), bufferOrder)); 49 | bufferOps.writeInt(input, offset + STORE_LENGTH_OFFSET, littleEndian(storeLength, bufferOrder)); 50 | bufferOps.writeInt(input, offset + RECORD_LENGTH_OFFSET, littleEndian(maxRecordLength, bufferOrder)); 51 | bufferOps.writeInt(input, offset + WRITE_OFFSET_OFFSET, littleEndian(nextWriteOffset, bufferOrder)); 52 | } 53 | 54 | Version version() 55 | { 56 | return version; 57 | } 58 | 59 | int storeLength() 60 | { 61 | return storeLength; 62 | } 63 | 64 | int maxRecordLength() 65 | { 66 | return maxRecordLength; 67 | } 68 | 69 | int nextWriteOffset() 70 | { 71 | return nextWriteOffset; 72 | } 73 | 74 | Header version(final Version version) 75 | { 76 | this.version = version; 77 | return this; 78 | } 79 | 80 | Header storeLength(final int storeLength) 81 | { 82 | this.storeLength = storeLength; 83 | return this; 84 | } 85 | 86 | Header maxRecordLength(final int maxRecordLength) 87 | { 88 | this.maxRecordLength = maxRecordLength; 89 | return this; 90 | } 91 | 92 | Header nextWriteOffset(final int nextWriteOffset) 93 | { 94 | this.nextWriteOffset = nextWriteOffset; 95 | return this; 96 | } 97 | 98 | private static int littleEndian(final int value, final ByteOrder byteOrder) 99 | { 100 | return byteOrder == ByteOrder.LITTLE_ENDIAN ? value : Integer.reverseBytes(value); 101 | } 102 | } -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/example/OrderByteBufferTranscoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.example; 19 | 20 | import com.aitusoftware.recall.persistence.AsciiCharSequence; 21 | import com.aitusoftware.recall.persistence.Decoder; 22 | import com.aitusoftware.recall.persistence.Encoder; 23 | 24 | import java.nio.ByteBuffer; 25 | 26 | public final class OrderByteBufferTranscoder implements Encoder, Decoder 27 | { 28 | @Override 29 | public void load(final ByteBuffer buffer, final int offset, final Order container) 30 | { 31 | container.setId(buffer.getLong(offset + ID_OFFSET)); 32 | container.setInstrumentId(buffer.getLong(offset + INSTRUMENT_ID_OFFSET)); 33 | container.setCreatedEpochSeconds(buffer.getLong(offset + CREATED_SECONDS_OFFSET)); 34 | container.setCreatedNanos(buffer.getInt(offset + CREATED_NANOS_OFFSET)); 35 | container.setExecutedAtEpochSeconds(buffer.getLong(offset + EXECUTED_SECONDS_OFFSET)); 36 | container.setExecutedAtNanos(buffer.getInt(offset + EXECUTED_NANOS_OFFSET)); 37 | final int charCount = buffer.getInt(offset + SYMBOL_LENGTH_OFFSET); 38 | final AsciiCharSequence symbolCharSequence = container.getSymbolCharSequence(); 39 | symbolCharSequence.reset(); 40 | for (int i = 0; i < charCount; i++) 41 | { 42 | symbolCharSequence.append((char)buffer.get(offset + SYMBOL_VALUE_OFFSET + i)); 43 | } 44 | } 45 | 46 | @Override 47 | public void store(final ByteBuffer buffer, final int offset, final Order value) 48 | { 49 | buffer.putLong(offset + ID_OFFSET, value.getId()); 50 | buffer.putLong(offset + INSTRUMENT_ID_OFFSET, value.getInstrumentId()); 51 | buffer.putLong(offset + CREATED_SECONDS_OFFSET, value.getCreatedEpochSeconds()); 52 | buffer.putInt(offset + CREATED_NANOS_OFFSET, value.getCreatedNanos()); 53 | buffer.putLong(offset + EXECUTED_SECONDS_OFFSET, value.getExecutedAtEpochSeconds()); 54 | buffer.putInt(offset + EXECUTED_NANOS_OFFSET, value.getExecutedAtNanos()); 55 | final CharSequence symbol = value.getSymbol(); 56 | buffer.putInt(offset + SYMBOL_LENGTH_OFFSET, symbol.length()); 57 | for (int i = 0; i < symbol.length(); i++) 58 | { 59 | buffer.put(offset + SYMBOL_VALUE_OFFSET + i, (byte)symbol.charAt(i)); 60 | } 61 | } 62 | 63 | private static final int ID_OFFSET = 0; 64 | private static final int INSTRUMENT_ID_OFFSET = ID_OFFSET + Long.BYTES; 65 | private static final int CREATED_SECONDS_OFFSET = INSTRUMENT_ID_OFFSET + Long.BYTES; 66 | private static final int CREATED_NANOS_OFFSET = CREATED_SECONDS_OFFSET + Long.BYTES; 67 | private static final int EXECUTED_SECONDS_OFFSET = CREATED_NANOS_OFFSET + Integer.BYTES; 68 | private static final int EXECUTED_NANOS_OFFSET = EXECUTED_SECONDS_OFFSET + Long.BYTES; 69 | private static final int SYMBOL_LENGTH_OFFSET = EXECUTED_NANOS_OFFSET + Integer.BYTES; 70 | private static final int SYMBOL_VALUE_OFFSET = SYMBOL_LENGTH_OFFSET + Integer.BYTES; 71 | } 72 | -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/example/OrderUnsafeBufferTranscoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.example; 19 | 20 | import com.aitusoftware.recall.persistence.AsciiCharSequence; 21 | import com.aitusoftware.recall.persistence.Decoder; 22 | import com.aitusoftware.recall.persistence.Encoder; 23 | import org.agrona.concurrent.UnsafeBuffer; 24 | 25 | public final class OrderUnsafeBufferTranscoder implements Encoder, Decoder 26 | { 27 | @Override 28 | public void load(final UnsafeBuffer buffer, final int offset, final Order container) 29 | { 30 | container.setId(buffer.getLong(offset + ID_OFFSET)); 31 | container.setInstrumentId(buffer.getLong(offset + INSTRUMENT_ID_OFFSET)); 32 | container.setCreatedEpochSeconds(buffer.getLong(offset + CREATED_SECONDS_OFFSET)); 33 | container.setCreatedNanos(buffer.getInt(offset + CREATED_NANOS_OFFSET)); 34 | container.setExecutedAtEpochSeconds(buffer.getLong(offset + EXECUTED_SECONDS_OFFSET)); 35 | container.setExecutedAtNanos(buffer.getInt(offset + EXECUTED_NANOS_OFFSET)); 36 | final int charCount = buffer.getInt(offset + SYMBOL_LENGTH_OFFSET); 37 | final AsciiCharSequence symbolCharSequence = container.getSymbolCharSequence(); 38 | symbolCharSequence.reset(); 39 | for (int i = 0; i < charCount; i++) 40 | { 41 | symbolCharSequence.append((char)buffer.getByte(offset + SYMBOL_VALUE_OFFSET + i)); 42 | } 43 | } 44 | 45 | @Override 46 | public void store(final UnsafeBuffer buffer, final int offset, final Order value) 47 | { 48 | buffer.putLong(offset + ID_OFFSET, value.getId()); 49 | buffer.putLong(offset + INSTRUMENT_ID_OFFSET, value.getInstrumentId()); 50 | buffer.putLong(offset + CREATED_SECONDS_OFFSET, value.getCreatedEpochSeconds()); 51 | buffer.putInt(offset + CREATED_NANOS_OFFSET, value.getCreatedNanos()); 52 | buffer.putLong(offset + EXECUTED_SECONDS_OFFSET, value.getExecutedAtEpochSeconds()); 53 | buffer.putInt(offset + EXECUTED_NANOS_OFFSET, value.getExecutedAtNanos()); 54 | final CharSequence symbol = value.getSymbol(); 55 | buffer.putInt(offset + SYMBOL_LENGTH_OFFSET, symbol.length()); 56 | for (int i = 0; i < symbol.length(); i++) 57 | { 58 | buffer.putByte(offset + SYMBOL_VALUE_OFFSET + i, (byte)symbol.charAt(i)); 59 | } 60 | } 61 | 62 | private static final int ID_OFFSET = 0; 63 | private static final int INSTRUMENT_ID_OFFSET = ID_OFFSET + Long.BYTES; 64 | private static final int CREATED_SECONDS_OFFSET = INSTRUMENT_ID_OFFSET + Long.BYTES; 65 | private static final int CREATED_NANOS_OFFSET = CREATED_SECONDS_OFFSET + Long.BYTES; 66 | private static final int EXECUTED_SECONDS_OFFSET = CREATED_NANOS_OFFSET + Integer.BYTES; 67 | private static final int EXECUTED_NANOS_OFFSET = EXECUTED_SECONDS_OFFSET + Long.BYTES; 68 | private static final int SYMBOL_LENGTH_OFFSET = EXECUTED_NANOS_OFFSET + Integer.BYTES; 69 | private static final int SYMBOL_VALUE_OFFSET = SYMBOL_LENGTH_OFFSET + Integer.BYTES; 70 | } 71 | -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/store/ByteBufferOps.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import java.io.IOException; 21 | import java.io.UncheckedIOException; 22 | import java.nio.ByteBuffer; 23 | import java.nio.ByteOrder; 24 | import java.nio.channels.FileChannel; 25 | 26 | /** 27 | * Utility class for performing operations on an {@link ByteBuffer}. 28 | */ 29 | public final class ByteBufferOps extends BufferOps 30 | { 31 | /** 32 | * {@inheritDoc} 33 | */ 34 | @Override 35 | ByteBuffer createFrom(final FileChannel fileChannel, final int offset, final int length) 36 | { 37 | final ByteBuffer buffer = ByteBuffer.allocateDirect(length); 38 | try 39 | { 40 | fileChannel.position(offset); 41 | while (buffer.remaining() != 0) 42 | { 43 | fileChannel.read(buffer); 44 | } 45 | } 46 | catch (final IOException e) 47 | { 48 | throw new UncheckedIOException(e); 49 | } 50 | buffer.flip(); 51 | return buffer; 52 | } 53 | 54 | /** 55 | * {@inheritDoc} 56 | */ 57 | @Override 58 | void storeTo(final FileChannel fileChannel, final ByteBuffer buffer, final int length) 59 | { 60 | int lengthRemaining = length; 61 | buffer.position(0).limit(length); 62 | while (lengthRemaining != 0) 63 | { 64 | try 65 | { 66 | lengthRemaining -= fileChannel.write(buffer); 67 | } 68 | catch (final IOException e) 69 | { 70 | throw new UncheckedIOException(e); 71 | } 72 | } 73 | } 74 | 75 | /** 76 | * {@inheritDoc} 77 | */ 78 | @Override 79 | void writeLong(final ByteBuffer buffer, final int offset, final long value) 80 | { 81 | buffer.putLong(offset, value); 82 | } 83 | 84 | /** 85 | * {@inheritDoc} 86 | */ 87 | @Override 88 | long readLong(final ByteBuffer buffer, final int offset) 89 | { 90 | return buffer.getLong(offset); 91 | } 92 | 93 | /** 94 | * {@inheritDoc} 95 | */ 96 | @Override 97 | void writeInt(final ByteBuffer buffer, final int offset, final int value) 98 | { 99 | buffer.putInt(offset, value); 100 | } 101 | 102 | /** 103 | * {@inheritDoc} 104 | */ 105 | @Override 106 | int readInt(final ByteBuffer buffer, final int offset) 107 | { 108 | return buffer.getInt(offset); 109 | } 110 | 111 | /** 112 | * {@inheritDoc} 113 | */ 114 | @Override 115 | void writeByte(final ByteBuffer buffer, final int offset, final byte value) 116 | { 117 | buffer.put(offset, value); 118 | } 119 | 120 | /** 121 | * {@inheritDoc} 122 | */ 123 | @Override 124 | byte readByte(final ByteBuffer buffer, final int offset) 125 | { 126 | return buffer.get(offset); 127 | } 128 | 129 | /** 130 | * {@inheritDoc} 131 | */ 132 | @Override 133 | ByteOrder byteOrder() 134 | { 135 | return ByteOrder.BIG_ENDIAN; 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/example/Order.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.example; 19 | 20 | import com.aitusoftware.recall.persistence.AsciiCharSequence; 21 | import com.aitusoftware.recall.persistence.IdAccessor; 22 | 23 | public final class Order implements IdAccessor 24 | { 25 | private long id; 26 | private long createdEpochSeconds; 27 | private int createdNanos; 28 | private long instrumentId; 29 | private long executedAtEpochSeconds; 30 | private int executedAtNanos; 31 | private AsciiCharSequence symbol; 32 | 33 | public Order( 34 | final long id, final long createdEpochSeconds, final int createdNanos, 35 | final long instrumentId, final long executedAtEpochSeconds, 36 | final int executedAtNanos, final String symbol) 37 | { 38 | this.id = id; 39 | this.createdEpochSeconds = createdEpochSeconds; 40 | this.createdNanos = createdNanos; 41 | this.instrumentId = instrumentId; 42 | this.executedAtEpochSeconds = executedAtEpochSeconds; 43 | this.executedAtNanos = executedAtNanos; 44 | this.symbol = new AsciiCharSequence(symbol); 45 | } 46 | 47 | public static Order of(final long id) 48 | { 49 | return new Order(id, System.currentTimeMillis() / 1000, 50 | (int)System.currentTimeMillis() % 1000, 37L, 51 | 0L, 0, "SYM_" + id); 52 | } 53 | 54 | public long getId() 55 | { 56 | return id; 57 | } 58 | 59 | public void setId(final long id) 60 | { 61 | this.id = id; 62 | } 63 | 64 | public long getCreatedEpochSeconds() 65 | { 66 | return createdEpochSeconds; 67 | } 68 | 69 | public void setCreatedEpochSeconds(final long createdEpochSeconds) 70 | { 71 | this.createdEpochSeconds = createdEpochSeconds; 72 | } 73 | 74 | public int getCreatedNanos() 75 | { 76 | return createdNanos; 77 | } 78 | 79 | public void setCreatedNanos(final int createdNanos) 80 | { 81 | this.createdNanos = createdNanos; 82 | } 83 | 84 | public long getInstrumentId() 85 | { 86 | return instrumentId; 87 | } 88 | 89 | public void setInstrumentId(final long instrumentId) 90 | { 91 | this.instrumentId = instrumentId; 92 | } 93 | 94 | public long getExecutedAtEpochSeconds() 95 | { 96 | return executedAtEpochSeconds; 97 | } 98 | 99 | public void setExecutedAtEpochSeconds(final long executedAtEpochSeconds) 100 | { 101 | this.executedAtEpochSeconds = executedAtEpochSeconds; 102 | } 103 | 104 | public int getExecutedAtNanos() 105 | { 106 | return executedAtNanos; 107 | } 108 | 109 | public void setExecutedAtNanos(final int executedAtNanos) 110 | { 111 | this.executedAtNanos = executedAtNanos; 112 | } 113 | 114 | public CharSequence getSymbol() 115 | { 116 | return symbol; 117 | } 118 | 119 | public AsciiCharSequence getSymbolCharSequence() 120 | { 121 | return symbol; 122 | } 123 | 124 | @Override 125 | public long getId(final Order value) 126 | { 127 | return value.getId(); 128 | } 129 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/store/SingleTypeStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import com.aitusoftware.recall.persistence.Decoder; 21 | import com.aitusoftware.recall.persistence.Encoder; 22 | import com.aitusoftware.recall.persistence.IdAccessor; 23 | 24 | import java.nio.channels.FileChannel; 25 | 26 | /** 27 | * A store that wraps a {@link Decoder}, {@link Encoder}, and {@link IdAccessor} to 28 | * provide storage for a single type. 29 | * 30 | * @param the type of the underlying buffer 31 | * @param the type that will be stored 32 | */ 33 | public final class SingleTypeStore 34 | { 35 | private final Store store; 36 | private final Decoder decoder; 37 | private final Encoder encoder; 38 | private final IdAccessor idAccessor; 39 | 40 | /** 41 | * Constructor for the store. 42 | * 43 | * @param store the underlying {@link Store} 44 | * @param decoder the decoder for deserialising 45 | * @param encoder the encode for serialising 46 | * @param idAccessor the accessor for retrieving the type's ID 47 | */ 48 | public SingleTypeStore( 49 | final Store store, final Decoder decoder, 50 | final Encoder encoder, final IdAccessor idAccessor) 51 | { 52 | this.store = store; 53 | this.decoder = decoder; 54 | this.encoder = encoder; 55 | this.idAccessor = idAccessor; 56 | } 57 | 58 | /** 59 | * Loads an entry into the specified container. 60 | * 61 | * @param id id to retrieve 62 | * @param container container to populate with data 63 | * @return indicates whether the ID was found 64 | */ 65 | public boolean load(final long id, final T container) 66 | { 67 | return store.load(id, decoder, container); 68 | } 69 | 70 | /** 71 | * Stores an entry. 72 | * 73 | * @param value the value to be stored 74 | */ 75 | public void store(final T value) 76 | { 77 | store.store(encoder, value, idAccessor); 78 | } 79 | 80 | /** 81 | * Removes an entry. 82 | * 83 | * @param id id to remove 84 | * @return indicates whether an entry was removed 85 | */ 86 | public boolean remove(final long id) 87 | { 88 | return store.remove(id); 89 | } 90 | 91 | /** 92 | * Delegates to the underlying {@link Store}. 93 | */ 94 | public void compact() 95 | { 96 | store.compact(); 97 | } 98 | 99 | /** 100 | * Delegates to the underlying {@link Store}. 101 | */ 102 | public void sync() 103 | { 104 | store.sync(); 105 | } 106 | 107 | /** 108 | * Delegates to the underlying {@link Store}. 109 | * 110 | * @param output the target file 111 | */ 112 | public void writeTo(final FileChannel output) 113 | { 114 | store.writeTo(output); 115 | } 116 | 117 | /** 118 | * Delegates to the underlying {@link Store}. 119 | * 120 | * @return store utilisation 121 | */ 122 | public float utilisation() 123 | { 124 | return store.utilisation(); 125 | } 126 | 127 | /** 128 | * Delegates to the underlying {@link Store}. 129 | */ 130 | public void clear() 131 | { 132 | store.clear(); 133 | } 134 | 135 | /** 136 | * Retrieve the underlying {@link Store}. 137 | * @return the underlying {@link Store} 138 | */ 139 | public Store store() 140 | { 141 | return store; 142 | } 143 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/store/BufferOps.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import java.nio.ByteOrder; 21 | import java.nio.channels.FileChannel; 22 | 23 | /** 24 | * Utility class for performing operations on a buffer. 25 | * 26 | * @param the type of the buffer 27 | */ 28 | public abstract class BufferOps 29 | { 30 | /** 31 | * Create a buffer populated with the contents of the supplied file. 32 | * 33 | * @param fileChannel input file 34 | * @param offset offset to start reading from 35 | * @param length length of data 36 | * @return the buffer 37 | */ 38 | abstract T createFrom(FileChannel fileChannel, int offset, int length); 39 | 40 | /** 41 | * Store a buffer to the supplied file. 42 | * 43 | * @param fileChannel output file 44 | * @param buffer data 45 | * @param length length of data 46 | */ 47 | abstract void storeTo(FileChannel fileChannel, T buffer, int length); 48 | 49 | /** 50 | * Write a long to the specified buffer. 51 | * 52 | * @param buffer the target buffer 53 | * @param offset the offset into the buffer 54 | * @param value the value to write 55 | */ 56 | abstract void writeLong(T buffer, int offset, long value); 57 | 58 | /** 59 | * Read a long from the specified buffer. 60 | * 61 | * @param buffer the source buffer 62 | * @param offset the offset into the buffer 63 | * @return the value at the specified offset 64 | */ 65 | abstract long readLong(T buffer, int offset); 66 | 67 | /** 68 | * Write an int to the specified buffer. 69 | * 70 | * @param buffer the target buffer 71 | * @param offset the offset into the buffer 72 | * @param value the value to write 73 | */ 74 | abstract void writeInt(T buffer, int offset, int value); 75 | 76 | /** 77 | * Read an int from the specified buffer. 78 | * 79 | * @param buffer the source buffer 80 | * @param offset the offset into the buffer 81 | * @return the value at the specified offset 82 | */ 83 | abstract int readInt(T buffer, int offset); 84 | 85 | /** 86 | * Write a byte to the specified buffer. 87 | * 88 | * @param buffer the target buffer 89 | * @param offset the offset into the buffer 90 | * @param value the value to write 91 | */ 92 | abstract void writeByte(T buffer, int offset, byte value); 93 | 94 | /** 95 | * Read a byte from the specified buffer. 96 | * 97 | * @param buffer the source buffer 98 | * @param offset the offset into the buffer 99 | * @return the value at the specified offset 100 | */ 101 | abstract byte readByte(T buffer, int offset); 102 | 103 | /** 104 | * Byte Order of the underlying buffer type. 105 | * 106 | * @return the byte order 107 | */ 108 | abstract ByteOrder byteOrder(); 109 | 110 | /** 111 | * Copy bytes between buffers. 112 | * 113 | * @param source source buffer 114 | * @param target target buffer 115 | * @param sourceOffset offset in source buffer 116 | * @param targetOffset offset in target buffer 117 | * @param length length of data to be copied 118 | */ 119 | protected void copyBytes( 120 | final T source, final T target, final int sourceOffset, final int targetOffset, final int length) 121 | { 122 | final int eightByteSegments = length >> 3; 123 | final int singleByteOffset = eightByteSegments << 3; 124 | final int trailingBytes = length & 7; 125 | for (int j = 0; j < eightByteSegments; j++) 126 | { 127 | final int subOffset = j << 3; 128 | writeLong(target, targetOffset + subOffset, 129 | readLong(source, sourceOffset + subOffset)); 130 | } 131 | 132 | for (int j = 0; j < trailingBytes; j++) 133 | { 134 | writeByte(target, targetOffset + singleByteOffset + j, 135 | readByte(source, sourceOffset + singleByteOffset + j)); 136 | } 137 | } 138 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/store/UnsafeBufferOps.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import org.agrona.concurrent.UnsafeBuffer; 21 | 22 | import java.io.IOException; 23 | import java.io.UncheckedIOException; 24 | import java.nio.ByteBuffer; 25 | import java.nio.ByteOrder; 26 | import java.nio.channels.FileChannel; 27 | 28 | /** 29 | * Utility class for performing operations on an {@link UnsafeBuffer}. 30 | */ 31 | public final class UnsafeBufferOps extends BufferOps 32 | { 33 | /** 34 | * {@inheritDoc} 35 | */ 36 | @Override 37 | UnsafeBuffer createFrom(final FileChannel fileChannel, final int offset, final int length) 38 | { 39 | final ByteBuffer content = ByteBuffer.allocateDirect(length); 40 | try 41 | { 42 | fileChannel.position(0); 43 | while (content.remaining() != 0) 44 | { 45 | fileChannel.read(content); 46 | } 47 | } 48 | catch (final IOException e) 49 | { 50 | throw new UncheckedIOException(e); 51 | } 52 | content.clear(); 53 | return new UnsafeBuffer(content); 54 | } 55 | 56 | /** 57 | * {@inheritDoc} 58 | */ 59 | @Override 60 | void storeTo(final FileChannel fileChannel, final UnsafeBuffer buffer, final int length) 61 | { 62 | final ByteBuffer tmp = ByteBuffer.allocateDirect(4096); 63 | int lengthRemaining = length; 64 | int offset = 0; 65 | try 66 | { 67 | fileChannel.position(0); 68 | while (lengthRemaining != 0) 69 | { 70 | tmp.clear(); 71 | final int copyLength = Math.min(tmp.capacity(), lengthRemaining); 72 | buffer.getBytes(offset, tmp, 0, copyLength); 73 | tmp.position(0).limit(copyLength); 74 | while (tmp.remaining() != 0) 75 | { 76 | fileChannel.write(tmp); 77 | } 78 | lengthRemaining -= copyLength; 79 | offset += copyLength; 80 | } 81 | } 82 | catch (final IOException e) 83 | { 84 | throw new UncheckedIOException(e); 85 | } 86 | 87 | } 88 | 89 | /** 90 | * {@inheritDoc} 91 | */ 92 | @Override 93 | void writeLong(final UnsafeBuffer buffer, final int offset, final long value) 94 | { 95 | buffer.putLong(offset, value); 96 | } 97 | 98 | /** 99 | * {@inheritDoc} 100 | */ 101 | @Override 102 | long readLong(final UnsafeBuffer buffer, final int offset) 103 | { 104 | return buffer.getLong(offset); 105 | } 106 | 107 | /** 108 | * {@inheritDoc} 109 | */ 110 | @Override 111 | void writeInt(final UnsafeBuffer buffer, final int offset, final int value) 112 | { 113 | buffer.putInt(offset, value); 114 | } 115 | 116 | /** 117 | * {@inheritDoc} 118 | */ 119 | @Override 120 | int readInt(final UnsafeBuffer buffer, final int offset) 121 | { 122 | return buffer.getInt(offset); 123 | } 124 | 125 | /** 126 | * {@inheritDoc} 127 | */ 128 | @Override 129 | void writeByte(final UnsafeBuffer buffer, final int offset, final byte value) 130 | { 131 | buffer.putByte(offset, value); 132 | } 133 | 134 | /** 135 | * {@inheritDoc} 136 | */ 137 | @Override 138 | byte readByte(final UnsafeBuffer buffer, final int offset) 139 | { 140 | return buffer.getByte(offset); 141 | } 142 | 143 | /** 144 | * {@inheritDoc} 145 | */ 146 | @Override 147 | protected void copyBytes( 148 | final UnsafeBuffer source, final UnsafeBuffer target, final int sourceOffset, 149 | final int targetOffset, final int length) 150 | { 151 | target.putBytes(targetOffset, source, sourceOffset, length); 152 | } 153 | 154 | /** 155 | * {@inheritDoc} 156 | */ 157 | @Override 158 | ByteOrder byteOrder() 159 | { 160 | return ByteOrder.LITTLE_ENDIAN; 161 | } 162 | } -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /recall-sbe/src/test/resources/sbe-schema.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 23 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | T 59 | S 60 | N 61 | K 62 | 63 | 64 | 65 | 66 | 67 | 68 | 9000 69 | 70 | Petrol 71 | 72 | 73 | 74 | 75 | 76 | 0 77 | 1 78 | 79 | 80 | A 81 | B 82 | C 83 | 84 | 85 | 0 86 | 1 87 | 2 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /config/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 155 | 157 | 158 | 159 | 160 | 161 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | -------------------------------------------------------------------------------- /recall-sbe/src/test/java/com/aitusoftware/recall/sbe/SbeObjectStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.sbe; 19 | 20 | 21 | import com.aitusoftware.recall.persistence.IdAccessor; 22 | import com.aitusoftware.recall.sbe.example.*; 23 | import com.aitusoftware.recall.store.BufferStore; 24 | import com.aitusoftware.recall.store.SingleTypeStore; 25 | import com.aitusoftware.recall.store.UnsafeBufferOps; 26 | import org.agrona.ExpandableArrayBuffer; 27 | import org.agrona.concurrent.UnsafeBuffer; 28 | import org.junit.jupiter.api.Test; 29 | 30 | import java.util.Random; 31 | import java.util.concurrent.ThreadLocalRandom; 32 | import java.util.function.IntFunction; 33 | 34 | import static com.google.common.truth.Truth.assertThat; 35 | 36 | class SbeObjectStoreTest 37 | { 38 | private static final MessageHeaderEncoder HEADER_ENCODER = new MessageHeaderEncoder(); 39 | private static final long ID = 37L; 40 | private static final String ACTIVATION_CODE = "ACTIVATION_CODE"; 41 | private static final BooleanType TRUE = BooleanType.T; 42 | private static final Model CODE = Model.A; 43 | private static final String MANUFACTURER = "Mitsubishi"; 44 | private static final int MODEL_YEAR = 1979; 45 | private static final String MODEL = "Mirage"; 46 | private static final int MAX_RECORD_LENGTH = 256; 47 | private static final int MAX_RECORDS = 2000; 48 | private final ExpandableArrayBuffer buffer = new ExpandableArrayBuffer(); 49 | private final BufferStore bufferStore = new BufferStore<>(MAX_RECORD_LENGTH, MAX_RECORDS, 50 | bufferFactory(), new UnsafeBufferOps()); 51 | private final SbeMessageBufferEncoder recallEncoder = new SbeMessageBufferEncoder<>(MAX_RECORD_LENGTH); 52 | private final CarIdAccessor idAccessor = new CarIdAccessor(); 53 | private final SbeMessageBufferDecoder recallDecoder = new SbeMessageBufferDecoder<>(); 54 | 55 | @Test 56 | void shouldConstructStoreForType() 57 | { 58 | final SingleTypeStore store = 59 | SbeMessageStoreFactory.forSbeMessage(new CarDecoder(), MAX_RECORD_LENGTH, MAX_RECORDS, 60 | bufferFactory(), new CarIdAccessor()); 61 | 62 | final CarEncoder encoder = new CarEncoder().wrapAndApplyHeader(buffer, 0, HEADER_ENCODER); 63 | encoder.id(ID).available(TRUE).code(CODE) 64 | .modelYear(MODEL_YEAR) 65 | .manufacturer(MANUFACTURER) 66 | .model(MODEL) 67 | .activationCode(ACTIVATION_CODE) 68 | .engine().boosterEnabled(TRUE); 69 | 70 | final CarDecoder decoder = new CarDecoder().wrap(buffer, MessageHeaderEncoder.ENCODED_LENGTH, 71 | encoder.encodedLength(), encoder.sbeSchemaVersion()); 72 | assertThat(decoder.id()).isEqualTo(ID); 73 | 74 | store.store(decoder); 75 | 76 | final CarDecoder loaded = new CarDecoder(); 77 | assertThat(store.load(ID, loaded)).isTrue(); 78 | } 79 | 80 | @Test 81 | void shouldStoreAndRetrieve() 82 | { 83 | final CarEncoder encoder = new CarEncoder().wrapAndApplyHeader(buffer, 0, HEADER_ENCODER); 84 | encoder.id(ID).available(TRUE).code(CODE) 85 | .modelYear(MODEL_YEAR) 86 | .manufacturer(MANUFACTURER) 87 | .model(MODEL) 88 | .activationCode(ACTIVATION_CODE) 89 | .engine().boosterEnabled(TRUE); 90 | 91 | final CarDecoder decoder = new CarDecoder().wrap(buffer, MessageHeaderEncoder.ENCODED_LENGTH, 92 | encoder.encodedLength(), encoder.sbeSchemaVersion()); 93 | assertThat(decoder.id()).isEqualTo(ID); 94 | 95 | bufferStore.store(recallEncoder, decoder, idAccessor); 96 | 97 | final CarDecoder loaded = new CarDecoder(); 98 | assertThat(bufferStore.load(ID, recallDecoder, loaded)).isTrue(); 99 | 100 | assertThat(loaded.id()).isEqualTo(ID); 101 | assertThat(loaded.available()).isEqualTo(TRUE); 102 | assertThat(loaded.code()).isEqualTo(CODE); 103 | assertThat(loaded.modelYear()).isEqualTo(MODEL_YEAR); 104 | assertThat(loaded.engine().boosterEnabled()).isEqualTo(TRUE); 105 | assertThat(loaded.manufacturer()).isEqualTo(MANUFACTURER); 106 | assertThat(loaded.model()).isEqualTo(MODEL); 107 | assertThat(loaded.activationCode()).isEqualTo(ACTIVATION_CODE); 108 | } 109 | 110 | @Test 111 | void shouldStoreAndRetrieveMultipleRecords() 112 | { 113 | final Random random = ThreadLocalRandom.current(); 114 | final long[] ids = new long[MAX_RECORDS]; 115 | for (int i = 0; i < MAX_RECORDS; i++) 116 | { 117 | final long id = random.nextLong(); 118 | ids[i] = id; 119 | final CarEncoder encoder = new CarEncoder().wrapAndApplyHeader(buffer, 0, HEADER_ENCODER); 120 | encoder.id(id).available(TRUE).code(CODE) 121 | .modelYear((int)(id * 17)) 122 | .manufacturer(MANUFACTURER) 123 | .model(MODEL) 124 | .activationCode(ACTIVATION_CODE) 125 | .engine().boosterEnabled(TRUE); 126 | 127 | final CarDecoder decoder = new CarDecoder().wrap(buffer, MessageHeaderEncoder.ENCODED_LENGTH, 128 | encoder.encodedLength(), encoder.sbeSchemaVersion()); 129 | bufferStore.store(recallEncoder, decoder, idAccessor); 130 | } 131 | 132 | for (int i = 0; i < MAX_RECORDS; i++) 133 | { 134 | final CarDecoder loaded = new CarDecoder(); 135 | final long id = ids[i]; 136 | assertThat(bufferStore.load(id, recallDecoder, loaded)).isTrue(); 137 | assertThat(id).isEqualTo(loaded.id()); 138 | } 139 | } 140 | 141 | private static final class CarIdAccessor implements IdAccessor 142 | { 143 | @Override 144 | public long getId(final CarDecoder value) 145 | { 146 | return value.id(); 147 | } 148 | } 149 | 150 | private static IntFunction bufferFactory() 151 | { 152 | return len -> new UnsafeBuffer(new byte[len]); 153 | } 154 | } -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/map/ByteSequenceMapTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.map; 19 | 20 | import org.junit.jupiter.api.Assertions; 21 | import org.junit.jupiter.api.Test; 22 | 23 | import java.nio.ByteBuffer; 24 | import java.nio.charset.StandardCharsets; 25 | import java.util.*; 26 | 27 | import static com.google.common.truth.Truth.assertThat; 28 | 29 | class ByteSequenceMapTest 30 | { 31 | private static final ByteBuffer SEARCH_TERM = toBuffer("searchTerm"); 32 | private static final long ID = 17L; 33 | private static final int INITIAL_SIZE = 16; 34 | private static final long MISSING_VALUE = Long.MIN_VALUE; 35 | private static final int MAX_KEY_LENGTH = 64; 36 | private final ByteSequenceMap map = new ByteSequenceMap(MAX_KEY_LENGTH, INITIAL_SIZE, MISSING_VALUE); 37 | private final List receivedList = new ArrayList<>(); 38 | 39 | @Test 40 | void shouldRejectKeyThatIsTooLong() 41 | { 42 | final byte[] key = new byte[MAX_KEY_LENGTH + 1]; 43 | for (int i = 0; i <= MAX_KEY_LENGTH; i++) 44 | { 45 | key[i] = (byte)'x'; 46 | } 47 | Assertions.assertThrows(IllegalArgumentException.class, () -> map.put(ByteBuffer.wrap(key), 7L)); 48 | } 49 | 50 | @Test 51 | void shouldStoreSingleValue() 52 | { 53 | map.put(SEARCH_TERM, ID); 54 | 55 | assertSearchResult(map, SEARCH_TERM, ID); 56 | } 57 | 58 | @Test 59 | void shouldNotRetrieveUnknownValue() 60 | { 61 | map.get(SEARCH_TERM); 62 | 63 | assertThat(receivedList).isEmpty(); 64 | } 65 | 66 | @Test 67 | void shouldRetrieveMultipleValuesWhenHashesCollide() 68 | { 69 | final ByteSequenceMap poorIndex = 70 | new ByteSequenceMap(16, 10, cs -> 7, Long.MIN_VALUE, ByteBuffer::allocate); 71 | 72 | final ByteBuffer otherTerm = toBuffer("otherTerm"); 73 | final long otherId = 99L; 74 | 75 | poorIndex.put(SEARCH_TERM, ID); 76 | poorIndex.put(otherTerm, otherId); 77 | 78 | assertSearchResult(poorIndex, SEARCH_TERM, ID); 79 | 80 | receivedList.clear(); 81 | 82 | assertSearchResult(poorIndex, otherTerm, otherId); 83 | } 84 | 85 | @Test 86 | void shouldHaveCorrectIndexAfterResize() 87 | { 88 | final int doubleInitialSize = INITIAL_SIZE * 2; 89 | for (int i = 0; i < doubleInitialSize; i++) 90 | { 91 | map.put(toBuffer("searchTerm_" + i), i); 92 | } 93 | 94 | for (int i = 0; i < doubleInitialSize; i++) 95 | { 96 | receivedList.clear(); 97 | assertSearchResult(map, toBuffer("searchTerm_" + i), i); 98 | } 99 | } 100 | 101 | @Test 102 | void shouldReplaceExistingValue() 103 | { 104 | final long otherId = 42L; 105 | map.put(SEARCH_TERM, ID); 106 | map.put(SEARCH_TERM, otherId); 107 | 108 | assertSearchResult(map, SEARCH_TERM, otherId); 109 | } 110 | 111 | @Test 112 | void shouldWrapBuffer() 113 | { 114 | final int initialSize = 20; 115 | final ByteSequenceMap map = new ByteSequenceMap(64, initialSize, Long.MIN_VALUE); 116 | final String prefix = "SYM_"; 117 | for (int i = 0; i < initialSize; i++) 118 | { 119 | map.put(ByteBuffer.wrap((prefix + i).getBytes(StandardCharsets.UTF_8)), i); 120 | } 121 | 122 | for (int i = 0; i < initialSize; i++) 123 | { 124 | assertThat(map.get(ByteBuffer.wrap((prefix + i).getBytes(StandardCharsets.UTF_8)))).isEqualTo(i); 125 | } 126 | } 127 | 128 | @Test 129 | void shouldRemoveValue() 130 | { 131 | map.put(SEARCH_TERM, ID); 132 | 133 | assertThat(map.remove(SEARCH_TERM)).isEqualTo(ID); 134 | assertThat(map.remove(SEARCH_TERM)).isEqualTo(MISSING_VALUE); 135 | assertThat(map.get(SEARCH_TERM)).isEqualTo(MISSING_VALUE); 136 | assertThat(map.size()).isEqualTo(0); 137 | } 138 | 139 | @Test 140 | void comparisonTest() 141 | { 142 | final Map control = new HashMap<>(); 143 | final long seed = System.nanoTime(); 144 | final Random random = new Random(seed); 145 | 146 | for (int i = 0; i < 10_000; i++) 147 | { 148 | final String key = new UUID(random.nextLong(), random.nextLong()).toString(); 149 | final long value = random.nextLong(); 150 | control.put(key, value); 151 | map.put(toBuffer(key), value); 152 | 153 | assertThat(map.get(toBuffer(key))).isEqualTo(control.get(key)); 154 | } 155 | 156 | assertThat(map.size()).isEqualTo(10_000); 157 | 158 | Set controlKeys = new HashSet<>(control.keySet()); 159 | int counter = 0; 160 | for (final String controlKey : controlKeys) 161 | { 162 | if ((counter++ & 7) == 0) 163 | { 164 | assertThat(map.remove(toBuffer(controlKey))) 165 | .isEqualTo(control.remove(controlKey)); 166 | } 167 | } 168 | 169 | assertThat(map.size()).isEqualTo(control.size()); 170 | 171 | controlKeys = new HashSet<>(control.keySet()); 172 | for (final String controlKey : controlKeys) 173 | { 174 | assertThat(map.get(toBuffer(controlKey))).isEqualTo(control.get(controlKey)); 175 | } 176 | 177 | for (int i = 0; i < 10_000; i++) 178 | { 179 | final String key = UUID.randomUUID().toString(); 180 | final long value = random.nextLong(); 181 | control.put(key, value); 182 | map.put(toBuffer(key), value); 183 | } 184 | 185 | controlKeys = new HashSet<>(control.keySet()); 186 | for (final String controlKey : controlKeys) 187 | { 188 | assertThat(map.get(toBuffer(controlKey))).isEqualTo(control.get(controlKey)); 189 | } 190 | 191 | map.rehash(); 192 | 193 | for (final String controlKey : controlKeys) 194 | { 195 | assertThat(map.get(toBuffer(controlKey))).isEqualTo(control.get(controlKey)); 196 | } 197 | } 198 | 199 | @Test 200 | void sizeShouldNotIncreaseWithRepeatedInserts() 201 | { 202 | for (int i = 0; i < 100; i++) 203 | { 204 | map.put(toBuffer("key_" + (i % 10)), i); 205 | assertThat(map.size()).isAtMost(10); 206 | } 207 | } 208 | 209 | 210 | private void assertSearchResult(final ByteSequenceMap index, final ByteBuffer searchTerm, final long retrievedId) 211 | { 212 | assertThat(index.get(searchTerm)).isEqualTo(retrievedId); 213 | } 214 | 215 | private static ByteBuffer toBuffer(final String term) 216 | { 217 | return ByteBuffer.wrap(term.getBytes(StandardCharsets.UTF_8)); 218 | } 219 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://circleci.com/gh/aitusoftware/recall.svg?style=svg)](https://circleci.com/gh/aitusoftware/recall) 2 | 3 | [![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/aitusoftware/recall.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/aitusoftware/recall/context:java) 4 | 5 | ![Recall Icon](https://github.com/aitusoftware/recall/raw/master/resources/img/RecallIcon.png) 6 | 7 | Recall is an off-heap, allocation-free object store for the JVM. 8 | 9 | ## Usage 10 | 11 | Recall is designed for use in allocation-free or low-garbage systems. Objects are expected to be 12 | mutable in order to reduce allocation costs. For this reason, domain objects should have 13 | mutator methods for any fields that need to be serialised. 14 | 15 | ### Dependency 16 | 17 | #### Gradle 18 | 19 | ``` 20 | dependencies { 21 | compile group: 'com.aitusoftware', name: 'recall-store', version: '0.2.0' 22 | } 23 | ``` 24 | 25 | #### Maven 26 | 27 | ```xml 28 | 29 | com.aitusoftware 30 | recall-store 31 | 0.2.0 32 | 33 | ``` 34 | 35 | Recall can use either a standard JDK `ByteBuffer` or an 36 | [Agrona](https://github.com/real-logic/Agrona) `UnsafeBuffer` for storage of 37 | objects outside of the Java heap. 38 | 39 | To use the Recall object store, implement the `Encoder`, `Decoder`, and `IdAccessor` interface for 40 | a given object and buffer type: 41 | 42 | ```java 43 | public class Order { 44 | private long id; 45 | private double quantity; 46 | private double price; 47 | 48 | // constructor omitted 49 | // getters and setters omitted 50 | } 51 | ``` 52 | 53 | ### Implement `Encoder` 54 | 55 | ```java 56 | public class OrderEncoder implements Encoder { 57 | public void store(ByteBuffer buffer, int offset, Order order) { 58 | buffer.putLong(offset, order.getId()); 59 | buffer.putDouble(offset + Long.BYTES, order.getQuantity()); 60 | buffer.putDouble(offset + Long.BYTES + Double.BYTES, order.getPrice()); 61 | } 62 | } 63 | ``` 64 | 65 | ### Implement `Decoder` 66 | 67 | ```java 68 | public class OrderDecoder implements Decoder { 69 | public void load(ByteBuffer buffer, int offset, Order target) { 70 | target.setId(buffer.getLong(offset)); 71 | target.setQuantity(buffer.getDouble(offset + Long.BYTES)); 72 | target.setPrice(buffer.getDouble(offset + Long.BYTES + Double.BYTES)); 73 | } 74 | } 75 | ``` 76 | 77 | ### Implement `IdAccessor` 78 | 79 | ```java 80 | public class OrderIdAccessor implements IdAccessor { 81 | public long getId(Order order) { 82 | return order.getId(); 83 | } 84 | } 85 | ``` 86 | 87 | ### Store 88 | 89 | Create a `Store`: 90 | 91 | ```java 92 | BufferStore store = 93 | new BufferStore<>(24, 100, ByteBuffer::allocateDirect, new ByteBufferOps()); 94 | ``` 95 | 96 | Optionally wrap it in a `SingleTypeStore` (if only one type is going to be stored): 97 | 98 | ```java 99 | SingleTypeStore typeStore = 100 | new SingleTypeStore<>(store, new OrderDecoder(), new OrderEncoder(), 101 | new OrderIdAccessor()); 102 | ``` 103 | 104 | ### Storage and Retrieval 105 | 106 | Domain objects can be serialised to off-heap storage, and retrieved at a later time: 107 | 108 | ```java 109 | long orderId = 42L; 110 | Order testOrder = new Order(orderId, 12.34D, 56.78D); 111 | typeStore.store(testOrder); 112 | 113 | Order container = new Order(-1, -1, -1); 114 | assert typeStore.load(orderId, container); 115 | assert container.getQuantity() == 12.34D; 116 | ``` 117 | 118 | ## SBE integration 119 | 120 | Recall is able to provide efficient off-heap storage of SBE-encoded messages. 121 | 122 | This example uses the canonical `Car` example from 123 | [SBE](https://github.com/real-logic/simple-binary-encoding/blob/master/sbe-tool/src/test/resources/example-schema.xml). 124 | 125 | ### SBE codecs 126 | 127 | SBE objects must be generated with: 128 | 129 | `-Dsbe.java.generate.interfaces=true` 130 | 131 | this causes the `Decoder` to implement `MessageDecoderFlyweight`. 132 | 133 | ### Implement `IdAccessor` 134 | 135 | It is necessary to implement the `IdAccessor` interface for the SBE `Decoder` type: 136 | 137 | ```java 138 | public class CarIdAccessor implements IdAccessor { 139 | public long getId(CarDecoder decoder) { 140 | return decoder.id(); 141 | } 142 | } 143 | ``` 144 | 145 | ### SBE Message Store 146 | 147 | Create a `SingleTypeStore` for the type of the `Decoder`: 148 | 149 | ```java 150 | SingleTypeStore messageStore = 151 | SbeMessageStoreFactory.forSbeMessage(new CarDecoder(), 152 | MAX_RECORD_LENGTH, 100, 153 | len -> new UnsafeBuffer(ByteBuffer.allocateDirect(len)), 154 | new CarIdAccessor()); 155 | ``` 156 | 157 | *Note*: it is up to the application developer to determine the maximum length 158 | of any given SBE message (even in the case of variable-length fields). 159 | 160 | If an encoded value exceeds the specified maximum record length, then the 161 | `store` method will throw an `IllegalArgumentException`. 162 | 163 | SBE messages can now be stored for later retrieval: 164 | 165 | #### Storage 166 | 167 | ```java 168 | public void receiveCar(ReadableByteChannel channel) { 169 | CarDecoder decoder = new CarDecoder(); 170 | UnsafeBuffer buffer = new UnsafeBuffer(); 171 | ByteBuffer inputData = ByteBuffer.allocateDirect(MAX_RECORD_LENGTH); 172 | channel.read(inputData); 173 | inputData.flip(); 174 | buffer.wrap(inputData); 175 | decoder.wrap(buffer, 0, BLOCK_LENGTH, VERSION); 176 | 177 | dispatchCarReceivedEvent(decoder); 178 | messageStore.store(decoder); 179 | } 180 | ``` 181 | 182 | #### Retrieval 183 | 184 | ```java 185 | public void notifyCarSold(long carId) { 186 | CarDecoder decoder = new CarDecoder(); 187 | messageStore.load(carId, decoder); 188 | 189 | dispatchCarSoldEvent(decoder); 190 | } 191 | ``` 192 | 193 | ## Non-integer keys 194 | 195 | Since it is sometimes useful to be able to store and retrieve objects by something other 196 | than an integer key, Recall also provides the ability to create mappings based on 197 | variable-length keys based on either strings, or byte-sequences. 198 | 199 | ### `CharSequenceMap` 200 | 201 | `CharSequenceMap` is an open-addressed hash map with that can be used to store a `CharSequence` 202 | against an integer identifier. 203 | 204 | Example usage: 205 | 206 | ```java 207 | private final OrderByteBufferTranscoder transcoder = 208 | new OrderByteBufferTranscoder(); 209 | private final SingleTypeStore store = 210 | new SingleTypeStore<>( 211 | new BufferStore<>(MAX_RECORD_LENGTH, INITIAL_SIZE, 212 | ByteBuffer::allocateDirect, new ByteBufferOps()), 213 | transcoder, transcoder, Order::getId); 214 | private final CharSequenceMap orderBySymbol = 215 | new CharSequenceMap(MAX_KEY_LENGTH, INITIAL_SIZE, Long.MIN_VALUE); 216 | 217 | private void execute() 218 | { 219 | final String[] symbols = new String[INITIAL_SIZE]; 220 | for (int i = 0; i < INITIAL_SIZE; i++) 221 | { 222 | final Order order = Order.of(i); 223 | 224 | store.store(order); 225 | orderBySymbol.insert(order.getSymbol(), order.getId()); 226 | symbols[i] = order.getSymbol().toString(); 227 | } 228 | 229 | final Order container = Order.of(-1L); 230 | for (int i = 0; i < INITIAL_SIZE; i++) 231 | { 232 | final String searchTerm = symbols[i]; 233 | final long id = orderBySymbol.search(searchTerm); 234 | 235 | assertThat(store.load(id, container)).isTrue(); 236 | System.out.printf("Order with symbol %s has id %d%n", searchTerm, id); 237 | } 238 | } 239 | ``` 240 | 241 | ### `ByteSequenceMap` 242 | 243 | `ByteSequenceMap` is an open-addressed hash map with that can be used to store a `ByteBuffer` 244 | against an integer identifier. 245 | -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/map/CharSequenceMapTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.map; 19 | 20 | import org.junit.jupiter.api.Assertions; 21 | import org.junit.jupiter.api.Disabled; 22 | import org.junit.jupiter.api.Test; 23 | 24 | import java.nio.ByteBuffer; 25 | import java.util.*; 26 | 27 | import static com.google.common.truth.Truth.assertThat; 28 | 29 | class CharSequenceMapTest 30 | { 31 | private static final String SEARCH_TERM = "searchTerm"; 32 | private static final long ID = 17L; 33 | private static final int INITIAL_SIZE = 16; 34 | private static final long MISSING_VALUE = Long.MIN_VALUE; 35 | private static final int MAX_KEY_LENGTH = 64; 36 | 37 | private final CharSequenceMap map = new CharSequenceMap( 38 | MAX_KEY_LENGTH, INITIAL_SIZE, MISSING_VALUE); 39 | 40 | @Test 41 | void shouldRejectKeyThatIsTooLong() 42 | { 43 | final StringBuilder key = new StringBuilder(); 44 | for (int i = 0; i <= MAX_KEY_LENGTH; i++) 45 | { 46 | key.append('x'); 47 | } 48 | Assertions.assertThrows(IllegalArgumentException.class, () -> map.put(key, 7L)); 49 | } 50 | 51 | @Test 52 | void shouldStoreSingleValue() 53 | { 54 | map.put(SEARCH_TERM, ID); 55 | 56 | assertSearchResult(map, SEARCH_TERM, ID); 57 | } 58 | 59 | @Test 60 | void shouldNotRetrieveUnknownValue() 61 | { 62 | assertThat(map.get(SEARCH_TERM)).isEqualTo(MISSING_VALUE); 63 | } 64 | 65 | @Test 66 | void shouldRetrieveMultipleValuesWhenHashesCollide() 67 | { 68 | final CharSequenceMap poorIndex = 69 | new CharSequenceMap(16, 10, cs -> 7, Long.MIN_VALUE, ByteBuffer::allocateDirect); 70 | 71 | final String otherTerm = "otherTerm"; 72 | final long otherId = 99L; 73 | 74 | poorIndex.put(SEARCH_TERM, ID); 75 | poorIndex.put(otherTerm, otherId); 76 | 77 | assertSearchResult(poorIndex, SEARCH_TERM, ID); 78 | assertSearchResult(poorIndex, otherTerm, otherId); 79 | } 80 | 81 | @Test 82 | void shouldHaveCorrectIndexAfterResize() 83 | { 84 | final int doubleInitialSize = INITIAL_SIZE * 2; 85 | for (int i = 0; i < doubleInitialSize; i++) 86 | { 87 | map.put("searchTerm_" + i, i); 88 | assertThat(map.size()).isEqualTo(i + 1); 89 | } 90 | 91 | for (int i = 0; i < doubleInitialSize; i++) 92 | { 93 | assertSearchResult(map, "searchTerm_" + i, i); 94 | } 95 | } 96 | 97 | @Test 98 | void shouldReplaceExistingValue() 99 | { 100 | final long otherId = 42L; 101 | map.put(SEARCH_TERM, ID); 102 | map.put(SEARCH_TERM, otherId); 103 | 104 | assertSearchResult(map, SEARCH_TERM, otherId); 105 | } 106 | 107 | @Test 108 | void shouldWrapBuffer() 109 | { 110 | final int initialSize = 20; 111 | final CharSequenceMap map = new CharSequenceMap(64, initialSize, Long.MIN_VALUE); 112 | final String prefix = "SYM_"; 113 | for (int i = 0; i < initialSize; i++) 114 | { 115 | map.put(prefix + i, i); 116 | } 117 | 118 | for (int i = 0; i < initialSize; i++) 119 | { 120 | assertThat(map.get(prefix + i)).isEqualTo(i); 121 | } 122 | } 123 | 124 | @Test 125 | void shouldRemoveValue() 126 | { 127 | map.put(SEARCH_TERM, ID); 128 | 129 | assertThat(map.remove(SEARCH_TERM)).isEqualTo(ID); 130 | assertThat(map.remove(SEARCH_TERM)).isEqualTo(MISSING_VALUE); 131 | assertThat(map.get(SEARCH_TERM)).isEqualTo(MISSING_VALUE); 132 | assertThat(map.size()).isEqualTo(0); 133 | } 134 | 135 | @Test 136 | void comparisonTest() 137 | { 138 | final Map control = new HashMap<>(); 139 | final Random random = new Random(1234567890L); 140 | 141 | for (int i = 0; i < 10_000; i++) 142 | { 143 | final String key = new UUID(random.nextLong(), random.nextLong()).toString(); 144 | final long value = random.nextLong(); 145 | control.put(key, value); 146 | map.put(key, value); 147 | 148 | assertThat(map.get(key)).isEqualTo(control.get(key)); 149 | } 150 | 151 | assertThat(map.size()).isEqualTo(10_000); 152 | 153 | Set controlKeys = new HashSet<>(control.keySet()); 154 | int counter = 0; 155 | for (final String controlKey : controlKeys) 156 | { 157 | if ((counter++ & 7) == 0) 158 | { 159 | final long controlValue = control.remove(controlKey); 160 | assertThat(map.remove(controlKey)) 161 | .isEqualTo(controlValue); 162 | } 163 | } 164 | 165 | assertThat(map.size()).isEqualTo(control.size()); 166 | 167 | controlKeys = new HashSet<>(control.keySet()); 168 | for (final String controlKey : controlKeys) 169 | { 170 | assertThat(map.get(controlKey)).isEqualTo(control.get(controlKey)); 171 | } 172 | 173 | for (int i = 0; i < 10_000; i++) 174 | { 175 | final String key = UUID.randomUUID().toString(); 176 | final long value = random.nextLong(); 177 | control.put(key, value); 178 | map.put(key, value); 179 | } 180 | 181 | controlKeys = new HashSet<>(control.keySet()); 182 | for (final String controlKey : controlKeys) 183 | { 184 | assertThat(map.get(controlKey)).isEqualTo(control.get(controlKey)); 185 | } 186 | 187 | map.rehash(); 188 | 189 | for (final String controlKey : controlKeys) 190 | { 191 | assertThat(map.get(controlKey)).isEqualTo(control.get(controlKey)); 192 | } 193 | } 194 | 195 | @Test 196 | void sizeShouldNotIncreaseWithRepeatedInserts() 197 | { 198 | for (int i = 0; i < 100; i++) 199 | { 200 | map.put("key_" + (i % 10), i); 201 | assertThat(map.size()).isAtMost(10); 202 | } 203 | } 204 | 205 | @Disabled("Requires more RAM than that provided by Travis container") 206 | @Test 207 | void shouldUseFullBufferCapactity() 208 | { 209 | final int perRecordOverhead = 4 * Integer.BYTES; 210 | final int maxKeyLength = 128; 211 | final int recordLength = perRecordOverhead + maxKeyLength; 212 | long maxBufferSize = recordLength * 8; 213 | while ((maxBufferSize * 2) < (long)Integer.MAX_VALUE) 214 | { 215 | maxBufferSize = maxBufferSize << 1; 216 | } 217 | maxBufferSize /= 2; 218 | 219 | final int maxRecordableValues = 220 | (int)((((int)maxBufferSize) / (perRecordOverhead + maxKeyLength) - 1) * 0.7f); 221 | final CharSequenceMap largeMap = new CharSequenceMap( 222 | maxKeyLength, maxRecordableValues / 4, Long.MIN_VALUE); 223 | 224 | for (int i = 0; i < maxRecordableValues; i++) 225 | { 226 | largeMap.put(Integer.toString(i), i); 227 | } 228 | 229 | Assertions.assertThrows(IllegalStateException.class, () -> 230 | { 231 | for (int i = 0; i < 5; i++) 232 | { 233 | largeMap.put(Integer.toString(i + maxRecordableValues), i); 234 | } 235 | }); 236 | } 237 | 238 | @Test 239 | void shouldBlowUpIfInitialAllocationIsTooLarge() 240 | { 241 | Assertions.assertThrows(IllegalArgumentException.class, () -> 242 | { 243 | new CharSequenceMap(64, Integer.MAX_VALUE, Long.MIN_VALUE); 244 | }); 245 | } 246 | 247 | private void assertSearchResult(final CharSequenceMap index, final String searchTerm, final long retrievedId) 248 | { 249 | assertThat(index.get(searchTerm)).isEqualTo(retrievedId); 250 | } 251 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/store/BufferStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | import com.aitusoftware.recall.persistence.Decoder; 21 | import com.aitusoftware.recall.persistence.Encoder; 22 | import com.aitusoftware.recall.persistence.IdAccessor; 23 | import org.agrona.collections.Hashing; 24 | import org.agrona.collections.Long2LongHashMap; 25 | 26 | import java.io.IOException; 27 | import java.io.UncheckedIOException; 28 | import java.nio.ByteBuffer; 29 | import java.nio.channels.FileChannel; 30 | import java.util.function.IntFunction; 31 | 32 | /** 33 | * Implementation of {@link Store} that serialises data to a buffer of type B. 34 | * 35 | * @param type of the underlying buffer 36 | */ 37 | public final class BufferStore implements Store 38 | { 39 | private static final long NOT_IN_MAP = Long.MIN_VALUE; 40 | private static final int DATA_OFFSET = Header.LENGTH; 41 | private static final int HEADER_OFFSET = 0; 42 | private final Long2LongHashMap index; 43 | private final int internalRecordLength; 44 | private final BufferOps bufferOps; 45 | private final IntFunction bufferFactory; 46 | private final Header header; 47 | private int bufferCapacity; 48 | private B buffer; 49 | private int nextWriteOffset; 50 | private int size; 51 | 52 | /** 53 | * Constructor for the BufferStore. 54 | * 55 | * @param maxRecordLength max length of any record 56 | * @param initialSize initial number of records that need to be stored 57 | * @param bufferFactory provider for the underlying buffer type 58 | * @param bufferOps provider of operations on the underlying buffer type 59 | */ 60 | public BufferStore( 61 | final int maxRecordLength, final int initialSize, 62 | final IntFunction bufferFactory, 63 | final BufferOps bufferOps) 64 | { 65 | internalRecordLength = maxRecordLength + Long.BYTES; 66 | bufferCapacity = internalRecordLength * initialSize; 67 | this.bufferOps = bufferOps; 68 | this.bufferFactory = bufferFactory; 69 | buffer = this.bufferFactory.apply(bufferCapacity + DATA_OFFSET); 70 | nextWriteOffset = DATA_OFFSET; 71 | header = new Header(); 72 | header.maxRecordLength(maxRecordLength).version(Version.ONE) 73 | .storeLength(bufferCapacity).nextWriteOffset(nextWriteOffset); 74 | header.writeTo(buffer, bufferOps, HEADER_OFFSET); 75 | index = new Long2LongHashMap(initialSize, Hashing.DEFAULT_LOAD_FACTOR, NOT_IN_MAP); 76 | } 77 | 78 | private BufferStore( 79 | final IntFunction bufferFactory, final BufferOps bufferOps, 80 | final B existingBuffer, final Header header) 81 | { 82 | internalRecordLength = header.maxRecordLength() + Long.BYTES; 83 | bufferCapacity = header.storeLength(); 84 | this.bufferOps = bufferOps; 85 | this.bufferFactory = bufferFactory; 86 | buffer = existingBuffer; 87 | this.nextWriteOffset = header.nextWriteOffset(); 88 | this.header = header; 89 | final int numberOfRecords = nextWriteOffset / internalRecordLength; 90 | index = new Long2LongHashMap(numberOfRecords, Hashing.DEFAULT_LOAD_FACTOR, NOT_IN_MAP); 91 | for (int i = 0; i < numberOfRecords; i++) 92 | { 93 | final int entryOffset = (i * internalRecordLength) + DATA_OFFSET; 94 | final long id = bufferOps.readLong(buffer, entryOffset); 95 | index.put(id, entryOffset); 96 | } 97 | } 98 | 99 | public static BufferStore loadFrom( 100 | final FileChannel input, final BufferOps bufferOps, final IntFunction bufferFactory) 101 | { 102 | final ByteBuffer headerBuffer = ByteBuffer.allocateDirect(Header.LENGTH); 103 | try 104 | { 105 | input.position(0); 106 | while (headerBuffer.remaining() != 0) 107 | { 108 | input.read(headerBuffer); 109 | } 110 | } 111 | catch (final IOException e) 112 | { 113 | throw new UncheckedIOException(e); 114 | } 115 | headerBuffer.flip(); 116 | final Header header = new Header(); 117 | header.readFrom(headerBuffer); 118 | 119 | final B buffer = bufferOps.createFrom(input, 0, header.storeLength() + Header.LENGTH); 120 | return new BufferStore<>(bufferFactory, bufferOps, buffer, header); 121 | } 122 | 123 | /** 124 | * {@inheritDoc} 125 | */ 126 | @Override 127 | public boolean load( 128 | final long id, final Decoder decoder, final T container) 129 | { 130 | final long recordOffset = index.get(id); 131 | if (recordOffset == NOT_IN_MAP) 132 | { 133 | return false; 134 | } 135 | final long storedId = bufferOps.readLong(buffer, (int)recordOffset); 136 | assert storedId == id : String.format("stored: %d, requested: %d, at %d", storedId, id, recordOffset); 137 | decoder.load(buffer, (int)recordOffset + Long.BYTES, container); 138 | 139 | return true; 140 | } 141 | 142 | /** 143 | * {@inheritDoc} 144 | */ 145 | @Override 146 | public void store( 147 | final Encoder encoder, final T value, final IdAccessor idAccessor) 148 | { 149 | final long valueId = idAccessor.getId(value); 150 | 151 | if (nextWriteOffset == bufferCapacity + DATA_OFFSET) 152 | { 153 | final B expandedBuffer = bufferFactory.apply((bufferCapacity << 1) + Header.LENGTH); 154 | bufferOps.copyBytes(buffer, expandedBuffer, DATA_OFFSET, DATA_OFFSET, bufferCapacity); 155 | buffer = expandedBuffer; 156 | bufferCapacity <<= 1; 157 | header.storeLength(bufferCapacity).writeTo(buffer, bufferOps, HEADER_OFFSET); 158 | } 159 | 160 | final long existingPosition = index.get(valueId); 161 | final int recordWriteOffset; 162 | if (existingPosition != NOT_IN_MAP) 163 | { 164 | recordWriteOffset = (int)existingPosition + Long.BYTES; 165 | } 166 | else 167 | { 168 | index.put(valueId, nextWriteOffset); 169 | bufferOps.writeLong(buffer, nextWriteOffset, valueId); 170 | recordWriteOffset = nextWriteOffset + Long.BYTES; 171 | nextWriteOffset += internalRecordLength; 172 | size++; 173 | } 174 | try 175 | { 176 | encoder.store(this.buffer, recordWriteOffset, value); 177 | } 178 | catch (final IllegalArgumentException e) 179 | { 180 | throw new IllegalArgumentException(String.format("Failed to store value with id %d at offset %d", 181 | valueId, recordWriteOffset), e); 182 | } 183 | } 184 | 185 | /** 186 | * {@inheritDoc} 187 | */ 188 | @Override 189 | public boolean remove(final long id) 190 | { 191 | final long writeOffset = index.remove(id); 192 | final boolean wasRemoved = writeOffset != NOT_IN_MAP; 193 | if (wasRemoved) 194 | { 195 | moveLastWrittenEntryTo(id, writeOffset); 196 | size--; 197 | } 198 | return wasRemoved; 199 | } 200 | 201 | /** 202 | * {@inheritDoc} 203 | */ 204 | @Override 205 | public void compact() 206 | { 207 | index.compact(); 208 | } 209 | 210 | /** 211 | * {@inheritDoc} 212 | */ 213 | @Override 214 | public void sync() 215 | { 216 | throw new UnsupportedOperationException(); 217 | } 218 | 219 | /** 220 | * {@inheritDoc} 221 | */ 222 | @Override 223 | public void writeTo(final FileChannel output) 224 | { 225 | header.nextWriteOffset(nextWriteOffset).writeTo(buffer, bufferOps, HEADER_OFFSET); 226 | 227 | bufferOps.storeTo(output, buffer, bufferCapacity + Header.LENGTH); 228 | } 229 | 230 | /** 231 | * {@inheritDoc} 232 | */ 233 | @Override 234 | public float utilisation() 235 | { 236 | return (nextWriteOffset - Header.LENGTH) / (float)bufferCapacity; 237 | } 238 | 239 | /** 240 | * {@inheritDoc} 241 | */ 242 | @Override 243 | public int size() 244 | { 245 | return size; 246 | } 247 | 248 | /** 249 | * {@inheritDoc} 250 | */ 251 | @Override 252 | public void clear() 253 | { 254 | nextWriteOffset = DATA_OFFSET; 255 | index.clear(); 256 | size = 0; 257 | } 258 | 259 | int nextWriteOffset() 260 | { 261 | return nextWriteOffset; 262 | } 263 | 264 | private void moveLastWrittenEntryTo(final long id, final long writeOffset) 265 | { 266 | final int sourcePosition = nextWriteOffset - internalRecordLength; 267 | final long retrievedId = bufferOps.readLong(buffer, sourcePosition); 268 | if (id != retrievedId) 269 | { 270 | moveRecord((int)writeOffset, sourcePosition); 271 | index.put(retrievedId, writeOffset); 272 | } 273 | 274 | nextWriteOffset -= internalRecordLength; 275 | } 276 | 277 | private void moveRecord(final int targetPosition, final int sourcePosition) 278 | { 279 | bufferOps.copyBytes(buffer, buffer, sourcePosition, targetPosition, internalRecordLength); 280 | } 281 | } -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/store/UnsafeBufferStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | 21 | import com.aitusoftware.recall.example.Order; 22 | import com.aitusoftware.recall.example.OrderUnsafeBufferTranscoder; 23 | import com.aitusoftware.recall.persistence.IdAccessor; 24 | import org.agrona.collections.LongHashSet; 25 | import org.agrona.concurrent.UnsafeBuffer; 26 | import org.junit.jupiter.api.Test; 27 | 28 | import java.io.IOException; 29 | import java.nio.ByteBuffer; 30 | import java.nio.channels.FileChannel; 31 | import java.nio.file.Files; 32 | import java.nio.file.Path; 33 | import java.nio.file.StandardOpenOption; 34 | import java.util.Random; 35 | import java.util.concurrent.ThreadLocalRandom; 36 | import java.util.function.IntFunction; 37 | 38 | import static com.google.common.truth.Truth.assertThat; 39 | 40 | class UnsafeBufferStoreTest 41 | { 42 | private static final long ID = 17L; 43 | private static final int INITIAL_RECORDS = 16; 44 | private static final int MAX_RECORD_LENGTH = 72; 45 | private final IntFunction bufferFactory = len -> new UnsafeBuffer(ByteBuffer.allocateDirect(len)); 46 | private final BufferOps bufferOps = new UnsafeBufferOps(); 47 | private final BufferStore store = 48 | new BufferStore<>(MAX_RECORD_LENGTH, INITIAL_RECORDS, bufferFactory, bufferOps); 49 | private final OrderUnsafeBufferTranscoder transcoder = new OrderUnsafeBufferTranscoder(); 50 | private final IdAccessor idAccessor = UnsafeBufferStoreTest::idOf; 51 | 52 | @Test 53 | void shouldStoreAndLoad() 54 | { 55 | final Order order = Order.of(ID); 56 | store.store(transcoder, order, order); 57 | 58 | final Order container = Order.of(-1L); 59 | assertThat(store.load(ID, transcoder, container)).isTrue(); 60 | 61 | assertEquality(order, container); 62 | } 63 | 64 | @Test 65 | void shouldDelete() 66 | { 67 | final Order order = Order.of(ID); 68 | store.store(transcoder, order, order); 69 | 70 | assertThat(store.remove(ID)).isTrue(); 71 | 72 | final Order container = Order.of(-1L); 73 | assertThat(store.load(ID, transcoder, container)).isFalse(); 74 | } 75 | 76 | @Test 77 | void shouldIndicateFailedDelete() 78 | { 79 | assertThat(store.remove(ID)).isFalse(); 80 | } 81 | 82 | @Test 83 | void shouldUpdateInPlace() 84 | { 85 | final Order order = Order.of(ID); 86 | store.store(transcoder, order, order); 87 | final int nextWriteOffset = store.nextWriteOffset(); 88 | 89 | final Order updated = new Order(ID, 17L, 37, 90 | 13L, 17L, 35, "Foo"); 91 | store.store(transcoder, updated, updated); 92 | 93 | assertThat(store.nextWriteOffset()).isEqualTo(nextWriteOffset); 94 | 95 | final Order container = Order.of(-1L); 96 | assertThat(store.load(ID, transcoder, container)).isTrue(); 97 | 98 | assertEquality(container, updated); 99 | } 100 | 101 | @Test 102 | void shouldStoreAfterRemoval() 103 | { 104 | final Order order = Order.of(ID); 105 | store.store(transcoder, order, order); 106 | 107 | store.remove(ID); 108 | 109 | store.store(transcoder, order, order); 110 | 111 | final Order container = Order.of(-1L); 112 | assertThat(store.load(ID, transcoder, container)).isTrue(); 113 | 114 | assertEquality(container, order); 115 | } 116 | 117 | @Test 118 | void shouldGrowIfInitialCapacityExceeded() 119 | { 120 | for (int i = 0; i < INITIAL_RECORDS; i++) 121 | { 122 | final Order order = Order.of(i); 123 | store.store(transcoder, order, order); 124 | } 125 | 126 | assertThat(store.utilisation() > 0.99).isTrue(); 127 | 128 | final Order order = Order.of(INITIAL_RECORDS); 129 | store.store(transcoder, order, order); 130 | 131 | assertThat(store.utilisation() < 0.6).isTrue(); 132 | assertThat(store.size()).isEqualTo(INITIAL_RECORDS + 1); 133 | 134 | for (int i = 0; i <= INITIAL_RECORDS; i++) 135 | { 136 | assertThat(store.load(i, transcoder, Order.of(-1))) 137 | .isTrue(); 138 | } 139 | } 140 | 141 | @Test 142 | void shouldCompactAfterRemoval() 143 | { 144 | for (int i = 0; i < INITIAL_RECORDS; i++) 145 | { 146 | final Order order = Order.of(i); 147 | store.store(transcoder, order, order); 148 | } 149 | 150 | for (int i = 0; i < INITIAL_RECORDS; i += 2) 151 | { 152 | store.remove(i); 153 | } 154 | 155 | store.compact(); 156 | 157 | for (int i = 0; i < (INITIAL_RECORDS / 2); i++) 158 | { 159 | final Order order = Order.of(i + INITIAL_RECORDS); 160 | store.store(transcoder, order, order); 161 | } 162 | 163 | final Order container = Order.of(-1L); 164 | 165 | for (int i = 1; i < INITIAL_RECORDS; i += 2) 166 | { 167 | assertThat(store.load(i, transcoder, container)).isTrue(); 168 | } 169 | for (int i = 0; i < (INITIAL_RECORDS / 2); i++) 170 | { 171 | assertThat(store.load(i + INITIAL_RECORDS, transcoder, container)).isTrue(); 172 | } 173 | } 174 | 175 | @Test 176 | void shouldPerformIdealCompaction() 177 | { 178 | for (int i = 0; i < 4; i++) 179 | { 180 | final Order order = Order.of(i); 181 | store.store(transcoder, order, order); 182 | } 183 | 184 | for (int i = 0; i < 4; i += 2) 185 | { 186 | store.remove(i); 187 | } 188 | 189 | store.compact(); 190 | 191 | assertThat(store.nextWriteOffset()).isEqualTo(160 + Header.LENGTH); 192 | } 193 | 194 | @Test 195 | void correctnessTest() 196 | { 197 | final long randomSeed = System.nanoTime(); 198 | final Random random = new Random(randomSeed); 199 | final int maxRecords = 50_000; 200 | final BufferStore store = new BufferStore<>(128, maxRecords, bufferFactory, bufferOps); 201 | final LongHashSet createdIds = new LongHashSet(); 202 | final LongHashSet removedIds = new LongHashSet(); 203 | 204 | for (int i = 0; i < maxRecords; i++) 205 | { 206 | final long id = random.nextLong(); 207 | store.store(transcoder, Order.of(id), idAccessor); 208 | createdIds.add(id); 209 | } 210 | 211 | final LongHashSet.LongIterator iterator = createdIds.iterator(); 212 | for (int i = 0; i < maxRecords / 10; i++) 213 | { 214 | final long idToRemove = iterator.nextValue(); 215 | createdIds.remove(idToRemove); 216 | removedIds.add(idToRemove); 217 | store.remove(idToRemove); 218 | if (i % 100 == 0) 219 | { 220 | store.compact(); 221 | } 222 | } 223 | try 224 | { 225 | assertContent(store, createdIds, removedIds); 226 | } 227 | catch (final AssertionError e) 228 | { 229 | throw new AssertionError("Test failed with random seed " + randomSeed, e); 230 | } 231 | } 232 | 233 | @Test 234 | void shouldPersistAndLoad() throws IOException 235 | { 236 | final LongHashSet createdIds = new LongHashSet(); 237 | final int recordCount = INITIAL_RECORDS * 4; 238 | for (int i = 0; i < recordCount; i++) 239 | { 240 | final long id = ThreadLocalRandom.current().nextLong(); 241 | createdIds.add(id); 242 | store.store(transcoder, Order.of(id), idAccessor); 243 | } 244 | final Path storeFile = Files.createTempFile("recall", ".store"); 245 | final FileChannel storeChannel = FileChannel.open(storeFile.toAbsolutePath(), 246 | StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.READ); 247 | store.writeTo(storeChannel); 248 | 249 | final BufferStore loadedStore = BufferStore.loadFrom(storeChannel, bufferOps, bufferFactory); 250 | 251 | final LongHashSet.LongIterator iterator = createdIds.iterator(); 252 | while (iterator.hasNext()) 253 | { 254 | assertThat(loadedStore.load(iterator.nextValue(), transcoder, Order.of(77))).isTrue(); 255 | } 256 | } 257 | 258 | 259 | private void assertContent( 260 | final BufferStore store, final LongHashSet createdIds, 261 | final LongHashSet removedIds) 262 | { 263 | final Order container = Order.of(99999L); 264 | 265 | for (final long id : createdIds) 266 | { 267 | assertThat(store.load(id, transcoder, container)).isTrue(); 268 | assertThat(container.getId()).isEqualTo(id); 269 | assertThat(container.getInstrumentId()).isEqualTo(37L); 270 | } 271 | 272 | for (final long id : removedIds) 273 | { 274 | assertThat(store.load(id, transcoder, container)).isFalse(); 275 | } 276 | } 277 | 278 | private static long idOf(final Order order) 279 | { 280 | return order.getId(); 281 | } 282 | 283 | private static void assertEquality(final Order actual, final Order expected) 284 | { 285 | assertThat(actual.getId()).isEqualTo(expected.getId()); 286 | assertCharSequenceEquality(actual.getSymbol(), expected.getSymbol()); 287 | assertThat(actual.getCreatedEpochSeconds()).isEqualTo(expected.getCreatedEpochSeconds()); 288 | } 289 | 290 | private static void assertCharSequenceEquality(final CharSequence actual, final CharSequence expected) 291 | { 292 | assertThat(actual.length()).isEqualTo(expected.length()); 293 | for (int i = 0; i < actual.length(); i++) 294 | { 295 | assertThat(actual.charAt(i)).isEqualTo(expected.charAt(i)); 296 | } 297 | } 298 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /recall-store/src/test/java/com/aitusoftware/recall/store/ByteBufferStoreTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.store; 19 | 20 | 21 | import com.aitusoftware.recall.example.Order; 22 | import com.aitusoftware.recall.example.OrderByteBufferTranscoder; 23 | import com.aitusoftware.recall.persistence.IdAccessor; 24 | import org.agrona.collections.LongHashSet; 25 | import org.junit.jupiter.api.Assertions; 26 | import org.junit.jupiter.api.Test; 27 | 28 | import java.io.File; 29 | import java.io.IOException; 30 | import java.nio.ByteBuffer; 31 | import java.nio.channels.FileChannel; 32 | import java.nio.file.Files; 33 | import java.nio.file.Path; 34 | import java.nio.file.StandardOpenOption; 35 | import java.util.Random; 36 | import java.util.concurrent.ThreadLocalRandom; 37 | import java.util.function.IntFunction; 38 | 39 | import static com.google.common.truth.Truth.assertThat; 40 | import static java.nio.file.StandardOpenOption.READ; 41 | import static java.nio.file.StandardOpenOption.WRITE; 42 | 43 | class ByteBufferStoreTest 44 | { 45 | private static final long ID = 17L; 46 | private static final int INITIAL_RECORDS = 16; 47 | private static final int MAX_RECORD_LENGTH = 72; 48 | private final IntFunction bufferFactory = ByteBuffer::allocate; 49 | private final ByteBufferOps bufferOps = new ByteBufferOps(); 50 | private final BufferStore store = 51 | new BufferStore<>(MAX_RECORD_LENGTH, INITIAL_RECORDS, bufferFactory, bufferOps); 52 | private final OrderByteBufferTranscoder transcoder = new OrderByteBufferTranscoder(); 53 | private final IdAccessor idAccessor = ByteBufferStoreTest::idOf; 54 | 55 | @Test 56 | void shouldStoreAndLoad() 57 | { 58 | final Order order = Order.of(ID); 59 | store.store(transcoder, order, order); 60 | 61 | final Order container = Order.of(-1L); 62 | assertThat(store.load(ID, transcoder, container)).isTrue(); 63 | 64 | assertEquality(order, container); 65 | } 66 | 67 | @Test 68 | void shouldDelete() 69 | { 70 | final Order order = Order.of(ID); 71 | store.store(transcoder, order, order); 72 | 73 | assertThat(store.remove(ID)).isTrue(); 74 | 75 | final Order container = Order.of(-1L); 76 | assertThat(store.load(ID, transcoder, container)).isFalse(); 77 | } 78 | 79 | @Test 80 | void shouldIndicateFailedDelete() 81 | { 82 | assertThat(store.remove(ID)).isFalse(); 83 | } 84 | 85 | @Test 86 | void shouldUpdateInPlace() 87 | { 88 | final Order order = Order.of(ID); 89 | store.store(transcoder, order, order); 90 | final int nextWriteOffset = store.nextWriteOffset(); 91 | 92 | final Order updated = new Order(ID, 17L, 37, 93 | 13L, 17L, 35, "Foo"); 94 | store.store(transcoder, updated, updated); 95 | 96 | assertThat(store.nextWriteOffset()).isEqualTo(nextWriteOffset); 97 | 98 | final Order container = Order.of(-1L); 99 | assertThat(store.load(ID, transcoder, container)).isTrue(); 100 | 101 | assertEquality(container, updated); 102 | } 103 | 104 | @Test 105 | void shouldStoreAfterRemoval() 106 | { 107 | final Order order = Order.of(ID); 108 | store.store(transcoder, order, order); 109 | 110 | store.remove(ID); 111 | 112 | store.store(transcoder, order, order); 113 | 114 | final Order container = Order.of(-1L); 115 | assertThat(store.load(ID, transcoder, container)).isTrue(); 116 | 117 | assertEquality(container, order); 118 | } 119 | 120 | @Test 121 | void shouldGrowIfInitialCapacityExceeded() 122 | { 123 | for (int i = 0; i < INITIAL_RECORDS; i++) 124 | { 125 | final Order order = Order.of(i); 126 | store.store(transcoder, order, order); 127 | } 128 | 129 | assertThat(store.utilisation() > 0.99).isTrue(); 130 | 131 | final Order order = Order.of(INITIAL_RECORDS); 132 | store.store(transcoder, order, order); 133 | 134 | assertThat(store.utilisation() < 0.6).isTrue(); 135 | assertThat(store.size()).isEqualTo(INITIAL_RECORDS + 1); 136 | 137 | for (int i = 0; i <= INITIAL_RECORDS; i++) 138 | { 139 | assertThat(store.load(i, transcoder, Order.of(-1))).isTrue(); 140 | } 141 | } 142 | 143 | @Test 144 | void shouldReportUtilisation() 145 | { 146 | assertThat(store.utilisation()).isEqualTo(0f); 147 | store.store(transcoder, Order.of(1), Order.of(1)); 148 | store.store(transcoder, Order.of(2), Order.of(1)); 149 | 150 | assertThat(store.utilisation()).isWithin(0.01f).of(2 / (float)INITIAL_RECORDS); 151 | 152 | for (int i = 0; i < 6; i++) 153 | { 154 | store.store(transcoder, Order.of(3 + i), Order.of(1)); 155 | } 156 | 157 | assertThat(store.utilisation()).isWithin(0.01f).of(0.5f); 158 | 159 | for (int i = 0; i < 8; i++) 160 | { 161 | store.store(transcoder, Order.of(13 + i), Order.of(1)); 162 | } 163 | 164 | assertThat(store.utilisation()).isWithin(0.01f).of(1f); 165 | } 166 | 167 | @Test 168 | void shouldCompactAfterRemoval() 169 | { 170 | for (int i = 0; i < INITIAL_RECORDS; i++) 171 | { 172 | final Order order = Order.of(i); 173 | store.store(transcoder, order, order); 174 | } 175 | 176 | int expectedSize = store.size(); 177 | for (int i = 0; i < INITIAL_RECORDS; i += 2) 178 | { 179 | assertThat(store.remove(i)).isTrue(); 180 | assertThat(store.size()).isEqualTo(--expectedSize); 181 | } 182 | 183 | store.compact(); 184 | 185 | for (int i = 0; i < (INITIAL_RECORDS / 2); i++) 186 | { 187 | final Order order = Order.of(i + INITIAL_RECORDS); 188 | store.store(transcoder, order, order); 189 | } 190 | 191 | final Order container = Order.of(-1L); 192 | 193 | for (int i = 1; i < INITIAL_RECORDS; i += 2) 194 | { 195 | assertThat(store.load(i, transcoder, container)).isTrue(); 196 | } 197 | for (int i = 0; i < (INITIAL_RECORDS / 2); i++) 198 | { 199 | assertThat(store.load(i + INITIAL_RECORDS, transcoder, container)).isTrue(); 200 | } 201 | } 202 | 203 | @Test 204 | void shouldPerformIdealCompaction() 205 | { 206 | for (int i = 0; i < 4; i++) 207 | { 208 | final Order order = Order.of(i); 209 | store.store(transcoder, order, order); 210 | } 211 | 212 | for (int i = 0; i < 4; i += 2) 213 | { 214 | store.remove(i); 215 | } 216 | 217 | store.compact(); 218 | 219 | assertThat(store.nextWriteOffset()).isEqualTo(160 + Header.LENGTH); 220 | } 221 | 222 | @Test 223 | void correctnessTest() 224 | { 225 | final long randomSeed = System.nanoTime(); 226 | final Random random = new Random(randomSeed); 227 | final int maxRecords = 50_000; 228 | final BufferStore store = new BufferStore<>(128, maxRecords, bufferFactory, bufferOps); 229 | final LongHashSet createdIds = new LongHashSet(); 230 | final LongHashSet removedIds = new LongHashSet(); 231 | 232 | for (int i = 0; i < maxRecords; i++) 233 | { 234 | final long id = random.nextLong(); 235 | store.store(transcoder, Order.of(id), ByteBufferStoreTest::idOf); 236 | createdIds.add(id); 237 | assertThat(store.size()).isEqualTo(i + 1); 238 | } 239 | 240 | final LongHashSet.LongIterator iterator = createdIds.iterator(); 241 | for (int i = 0; i < maxRecords / 10; i++) 242 | { 243 | final long idToRemove = iterator.nextValue(); 244 | createdIds.remove(idToRemove); 245 | removedIds.add(idToRemove); 246 | store.remove(idToRemove); 247 | if (i % 100 == 0) 248 | { 249 | store.compact(); 250 | } 251 | } 252 | try 253 | { 254 | assertContent(store, createdIds, removedIds); 255 | } 256 | catch (final AssertionError e) 257 | { 258 | throw new AssertionError("Test failed with random seed " + randomSeed, e); 259 | } 260 | 261 | store.clear(); 262 | 263 | assertThat(store.size()).isEqualTo(0); 264 | final LongHashSet.LongIterator createdIdIterator = createdIds.iterator(); 265 | final Order container = Order.of(-1L); 266 | while (createdIdIterator.hasNext()) 267 | { 268 | assertThat(store.load(createdIdIterator.nextValue(), transcoder, container)).isFalse(); 269 | } 270 | } 271 | 272 | @Test 273 | void shouldPersistAndLoad() throws IOException 274 | { 275 | final LongHashSet createdIds = new LongHashSet(); 276 | final int recordCount = INITIAL_RECORDS * 4; 277 | for (int i = 0; i < recordCount; i++) 278 | { 279 | final long id = ThreadLocalRandom.current().nextLong(); 280 | createdIds.add(id); 281 | store.store(transcoder, Order.of(id), idAccessor); 282 | } 283 | final Path storeFile = Files.createTempFile("recall", ".store"); 284 | final FileChannel storeChannel = FileChannel.open(storeFile.toAbsolutePath(), 285 | StandardOpenOption.CREATE, WRITE, READ); 286 | store.writeTo(storeChannel); 287 | 288 | final BufferStore loadedStore = BufferStore.loadFrom(storeChannel, bufferOps, bufferFactory); 289 | 290 | final LongHashSet.LongIterator iterator = createdIds.iterator(); 291 | while (iterator.hasNext()) 292 | { 293 | assertThat(loadedStore.load(iterator.nextValue(), transcoder, Order.of(77))).isTrue(); 294 | } 295 | } 296 | 297 | @Test 298 | void shouldBlowUpIfVersionIsUnknown() throws IOException 299 | { 300 | store.store(transcoder, Order.of(17), idAccessor); 301 | final File tempFile = File.createTempFile("BufferStoreTest", "bin"); 302 | store.writeTo(FileChannel.open(tempFile.toPath(), WRITE)); 303 | 304 | final FileChannel serialisedStore = FileChannel.open(tempFile.toPath(), READ, WRITE); 305 | final ByteBuffer headerBuffer = ByteBuffer.allocateDirect(Header.LENGTH); 306 | serialisedStore.position(0).read(headerBuffer); 307 | headerBuffer.flip(); 308 | headerBuffer.putInt(0, 234445); 309 | serialisedStore.position(0).write(headerBuffer); 310 | 311 | serialisedStore.position(0); 312 | Assertions.assertThrows(IllegalArgumentException.class, () -> 313 | BufferStore.loadFrom(serialisedStore, bufferOps, bufferFactory)); 314 | } 315 | 316 | private void assertContent( 317 | final BufferStore store, final LongHashSet createdIds, 318 | final LongHashSet removedIds) 319 | { 320 | final Order container = Order.of(99999L); 321 | 322 | for (final long id : createdIds) 323 | { 324 | assertThat(store.load(id, transcoder, container)).isTrue(); 325 | assertThat(container.getId()).isEqualTo(id); 326 | assertThat(container.getInstrumentId()).isEqualTo(37L); 327 | } 328 | 329 | for (final long id : removedIds) 330 | { 331 | assertThat(store.load(id, transcoder, container)).isFalse(); 332 | } 333 | } 334 | 335 | private static long idOf(final Order order) 336 | { 337 | return order.getId(); 338 | } 339 | 340 | private static void assertEquality(final Order actual, final Order expected) 341 | { 342 | assertThat(actual.getId()).isEqualTo(expected.getId()); 343 | assertCharSequenceEquality(actual.getSymbol(), expected.getSymbol()); 344 | assertThat(actual.getCreatedEpochSeconds()).isEqualTo(expected.getCreatedEpochSeconds()); 345 | } 346 | 347 | private static void assertCharSequenceEquality(final CharSequence actual, final CharSequence expected) 348 | { 349 | assertThat(actual.length()).isEqualTo(expected.length()); 350 | for (int i = 0; i < actual.length(); i++) 351 | { 352 | assertThat(actual.charAt(i)).isEqualTo(expected.charAt(i)); 353 | } 354 | } 355 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/map/ByteSequenceMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.map; 19 | 20 | import org.agrona.BitUtil; 21 | 22 | import java.nio.ByteBuffer; 23 | import java.util.function.IntFunction; 24 | import java.util.function.ToIntFunction; 25 | 26 | /** 27 | * Map for storing a byte-sequence against a long value. 28 | */ 29 | public final class ByteSequenceMap implements SequenceMap 30 | { 31 | private static final int ID_OFFSET = Integer.BYTES; 32 | private static final int KEY_OFFSET = Integer.BYTES * 4; 33 | private static final int HASH_OFFSET = Integer.BYTES * 3; 34 | private final ToIntFunction hash; 35 | private final float loadFactor = 0.7f; 36 | private final IntFunction bufferFactory; 37 | private final long missingValue; 38 | private final int maxKeyLength; 39 | private final EntryHandler getEntryHandler = (b, i) -> {}; 40 | private final EntryHandler removeEntryHandler; 41 | private ByteBuffer dataBuffer; 42 | private int totalEntryCount; 43 | private int liveEntryCount; 44 | private int entryCountToTriggerRehash; 45 | private int entryMask; 46 | private int entrySizeInBytes; 47 | private boolean noDeletes = true; 48 | 49 | /** 50 | * Constructor for the map. 51 | * 52 | * @param maxKeyLength max length of any key 53 | * @param initialSize initial size of the map 54 | * @param missingValue initial size of the map 55 | */ 56 | public ByteSequenceMap(final int maxKeyLength, final int initialSize, final long missingValue) 57 | { 58 | this(maxKeyLength, initialSize, ByteSequenceMap::defaultHash, 59 | missingValue, ByteBuffer::allocateDirect); 60 | } 61 | 62 | /** 63 | * Constructor for the map. 64 | * 65 | * @param maxKeyLength max length of any key 66 | * @param initialSize initial size of the map 67 | * @param missingValue initial size of the map 68 | * @param bufferFactory factory method for creating new {@code ByteBuffer} instances 69 | */ 70 | public ByteSequenceMap( 71 | final int maxKeyLength, final int initialSize, 72 | final long missingValue, final IntFunction bufferFactory) 73 | { 74 | this(maxKeyLength, initialSize, ByteSequenceMap::defaultHash, 75 | missingValue, bufferFactory); 76 | } 77 | 78 | ByteSequenceMap( 79 | final int maxKeyLength, final int initialSize, 80 | final ToIntFunction hash, final long missingValue, 81 | final IntFunction bufferFactory) 82 | { 83 | totalEntryCount = BitUtil.findNextPositivePowerOfTwo(initialSize); 84 | entryMask = totalEntryCount - 1; 85 | entrySizeInBytes = (maxKeyLength + Long.BYTES + Long.BYTES); 86 | final long bufferSize = entrySizeInBytes * (long)totalEntryCount; 87 | if (bufferSize > ((long)Integer.MAX_VALUE) || totalEntryCount < 0) 88 | { 89 | throw new IllegalArgumentException("Requested buffer size too large"); 90 | } 91 | this.bufferFactory = bufferFactory; 92 | dataBuffer = this.bufferFactory.apply((int)bufferSize); 93 | entryCountToTriggerRehash = (int)(loadFactor * totalEntryCount); 94 | this.hash = hash; 95 | this.missingValue = missingValue; 96 | this.maxKeyLength = maxKeyLength; 97 | removeEntryHandler = new RemoveEntryHandler(entrySizeInBytes); 98 | } 99 | 100 | /** 101 | * {@inheritDoc}. 102 | */ 103 | public void put(final ByteBuffer value, final long id) 104 | { 105 | if (value.remaining() > maxKeyLength) 106 | { 107 | throw new IllegalArgumentException("Key too long"); 108 | } 109 | if (liveEntryCount > entryCountToTriggerRehash) 110 | { 111 | rehash(true); 112 | } 113 | final int hashValue = hash.applyAsInt(value); 114 | final int entryIndex = (hashValue & entryMask); 115 | final boolean existingEntryAt = isExistingEntryAt(value, entryIndex, hashValue); 116 | if (!isValuePresent(entryIndex, dataBuffer) || existingEntryAt) 117 | { 118 | insertEntry(value, id, entryIndex, !existingEntryAt, hashValue); 119 | } 120 | else 121 | { 122 | for (int i = 1; i < totalEntryCount; i++) 123 | { 124 | int candidateIndex = (entryIndex + (i)); 125 | if (candidateIndex >= totalEntryCount) 126 | { 127 | candidateIndex -= totalEntryCount; 128 | } 129 | final boolean innerExistingEntryAt = isExistingEntryAt(value, candidateIndex, hashValue); 130 | if (!isValuePresent(candidateIndex, dataBuffer) || innerExistingEntryAt) 131 | { 132 | insertEntry(value, id, candidateIndex, !innerExistingEntryAt, hashValue); 133 | return; 134 | } 135 | } 136 | 137 | put(value, id); 138 | } 139 | } 140 | 141 | /** 142 | * {@inheritDoc}. 143 | */ 144 | public long get(final ByteBuffer value) 145 | { 146 | return search(value, getEntryHandler); 147 | } 148 | 149 | /** 150 | * {@inheritDoc}. 151 | */ 152 | public long remove(final ByteBuffer value) 153 | { 154 | return search(value, removeEntryHandler); 155 | } 156 | 157 | /** 158 | * {@inheritDoc}. 159 | */ 160 | public int size() 161 | { 162 | return liveEntryCount; 163 | } 164 | 165 | /** 166 | * {@inheritDoc}. 167 | */ 168 | public void rehash() 169 | { 170 | rehash(false); 171 | } 172 | 173 | private long search(final ByteBuffer value, final EntryHandler entryHandler) 174 | { 175 | final int hashValue = hash.applyAsInt(value); 176 | int entryIndex = (hashValue & entryMask); 177 | int entry = 0; 178 | while (entry < totalEntryCount) 179 | { 180 | if (!isValuePresent(entryIndex, dataBuffer) && noDeletes) 181 | { 182 | break; 183 | } 184 | 185 | if (isExistingEntryAt(value, entryIndex, getHash(entryIndex, dataBuffer))) 186 | { 187 | final long storedId = getId(entryIndex, dataBuffer); 188 | entryHandler.onEntryFound(dataBuffer, entryIndex); 189 | return storedId; 190 | } 191 | entryIndex++; 192 | entryIndex = entryIndex & entryMask; 193 | entry++; 194 | } 195 | 196 | return missingValue; 197 | } 198 | 199 | private interface EntryHandler 200 | { 201 | void onEntryFound(ByteBuffer dataBuffer, int index); 202 | } 203 | 204 | private void rehash(final boolean shouldResize) 205 | { 206 | final ByteBuffer oldBuffer = dataBuffer; 207 | final int oldEntryCount = totalEntryCount; 208 | final int newSize = oldBuffer.capacity() * 2; 209 | if (newSize < 0) 210 | { 211 | throw new IllegalStateException( 212 | String.format( 213 | "Maximum map capacity exceeded. Entry count: %d, entrySize: %d, newSize: %d", 214 | size(), entrySizeInBytes, ((long)oldBuffer.capacity()) * 2)); 215 | } 216 | 217 | if (shouldResize) 218 | { 219 | dataBuffer = bufferFactory.apply(newSize); 220 | totalEntryCount *= 2; 221 | entryMask = totalEntryCount - 1; 222 | entryCountToTriggerRehash = (int)(loadFactor * totalEntryCount); 223 | } 224 | else 225 | { 226 | dataBuffer = bufferFactory.apply(oldBuffer.capacity()); 227 | } 228 | 229 | liveEntryCount = 0; 230 | 231 | for (int i = 0; i < oldEntryCount; i++) 232 | { 233 | if (isValuePresent(i, oldBuffer)) 234 | { 235 | final long id = getId(i, oldBuffer); 236 | final int sourceOffset = keyOffset(i); 237 | final int valueLength = getValueLength(i, oldBuffer); 238 | final int endPosition = sourceOffset + valueLength; 239 | oldBuffer.limit(endPosition).position(sourceOffset); 240 | put(oldBuffer, id); 241 | oldBuffer.limit(oldBuffer.capacity()).position(0); 242 | } 243 | } 244 | } 245 | 246 | private void insertEntry( 247 | final ByteBuffer value, final long id, 248 | final int entryIndex, final boolean isInsert, final int hashValue) 249 | { 250 | setValueLength(entryIndex, value.remaining(), dataBuffer); 251 | final int keyOffset = keyOffset(entryIndex); 252 | for (int i = 0; i < value.remaining(); i++) 253 | { 254 | dataBuffer.put(keyOffset + (i), value.get(i + value.position())); 255 | } 256 | setId(entryIndex, id, dataBuffer); 257 | setHash(entryIndex, hashValue, dataBuffer); 258 | if (isInsert) 259 | { 260 | liveEntryCount++; 261 | } 262 | } 263 | 264 | private boolean isExistingEntryAt(final ByteBuffer value, final int entryIndex, final int hashValue) 265 | { 266 | final int keyOffset = keyOffset(entryIndex); 267 | if (hashValue != getHash(entryIndex, dataBuffer)) 268 | { 269 | return false; 270 | } 271 | for (int i = 0; i < value.remaining(); i++) 272 | { 273 | if (dataBuffer.get(keyOffset + (i)) != value.get(value.position() + i)) 274 | { 275 | return false; 276 | } 277 | } 278 | return true; 279 | } 280 | 281 | private static int defaultHash(final ByteBuffer value) 282 | { 283 | final int endOfData = value.remaining() + value.position(); 284 | int hash = 0; 285 | for (int i = value.position(); i < endOfData; i++) 286 | { 287 | hash = (31 * hash) + value.get(i); 288 | } 289 | return hash; 290 | } 291 | 292 | private int byteOffset(final int entryIndex) 293 | { 294 | return entryIndex * entrySizeInBytes; 295 | } 296 | 297 | private boolean isValuePresent(final int entryIndex, final ByteBuffer dataBuffer) 298 | { 299 | return (0b1000_0000_0000_0000_0000_0000_0000_0000 & dataBuffer.getInt(byteOffset(entryIndex))) != 0; 300 | } 301 | 302 | private void setValueLength(final int entryIndex, final int valueLength, final ByteBuffer dataBuffer) 303 | { 304 | dataBuffer.putInt(byteOffset(entryIndex), 0b1000_0000_0000_0000_0000_0000_0000_0000 | valueLength); 305 | } 306 | 307 | private int getValueLength(final int entryIndex, final ByteBuffer dataBuffer) 308 | { 309 | return (0b0111_1111_1111_1111_1111_1111_1111_1111 & dataBuffer.getInt(byteOffset(entryIndex))); 310 | } 311 | 312 | private void setId(final int entryIndex, final long id, final ByteBuffer dataBuffer) 313 | { 314 | dataBuffer.putLong(byteOffset(entryIndex) + ID_OFFSET, id); 315 | } 316 | 317 | private long getId(final int entryIndex, final ByteBuffer dataBuffer) 318 | { 319 | return dataBuffer.getLong(byteOffset(entryIndex) + ID_OFFSET); 320 | } 321 | 322 | private void setHash(final int entryIndex, final int hash, final ByteBuffer dataBuffer) 323 | { 324 | dataBuffer.putInt(byteOffset(entryIndex) + HASH_OFFSET, hash); 325 | } 326 | 327 | private int getHash(final int entryIndex, final ByteBuffer dataBuffer) 328 | { 329 | return dataBuffer.getInt(byteOffset(entryIndex) + HASH_OFFSET); 330 | } 331 | 332 | private int keyOffset(final int entryIndex) 333 | { 334 | return byteOffset(entryIndex) + KEY_OFFSET; 335 | } 336 | 337 | private final class RemoveEntryHandler implements EntryHandler 338 | { 339 | private final int longsInEntry; 340 | private final int remainingBytesInEntry; 341 | 342 | private RemoveEntryHandler(final int entrySizeInBytes) 343 | { 344 | longsInEntry = entrySizeInBytes / Long.BYTES; 345 | remainingBytesInEntry = entrySizeInBytes & (Long.BYTES - 1); 346 | } 347 | 348 | @Override 349 | public void onEntryFound(final ByteBuffer buffer, final int entryIndex) 350 | { 351 | int byteOffset = byteOffset(entryIndex); 352 | for (int i = 0; i < longsInEntry; i++) 353 | { 354 | buffer.putLong(byteOffset, 0L); 355 | byteOffset += Long.BYTES; 356 | } 357 | for (int i = 0; i < remainingBytesInEntry; i++) 358 | { 359 | buffer.put(byteOffset++, (byte)0); 360 | } 361 | liveEntryCount--; 362 | noDeletes = false; 363 | } 364 | } 365 | } -------------------------------------------------------------------------------- /recall-store/src/main/java/com/aitusoftware/recall/map/CharSequenceMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Aitu Software Limited. 3 | * 4 | * https://aitusoftware.com 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package com.aitusoftware.recall.map; 19 | 20 | import org.agrona.BitUtil; 21 | 22 | import java.nio.ByteBuffer; 23 | import java.util.function.IntFunction; 24 | import java.util.function.ToIntFunction; 25 | 26 | /** 27 | * Map for storing a char-sequence against a long value. 28 | */ 29 | public final class CharSequenceMap implements SequenceMap 30 | { 31 | private static final int KEY_OFFSET = Integer.BYTES * 4; 32 | private static final int HASH_OFFSET = Integer.BYTES * 3; 33 | private static final int ID_OFFSET = Integer.BYTES; 34 | private final ToIntFunction hash; 35 | private final CharArrayCharSequence charBuffer = new CharArrayCharSequence(); 36 | private final float loadFactor = 0.7f; 37 | private final IntFunction bufferFactory; 38 | private final long missingValue; 39 | private final int maxKeyLength; 40 | private final RemoveEntryHandler removeEntryHandler; 41 | private final EntryHandler searchEntryHandler = (b, i) -> {}; 42 | private final int entrySizeInBytes; 43 | private ByteBuffer dataBuffer; 44 | private int totalEntryCount; 45 | private int liveEntryCount; 46 | private int entryCountToTriggerRehash; 47 | private int entryMask; 48 | private boolean noDeletes = true; 49 | 50 | /** 51 | * Constructor for the map. 52 | * 53 | * @param maxKeyLength max length of any key 54 | * @param initialSize initial size of the map 55 | * @param missingValue value to return if the key is not present 56 | */ 57 | public CharSequenceMap(final int maxKeyLength, final int initialSize, final long missingValue) 58 | { 59 | this(maxKeyLength, initialSize, CharSequenceMap::defaultHash, 60 | missingValue, ByteBuffer::allocateDirect); 61 | } 62 | 63 | /** 64 | * Constructor for the map. 65 | * 66 | * @param maxKeyLength max length of any key 67 | * @param initialSize initial size of the map 68 | * @param missingValue value to return if the key is not present 69 | * @param bufferFactory factory method for creating new {@code ByteBuffer} instances 70 | */ 71 | public CharSequenceMap( 72 | final int maxKeyLength, final int initialSize, 73 | final long missingValue, final IntFunction bufferFactory) 74 | { 75 | this(maxKeyLength, initialSize, CharSequenceMap::defaultHash, 76 | missingValue, bufferFactory); 77 | } 78 | 79 | CharSequenceMap( 80 | final int maxKeyLength, final int initialSize, 81 | final ToIntFunction hash, final long missingValue, 82 | final IntFunction bufferFactory) 83 | { 84 | if (maxKeyLength > (1 << 28)) 85 | { 86 | throw new IllegalArgumentException("Key too long"); 87 | } 88 | totalEntryCount = BitUtil.findNextPositivePowerOfTwo(initialSize); 89 | entryMask = totalEntryCount - 1; 90 | entrySizeInBytes = (maxKeyLength * Character.BYTES) + (4 * Integer.BYTES); 91 | final long bufferSize = entrySizeInBytes * (long)totalEntryCount; 92 | if (bufferSize > ((long)Integer.MAX_VALUE) || totalEntryCount < 0) 93 | { 94 | throw new IllegalArgumentException("Requested buffer size too large"); 95 | } 96 | this.bufferFactory = bufferFactory; 97 | dataBuffer = this.bufferFactory.apply((int)bufferSize); 98 | entryCountToTriggerRehash = (int)(loadFactor * totalEntryCount); 99 | this.maxKeyLength = maxKeyLength; 100 | this.hash = hash; 101 | this.missingValue = missingValue; 102 | removeEntryHandler = new RemoveEntryHandler(entrySizeInBytes); 103 | } 104 | 105 | /** 106 | * {@inheritDoc}. 107 | */ 108 | public void put(final CharSequence value, final long id) 109 | { 110 | if (value.length() > maxKeyLength) 111 | { 112 | throw new IllegalArgumentException("Key too long"); 113 | } 114 | if (liveEntryCount > entryCountToTriggerRehash) 115 | { 116 | rehash(true); 117 | } 118 | final int hashValue = hash.applyAsInt(value); 119 | final int entryIndex = (hashValue & entryMask); 120 | 121 | final boolean existingEntryAt = isExistingEntryAt(value, entryIndex, hashValue); 122 | if (isEmptyEntrySlot(entryIndex) || existingEntryAt) 123 | { 124 | insertEntry(value, id, entryIndex, !existingEntryAt, hashValue); 125 | } 126 | else 127 | { 128 | for (int i = 1; i < totalEntryCount; i++) 129 | { 130 | int candidateIndex = (entryIndex + (i)); 131 | 132 | if (candidateIndex >= totalEntryCount) 133 | { 134 | candidateIndex -= totalEntryCount; 135 | } 136 | final boolean innerExistingEntryAt = isExistingEntryAt(value, candidateIndex, hashValue); 137 | if (isEmptyEntrySlot(candidateIndex) || innerExistingEntryAt) 138 | { 139 | insertEntry(value, id, candidateIndex, !innerExistingEntryAt, hashValue); 140 | return; 141 | } 142 | } 143 | 144 | put(value, id); 145 | } 146 | } 147 | 148 | /** 149 | * {@inheritDoc}. 150 | */ 151 | public long get(final CharSequence value) 152 | { 153 | return search(value, searchEntryHandler); 154 | } 155 | 156 | /** 157 | * {@inheritDoc}. 158 | */ 159 | public long remove(final CharSequence value) 160 | { 161 | return search(value, removeEntryHandler); 162 | } 163 | 164 | /** 165 | * {@inheritDoc}. 166 | */ 167 | public int size() 168 | { 169 | return liveEntryCount; 170 | } 171 | 172 | /** 173 | * {@inheritDoc}. 174 | */ 175 | public void rehash() 176 | { 177 | rehash(false); 178 | } 179 | 180 | private void insertEntry( 181 | final CharSequence value, final long id, 182 | final int entryIndex, final boolean isInsert, final int hashValue) 183 | { 184 | setValueLength(entryIndex, value.length(), dataBuffer); 185 | final int keyOffset = keyOffset(entryIndex); 186 | for (int i = 0; i < value.length(); i++) 187 | { 188 | dataBuffer.putChar(keyOffset + (i * Character.BYTES), value.charAt(i)); 189 | } 190 | setId(entryIndex, id, dataBuffer); 191 | setHash(entryIndex, hashValue, dataBuffer); 192 | if (isInsert) 193 | { 194 | liveEntryCount++; 195 | } 196 | } 197 | 198 | private boolean isEmptyEntrySlot(final int entryIndex) 199 | { 200 | return !isValuePresent(entryIndex, dataBuffer); 201 | } 202 | 203 | private void rehash(final boolean shouldResize) 204 | { 205 | final ByteBuffer oldBuffer = dataBuffer; 206 | final int oldEntryCount = totalEntryCount; 207 | 208 | if (shouldResize) 209 | { 210 | final int newSize = oldBuffer.capacity() * 2; 211 | if (newSize < 0) 212 | { 213 | throw new IllegalStateException( 214 | String.format( 215 | "Maximum map capacity exceeded. Entry count: %d, entrySize: %d, newSize: %d", 216 | size(), entrySizeInBytes, ((long)oldBuffer.capacity()) * 2)); 217 | } 218 | dataBuffer = bufferFactory.apply(newSize); 219 | totalEntryCount *= 2; 220 | entryMask = totalEntryCount - 1; 221 | entryCountToTriggerRehash = (int)(loadFactor * totalEntryCount); 222 | } 223 | else 224 | { 225 | dataBuffer = bufferFactory.apply(oldBuffer.capacity()); 226 | } 227 | 228 | liveEntryCount = 0; 229 | 230 | for (int i = 0; i < oldEntryCount; i++) 231 | { 232 | if (isValuePresent(i, oldBuffer)) 233 | { 234 | final long id = getId(i, oldBuffer); 235 | 236 | charBuffer.reset(oldBuffer, keyOffset(i), getValueLength(i, oldBuffer)); 237 | put(charBuffer, id); 238 | } 239 | } 240 | } 241 | 242 | private long search(final CharSequence value, final EntryHandler entryHandler) 243 | { 244 | final int hashValue = hash.applyAsInt(value); 245 | int entryIndex = (hashValue & entryMask); 246 | int entry = 0; 247 | while (entry < totalEntryCount) 248 | { 249 | if (isEmptyEntrySlot(entryIndex) && noDeletes) 250 | { 251 | break; 252 | } 253 | 254 | if (isExistingEntryAt(value, entryIndex, getHash(entryIndex, dataBuffer))) 255 | { 256 | final long storedId = getId(entryIndex, dataBuffer); 257 | entryHandler.onEntryFound(dataBuffer, entryIndex); 258 | return storedId; 259 | } 260 | 261 | entryIndex++; 262 | entryIndex = entryIndex & entryMask; 263 | entry++; 264 | } 265 | 266 | return missingValue; 267 | } 268 | 269 | private interface EntryHandler 270 | { 271 | void onEntryFound(ByteBuffer dataBuffer, int index); 272 | } 273 | 274 | private boolean isExistingEntryAt(final CharSequence value, final int entryIndex, final int hashValue) 275 | { 276 | final int keyOffset = keyOffset(entryIndex); 277 | final int hash = getHash(entryIndex, dataBuffer); 278 | if (hash != hashValue) 279 | { 280 | return false; 281 | } 282 | for (int i = 0; i < value.length(); i++) 283 | { 284 | if (dataBuffer.getChar(keyOffset + (i * Character.BYTES)) != value.charAt(i)) 285 | { 286 | return false; 287 | } 288 | } 289 | return true; 290 | } 291 | 292 | private static int defaultHash(final CharSequence value) 293 | { 294 | int hash = 0; 295 | for (int i = 0; i < value.length(); i++) 296 | { 297 | hash = (31 * hash) + value.charAt(i); 298 | } 299 | return hash; 300 | } 301 | 302 | private int byteOffset(final int entryIndex) 303 | { 304 | return entryIndex * entrySizeInBytes; 305 | } 306 | 307 | private boolean isValuePresent(final int entryIndex, final ByteBuffer dataBuffer) 308 | { 309 | return (0b1000_0000_0000_0000_0000_0000_0000_0000 & dataBuffer.getInt(byteOffset(entryIndex))) != 0; 310 | } 311 | 312 | private void setValueLength(final int entryIndex, final int valueLength, final ByteBuffer dataBuffer) 313 | { 314 | dataBuffer.putInt(byteOffset(entryIndex), 0b1000_0000_0000_0000_0000_0000_0000_0000 | valueLength); 315 | } 316 | 317 | private int getValueLength(final int entryIndex, final ByteBuffer dataBuffer) 318 | { 319 | return (0b0111_1111_1111_1111_1111_1111_1111_1111 & dataBuffer.getInt(byteOffset(entryIndex))); 320 | } 321 | 322 | private void setId(final int entryIndex, final long id, final ByteBuffer dataBuffer) 323 | { 324 | dataBuffer.putLong(byteOffset(entryIndex) + ID_OFFSET, id); 325 | } 326 | 327 | private long getId(final int entryIndex, final ByteBuffer dataBuffer) 328 | { 329 | return dataBuffer.getLong(byteOffset(entryIndex) + ID_OFFSET); 330 | } 331 | 332 | private void setHash(final int entryIndex, final int hashValue, final ByteBuffer dataBuffer) 333 | { 334 | dataBuffer.putInt(byteOffset(entryIndex) + HASH_OFFSET, hashValue); 335 | } 336 | 337 | private int getHash(final int entryIndex, final ByteBuffer dataBuffer) 338 | { 339 | return dataBuffer.getInt(byteOffset(entryIndex) + HASH_OFFSET); 340 | } 341 | 342 | private int keyOffset(final int entryIndex) 343 | { 344 | return byteOffset(entryIndex) + KEY_OFFSET; 345 | } 346 | 347 | private static final class CharArrayCharSequence implements CharSequence 348 | { 349 | private ByteBuffer dataBuffer; 350 | private int offset; 351 | private int length; 352 | 353 | void reset(final ByteBuffer dataBuffer, final int offset, final int length) 354 | { 355 | this.dataBuffer = dataBuffer; 356 | this.offset = offset; 357 | this.length = length; 358 | } 359 | 360 | @Override 361 | public int length() 362 | { 363 | return length; 364 | } 365 | 366 | @Override 367 | public char charAt(final int i) 368 | { 369 | return dataBuffer.getChar((offset) + (i * Character.BYTES)); 370 | } 371 | 372 | @Override 373 | public CharSequence subSequence(final int offset, final int length) 374 | { 375 | throw new UnsupportedOperationException(); 376 | } 377 | 378 | @Override 379 | public String toString() 380 | { 381 | final StringBuilder builder = new StringBuilder(); 382 | for (int i = 0; i < length; i++) 383 | { 384 | builder.append(charAt(i)); 385 | } 386 | return builder.toString(); 387 | } 388 | } 389 | 390 | private final class RemoveEntryHandler implements EntryHandler 391 | { 392 | private final int longsInEntry; 393 | private final int remainingBytesInEntry; 394 | 395 | private RemoveEntryHandler(final int entrySizeInBytes) 396 | { 397 | longsInEntry = entrySizeInBytes / Long.BYTES; 398 | remainingBytesInEntry = entrySizeInBytes & (Long.BYTES - 1); 399 | } 400 | 401 | @Override 402 | public void onEntryFound(final ByteBuffer buffer, final int entryIndex) 403 | { 404 | int byteOffset = byteOffset(entryIndex); 405 | for (int i = 0; i < longsInEntry; i++) 406 | { 407 | buffer.putLong(byteOffset, 0L); 408 | byteOffset += Long.BYTES; 409 | } 410 | for (int i = 0; i < remainingBytesInEntry; i++) 411 | { 412 | buffer.put(byteOffset++, (byte)0); 413 | } 414 | liveEntryCount--; 415 | noDeletes = false; 416 | } 417 | } 418 | } --------------------------------------------------------------------------------