├── .gitignore ├── README.md ├── pom-package.xml ├── pom.xml └── src ├── main └── java │ └── com │ └── nanosai │ └── streamops │ ├── StreamOps.java │ ├── StreamOpsException.java │ ├── navigation │ ├── RecordIterator.java │ └── StreamNavigator.java │ ├── rion │ ├── RionUtil.java │ └── StreamOpsRionFieldTypes.java │ ├── storage │ ├── StreamStorageFactory.java │ └── file │ │ ├── IAppendListener.java │ │ ├── IOutputStreamFactory.java │ │ ├── IRecordProcessor.java │ │ ├── StreamStorageBlockFS.java │ │ ├── StreamStorageFS.java │ │ ├── StreamStorageFSOld.java │ │ ├── StreamStorageRootFS.java │ │ └── index │ │ └── StreamStorageIndexFS.java │ └── util │ ├── FileUtil.java │ └── HexUtil.java └── test └── java └── com └── nanosai └── streamops ├── examples ├── FullExternalStreamIterationExample.java ├── FullInternalStreamIterationExample.java ├── RecordIterationExample.java ├── StreamNavigatorExample.java ├── StreamOpsExamples.java ├── StreamStorageFSExample.java └── ecommerce │ ├── Customer.java │ ├── ECommerceReport.java │ ├── ECommerceStreamIterationExample.java │ ├── Product.java │ ├── ProductRevenueSum.java │ └── StreamWriteExample.java ├── navigation ├── RecordIteratorTest.java └── StreamIteratorTest.java └── storage └── file ├── StreamStorageFSOldTest.java ├── StreamStorageFSTest.java ├── StreamStorageRootFSTest.java └── index └── StreamStorageIndexFSTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | target/ 3 | *.iml 4 | 5 | data/ 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Stream Ops for Java 2 | 3 | - [Introduction](#introduction) 4 | - [Stream Ops Tutorial](#stream-ops-tutorial) 5 | - [The 1 Billion Records Per Second Challenge](#1brs) 6 | - [License](#license) 7 | - [Getting Started](#getting-started) 8 | - [Maven Dependency](#maven-dependency) 9 | - [Dependencies](#dependencies) 10 | 11 | 12 | 13 | 14 | # Introduction 15 | Stream Ops for Java is a fully embeddable data streaming engine and stream processing API for Java. 16 | The data streaming engine and the stream processing API can be used together, or separately. 17 | 18 | Stream Ops addresses the same use cases as Kafka, but with some significant design differences which gives you 19 | a higher degree of architectural freedom. You can embed Stream Ops in a mobile app, desktop app, microservice, 20 | web app backend, and possibly also inside serverless functions like AWS Lambda etc. (we are still looking into 21 | how this can be done) and Google App Engine apps. 22 | 23 | Stream Ops design differences also supports a wider set of use cases than Kafka does out-of-the-box. We will 24 | describe these differences as the code that provides them is cleaned up and released. 25 | 26 | Stream Ops is developed by Nanosai: [http://nanosai.com](https://nanosai.com) 27 | 28 | 29 | 30 | 31 | # Stream Ops Tutorial 32 | We have a Stream Ops tutorial in progress here: 33 | 34 | [http://tutorials.jenkov.com/stream-ops-java/index.html](http://tutorials.jenkov.com/stream-ops-java/index.html) 35 | 36 | 37 | 38 | 39 | 40 | 41 | # The 1 Billion Records Per Second Challenge 42 | We (Nanosai) will try to get Stream Ops to be able to process 1 billion records per second. You can read more about 43 | that challenge at [The 1BRS Challenge](https://nanosai.com/1brs-challenge). 44 | 45 | 46 | 47 | 48 | # License 49 | The intention is to release Stream Ops under the Apache License 2.0 . We need to discuss this a bit further internally 50 | in Nanosai, but in any case, it will be something close to that. 51 | 52 | 53 | 54 | 55 | # Getting Started 56 | 57 | The easiest way to use Stream Ops is via Maven. See the Maven dependency for Stream Ops in the next section. 58 | 59 | Alternatively you can clone Stream Ops and build it yourself. Stream Ops have 2 small dependencies which you may 60 | (or may not) have to clone too. That depends on what you are trying to do. The dependencies are also available 61 | from the central Maven repository though, so it is not necessary to clone and build them. Not unless you want 62 | to compile them all using a different Java version than the Java version they are currently built with. 63 | 64 | 65 | 66 | # Maven Dependency 67 | 68 | If you want to use Stream Ops with Maven, the Maven dependency for Stream Ops looks like this: 69 | 70 | 71 | com.nanosai 72 | stream-ops 73 | 0.7.0 74 | 75 | 76 | Remember to substitute the version number with the version of Stream Ops you want to use. You can see the 77 | version history of Stream Ops further down this page. 78 | 79 | 80 | 81 | 82 | # Dependencies 83 | Stream Ops depends on these two other Nanosai projects: 84 | 85 | - [Mem Ops for Java](https://github.com/nanosai/mem-ops-java) 86 | - [RION Ops for Java](https://github.com/nanosai/rion-ops-java) 87 | 88 | 89 | 90 | 91 | 92 | # Version History 93 | 94 | | Version | Java Version | Change | 95 | |---------|--------------|--------| 96 | | 0.7.0 | Java 8 | First release | 97 | 98 | 99 | -------------------------------------------------------------------------------- /pom-package.xml: -------------------------------------------------------------------------------- 1 | 5 | 4.0.0 6 | 7 | com.nanosai 8 | stream-ops 9 | 0.6.4 10 | jar 11 | 12 | Stream Ops for Java - Fat JAR Packaging 13 | 14 | Stream Ops for Java is an open source toolkit containing an embeddable data streaming engine and a stream processing API. 15 | 16 | 17 | https://github.com/nanosai/stream-ops-java 18 | 19 | 20 | 21 | The Apache License, Version 2.0 22 | http://www.apache.org/licenses/LICENSE-2.0.txt 23 | 24 | 25 | 26 | 27 | 28 | Jakob Jenkov 29 | jakob@jenkov.com 30 | Jenkov Aps (on behalf of Nanosai.com) 31 | http://www.nanosai.com 32 | 33 | 34 | 35 | 36 | scm:git:git://github.com/nanosai/stream-ops-java.git 37 | scm:git:ssh://github.com:nanosai/rion-ops-java.git 38 | http://github.com/nanosai/rion-ops-java/tree/master 39 | 40 | 41 | 42 | 43 | 44 | 45 | com.nanosai 46 | rion-ops 47 | 0.5.2 48 | 49 | 50 | 51 | org.junit.jupiter 52 | junit-jupiter 53 | 5.4.2 54 | test 55 | 56 | 57 | 58 | 59 | 60 | 61 | ossrh 62 | https://oss.sonatype.org/content/repositories/snapshots 63 | 64 | 65 | ossrh 66 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 67 | 68 | 69 | 70 | 71 | stream-ops 72 | 73 | 74 | 75 | org.apache.maven.plugins 76 | maven-compiler-plugin 77 | 3.6.1 78 | 79 | 1.8 80 | 1.8 81 | 82 | 83 | 84 | 85 | org.apache.maven.plugins 86 | maven-assembly-plugin 87 | 3.1.1 88 | 89 | 90 | jar-with-dependencies 91 | 92 | 93 | 94 | 95 | make-assembly 96 | package 97 | 98 | single 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | UTF-8 108 | 109 | 113 | 114 | 115 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 5 | 4.0.0 6 | 7 | com.nanosai 8 | stream-ops 9 | 0.7.0 10 | jar 11 | 12 | Stream Ops for Java 13 | 14 | Stream Ops for Java is an open source toolkit containing an embeddable data streaming engine and a stream processing API. 15 | 16 | 17 | https://github.com/nanosai/stream-ops-java 18 | 19 | 20 | 21 | The Apache License, Version 2.0 22 | http://www.apache.org/licenses/LICENSE-2.0.txt 23 | 24 | 25 | 26 | 27 | 28 | Jakob Jenkov 29 | jakob@jenkov.com 30 | Jenkov Aps (on behalf of Nanosai.com) 31 | http://www.nanosai.com 32 | 33 | 34 | 35 | 36 | scm:git:git://github.com/nanosai/stream-ops-java.git 37 | scm:git:ssh://github.com:nanosai/stream-ops-java.git 38 | http://github.com/nanosai/stream-ops-java/tree/master 39 | 40 | 41 | 42 | 43 | 44 | 45 | com.nanosai 46 | rion-ops 47 | 0.5.2 48 | 49 | 50 | 51 | org.junit.jupiter 52 | junit-jupiter 53 | 5.4.2 54 | test 55 | 56 | 57 | 58 | 59 | 60 | 61 | ossrh 62 | https://oss.sonatype.org/content/repositories/snapshots 63 | 64 | 65 | ossrh 66 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | org.apache.maven.plugins 75 | maven-compiler-plugin 76 | 3.6.1 77 | 78 | 1.8 79 | 1.8 80 | 81 | 82 | 83 | 84 | 85 | org.sonatype.plugins 86 | nexus-staging-maven-plugin 87 | 1.6.8 88 | true 89 | 90 | ossrh 91 | https://oss.sonatype.org/ 92 | true 93 | 94 | 95 | 96 | 97 | 98 | 99 | org.apache.maven.plugins 100 | maven-source-plugin 101 | 2.2.1 102 | 103 | 104 | attach-sources 105 | 106 | jar-no-fork 107 | 108 | 109 | 110 | 111 | 112 | 113 | org.apache.maven.plugins 114 | maven-javadoc-plugin 115 | 2.9.1 116 | 117 | 118 | attach-javadocs 119 | 120 | jar 121 | 122 | 123 | 124 | 125 | 126 | 127 | org.apache.maven.plugins 128 | maven-gpg-plugin 129 | 1.6 130 | 131 | 132 | sign-artifacts 133 | verify 134 | 135 | sign 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | UTF-8 147 | 148 | 152 | 153 | 154 | -------------------------------------------------------------------------------- /src/main/java/com/nanosai/streamops/StreamOps.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops; 2 | 3 | import com.nanosai.streamops.navigation.RecordIterator; 4 | import com.nanosai.streamops.storage.StreamStorageFactory; 5 | 6 | public class StreamOps { 7 | 8 | 9 | public static RecordIterator createRecordIterator(){ 10 | return new RecordIterator(); 11 | } 12 | 13 | public static StreamStorageFactory createStreamStorageFactory() { 14 | return new StreamStorageFactory(); 15 | } 16 | 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/nanosai/streamops/StreamOpsException.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops; 2 | 3 | public class StreamOpsException extends RuntimeException { 4 | 5 | public StreamOpsException() { 6 | super(); 7 | } 8 | 9 | public StreamOpsException(String message) { 10 | super(message); 11 | } 12 | 13 | public StreamOpsException(String message, Throwable cause) { 14 | super(message, cause); 15 | } 16 | 17 | public StreamOpsException(Throwable cause) { 18 | super(cause); 19 | } 20 | 21 | protected StreamOpsException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { 22 | super(message, cause, enableSuppression, writableStackTrace); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/nanosai/streamops/navigation/RecordIterator.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.navigation; 2 | 3 | import com.nanosai.rionops.rion.RionFieldTypes; 4 | import com.nanosai.rionops.rion.read.RionReader; 5 | import com.nanosai.streamops.rion.StreamOpsRionFieldTypes; 6 | import com.nanosai.streamops.storage.file.StreamStorageFS; 7 | 8 | public class RecordIterator { 9 | 10 | public long offset = -1; 11 | 12 | protected RionReader rionReader = new RionReader(); 13 | 14 | public RecordIterator setSource(byte[] source, int sourceOffset, int sourceLength){ 15 | this.rionReader.setSource(source, sourceOffset, sourceLength); 16 | return this; 17 | } 18 | 19 | public RecordIterator resetSource(int sourceOffset, int sourceLength){ 20 | this.rionReader.setSource(this.rionReader.source, sourceOffset, sourceLength); 21 | return this; 22 | } 23 | 24 | public boolean hasNext() { 25 | return this.rionReader.hasNext(); 26 | } 27 | 28 | public RionReader getRionReader() { 29 | return this.rionReader; 30 | } 31 | 32 | public void next() { 33 | this.rionReader.nextParse(); 34 | this.offset++; 35 | 36 | while(this.rionReader.fieldType == RionFieldTypes.EXTENDED && 37 | this.rionReader.fieldTypeExtended == StreamOpsRionFieldTypes.OFFSET_EXTENDED_RION_TYPE){ 38 | 39 | //read offset value of extended field as long 40 | this.offset = this.rionReader.readInt64(); 41 | this.rionReader.nextParse(); 42 | } 43 | 44 | // if field type == extended && extended field type == offset 45 | // read offset into iterators offset variable. 46 | // nextParse() 47 | // repeat until not field type == extended && field type == offset 48 | // else increment offset variable by 1. 49 | } 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/nanosai/streamops/navigation/StreamNavigator.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.navigation; 2 | 3 | import com.nanosai.streamops.StreamOpsException; 4 | import com.nanosai.streamops.storage.file.StreamStorageBlockFS; 5 | import com.nanosai.streamops.storage.file.StreamStorageFS; 6 | 7 | import java.io.IOException; 8 | 9 | public class StreamNavigator { 10 | 11 | 12 | public void navigateTo(StreamStorageFS streamStorageFS, long toOffset, RecordIterator iterator){ 13 | 14 | // 1. Find the correct block file. 15 | StreamStorageBlockFS closestStorageBlock = streamStorageFS.getStorageBlockContainingOffset(toOffset); 16 | if(closestStorageBlock == null){ 17 | //no storage block contains the request toOffset - stop navigation attempt. 18 | return; 19 | } 20 | 21 | System.out.println(closestStorageBlock.getFileName()); 22 | 23 | // 2. Read the whole block into byte array. 24 | if(iterator.getRionReader().source.length < closestStorageBlock.getFileLength()){ 25 | throw new StreamOpsException("Buffer inside RecordIterator too small to hold content of stream block file. " + 26 | "Buffer was " + iterator.getRionReader().source.length + " bytes in length. " + 27 | "Stream block file was " + closestStorageBlock.getFileLength() + " bytes in length. " 28 | ); 29 | } 30 | int bytesRead = 0; 31 | try { 32 | bytesRead = streamStorageFS.readFromBlock(closestStorageBlock, 0, iterator.getRionReader().source, 0, (int) closestStorageBlock.getFileLength()); 33 | } catch (IOException e) { 34 | throw new StreamOpsException("Error reading stream block file [" + closestStorageBlock.getFileName() + "]: " 35 | + e.getMessage(), e); 36 | } 37 | 38 | iterator.resetSource(0, bytesRead); 39 | 40 | // 3. Iterate through RecordIterator until toOffset is found 41 | 42 | while(iterator.hasNext() && iterator.offset != toOffset){ 43 | iterator.next(); 44 | } 45 | 46 | 47 | } 48 | 49 | /* 50 | protected StreamStorageBlockFS findClosestStorageBlock(StreamStorageFS streamStorageFS, long toOffset) { 51 | List storageBlocks = streamStorageFS.getStorageBlocks(); 52 | StreamStorageBlockFS closestStorageBlock = null; 53 | for(int i=0; i storageBlocks = new ArrayList<>(); 26 | 27 | private StreamStorageBlockFS latestBlock = null; 28 | private OutputStream latestBlockOutputStream = null; 29 | 30 | private long storageFileBlockMaxSize = 8 * 1024 * 1024; 31 | 32 | public long nextRecordOffset = 0; 33 | protected byte[] offsetRionBuffer = new byte[16]; 34 | 35 | private IOutputStreamFactory outputStreamFactory = null; 36 | 37 | private IAppendListener appendListener = null; 38 | 39 | public StreamStorageFS(String streamId, String rootDirPath) throws IOException { 40 | this.streamId = streamId; 41 | this.rootDirPath = rootDirPath; 42 | 43 | syncFromDisk(); 44 | } 45 | 46 | public StreamStorageFS(String streamId, String rootDirPath, long storageFileBlockMaxSize) throws IOException { 47 | this.streamId = streamId; 48 | this.rootDirPath = rootDirPath; 49 | this.storageFileBlockMaxSize = storageFileBlockMaxSize; 50 | 51 | syncFromDisk(); 52 | } 53 | 54 | public StreamStorageFS setOutputStreamFactory(IOutputStreamFactory outputStreamFactory) { 55 | this.outputStreamFactory = outputStreamFactory; 56 | return this; 57 | } 58 | 59 | public StreamStorageFS setAppendListener(IAppendListener listener){ 60 | this.appendListener = listener; 61 | return this; 62 | } 63 | 64 | public String getStreamId() { 65 | return streamId; 66 | } 67 | 68 | public String getRootDirPath() { 69 | return rootDirPath; 70 | } 71 | 72 | public long getStorageFileBlockMaxSize() { 73 | return this.storageFileBlockMaxSize; 74 | } 75 | 76 | public StreamStorageBlockFS getLatestBlock() { 77 | return this.latestBlock; 78 | } 79 | 80 | public List getStorageBlocks() { 81 | return storageBlocks; 82 | } 83 | 84 | public StreamStorageBlockFS getStorageBlockContainingOffset(long offset) { 85 | //0. check if the requested start offset even exists in the stream 86 | if(offset >= nextRecordOffset){ 87 | return null; //No, it doesn't - nothing to iterate to. 88 | } 89 | 90 | List storageBlocks = this.storageBlocks; 91 | StreamStorageBlockFS closestStorageBlock = null; 92 | for(int i=0; i() { 131 | @Override 132 | public int compare(StreamStorageBlockFS block0, StreamStorageBlockFS block1) { 133 | if (block0.getFirstOffset() < block1.getFirstOffset()) { 134 | return -1; 135 | } 136 | if (block0.getFirstOffset() > block1.getFirstOffset()) { 137 | return 1; 138 | } 139 | return 0; 140 | } 141 | }); 142 | 143 | if(this.storageBlocks.size() > 0) { 144 | this.latestBlock = storageBlocks.get(storageBlocks.size()-1); 145 | //todo find latest offset in that storage block, and set nextRecordOffset to that latest offset + 1 146 | byte[] latestBlockBytes = new byte[(int) this.latestBlock.getFileLength()]; 147 | readFromBlock(this.latestBlock, 0, latestBlockBytes, 0, latestBlockBytes.length); 148 | 149 | RecordIterator recordIterator = new RecordIterator(); 150 | recordIterator.setSource(latestBlockBytes, 0, latestBlockBytes.length); 151 | 152 | while(recordIterator.hasNext()){ 153 | recordIterator.next(); 154 | } 155 | this.nextRecordOffset = recordIterator.offset + 1; 156 | 157 | } else { 158 | this.latestBlock = new StreamStorageBlockFS(createNewStreamBlockFileName(), 0, 0); 159 | //openForAppend(); 160 | //appendOffset(); 161 | //closeForAppend(); 162 | this.storageBlocks.add(this.latestBlock); 163 | } 164 | 165 | } 166 | 167 | public void openForAppend() throws IOException { 168 | String latestBlockFilePath = this.rootDirPath + "/" + this.latestBlock.getFileName(); 169 | this.latestBlockOutputStream = 170 | this.outputStreamFactory != null ? 171 | this.outputStreamFactory.createOutputStream(latestBlockFilePath) : 172 | new FileOutputStream(latestBlockFilePath, true); 173 | } 174 | 175 | /* 176 | public void incNextRecordOffset() { 177 | this.nextRecordOffset++; 178 | } 179 | */ 180 | 181 | /* 182 | public void appendOffset() throws IOException { 183 | int lengthOfOffset = (255 & RionUtil.lengthOfInt64Value(this.nextRecordOffset)); 184 | int lengthOfLengthOfOffset = (255 & RionUtil.lengthOfInt64Value(lengthOfOffset)); 185 | 186 | byte rionExtendedFieldLeadByte = (byte) ((255 & (15 << 4)) | lengthOfLengthOfOffset); 187 | byte rionStreamOffsetType = (byte) StreamOpsRionFieldTypes.OFFSET_EXTENDED_RION_TYPE; 188 | 189 | this.offsetRionBuffer[0] = rionExtendedFieldLeadByte; 190 | this.offsetRionBuffer[1] = rionStreamOffsetType; 191 | this.offsetRionBuffer[2] = (byte) (255 & lengthOfOffset); 192 | 193 | int index = 3; 194 | for(int i=(lengthOfOffset-1)*8; i >= 0; i-=8){ 195 | this.offsetRionBuffer[index++] = (byte) (255 & (this.nextRecordOffset >> i)); 196 | } 197 | this.latestBlockOutputStream.write(this.offsetRionBuffer, 0, lengthOfOffset + OFFSET_FIELD_LEAD_TYPE_LENGTH_BYTES_LENGTH); 198 | this.latestBlock.fileLength += lengthOfOffset + OFFSET_FIELD_LEAD_TYPE_LENGTH_BYTES_LENGTH; 199 | } 200 | */ 201 | 202 | 203 | public void appendRecord(byte[] source, int sourceOffset, int sourceLength) throws IOException { 204 | if(this.latestBlock.fileLength + sourceLength > storageFileBlockMaxSize){ 205 | //create a new latest block 206 | closeForAppend(); 207 | StreamStorageBlockFS newStreamStorageBlockFS = new StreamStorageBlockFS(createNewStreamBlockFileName(), 0, this.nextRecordOffset); 208 | this.storageBlocks.add(newStreamStorageBlockFS); 209 | this.latestBlock = newStreamStorageBlockFS; 210 | openForAppend(); 211 | } 212 | 213 | /* 214 | if(this.latestBlock.fileLength == 0){ 215 | appendOffset(); 216 | } 217 | */ 218 | 219 | // appendRecord data to file 220 | this.latestBlockOutputStream.write(source, sourceOffset, sourceLength); 221 | this.latestBlock.fileLength += sourceLength; 222 | this.nextRecordOffset++; 223 | 224 | if(this.appendListener != null) { 225 | this.appendListener.recordAppended(source, sourceOffset, sourceLength,this.nextRecordOffset - 1); 226 | } 227 | 228 | } 229 | 230 | 231 | public void flush() throws IOException { 232 | this.latestBlockOutputStream.flush(); 233 | } 234 | 235 | public void closeForAppend() throws IOException { 236 | this.latestBlockOutputStream.close(); 237 | } 238 | 239 | 240 | public int readFromBlock(StreamStorageBlockFS streamStorageBlockFS, long fromByteOffset, byte[] dest, int destOffset, int length) throws IOException { 241 | 242 | try(RandomAccessFile randomAccessFile = new RandomAccessFile(this.rootDirPath + "/" + streamStorageBlockFS.getFileName(), "r")){ 243 | randomAccessFile.seek(fromByteOffset); 244 | 245 | return randomAccessFile.read(dest, destOffset, length); 246 | } 247 | } 248 | 249 | public int readFromBlockWithIndex(int streamStorageBlockIndex, long fromByteOffset, byte[] dest, int destOffset, int length) throws IOException { 250 | if(streamStorageBlockIndex >= this.storageBlocks.size()){ 251 | return 0; 252 | } 253 | 254 | return readFromBlock(this.storageBlocks.get(streamStorageBlockIndex), fromByteOffset, dest, destOffset, length); 255 | } 256 | 257 | public int readFromBlockContainingOffset(long recordOffset, long fromByteOffset, byte[] dest, int destOffset, int length) throws IOException { 258 | 259 | StreamStorageBlockFS storageBlockContainingOffset = getStorageBlockContainingOffset(recordOffset); 260 | if(storageBlockContainingOffset == null){ 261 | return 0; 262 | } 263 | 264 | return readFromBlock(storageBlockContainingOffset, fromByteOffset, dest, destOffset, length); 265 | } 266 | 267 | 268 | 269 | public void iterate(byte[] recordBuffer, IRecordProcessor recordProcessor) throws IOException { 270 | 271 | RionReader rionReader = new RionReader(); //todo reuse a RionReader instead? 272 | long recordOffset = 0; 273 | 274 | for(int i=0, n = getStorageBlocks().size(); i= this.nextRecordOffset){ 313 | return; // no records from given fromOffset 314 | } 315 | 316 | //find block file containing fromOffset 317 | int blockIndex = 0; 318 | int noOfBlocks = this.storageBlocks.size(); 319 | while(blockIndex < noOfBlocks && fromOffset > this.storageBlocks.get(blockIndex).getFirstOffset()){ 320 | blockIndex++; 321 | } 322 | blockIndex--; 323 | 324 | 325 | RionReader rionReader = new RionReader(); //todo reuse a RionReader instead? 326 | long recordOffset = 0; 327 | 328 | for(int i=0, n = getStorageBlocks().size(); i= fromOffset) { 338 | boolean continueIteration = recordProcessor.process(recordOffset, rionReader); 339 | if(!continueIteration) { 340 | return; 341 | } 342 | } 343 | 344 | //after processing this record, increment recordOffset for next record - in case no explicit record offset field is found 345 | recordOffset++; 346 | 347 | /* 348 | if(rionReader.fieldType == RionFieldTypes.EXTENDED && 349 | rionReader.fieldTypeExtended == StreamOpsRionFieldTypes.OFFSET_EXTENDED_RION_TYPE){ 350 | //this is a record offset field - read record offset value as int 64 (Java long) 351 | recordOffset = rionReader.readInt64(); 352 | } else { 353 | //this is a record field - read record value 354 | if(recordOffset >= fromOffset) { 355 | boolean continueIteration = recordProcessor.process(recordOffset, rionReader); 356 | if(!continueIteration) { 357 | return; 358 | } 359 | } 360 | 361 | //after processing this record, increment recordOffset for next record - in case no explicit record offset field is found 362 | recordOffset++; 363 | } 364 | */ 365 | } 366 | } 367 | } 368 | 369 | 370 | 371 | 372 | 373 | public static class ReadResult { 374 | public int firstRecordByteOffset; // byte offset into block of bytes where first record with requested offset starts. 375 | public long lastRecordOffset; // record offset in stream of last record read into block 376 | 377 | } 378 | } 379 | -------------------------------------------------------------------------------- /src/main/java/com/nanosai/streamops/storage/file/StreamStorageFSOld.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.storage.file; 2 | 3 | import com.nanosai.rionops.rion.RionFieldTypes; 4 | import com.nanosai.rionops.rion.read.RionReader; 5 | import com.nanosai.streamops.navigation.RecordIterator; 6 | import com.nanosai.streamops.rion.RionUtil; 7 | import com.nanosai.streamops.rion.StreamOpsRionFieldTypes; 8 | import com.nanosai.streamops.util.HexUtil; 9 | 10 | import java.io.*; 11 | import java.util.ArrayList; 12 | import java.util.Collections; 13 | import java.util.Comparator; 14 | import java.util.List; 15 | 16 | public class StreamStorageFSOld { 17 | public static final int OFFSET_FIELD_LEAD_TYPE_LENGTH_BYTES_LENGTH = 3; 18 | 19 | //public static final int OFFSET_EXTENDED_RION_TYPE = 1; 20 | 21 | private String streamId = null; 22 | private String rootDirPath = null; 23 | 24 | private List storageBlocks = new ArrayList<>(); 25 | 26 | private StreamStorageBlockFS latestBlock = null; 27 | private OutputStream latestBlockOutputStream = null; 28 | 29 | private long storageFileBlockMaxSize = 8 * 1024 * 1024; 30 | 31 | public long nextRecordOffset = 0; 32 | protected byte[] offsetRionBuffer = new byte[16]; 33 | 34 | private IOutputStreamFactory outputStreamFactory = null; 35 | 36 | private IAppendListener appendListener = null; 37 | 38 | public StreamStorageFSOld(String streamId, String rootDirPath) throws IOException { 39 | this.streamId = streamId; 40 | this.rootDirPath = rootDirPath; 41 | 42 | syncFromDisk(); 43 | } 44 | 45 | public StreamStorageFSOld(String streamId, String rootDirPath, long storageFileBlockMaxSize) throws IOException { 46 | this.streamId = streamId; 47 | this.rootDirPath = rootDirPath; 48 | this.storageFileBlockMaxSize = storageFileBlockMaxSize; 49 | 50 | syncFromDisk(); 51 | } 52 | 53 | public StreamStorageFSOld setOutputStreamFactory(IOutputStreamFactory outputStreamFactory) { 54 | this.outputStreamFactory = outputStreamFactory; 55 | return this; 56 | } 57 | 58 | public StreamStorageFSOld setAppendListener(IAppendListener listener){ 59 | this.appendListener = listener; 60 | return this; 61 | } 62 | 63 | public String getStreamId() { 64 | return streamId; 65 | } 66 | 67 | public String getRootDirPath() { 68 | return rootDirPath; 69 | } 70 | 71 | public long getStorageFileBlockMaxSize() { 72 | return this.storageFileBlockMaxSize; 73 | } 74 | 75 | public StreamStorageBlockFS getLatestBlock() { 76 | return this.latestBlock; 77 | } 78 | 79 | public List getStorageBlocks() { 80 | return storageBlocks; 81 | } 82 | 83 | public StreamStorageBlockFS getStorageBlockContainingOffset(long offset) { 84 | //0. check if the requested start offset even exists in the stream 85 | if(offset >= nextRecordOffset){ 86 | return null; //No, it doesn't - nothing to iterate to. 87 | } 88 | 89 | List storageBlocks = this.storageBlocks; 90 | StreamStorageBlockFS closestStorageBlock = null; 91 | for(int i=0; i() { 125 | @Override 126 | public int compare(StreamStorageBlockFS block0, StreamStorageBlockFS block1) { 127 | if (block0.getFirstOffset() < block1.getFirstOffset()) { 128 | return -1; 129 | } 130 | if (block0.getFirstOffset() > block1.getFirstOffset()) { 131 | return 1; 132 | } 133 | return 0; 134 | } 135 | }); 136 | 137 | if(this.storageBlocks.size() > 0) { 138 | this.latestBlock = storageBlocks.get(storageBlocks.size()-1); 139 | //todo find latest offset in that storage block, and set nextRecordOffset to that latest offset + 1 140 | byte[] latestBlockBytes = new byte[(int) this.latestBlock.getFileLength()]; 141 | readFromBlock(this.latestBlock, 0, latestBlockBytes, 0, latestBlockBytes.length); 142 | 143 | RecordIterator recordIterator = new RecordIterator(); 144 | recordIterator.setSource(latestBlockBytes, 0, latestBlockBytes.length); 145 | 146 | while(recordIterator.hasNext()){ 147 | recordIterator.next(); 148 | } 149 | this.nextRecordOffset = recordIterator.offset + 1; 150 | 151 | } else { 152 | this.latestBlock = new StreamStorageBlockFS(createNewStreamBlockFileName(), 0, 0); 153 | openForAppend(); 154 | appendOffset(); 155 | closeForAppend(); 156 | this.storageBlocks.add(this.latestBlock); 157 | } 158 | 159 | } 160 | 161 | public void openForAppend() throws IOException { 162 | String latestBlockFilePath = this.rootDirPath + "/" + this.latestBlock.getFileName(); 163 | this.latestBlockOutputStream = 164 | this.outputStreamFactory != null ? 165 | this.outputStreamFactory.createOutputStream(latestBlockFilePath) : 166 | new FileOutputStream(latestBlockFilePath, true); 167 | } 168 | 169 | public void incNextRecordOffset() { 170 | this.nextRecordOffset++; 171 | } 172 | 173 | public void appendOffset() throws IOException { 174 | int lengthOfOffset = (255 & RionUtil.lengthOfInt64Value(this.nextRecordOffset)); 175 | int lengthOfLengthOfOffset = (255 & RionUtil.lengthOfInt64Value(lengthOfOffset)); 176 | 177 | byte rionExtendedFieldLeadByte = (byte) ((255 & (15 << 4)) | lengthOfLengthOfOffset); 178 | byte rionStreamOffsetType = (byte) StreamOpsRionFieldTypes.OFFSET_EXTENDED_RION_TYPE; 179 | 180 | this.offsetRionBuffer[0] = rionExtendedFieldLeadByte; 181 | this.offsetRionBuffer[1] = rionStreamOffsetType; 182 | this.offsetRionBuffer[2] = (byte) (255 & lengthOfOffset); 183 | 184 | int index = 3; 185 | for(int i=(lengthOfOffset-1)*8; i >= 0; i-=8){ 186 | this.offsetRionBuffer[index++] = (byte) (255 & (this.nextRecordOffset >> i)); 187 | } 188 | this.latestBlockOutputStream.write(this.offsetRionBuffer, 0, lengthOfOffset + OFFSET_FIELD_LEAD_TYPE_LENGTH_BYTES_LENGTH); 189 | this.latestBlock.fileLength += lengthOfOffset + OFFSET_FIELD_LEAD_TYPE_LENGTH_BYTES_LENGTH; 190 | } 191 | 192 | public void appendRecord(byte[] source, int sourceOffset, int sourceLength) throws IOException { 193 | if(this.latestBlock.fileLength + sourceLength > storageFileBlockMaxSize){ 194 | //create a new latest block 195 | closeForAppend(); 196 | StreamStorageBlockFS newStreamStorageBlockFS = new StreamStorageBlockFS(createNewStreamBlockFileName(), 0, this.nextRecordOffset); 197 | this.storageBlocks.add(newStreamStorageBlockFS); 198 | this.latestBlock = newStreamStorageBlockFS; 199 | openForAppend(); 200 | } 201 | if(this.latestBlock.fileLength == 0){ 202 | appendOffset(); 203 | } 204 | 205 | // appendRecord data to file 206 | this.latestBlockOutputStream.write(source, sourceOffset, sourceLength); 207 | this.latestBlock.fileLength += sourceLength; 208 | this.nextRecordOffset++; 209 | 210 | if(this.appendListener != null) { 211 | this.appendListener.recordAppended(source, sourceOffset, sourceLength,this.nextRecordOffset - 1); 212 | } 213 | 214 | } 215 | 216 | 217 | public void flush() throws IOException { 218 | this.latestBlockOutputStream.flush(); 219 | } 220 | 221 | public void closeForAppend() throws IOException { 222 | this.latestBlockOutputStream.close(); 223 | } 224 | 225 | 226 | public int readFromBlock(StreamStorageBlockFS streamStorageBlockFS, long fromByteOffset, byte[] dest, int destOffset, int length) throws IOException { 227 | 228 | try(RandomAccessFile randomAccessFile = new RandomAccessFile(this.rootDirPath + "/" + streamStorageBlockFS.getFileName(), "r")){ 229 | randomAccessFile.seek(fromByteOffset); 230 | 231 | return randomAccessFile.read(dest, destOffset, length); 232 | } 233 | } 234 | 235 | public int readFromBlockWithIndex(int streamStorageBlockIndex, long fromByteOffset, byte[] dest, int destOffset, int length) throws IOException { 236 | if(streamStorageBlockIndex >= this.storageBlocks.size()){ 237 | return 0; 238 | } 239 | 240 | return readFromBlock(this.storageBlocks.get(streamStorageBlockIndex), fromByteOffset, dest, destOffset, length); 241 | } 242 | 243 | public int readFromBlockContainingOffset(long recordOffset, long fromByteOffset, byte[] dest, int destOffset, int length) throws IOException { 244 | 245 | StreamStorageBlockFS storageBlockContainingOffset = getStorageBlockContainingOffset(recordOffset); 246 | if(storageBlockContainingOffset == null){ 247 | return 0; 248 | } 249 | 250 | return readFromBlock(storageBlockContainingOffset, fromByteOffset, dest, destOffset, length); 251 | } 252 | 253 | 254 | 255 | public void iterate(byte[] recordBuffer, IRecordProcessor recordProcessor) throws IOException { 256 | 257 | RionReader rionReader = new RionReader(); //todo reuse a RionReader instead? 258 | long recordOffset = 0; 259 | 260 | for(int i=0, n = getStorageBlocks().size(); i= this.nextRecordOffset){ 289 | return; // no records from given fromOffset 290 | } 291 | 292 | //find block file containing fromOffset 293 | int blockIndex = 0; 294 | int noOfBlocks = this.storageBlocks.size(); 295 | while(blockIndex < noOfBlocks && fromOffset > this.storageBlocks.get(blockIndex).getFirstOffset()){ 296 | blockIndex++; 297 | } 298 | blockIndex--; 299 | 300 | 301 | RionReader rionReader = new RionReader(); //todo reuse a RionReader instead? 302 | long recordOffset = 0; 303 | 304 | for(int i=0, n = getStorageBlocks().size(); i= fromOffset) { 319 | boolean continueIteration = recordProcessor.process(recordOffset, rionReader); 320 | if(!continueIteration) { 321 | return; 322 | } 323 | } 324 | 325 | //after processing this record, increment recordOffset for next record - in case no explicit record offset field is found 326 | recordOffset++; 327 | } 328 | } 329 | 330 | } 331 | 332 | 333 | 334 | 335 | 336 | } 337 | 338 | 339 | 340 | 341 | 342 | public static class ReadResult { 343 | public int firstRecordByteOffset; // byte offset into block of bytes where first record with requested offset starts. 344 | public long lastRecordOffset; // record offset in stream of last record read into block 345 | 346 | } 347 | } 348 | -------------------------------------------------------------------------------- /src/main/java/com/nanosai/streamops/storage/file/StreamStorageRootFS.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.storage.file; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | 6 | public class StreamStorageRootFS { 7 | 8 | private String rootDirPath = null; 9 | 10 | public StreamStorageRootFS(String rootDirPath){ 11 | this.rootDirPath = rootDirPath; 12 | new File(this.rootDirPath).mkdirs(); 13 | } 14 | 15 | 16 | public String getRootDirPath() { 17 | return this.rootDirPath; 18 | } 19 | 20 | 21 | public StreamStorageFS createStreamStorage(String streamId) throws IOException { 22 | String streamFileStorageRootDir = createStreamFileStorageRootDirPath(streamId); 23 | new File(streamFileStorageRootDir).mkdirs(); 24 | 25 | return new StreamStorageFS(streamId, streamFileStorageRootDir); 26 | } 27 | 28 | public StreamStorageFS createStreamStorage(String streamId, long storageFileBlockFileMaxSize) throws IOException { 29 | String streamFileStorageRootDir = createStreamFileStorageRootDirPath(streamId); 30 | new File(streamFileStorageRootDir).mkdirs(); 31 | 32 | return new StreamStorageFS(streamId, streamFileStorageRootDir, storageFileBlockFileMaxSize); 33 | } 34 | 35 | private String createStreamFileStorageRootDirPath(String streamId) { 36 | return this.rootDirPath + "/" + streamId; 37 | } 38 | 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/nanosai/streamops/storage/file/index/StreamStorageIndexFS.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.storage.file.index; 2 | 3 | public class StreamStorageIndexFS { 4 | 5 | private final long STREAM_FILE_BYTE_OFFSET_MASK = 0x00000000_FFFFFFFFL; 6 | 7 | private long offsetInterval = 1024; 8 | 9 | private long[] records = null; 10 | private int usedLength = 0; 11 | 12 | public StreamStorageIndexFS(int indexLength, long offsetInterval) { 13 | this.records = new long[indexLength]; 14 | this.offsetInterval = offsetInterval; 15 | } 16 | 17 | 18 | public void append(long streamRecordOffset, int streamFileStorageBlockIndex, long streamFileByteOffset) { 19 | if (streamRecordOffset % this.offsetInterval == 0) { 20 | long indexRecord = streamFileStorageBlockIndex; 21 | indexRecord <<= 32; 22 | indexRecord |= STREAM_FILE_BYTE_OFFSET_MASK & streamFileByteOffset; 23 | this.records[this.usedLength++] = indexRecord; 24 | } 25 | } 26 | 27 | 28 | public long lookup(long streamRecordOffset){ 29 | int closestRecordIndex = (int) (streamRecordOffset / this.offsetInterval); 30 | 31 | if(closestRecordIndex >= usedLength){ 32 | return -1; 33 | } 34 | 35 | return this.records[closestRecordIndex]; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/nanosai/streamops/util/FileUtil.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.util; 2 | 3 | import java.awt.image.DirectColorModel; 4 | import java.io.File; 5 | 6 | public class FileUtil { 7 | 8 | public static boolean deleteDir(String dirPath){ 9 | return deleteDir(new File(dirPath)); 10 | } 11 | 12 | public static boolean deleteDir(File dir){ 13 | File[] files = dir.listFiles(); 14 | if(files != null){ 15 | for(File file : files){ 16 | if(file.isDirectory()){ 17 | deleteDir(file); 18 | } else { 19 | file.delete(); 20 | } 21 | } 22 | } 23 | return dir.delete(); 24 | } 25 | 26 | public static boolean resetDir(File dir){ 27 | if(!deleteDir(dir)){ 28 | return false; 29 | } 30 | return dir.mkdirs(); 31 | 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/nanosai/streamops/util/HexUtil.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.util; 2 | 3 | public class HexUtil { 4 | 5 | public static String toHex(long offset) { 6 | StringBuilder builder = new StringBuilder(); 7 | 8 | for(int i=60; i >= 0; i-=4){ 9 | char digitChar='0'; 10 | long digit = offset >> i; 11 | digit &= 0xF; 12 | switch((int) digit) { 13 | case 0 : digitChar = '0'; break; 14 | case 1 : digitChar = '1'; break; 15 | case 2 : digitChar = '2'; break; 16 | case 3 : digitChar = '3'; break; 17 | case 4 : digitChar = '4'; break; 18 | case 5 : digitChar = '5'; break; 19 | case 6 : digitChar = '6'; break; 20 | case 7 : digitChar = '7'; break; 21 | case 8 : digitChar = '8'; break; 22 | case 9 : digitChar = '9'; break; 23 | case 10 : digitChar = 'A'; break; 24 | case 11 : digitChar = 'B'; break; 25 | case 12 : digitChar = 'C'; break; 26 | case 13 : digitChar = 'D'; break; 27 | case 14 : digitChar = 'E'; break; 28 | case 15 : digitChar = 'F'; break; 29 | } 30 | builder.append(digitChar); 31 | } 32 | 33 | return builder.toString(); 34 | } 35 | 36 | 37 | public static long fromHex(String hexNumber){ 38 | long returnValue = 0; 39 | 40 | for(int i=0; i < hexNumber.length(); i++){ 41 | char digit = hexNumber.charAt(i); 42 | 43 | long digitValue = 0; 44 | switch(digit){ 45 | case '0': digitValue = 0; break; 46 | case '1': digitValue = 1; break; 47 | case '2': digitValue = 2; break; 48 | case '3': digitValue = 3; break; 49 | case '4': digitValue = 4; break; 50 | case '5': digitValue = 5; break; 51 | case '6': digitValue = 6; break; 52 | case '7': digitValue = 7; break; 53 | case '8': digitValue = 8; break; 54 | case '9': digitValue = 9; break; 55 | case 'A': digitValue = 10; break; 56 | case 'B': digitValue = 11; break; 57 | case 'C': digitValue = 12; break; 58 | case 'D': digitValue = 13; break; 59 | case 'E': digitValue = 14; break; 60 | case 'F': digitValue = 15; break; 61 | } 62 | 63 | returnValue <<=4; 64 | returnValue |=digitValue; 65 | } 66 | 67 | return returnValue; 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/FullExternalStreamIterationExample.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples; 2 | 3 | import com.nanosai.rionops.rion.RionFieldTypes; 4 | import com.nanosai.rionops.rion.read.RionReader; 5 | import com.nanosai.rionops.rion.write.RionWriter; 6 | import com.nanosai.streamops.rion.StreamOpsRionFieldTypes; 7 | import com.nanosai.streamops.storage.file.StreamStorageFS; 8 | import com.nanosai.streamops.util.FileUtil; 9 | 10 | import java.io.File; 11 | import java.io.IOException; 12 | import java.util.GregorianCalendar; 13 | 14 | public class FullExternalStreamIterationExample { 15 | 16 | public static void main(String[] args) throws IOException { 17 | String streamId = "full-stream-iteration-example-1"; 18 | String streamDir = "data/" + streamId; 19 | FileUtil.resetDir(new File(streamDir)); 20 | 21 | byte[] rionRecord = new byte[1024]; 22 | RionWriter rionWriter = new RionWriter() 23 | .setNestedFieldStack(new int[16]) 24 | .setDestination(rionRecord,0); 25 | 26 | //create an order item record 27 | rionWriter.writeObjectBeginPush(2); 28 | rionWriter.writeInt64( 987654321); //orderItemId 29 | rionWriter.writeInt64(99999); //productId 30 | rionWriter.writeInt64(9999); //orderId 31 | rionWriter.writeInt64( 123); //customerId 32 | rionWriter.writeInt64( 99); //price - integer part (e.g. whole dollars) 33 | rionWriter.writeInt64(95); //price - fraction part (e.g. cents) 34 | rionWriter.writeInt64( 69); //cost - integer part (e.g. whole dollars) 35 | rionWriter.writeInt64(45); //cost - fraction part (e.g. cents) 36 | GregorianCalendar dateTime = new GregorianCalendar(); 37 | rionWriter.writeUtc( dateTime, 9); //date-time in UTC, 9 bytes = including milliseconds 38 | rionWriter.writeObjectEndPop(); 39 | 40 | //byte[] rionBytesRecord1 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 41 | 42 | StreamStorageFS streamStorage = new StreamStorageFS(streamId, streamDir, 1024); 43 | appendRecordsToStream(rionRecord, rionWriter.index, streamStorage); 44 | 45 | 46 | byte[] recordBuffer = new byte[(int) streamStorage.getStorageFileBlockMaxSize()]; 47 | RionReader rionReader = new RionReader(); 48 | long recordOffset = 0; 49 | 50 | for(int i=0, n = streamStorage.getStorageBlocks().size(); i { 49 | System.out.println("[" + recordOffset + "][" + rionReader.fieldType +"]["+rionReader.fieldLength +"]"); 50 | 51 | rionReader.moveInto(); 52 | while(rionReader.hasNext()){ 53 | rionReader.nextParse(); 54 | if(rionReader.fieldType == RionFieldTypes.UTF_8) { 55 | System.out.println(" [" + recordOffset + "][" + rionReader.fieldType + "][" + rionReader.readUtf8String() + "]"); 56 | } else if (rionReader.fieldType == RionFieldTypes.INT_POS){ 57 | System.out.println(" [" + recordOffset + "][" + rionReader.fieldType + "][" + rionReader.readInt64() + "]"); 58 | } 59 | } 60 | rionReader.moveOutOf(); 61 | 62 | return true; 63 | }); 64 | 65 | } 66 | 67 | private static void appendRecordsToStream(byte[] rionBytesRecord1, int length, StreamStorageFS streamStorageFS) throws IOException { 68 | streamStorageFS.openForAppend(); 69 | streamStorageFS.appendRecord(rionBytesRecord1, 0, length); 70 | streamStorageFS.appendRecord(rionBytesRecord1, 0, length); 71 | streamStorageFS.appendRecord(rionBytesRecord1, 0, length); 72 | 73 | //streamStorageFS.nextRecordOffset += 10; 74 | //streamStorageFS.appendOffset(); 75 | streamStorageFS.appendRecord(rionBytesRecord1, 0, length); 76 | streamStorageFS.appendRecord(rionBytesRecord1, 0, length); 77 | streamStorageFS.appendRecord(rionBytesRecord1, 0, length); 78 | 79 | streamStorageFS.closeForAppend(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/RecordIterationExample.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples; 2 | 3 | import com.nanosai.streamops.navigation.RecordIterator; 4 | import com.nanosai.streamops.storage.file.StreamStorageBlockFS; 5 | import com.nanosai.streamops.storage.file.StreamStorageFS; 6 | import com.nanosai.streamops.util.FileUtil; 7 | 8 | import java.io.File; 9 | import java.io.IOException; 10 | import java.util.List; 11 | 12 | public class RecordIterationExample { 13 | 14 | public static void main(String[] args) throws IOException { 15 | 16 | FileUtil.resetDir(new File("data/example-stream-1")); 17 | 18 | // write some records to a stream 19 | byte[] rionBytesRecord1 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 20 | 21 | StreamStorageFS streamStorageFS = new StreamStorageFS("example-stream-1", "data/example-stream-1", 1024); 22 | appendRecordsToStream(rionBytesRecord1, streamStorageFS); 23 | 24 | // read all records into a byte array 25 | byte[] records = new byte[1024]; 26 | 27 | List storageBlocks = streamStorageFS.getStorageBlocks(); 28 | StreamStorageBlockFS streamStorageBlockFS = storageBlocks.get(0); 29 | 30 | long fileLength = streamStorageBlockFS.getFileLength(); 31 | 32 | int bytesRead = streamStorageFS.readFromBlock(streamStorageBlockFS, 0, records, 0, (int) fileLength); 33 | 34 | 35 | // attach a RionReader to the byte array 36 | //RecordIterator recordIterator= StreamOps.createRecordIterator(); 37 | RecordIterator recordIterator = new RecordIterator(); 38 | recordIterator.setSource(records, 0, bytesRead); 39 | 40 | // iterate records 41 | while(recordIterator.hasNext()){ 42 | recordIterator.next(); 43 | 44 | System.out.println("record offset: " + recordIterator.offset); 45 | 46 | } 47 | } 48 | 49 | private static void appendRecordsToStream(byte[] rionBytesRecord1, StreamStorageFS streamStorageFS) throws IOException { 50 | streamStorageFS.openForAppend(); 51 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 52 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 53 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 54 | streamStorageFS.closeForAppend(); 55 | } 56 | 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/StreamNavigatorExample.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples; 2 | 3 | import com.nanosai.streamops.navigation.RecordIterator; 4 | import com.nanosai.streamops.navigation.StreamNavigator; 5 | import com.nanosai.streamops.storage.file.StreamStorageFS; 6 | 7 | import java.io.IOException; 8 | 9 | public class StreamNavigatorExample { 10 | 11 | public static void main(String[] args) throws IOException { 12 | StreamStorageFS streamStorage = new StreamStorageFS("navigator-example-stream","data/navigator-example-stream", 24); 13 | 14 | writeRecordsToStream(streamStorage); 15 | 16 | StreamNavigator streamNavigator = new StreamNavigator(); 17 | 18 | //source offset and length will be overridden in navigateTo() call later. 19 | RecordIterator recordIterator = new RecordIterator().setSource(new byte[1024], 0, 0); 20 | 21 | long currentOffset = 0; 22 | while(currentOffset < streamStorage.nextRecordOffset){ 23 | System.out.println("Navigating to: " + currentOffset); 24 | streamNavigator.navigateTo(streamStorage, currentOffset, recordIterator); 25 | 26 | System.out.println("-- Record offset: " + recordIterator.offset); 27 | //process first record 28 | while(recordIterator.hasNext()){ 29 | recordIterator.next(); 30 | System.out.println("-- Record offset: " + recordIterator.offset); 31 | //process following records 32 | } 33 | currentOffset = recordIterator.offset + 1; 34 | } 35 | 36 | 37 | 38 | } 39 | 40 | private static void writeRecordsToStream(StreamStorageFS streamStorage) throws IOException { 41 | System.out.println("streamStorage.nextRecordOffset = " + streamStorage.nextRecordOffset); 42 | byte[] rionBytesRecord1 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; 43 | 44 | streamStorage.openForAppend(); 45 | for(int i=0; i<6; i++){ 46 | streamStorage.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 47 | } 48 | streamStorage.closeForAppend(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/StreamOpsExamples.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples; 2 | 3 | import com.nanosai.streamops.StreamOps; 4 | import com.nanosai.streamops.storage.StreamStorageFactory; 5 | import com.nanosai.streamops.storage.file.StreamStorageRootFS; 6 | 7 | import java.io.IOException; 8 | 9 | public class StreamOpsExamples { 10 | 11 | public static void main(String[] args) throws IOException { 12 | StreamStorageFactory streamStorageFactory = StreamOps.createStreamStorageFactory(); 13 | 14 | StreamStorageRootFS streamStorageRootFs = 15 | streamStorageFactory.createStreamStorageRootFs("data/test-root2"); 16 | 17 | streamStorageRootFs.createStreamStorage("test-stream-1"); 18 | 19 | 20 | 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/StreamStorageFSExample.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples; 2 | 3 | import com.nanosai.streamops.navigation.RecordIterator; 4 | import com.nanosai.streamops.storage.file.StreamStorageBlockFS; 5 | import com.nanosai.streamops.storage.file.StreamStorageFS; 6 | import com.nanosai.streamops.util.FileUtil; 7 | 8 | import java.io.IOException; 9 | 10 | public class StreamStorageFSExample { 11 | 12 | public static void main(String[] args) throws IOException { 13 | FileUtil.deleteDir("data/stream-id"); 14 | 15 | //StreamStorageFactory streamStorageFactory = StreamOps.createStreamStorageFactory(); 16 | //StreamStorageFS streamStorageFS = streamStorageFactory.createStreamStorageFS("stream-id", "data/stream-id"); 17 | StreamStorageFS streamStorageFS = new StreamStorageFS("stream-id", "data/stream-id"); 18 | 19 | writeRecordsToStream(streamStorageFS); 20 | readRecordsFromStream(streamStorageFS); 21 | } 22 | 23 | private static void readRecordsFromStream(StreamStorageFS streamStorageFS) throws IOException { 24 | 25 | byte[] recordBuffer = new byte[1024]; 26 | 27 | StreamStorageBlockFS streamStorageBlockFS = streamStorageFS.getStorageBlocks().get(0); 28 | 29 | int lengthRead = streamStorageFS.readFromBlock(streamStorageBlockFS, 0, 30 | recordBuffer, 0, (int) streamStorageBlockFS.getFileLength()); 31 | 32 | RecordIterator recordIterator = new RecordIterator(); 33 | recordIterator.setSource(recordBuffer, 0, lengthRead); 34 | 35 | while(recordIterator.hasNext()){ 36 | recordIterator.next(); 37 | 38 | System.out.println(recordIterator.offset); 39 | } 40 | 41 | } 42 | 43 | 44 | private static void writeRecordsToStream(StreamStorageFS streamStorageFS) throws IOException { 45 | //10 bytes in total, 8 bytes in the RION Bytes field body 46 | byte[] rionBytesRecord1 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; 47 | byte[] rionBytesRecord2 = new byte[]{0x01, 0x08, 7,6,5,4,3,2,1,0}; 48 | byte[] rionBytesRecord3 = new byte[]{0x01, 0x08, 0,1,2,3,3,2,1,0}; 49 | 50 | 51 | streamStorageFS.openForAppend(); 52 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 53 | streamStorageFS.appendRecord(rionBytesRecord2, 0, rionBytesRecord2.length); 54 | streamStorageFS.appendRecord(rionBytesRecord3, 0, rionBytesRecord3.length); 55 | streamStorageFS.closeForAppend(); 56 | } 57 | 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/ecommerce/Customer.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples.ecommerce; 2 | 3 | public class Customer { 4 | public long customerId = 0; 5 | public String name = null; 6 | 7 | public Customer() {} 8 | 9 | public Customer(long customerId, String name) { 10 | this.customerId = customerId; 11 | this.name = name; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/ecommerce/ECommerceReport.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples.ecommerce; 2 | 3 | import com.nanosai.rionops.rion.read.RionReader; 4 | import com.nanosai.streamops.storage.file.IRecordProcessor; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | public class ECommerceReport implements IRecordProcessor { 10 | 11 | private List products = null; 12 | private List customers = null; 13 | 14 | private List productRevenueSums = new ArrayList<>(); 15 | 16 | 17 | 18 | private long totalRevenue = 0; 19 | public long recordCount = 0; 20 | 21 | 22 | public ECommerceReport(List products, List customers) { 23 | this.products = products; 24 | this.customers = customers; 25 | 26 | for(int i=0; i< this.products.size(); i++) { 27 | this.productRevenueSums.add(new ProductRevenueSum()); 28 | } 29 | } 30 | 31 | @Override 32 | public boolean process(long recordOffset, RionReader rionReader) { 33 | 34 | //System.out.println("[" + recordOffset + "][" + rionReader.fieldType +"]["+rionReader.fieldLength +"]"); 35 | 36 | this.recordCount++; 37 | 38 | rionReader.moveInto(); 39 | while(rionReader.hasNext()){ 40 | //rionReader.nextParse(); 41 | //long orderItemId = rionReader.readInt64(); 42 | 43 | rionReader.nextParse(); 44 | long productId = rionReader.readInt64(); 45 | 46 | rionReader.nextParse(); 47 | long orderId = rionReader.readInt64(); 48 | 49 | //rionReader.nextParse(); 50 | //long customerId = rionReader.readInt64(); 51 | 52 | //System.out.println(" {[" + orderItemId + "][" + productId + "][" + orderId + "][" + customerId + "]}"); 53 | //System.out.println(" {[" + productId + "][" + orderId + "][" + customerId + "]}"); 54 | 55 | Product product = this.products.get((int) productId); 56 | ProductRevenueSum productRevenueSum = this.productRevenueSums.get((int) productId); 57 | productRevenueSum.revenueSum += product.price; 58 | 59 | this.totalRevenue += product.price; 60 | } 61 | rionReader.moveOutOf(); 62 | 63 | 64 | return true; 65 | } 66 | 67 | 68 | 69 | public void printResult() { 70 | System.out.println("E-Commerce Report:"); 71 | 72 | System.out.println("Total revenue: " + this.totalRevenue); 73 | 74 | for(int i=0; i products = new ArrayList<>(); 14 | products.add(new Product(0, "Soda ", "Cold softdrink", 100, 50)); 15 | products.add(new Product(1, "Choc ", "Chocolate bar", 90, 47)); 16 | products.add(new Product(2, "Chips", "Potatoe chips", 150, 83)); 17 | 18 | List customers = new ArrayList<>(); 19 | customers.add(new Customer(0, "John Doe")); 20 | customers.add(new Customer(1, "Jane Dee")); 21 | 22 | 23 | String streamId = "e-commerce-example-2"; 24 | String streamDir = "data/" + streamId; 25 | 26 | StreamStorageFS streamStorage = new StreamStorageFS(streamId, streamDir, 8 * 1024 * 1024); 27 | 28 | byte[] recordBuffer = new byte[(int) streamStorage.getStorageFileBlockMaxSize()]; 29 | 30 | ECommerceReport report = new ECommerceReport(products, customers); 31 | 32 | long startTime = System.currentTimeMillis(); 33 | streamStorage.iterate(recordBuffer, report); 34 | long endTime = System.currentTimeMillis(); 35 | long fullTime = endTime - startTime; 36 | 37 | System.out.println("Time : " + fullTime); 38 | System.out.println("Records: " + report.recordCount); 39 | 40 | System.out.println("Speed :" + report.recordCount * 1000 / fullTime); 41 | 42 | report.printResult(); 43 | 44 | } 45 | 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/ecommerce/Product.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples.ecommerce; 2 | 3 | public class Product { 4 | public long productId = 0; 5 | public String productName = null; 6 | public String productDesc = null; 7 | public long price = 0; // price in smallest currency units - e.g. cents 8 | public long cost = 0; // cost in smallest currency units - e.g. cents 9 | 10 | public Product() {} 11 | 12 | public Product( 13 | long productId, String productName, String productDesc, 14 | long price, long cost) { 15 | this.productId = productId; 16 | this.productName = productName; 17 | this.productDesc = productDesc; 18 | this.price = price; 19 | this.cost = cost; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/ecommerce/ProductRevenueSum.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples.ecommerce; 2 | 3 | public class ProductRevenueSum { 4 | 5 | public long productId = 0; 6 | public long revenueSum = 0; 7 | 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/examples/ecommerce/StreamWriteExample.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.examples.ecommerce; 2 | 3 | import com.nanosai.rionops.rion.write.RionWriter; 4 | import com.nanosai.streamops.storage.file.StreamStorageFS; 5 | import com.nanosai.streamops.util.FileUtil; 6 | 7 | import java.io.BufferedOutputStream; 8 | import java.io.File; 9 | import java.io.FileOutputStream; 10 | import java.io.IOException; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | public class StreamWriteExample { 15 | 16 | public static void main(String[] args) throws IOException { 17 | 18 | List products = new ArrayList<>(); 19 | products.add(new Product(0, "Soda ", "Cold softdrink", 100, 50)); 20 | products.add(new Product(1, "Choc ", "Chocolate bar", 90, 47)); 21 | products.add(new Product(2, "Chips", "Potatoe chips", 150, 83)); 22 | 23 | List customers = new ArrayList<>(); 24 | customers.add(new Customer(0, "John Doe")); 25 | customers.add(new Customer(1, "Jane Dee")); 26 | 27 | long[] orderCustomerIds = new long[10]; 28 | 29 | 30 | 31 | String streamId = "e-commerce-example-2"; 32 | String streamDir = "data/" + streamId; 33 | FileUtil.resetDir(new File(streamDir)); 34 | 35 | StreamStorageFS streamStorage = new StreamStorageFS(streamId, streamDir, 8 * 1024 * 1024); 36 | streamStorage.setOutputStreamFactory((String filePath) -> { 37 | return new BufferedOutputStream(new FileOutputStream(filePath), 8 * 1024); 38 | }); 39 | 40 | streamStorage.openForAppend(); 41 | 42 | long noOfOrders = orderCustomerIds.length; 43 | int noOfOrderItems = 100_000_000; 44 | 45 | long startTime = System.currentTimeMillis(); 46 | 47 | byte[] rionRecord = new byte[1024]; 48 | RionWriter rionWriter = new RionWriter() 49 | .setNestedFieldStack(new int[16]) 50 | .setDestination(rionRecord,0); 51 | 52 | for(int i=0; i { 61 | appendListenerCalled.set(true); 62 | }); 63 | streamStorageFS.openForAppend(); 64 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 65 | 66 | assertTrue(appendListenerCalled.get()); 67 | 68 | streamStorageFS.closeForAppend(); 69 | 70 | } 71 | 72 | 73 | @Test 74 | public void testAppendSplitsIntoMultipleFiles() throws IOException { 75 | FileUtil.resetDir(new File("data/test-stream-1")); 76 | //initStreamDir("data/test-stream-1"); 77 | 78 | StreamStorageFSOld streamStorageFS = new StreamStorageFSOld("test-stream-1", "data/test-stream-1", 20); 79 | 80 | byte[] rionBytesRecord1 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 81 | byte[] rionBytesRecord2 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 82 | byte[] rionBytesRecord3 = new byte[]{0x01, 0x04, 0,1,2,3}; //10 bytes in total, 8 bytes in the RION Bytes field body 83 | byte[] rionBytesRecord4 = new byte[]{0x01, 0x04, 0,2,4,6}; //10 bytes in total, 8 bytes in the RION Bytes field body 84 | 85 | streamStorageFS.openForAppend(); 86 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 87 | streamStorageFS.appendRecord(rionBytesRecord2, 0, rionBytesRecord2.length); 88 | streamStorageFS.appendRecord(rionBytesRecord3, 0, rionBytesRecord3.length); 89 | streamStorageFS.appendRecord(rionBytesRecord4, 0, rionBytesRecord4.length); 90 | streamStorageFS.closeForAppend(); 91 | 92 | assertEquals( 3, streamStorageFS.getStorageBlocks().size()); 93 | assertEquals(14, streamStorageFS.getStorageBlocks().get(0).fileLength); 94 | assertEquals(20, streamStorageFS.getStorageBlocks().get(1).fileLength); 95 | assertEquals(10, streamStorageFS.getStorageBlocks().get(2).fileLength); 96 | 97 | byte[] dest1 = new byte[14]; 98 | 99 | int bytesRead1 = streamStorageFS.readFromBlock( 100 | streamStorageFS.getStorageBlocks().get(0), 0, dest1, 0, 14); 101 | 102 | assertEquals(14, bytesRead1); 103 | 104 | assertEquals(0xF1, 255 & dest1[0]); // first byte of a RION Extended Field with field byte length 1 105 | assertEquals(0x01, 255 & dest1[1]); // second byte of a RION Extended Field with field type 1 ( Extended 1 = stream record offset). 106 | assertEquals( 1, 255 & dest1[2]); // length byte of a RION Extended Field - expected to be 1 byte. 107 | assertEquals(0x00, 255 & dest1[3]); // record offset is expected to be 0. 108 | 109 | assertEquals(0x01, 255 & dest1[4]); // lead byte of RION Bytes field - with length byte count 1 110 | assertEquals(0x08, 255 & dest1[5]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 111 | 112 | assertEquals(0x00, 255 & dest1[6]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 113 | assertEquals(0x01, 255 & dest1[7]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 114 | assertEquals(0x02, 255 & dest1[8]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 115 | assertEquals(0x03, 255 & dest1[9]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 116 | assertEquals(0x04, 255 & dest1[10]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 117 | assertEquals(0x05, 255 & dest1[11]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 118 | assertEquals(0x06, 255 & dest1[12]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 119 | assertEquals(0x07, 255 & dest1[13]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 120 | 121 | byte[] dest2 = new byte[20]; 122 | 123 | int bytesRead2 = streamStorageFS.readFromBlock( 124 | streamStorageFS.getStorageBlocks().get(1), 0, dest2, 0, 20); 125 | 126 | assertEquals(20, bytesRead2); 127 | 128 | assertEquals(0xF1, 255 & dest2[0]); // first byte of a RION Extended Field with field byte length 1 129 | assertEquals(0x01, 255 & dest2[1]); // second byte of a RION Extended Field with field type 1 ( Extended 1 = stream record offset). 130 | assertEquals(0x01, 255 & dest2[2]); // third byte of a RION Extended Field - length byte with value 1. 131 | assertEquals(0x01, 255 & dest2[3]); // record offset is expected to be 1. 132 | 133 | assertEquals(0x01, 255 & dest2[4]); // lead byte of RION Bytes field - with length byte count 1 134 | assertEquals(0x08, 255 & dest2[5]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 135 | 136 | assertEquals(0x00, 255 & dest2[6]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 137 | assertEquals(0x01, 255 & dest2[7]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 138 | assertEquals(0x02, 255 & dest2[8]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 139 | assertEquals(0x03, 255 & dest2[9]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 140 | assertEquals(0x04, 255 & dest2[10]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 141 | assertEquals(0x05, 255 & dest2[11]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 142 | assertEquals(0x06, 255 & dest2[12]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 143 | assertEquals(0x07, 255 & dest2[13]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 144 | 145 | assertEquals(0x01, 255 & dest2[14]); // lead byte of RION Bytes field - with length byte count 1 146 | assertEquals(0x04, 255 & dest2[15]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 147 | 148 | assertEquals(0x00, 255 & dest2[16]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 149 | assertEquals(0x01, 255 & dest2[17]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 150 | assertEquals(0x02, 255 & dest2[18]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 151 | assertEquals(0x03, 255 & dest2[19]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 152 | 153 | byte[] dest3 = new byte[10]; 154 | 155 | int bytesRead3 = streamStorageFS.readFromBlock( 156 | streamStorageFS.getStorageBlocks().get(2), 0, dest3, 0, 10); 157 | 158 | assertEquals(10, bytesRead3); 159 | 160 | assertEquals(0xF1, 255 & dest3[0]); // first byte of a RION Extended Field with field byte length 1 161 | assertEquals(0x01, 255 & dest3[1]); // second byte of a RION Extended Field with field type 1 ( Extended 1 = stream record offset). 162 | assertEquals( 1, 255 & dest3[2]); // third byte of a RION Extended Field -length byte with a value of 1. 163 | assertEquals(0x03, 255 & dest3[3]); // record offset is expected to be 1. 164 | 165 | assertEquals(0x01, 255 & dest3[4]); // lead byte of RION Bytes field - with length byte count 1 166 | assertEquals(0x04, 255 & dest3[5]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 167 | 168 | assertEquals(0x00, 255 & dest3[6]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 169 | assertEquals(0x02, 255 & dest3[7]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 170 | assertEquals(0x04, 255 & dest3[8]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 171 | assertEquals(0x06, 255 & dest3[9]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 172 | 173 | } 174 | 175 | 176 | @Test 177 | public void testIterateFrom() throws IOException { 178 | FileUtil.resetDir(new File("data/test-stream-1")); 179 | StreamStorageFS streamStorageFS = new StreamStorageFS("test-stream-1", "data/test-stream-1", 34); 180 | 181 | byte[] rionBytesRecord1 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 182 | 183 | streamStorageFS.openForAppend(); 184 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 185 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 186 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 187 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 188 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 189 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 190 | streamStorageFS.closeForAppend(); 191 | 192 | assertEquals(2, streamStorageFS.getStorageBlocks().size()); 193 | 194 | 195 | byte[] recordBuffer = new byte[34]; 196 | AtomicLong firstOffset = new AtomicLong(); 197 | AtomicLong numberOfIterations = new AtomicLong(); 198 | 199 | streamStorageFS.iterateFromOffset(recordBuffer, 4, (offset, rionReader) -> { 200 | firstOffset.set(offset); 201 | numberOfIterations.incrementAndGet(); 202 | return false; 203 | }); 204 | 205 | assertEquals(4, firstOffset.get()); 206 | assertEquals(1, numberOfIterations.get()); 207 | 208 | } 209 | 210 | 211 | private void initStreamDir(String streamRootDirPath) { 212 | deleteDir(streamRootDirPath); 213 | new File(streamRootDirPath).mkdirs(); 214 | } 215 | 216 | private void deleteDir(String dirPath) { 217 | File streamRootDir = new File(dirPath); 218 | if(streamRootDir.exists()) { 219 | for(File file : streamRootDir.listFiles()){ 220 | boolean deleted = file.delete(); 221 | System.out.println("Deleted " + file. getName() + " : " + deleted); 222 | } 223 | boolean deleted = streamRootDir.delete(); 224 | System.out.println("Deleted " + streamRootDir. getName() + " : " + deleted); 225 | 226 | } 227 | } 228 | 229 | @Test 230 | public void testToHex() { 231 | assertEquals("0000000000000000", HexUtil.toHex(0x0)); 232 | assertEquals("000000000000000F", HexUtil.toHex(0xF)); 233 | assertEquals("00000000000000FF", HexUtil.toHex(0xFF)); 234 | assertEquals("FEDCBA9876543210", HexUtil.toHex(0xFEDCBA9876543210L)); 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/storage/file/StreamStorageFSTest.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.storage.file; 2 | 3 | import com.nanosai.streamops.util.FileUtil; 4 | import com.nanosai.streamops.util.HexUtil; 5 | import org.junit.jupiter.api.Test; 6 | 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.util.concurrent.atomic.AtomicBoolean; 10 | import java.util.concurrent.atomic.AtomicLong; 11 | 12 | import static org.junit.jupiter.api.Assertions.*; 13 | 14 | public class StreamStorageFSTest { 15 | 16 | 17 | @Test 18 | public void testCreateNewStreamFileStorage() throws IOException { 19 | 20 | String streamRootDir = "data/test-stream-fs2-1"; 21 | FileUtil.resetDir(new File(streamRootDir)); 22 | 23 | StreamStorageFS streamStorageFS = new StreamStorageFS("test-stream-fs2-1", streamRootDir); 24 | 25 | assertNotNull(streamStorageFS.getLatestBlock()); 26 | assertEquals("test-stream-fs2-1-0000000000000000", streamStorageFS.getLatestBlock().getFileName()); 27 | assertEquals(1, streamStorageFS.getStorageBlocks().size()); 28 | 29 | assertTrue(new File(streamRootDir).exists()); 30 | } 31 | 32 | @Test 33 | public void testAppendToStreamFileStorage() throws IOException { 34 | String streamRootDir = "data/test-stream-fs2-1"; 35 | FileUtil.resetDir(new File(streamRootDir)); 36 | //initStreamDir(streamRootDir); 37 | 38 | StreamStorageFS streamStorageFS = new StreamStorageFS("test-stream-fs2-1", streamRootDir); 39 | 40 | byte[] rionBytesRecord = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 41 | 42 | //todo next step is to test append, and see that files are created and data written to it. 43 | streamStorageFS.openForAppend(); 44 | streamStorageFS.appendRecord(rionBytesRecord, 0, rionBytesRecord.length); 45 | streamStorageFS.closeForAppend(); 46 | 47 | assertEquals(10, streamStorageFS.getLatestBlock().fileLength); 48 | 49 | streamStorageFS.openForAppend(); 50 | streamStorageFS.appendRecord(rionBytesRecord, 0, rionBytesRecord.length); 51 | streamStorageFS.closeForAppend(); 52 | 53 | assertEquals(20, streamStorageFS.getLatestBlock().fileLength); 54 | 55 | } 56 | 57 | @Test 58 | public void testAppendListenerCalled() throws IOException { 59 | String streamRootDir = "data/test-stream-fs2-1"; 60 | FileUtil.resetDir(new File(streamRootDir)); 61 | 62 | StreamStorageFS streamStorageFS = new StreamStorageFS("test-stream-fs2-1", streamRootDir, 20); 63 | 64 | byte[] rionBytesRecord1 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 65 | 66 | AtomicBoolean appendListenerCalled = new AtomicBoolean(); 67 | appendListenerCalled.set(false); 68 | streamStorageFS.setAppendListener((byteArray, offset, length, recordOffset) -> { 69 | appendListenerCalled.set(true); 70 | }); 71 | streamStorageFS.openForAppend(); 72 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 73 | 74 | assertTrue(appendListenerCalled.get()); 75 | 76 | streamStorageFS.closeForAppend(); 77 | 78 | } 79 | 80 | 81 | @Test 82 | public void testAppendSplitsIntoMultipleFiles() throws IOException { 83 | String streamRootDir = "data/test-stream-fs2-1"; 84 | FileUtil.resetDir(new File(streamRootDir)); 85 | //initStreamDir("data/test-stream-1"); 86 | 87 | StreamStorageFS streamStorageFS = new StreamStorageFS("test-stream-1", streamRootDir, 20); 88 | 89 | byte[] rionBytesRecord1 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 90 | byte[] rionBytesRecord2 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 91 | byte[] rionBytesRecord3 = new byte[]{0x01, 0x04, 0,1,2,3}; //10 bytes in total, 8 bytes in the RION Bytes field body 92 | byte[] rionBytesRecord4 = new byte[]{0x01, 0x04, 0,2,4,6}; //10 bytes in total, 8 bytes in the RION Bytes field body 93 | 94 | streamStorageFS.openForAppend(); 95 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 96 | streamStorageFS.appendRecord(rionBytesRecord2, 0, rionBytesRecord2.length); 97 | streamStorageFS.appendRecord(rionBytesRecord3, 0, rionBytesRecord3.length); 98 | streamStorageFS.appendRecord(rionBytesRecord4, 0, rionBytesRecord4.length); 99 | streamStorageFS.closeForAppend(); 100 | 101 | assertEquals(2, streamStorageFS.getStorageBlocks().size()); 102 | assertEquals(20, streamStorageFS.getStorageBlocks().get(0).fileLength); 103 | assertEquals(12, streamStorageFS.getStorageBlocks().get(1).fileLength); 104 | 105 | byte[] dest1 = new byte[20]; 106 | 107 | int bytesRead1 = streamStorageFS.readFromBlock( 108 | streamStorageFS.getStorageBlocks().get(0), 0, dest1, 0, 20); 109 | 110 | assertEquals(20, bytesRead1); 111 | 112 | assertEquals(0x01, 255 & dest1[0]); // lead byte of RION Bytes field - with length byte count 1 113 | assertEquals(0x08, 255 & dest1[1]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 114 | 115 | assertEquals(0x00, 255 & dest1[2]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 116 | assertEquals(0x01, 255 & dest1[3]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 117 | assertEquals(0x02, 255 & dest1[4]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 118 | assertEquals(0x03, 255 & dest1[5]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 119 | assertEquals(0x04, 255 & dest1[6]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 120 | assertEquals(0x05, 255 & dest1[7]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 121 | assertEquals(0x06, 255 & dest1[8]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 122 | assertEquals(0x07, 255 & dest1[9]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 123 | 124 | assertEquals(0x01, 255 & dest1[10]); // lead byte of RION Bytes field - with length byte count 1 125 | assertEquals(0x08, 255 & dest1[11]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 126 | 127 | assertEquals(0x00, 255 & dest1[12]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 128 | assertEquals(0x01, 255 & dest1[13]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 129 | assertEquals(0x02, 255 & dest1[14]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 130 | assertEquals(0x03, 255 & dest1[15]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 131 | assertEquals(0x04, 255 & dest1[16]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 132 | assertEquals(0x05, 255 & dest1[17]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 133 | assertEquals(0x06, 255 & dest1[18]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 134 | assertEquals(0x07, 255 & dest1[19]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 135 | 136 | byte[] dest2 = new byte[20]; 137 | 138 | int bytesRead2 = streamStorageFS.readFromBlock( 139 | streamStorageFS.getStorageBlocks().get(1), 0, dest2, 0, 20); 140 | 141 | assertEquals(12, bytesRead2); 142 | 143 | 144 | assertEquals(0x01, 255 & dest2[0]); // lead byte of RION Bytes field - with length byte count 1 145 | assertEquals(0x04, 255 & dest2[1]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 146 | 147 | assertEquals(0x00, 255 & dest2[2]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 148 | assertEquals(0x01, 255 & dest2[3]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 149 | assertEquals(0x02, 255 & dest2[4]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 150 | assertEquals(0x03, 255 & dest2[5]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 151 | 152 | assertEquals(0x01, 255 & dest2[6]); // lead byte of RION Bytes field - with length byte count 1 153 | assertEquals(0x04, 255 & dest2[7]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 154 | 155 | assertEquals(0x00, 255 & dest2[8]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 156 | assertEquals(0x02, 255 & dest2[9]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 157 | assertEquals(0x04, 255 & dest2[10]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 158 | assertEquals(0x06, 255 & dest2[11]); // length byte of RION Bytes field. Total length of RION Bytes field body is 8 bytes 159 | 160 | 161 | } 162 | 163 | 164 | @Test 165 | public void testIterateFrom() throws IOException { 166 | FileUtil.resetDir(new File("data/test-stream-1")); 167 | StreamStorageFS streamStorageFS = new StreamStorageFS("test-stream-1", "data/test-stream-1", 30); 168 | 169 | byte[] rionBytesRecord1 = new byte[]{0x01, 0x08, 0,1,2,3,4,5,6,7}; //10 bytes in total, 8 bytes in the RION Bytes field body 170 | 171 | streamStorageFS.openForAppend(); 172 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 173 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 174 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 175 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 176 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 177 | streamStorageFS.appendRecord(rionBytesRecord1, 0, rionBytesRecord1.length); 178 | streamStorageFS.closeForAppend(); 179 | 180 | assertEquals(2, streamStorageFS.getStorageBlocks().size()); 181 | 182 | 183 | byte[] recordBuffer = new byte[30]; 184 | AtomicLong firstOffset = new AtomicLong(); 185 | AtomicLong numberOfIterations = new AtomicLong(); 186 | 187 | streamStorageFS.iterateFromOffset(recordBuffer, 4, (offset, rionReader) -> { 188 | firstOffset.set(offset); 189 | numberOfIterations.incrementAndGet(); 190 | return false; 191 | }); 192 | 193 | assertEquals(4, firstOffset.get()); 194 | assertEquals(1, numberOfIterations.get()); 195 | 196 | } 197 | 198 | 199 | private void initStreamDir(String streamRootDirPath) { 200 | deleteDir(streamRootDirPath); 201 | new File(streamRootDirPath).mkdirs(); 202 | } 203 | 204 | private void deleteDir(String dirPath) { 205 | File streamRootDir = new File(dirPath); 206 | if(streamRootDir.exists()) { 207 | for(File file : streamRootDir.listFiles()){ 208 | boolean deleted = file.delete(); 209 | System.out.println("Deleted " + file. getName() + " : " + deleted); 210 | } 211 | boolean deleted = streamRootDir.delete(); 212 | System.out.println("Deleted " + streamRootDir. getName() + " : " + deleted); 213 | 214 | } 215 | } 216 | 217 | @Test 218 | public void testToHex() { 219 | assertEquals("0000000000000000", HexUtil.toHex(0x0)); 220 | assertEquals("000000000000000F", HexUtil.toHex(0xF)); 221 | assertEquals("00000000000000FF", HexUtil.toHex(0xFF)); 222 | assertEquals("FEDCBA9876543210", HexUtil.toHex(0xFEDCBA9876543210L)); 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/storage/file/StreamStorageRootFSTest.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.storage.file; 2 | 3 | import com.nanosai.streamops.util.FileUtil; 4 | import org.junit.jupiter.api.Test; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | 9 | import static org.junit.jupiter.api.Assertions.*; 10 | 11 | public class StreamStorageRootFSTest { 12 | 13 | 14 | @Test 15 | public void testConstructor() { 16 | StreamStorageRootFS streamStorageRootFS = new StreamStorageRootFS("data/test-root"); 17 | 18 | assertEquals("data/test-root", streamStorageRootFS.getRootDirPath()); 19 | } 20 | 21 | 22 | @Test 23 | public void testCreateStreamFileStorage() throws IOException { 24 | FileUtil.resetDir(new File("data/test-root/stream-1")); 25 | FileUtil.resetDir(new File("data/test-root")); 26 | 27 | StreamStorageRootFS streamStorageRootFS = new StreamStorageRootFS("data/test-root"); 28 | 29 | assertTrue(new File("data/test-root").exists()); 30 | 31 | StreamStorageFS streamStorageFS = streamStorageRootFS.createStreamStorage("stream-1"); 32 | assertTrue(new File("data/test-root/stream-1").exists()); 33 | 34 | assertNotNull(streamStorageFS); 35 | assertEquals("stream-1", streamStorageFS.getStreamId()); 36 | assertEquals("data/test-root/stream-1", streamStorageFS.getRootDirPath()); 37 | } 38 | 39 | @Test 40 | public void testCreateStreamFileStorageWithMaxBlockSize() throws IOException { 41 | FileUtil.resetDir(new File("data/test-root/stream-1")); 42 | FileUtil.resetDir(new File("data/test-root")); 43 | 44 | StreamStorageRootFS streamStorageRootFS = new StreamStorageRootFS("data/test-root"); 45 | assertTrue(new File("data/test-root").exists()); 46 | 47 | StreamStorageFS streamStorageFS = streamStorageRootFS.createStreamStorage("stream-1", 2 * 1024 * 1024); 48 | assertTrue(new File("data/test-root/stream-1").exists()); 49 | 50 | assertEquals("stream-1", streamStorageFS.getStreamId()); 51 | assertEquals("data/test-root/stream-1", streamStorageFS.getRootDirPath()); 52 | assertEquals(2 * 1024 * 1024, streamStorageFS.getStorageFileBlockMaxSize()); 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/nanosai/streamops/storage/file/index/StreamStorageIndexFSTest.java: -------------------------------------------------------------------------------- 1 | package com.nanosai.streamops.storage.file.index; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.junit.jupiter.api.Assertions.assertEquals; 6 | 7 | public class StreamStorageIndexFSTest { 8 | 9 | 10 | @Test 11 | public void test() { 12 | int offsetInterval = 1024; 13 | StreamStorageIndexFS index = new StreamStorageIndexFS(1024 * 1024, offsetInterval); 14 | 15 | int recordsPerFile = 4096; 16 | long recordOffsetMax = 1024L * 1024L; 17 | 18 | index.append(800, 0, 1600); 19 | index.append(800, 0, 1600); 20 | 21 | for(long recordOffset = 0; recordOffset < recordOffsetMax; recordOffset++){ 22 | int streamFileStorageBlockIndex = (int) (recordOffset / recordsPerFile); //doesn't make sense for real records, but fine for testing 23 | int streamFileByteOffset = (int) (recordOffset % recordsPerFile); //doesn't make sense for real records, but fine for testing 24 | 25 | index.append(recordOffset, streamFileStorageBlockIndex, streamFileByteOffset); 26 | } 27 | 28 | for(long recordOffset = 0; recordOffset < recordOffsetMax; recordOffset++){ 29 | long indexRecord = index.lookup(recordOffset); 30 | 31 | int streamFileStorageBlockIndex = (int) (indexRecord >> 32); 32 | int streamFileByteOffset = (int) (0x0000_0000_FFFF_FFFFL & indexRecord); 33 | 34 | int expectedStreamFileStorageBlockIndex = (int) (recordOffset / recordsPerFile); 35 | assertEquals(expectedStreamFileStorageBlockIndex, streamFileStorageBlockIndex); 36 | 37 | int expectedStreamFileByteOffset = (int) (((recordOffset / offsetInterval) * offsetInterval) % recordsPerFile); 38 | 39 | assertEquals(expectedStreamFileByteOffset, streamFileByteOffset); 40 | } 41 | 42 | 43 | //sample tests to make sure I have not made logical mistakes in the loops above 44 | 45 | long recordOffset = recordsPerFile + offsetInterval + 1; 46 | long indexRecord = index.lookup(recordOffset); 47 | 48 | int streamFileStorageBlockIndex = (int) (indexRecord >> 32); 49 | assertEquals(1, streamFileStorageBlockIndex); 50 | 51 | int streamFileByteOffset = (int) (0x0000_0000_FFFF_FFFF & indexRecord); 52 | assertEquals(offsetInterval, streamFileByteOffset); 53 | } 54 | } 55 | --------------------------------------------------------------------------------