├── .github └── workflows │ ├── build.yml │ └── publish.yml ├── .gitignore ├── LICENSE ├── README.md ├── build.gradle ├── chronos-api ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── ds │ │ └── chronos │ │ ├── api │ │ ├── Chronicle.java │ │ ├── ChronicleBatch.java │ │ ├── ChronologicalRecord.java │ │ ├── ChronosException.java │ │ ├── PartitionIterator.java │ │ ├── PartitionPeriod.java │ │ ├── PartitionedChronicle.java │ │ ├── Temporal.java │ │ ├── Timeline.java │ │ ├── TimelineDecoder.java │ │ ├── TimelineEncoder.java │ │ └── chronicle │ │ │ └── MemoryChronicle.java │ │ ├── streams │ │ ├── TemporalIterator.java │ │ ├── TemporalStream.java │ │ └── WindowIterator.java │ │ ├── timeline │ │ ├── ConcatenatedTimeline.java │ │ ├── SimpleTimeline.java │ │ ├── TimelineCopy.java │ │ └── stream │ │ │ ├── DataStream.java │ │ │ ├── DataStreamJoin.java │ │ │ └── partitioned │ │ │ ├── BucketSizePredicate.java │ │ │ ├── DurationPredicate.java │ │ │ ├── PartitionPredicate.java │ │ │ └── PartitionedDataStream.java │ │ └── util │ │ ├── Duration.java │ │ ├── JoiningIterator.java │ │ └── TimeFrame.java │ └── test │ └── java │ └── org │ └── ds │ └── chronos │ ├── api │ ├── ConcatenatedTimelineTest.java │ ├── DataStreamTest.java │ ├── DurationTest.java │ ├── MemoryChronicleTest.java │ ├── PartitionPeriodTest.java │ ├── TimeFrameTest.java │ ├── TimelineTest.java │ └── WindowTest.java │ └── support │ ├── TestBase.java │ ├── TestData.java │ ├── TestDecoder.java │ └── TestEncoder.java ├── chronos-aws ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── ds │ │ └── chronos │ │ └── aws │ │ ├── S3Chronicle.java │ │ ├── S3ParitionedChronicle.java │ │ ├── SimpleDBChronicle.java │ │ └── SimpleDBIterator.java │ └── test │ └── java │ └── org │ └── ds │ └── chronos │ └── aws │ ├── S3ChronicleTest.java │ └── SimpleDBChronicleTest.java ├── chronos-datastax ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── ds │ │ └── chronos │ │ └── chronicle │ │ ├── DatastaxChronicle.java │ │ └── DatastaxPartitionedChronicle.java │ └── test │ └── java │ └── org │ └── ds │ └── chronos │ └── chronicle │ ├── ChronicleTest.java │ └── TestBase.java ├── chronos-jackson ├── README.md ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── ds │ │ └── chronos │ │ └── timeline │ │ └── json │ │ ├── JsonTimeline.java │ │ ├── JsonTimelineDecoder.java │ │ ├── JsonTimelineEncoder.java │ │ └── Timestamped.java │ └── test │ └── java │ └── org │ └── ds │ └── chronos │ └── timeline │ └── json │ ├── JsonTest.java │ └── JsonTestObject.java ├── chronos-msgpack ├── README.md ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── ds │ │ └── chronos │ │ └── timeline │ │ └── msgpack │ │ ├── MsgPackTimeline.java │ │ ├── MsgPackTimelineDecoder.java │ │ ├── MsgPackTimelineEncoder.java │ │ └── Timestamped.java │ └── test │ └── java │ └── org │ └── ds │ └── chronos │ └── timeline │ └── msgpack │ ├── MsgPackTest.java │ └── TestObject.java ├── chronos-redis ├── README.md ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── ds │ │ └── chronos │ │ └── chronicle │ │ └── RedisChronicle.java │ └── test │ └── java │ └── org │ └── ds │ └── chronos │ ├── RedisChronicleTest.java │ └── TestBase.java ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew └── settings.gradle /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: run tests 2 | on: [push] 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | 7 | services: 8 | casandra: 9 | image: cassandra 10 | ports: 11 | - 9042:9042 12 | - 7000:7000 13 | - 7199:7199 14 | redis: 15 | image: redis 16 | ports: 17 | - 6379:6379 18 | 19 | strategy: 20 | matrix: 21 | java: [ 8.0.x ] 22 | name: Java ${{ matrix.java }} sample 23 | steps: 24 | - uses: actions/checkout@master 25 | - name: Setup java 26 | uses: actions/setup-java@v1 27 | with: 28 | java-version: ${{ matrix.java }} 29 | - uses: eskatos/gradle-command-action@v1 30 | with: 31 | arguments: clean test 32 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: publish artifact 2 | on: 3 | push: 4 | branches: 5 | - master 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | name: Publish 10 | steps: 11 | - uses: actions/checkout@master 12 | - name: Setup java 13 | uses: actions/setup-java@v1 14 | with: 15 | java-version: 8.0.x 16 | - uses: eskatos/gradle-command-action@v1 17 | env: 18 | GPR_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} 19 | with: 20 | arguments: clean publish 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .settings 3 | .project 4 | release.properties 5 | *.releaseBackup 6 | .classpath 7 | build/ 8 | .gradle/ 9 | */.gradle/ 10 | */build/ 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2013 Dan Simpson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Chronos 2 | 3 | A small API for storing, traversing, and processing 4 | timeseries data. 5 | 6 | #### APIS 7 | 8 | * Storage 9 | * Iteration, counting, and deletion for time ranges 10 | * Lazy processing 11 | * Filters, transformations, partitioning, and reducing 12 | * Customized serialization 13 | 14 | ## General Concept 15 | 16 | Creating timelines and adding data: 17 | 18 | ```java 19 | Timeline timeline = new MetricStore(chronicle); 20 | timeline.add(metric); 21 | timeline.add(metrics); 22 | ``` 23 | 24 | Lazily load events and process them through a pipeline 25 | with a simple moving average, threshold filter, and then 26 | reduce the data to into summary for every hour (1h), and finally 27 | iterate over the results. Example from chronos-metrics: 28 | 29 | ```java 30 | Iterable stream = timeline.query(begin, end) 31 | .map(sma(30)) 32 | .filter(gte(0f)) 33 | .partition(new DurationPredicate("1h")) 34 | .reduceAll(MetricFilters.summarize) 35 | .streamAs(MetricSummary.class); 36 | 37 | for(MetricSummary metric: stream) { 38 | System.out.println(metric.getStandardDeviation()); 39 | } 40 | ``` 41 | 42 | 43 | ### Chronicles 44 | 45 | A Chronicle is a the base unit of storage for timeseries. If you 46 | want a higher level API, see the Timeline class. 47 | 48 | Chronicle Types: 49 | 50 | * MemoryChronicle - Used for tests, etc 51 | * CassandraChronicle - A single row chronicle (see chronos-cassandra) 52 | * PartitionedChronicle - A chronicle partitioned over many chronicles. 53 | Each event falls on a key based on the event date, for example 54 | an event day 2013-01-15 would land on the key: site-445-2013-01. 55 | Queries iterate over keys and columns to stream the partitioned time series 56 | in order (see chronos-cassandra) 57 | * RedisChronicle - Redis backed with Jedis 58 | 59 | ##### Adding data 60 | 61 | ```java 62 | chronicle.add(new Date(), "log entry"); 63 | chronicle.add(System.currentTimeMillis(), new byte[] { 0x10, 0x50 }); 64 | ``` 65 | 66 | Add in bulk: 67 | 68 | ```java 69 | ChronicleBatch batch = new ChronicleBatch(); 70 | batch.add(System.currentTimeMillis() - 100, "1"); 71 | batch.add(System.currentTimeMillis() - 50, "2"); 72 | batch.add(System.currentTimeMillis() - 20, "1"); 73 | chronicle.add(batch); 74 | ``` 75 | 76 | ##### Streaming data 77 | 78 | ```java 79 | long t1 = System.currentTimeMillis() - 500; 80 | long t2 = System.currentTimeMillis(); 81 | 82 | Iterator stream = chronicle.getRange(t1, t2); 83 | while(stream.hasNext()) { 84 | ChronologicalRecord record = stream.next(); 85 | long time = record.getTimestamp(); 86 | byte[] data = record.getData(); 87 | } 88 | ``` 89 | 90 | ##### Counting 91 | 92 | ```java 93 | long count = chronicle.getNumEvents(t1, t2); 94 | ``` 95 | 96 | ##### Deleting 97 | 98 | ```java 99 | chronicle.deleteRange(t1, t2); 100 | ``` 101 | 102 | 103 | ##### Record check 104 | 105 | ```java 106 | boolean recorded = chronicle.isEventRecorded(t1); 107 | ``` 108 | 109 | ### Timelines and DataStreams 110 | 111 | A Timeline wraps a Chronicle and provides streaming encoding/decoding 112 | from ChronologicalRecord objects to a class of your choice and back. It abstracts away 113 | some of the complexity with pure Chronicles. A DataStream wraps the FluentIterable class 114 | from Guava and supports filters, transforms, partitioning, and reducers. This model supports 115 | lazy loading from the underlying data source. 116 | 117 | ##### Create an encoder/decoder 118 | 119 | Example class: 120 | 121 | ```java 122 | public class TestData { 123 | public long time; 124 | public byte type; 125 | public double value; 126 | } 127 | ``` 128 | 129 | Create a decoder which streams columns from a chronicle 130 | and outputs TestData objects: 131 | 132 | ```java 133 | public class TestDecoder implements TimelineDecoder { 134 | 135 | private Iterator input; 136 | 137 | @Override 138 | public void setInputStream(Iterator input) { 139 | this.input = input; 140 | } 141 | 142 | @Override 143 | public boolean hasNext() { 144 | return input.hasNext(); 145 | } 146 | 147 | @Override 148 | public TestData next() { 149 | ChronologicalRecord column = input.next(); 150 | ByteBuffer buffer = column.getValueBytes(); 151 | 152 | TestData data = new TestData(); 153 | data.time = column.getName(); 154 | data.type = buffer.get(); 155 | data.value = buffer.getDouble(); 156 | return data; 157 | } 158 | 159 | @Override 160 | public void remove() { 161 | } 162 | 163 | } 164 | ``` 165 | 166 | Create an encoder which encodes TestData objects as columns: 167 | 168 | ```java 169 | public class TestEncoder implements TimelineEncoder { 170 | 171 | private Iterator input; 172 | 173 | @Override 174 | public boolean hasNext() { 175 | return input.hasNext(); 176 | } 177 | 178 | @Override 179 | public ChronologicalRecord next() { 180 | TestData data = input.next(); 181 | ByteBuffer buffer = ByteBuffer.allocate(9); 182 | buffer.put(data.type); 183 | buffer.putDouble(data.value); 184 | buffer.rewind(); 185 | return new ChronologicalRecord(data.time, buffer.array()); 186 | } 187 | 188 | @Override 189 | public void remove() { 190 | } 191 | 192 | @Override 193 | public void setInputStream(Iterator input) { 194 | this.input = input; 195 | } 196 | } 197 | ``` 198 | 199 | ##### Create a Timeline 200 | 201 | ```java 202 | Timeline timeline 203 | = new Timeline(getChronicle(), new TestEncoder(), new TestDecoder()); 204 | ``` 205 | 206 | ##### Add objects 207 | 208 | Add an object: 209 | 210 | ```java 211 | timeline.add(buildTestData()); 212 | ``` 213 | 214 | Bulk add objects: 215 | 216 | ```java 217 | timeline.add(buildTestDataCollection()); 218 | ``` 219 | 220 | Alternate bulk add with specified batch size: 221 | 222 | ```java 223 | timeline.add(buildTestDataCollection().iterator(), 5); 224 | ``` 225 | The above shows the use of iterators, which we can take advantage 226 | of using timelines as an input for transfer. 227 | 228 | ##### Stream processing 229 | 230 | Processing the data stream is possible using 3 functional(ish) 231 | interfaces: 232 | 233 | * map(function -> O) 234 | * filter(predicate -> boolean) 235 | * partition(predicate) -> PartitionedDataStream 236 | * reduceAll(function, T>) -> DataStream 237 | 238 | Example of a map function which plucks just the tag from the stream 239 | of Observation objects. 240 | 241 | ```java 242 | new Function() { 243 | public String apply(Observation o) { 244 | return p.getTag(); 245 | } 246 | }; 247 | ``` 248 | 249 | Example of a filter, which only emits observations where the tag = "spike" 250 | 251 | ```java 252 | new Predicate() { 253 | public boolean apply(Observation o) { 254 | return "spike".equals(p.getTag()); 255 | } 256 | }; 257 | ``` 258 | 259 | #### Stiching it together 260 | 261 | With the DataStream, it's possible to compose multi stage pipelines 262 | of iterators which apply the map functions and emit a result stream. 263 | 264 | ```java 265 | Iterable stream = timeline.query(begin, end) 266 | .filter(tagFilter) 267 | .map(multBy2) 268 | .streamAs(Double.class); 269 | 270 | for(Double d: stream) { 271 | out.write(d); 272 | } 273 | ``` 274 | 275 | 276 | Maven 277 | ----- 278 | 279 | Maven central or other repo is planned. 280 | 281 | ```xml 282 | 283 | org.ds.chronos 284 | chronos-api 285 | ${chronos.version} 286 | 287 | ``` 288 | 289 | Modules 290 | ------- 291 | * chronos-api - Timeseries processing and storage API 292 | * chronos-jackson - Storing timeseries objects as json 293 | * chronos-metrics - Storing timeseries as numeric values with some transformation utilities 294 | * chronos-cassandra - Scalable timeseries storage and retreival with hector and cassandra 295 | * chronos-redis - Storage with redis 296 | * chronos-aws - Storage with S3/SimpleDB for archiving 297 | 298 | Contributing 299 | ------------ 300 | 301 | Just fork it, change it, rebase, and send a pull request. 302 | 303 | License 304 | ------- 305 | MIT License -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | allprojects { 2 | apply plugin: 'maven' 3 | apply plugin: 'maven-publish' 4 | group = 'org.ds.chronos' 5 | version = '1.0.13' 6 | } 7 | 8 | subprojects { 9 | apply plugin: 'java' 10 | sourceCompatibility = 1.8 11 | targetCompatibility = 1.8 12 | 13 | tasks.withType(JavaCompile) { 14 | options.encoding = 'UTF-8' 15 | } 16 | 17 | task packageSources(type: Jar) { 18 | classifier = 'sources' 19 | from sourceSets.main.allSource 20 | } 21 | 22 | artifacts.archives packageSources 23 | 24 | repositories { 25 | mavenLocal() 26 | mavenCentral() 27 | jcenter() 28 | } 29 | 30 | dependencies { 31 | testCompile group: 'junit', name: 'junit', version:'4.10' 32 | } 33 | 34 | publishing { 35 | repositories { 36 | maven { 37 | name = "GitHubPackages" 38 | url = uri("https://maven.pkg.github.com/dansimpson/chronos") 39 | credentials { 40 | username = System.getenv("GPR_USER") ?: 'dansimpson' 41 | password = System.getenv("GPR_AUTH_TOKEN") 42 | } 43 | } 44 | } 45 | publications { 46 | gpr(MavenPublication) { 47 | groupId group 48 | // artifactId "influx-protocol" 49 | version version 50 | from components.java 51 | } 52 | } 53 | } 54 | 55 | 56 | } 57 | -------------------------------------------------------------------------------- /chronos-api/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | description = 'chronos api' 3 | dependencies { 4 | compile group: 'com.google.guava', name: 'guava', version:'19.0' 5 | } 6 | 7 | task packageTests(type: Jar) { 8 | from sourceSets.test.output 9 | classifier = 'tests' 10 | } 11 | artifacts.archives packageTests 12 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/Chronicle.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.nio.charset.Charset; 5 | import java.util.Date; 6 | import java.util.Iterator; 7 | import java.util.Spliterator; 8 | import java.util.Spliterators; 9 | import java.util.stream.Stream; 10 | import java.util.stream.StreamSupport; 11 | 12 | import org.ds.chronos.timeline.SimpleTimeline; 13 | 14 | /** 15 | * 16 | * Chronicle 17 | *

18 | * A timeseries storage API with operations for storing columns, batches of columns, fetching ranges, deleting ranges, and counting ranges. 19 | * Events are composed of a long timestamp value and a byte[] of data. 20 | *

21 | * For a higher level of abstraction, see {@link SimpleTimeline} 22 | *

23 | * 24 | * @see CassandraChronicle 25 | * @see PartitionedChronicle 26 | * @see Chronos#getChronicle(String) 27 | * @see Chronos#getChronicle(String, PartitionPeriod) 28 | * 29 | * @author Dan Simpson 30 | * 31 | */ 32 | public abstract class Chronicle { 33 | 34 | private static final int STREAM_CHARACTERISTICS = Spliterator.DISTINCT | Spliterator.SORTED | Spliterator.ORDERED; 35 | public static final Charset CHARSET = Charset.forName("UTF8"); 36 | public static int WRITE_PAGE_SIZE = 1024; 37 | public static int READ_PAGE_SIZE = 2048; 38 | 39 | /** 40 | * Add a single column to the Chronicle 41 | * 42 | * @param column 43 | */ 44 | public abstract void add(ChronologicalRecord item); 45 | 46 | /** 47 | * Add columns in batches 48 | * 49 | * @param items 50 | * columns 51 | * @param pageSize 52 | * The number of columns to write per batch 53 | */ 54 | public abstract void add(Iterator items, int pageSize); 55 | 56 | /** 57 | * Add data with a given time stamp 58 | * 59 | * @param timestamp 60 | * @param data 61 | */ 62 | public void add(long timestamp, byte[] data) { 63 | add(new ChronologicalRecord(timestamp, data)); 64 | } 65 | 66 | /** 67 | * Add byte buffer with a given time stamp 68 | * 69 | * @param timestamp 70 | * @param data 71 | */ 72 | public void add(long timestamp, ByteBuffer data) { 73 | add(new ChronologicalRecord(timestamp, data.array())); 74 | } 75 | 76 | /** 77 | * Add string bytes with a given time stamp 78 | * 79 | * @param timestamp 80 | * @param data 81 | */ 82 | public void add(long timestamp, String data) { 83 | add(new ChronologicalRecord(timestamp, data.getBytes(CHARSET))); 84 | } 85 | 86 | /** 87 | * Add data with a given date time 88 | * 89 | * @param time 90 | * @param data 91 | */ 92 | public void add(Date time, byte[] data) { 93 | add(time.getTime(), data); 94 | } 95 | 96 | /** 97 | * Add data with a given date time 98 | * 99 | * @param time 100 | * @param data 101 | */ 102 | public void add(Date time, ByteBuffer data) { 103 | add(time.getTime(), data); 104 | } 105 | 106 | /** 107 | * Add data with a given date time 108 | * 109 | * @param time 110 | * @param data 111 | */ 112 | public void add(Date time, String data) { 113 | add(time.getTime(), data); 114 | } 115 | 116 | /** 117 | * 118 | * @param items 119 | */ 120 | public void add(Iterator items) { 121 | add(items, WRITE_PAGE_SIZE); 122 | } 123 | 124 | /** 125 | * 126 | * @param items 127 | */ 128 | public void add(Iterable items) { 129 | add(items.iterator(), WRITE_PAGE_SIZE); 130 | } 131 | 132 | /** 133 | * 134 | * @param items 135 | */ 136 | public void add(ChronicleBatch batch) { 137 | add(batch.iterator(), WRITE_PAGE_SIZE); 138 | } 139 | 140 | /** 141 | * Get lazy loading stream of events. If t1 > t2, then the list will be iterated in reverse. 142 | * 143 | * @param t1 144 | * begin time 145 | * @param t1 146 | * end time 147 | * @param pageSize 148 | * the number of columns to fetch per batch 149 | * @return 150 | */ 151 | public abstract Iterator getRange(long t1, long t2, int pageSize); 152 | 153 | /** 154 | * 155 | * Get lazy loaded range of events 156 | * 157 | * @param t1 158 | * @param t2 159 | * @return 160 | */ 161 | public Iterator getRange(long t1, long t2) { 162 | return getRange(t1, t2, READ_PAGE_SIZE); 163 | } 164 | 165 | /** 166 | * 167 | * Get lazy loaded range of events 168 | * 169 | * @param beginTime 170 | * @param endTime 171 | * @return 172 | */ 173 | public Iterator getRange(Date beginTime, Date endTime) { 174 | return getRange(beginTime.getTime(), endTime.getTime()); 175 | } 176 | 177 | /** 178 | * Return a java stream of records for the time range 179 | * 180 | * @param t1 181 | * @param t2 182 | * @return 183 | */ 184 | public Stream getStreamRange(long t1, long t2) { 185 | return StreamSupport 186 | .stream(Spliterators.spliteratorUnknownSize(getRange(t1, t2, READ_PAGE_SIZE), STREAM_CHARACTERISTICS), false); 187 | } 188 | 189 | /** 190 | * Return a java stream of records for the time range 191 | * 192 | * @param beginTime 193 | * @param endTime 194 | * @return 195 | */ 196 | public Stream getStreamRange(Date beginTime, Date endTime) { 197 | return getStreamRange(beginTime.getTime(), endTime.getTime()); 198 | } 199 | 200 | /** 201 | * 202 | * Get lazy loaded range of events 203 | * 204 | * @param beginTime 205 | * @param endTime 206 | * @param pageSize 207 | * number of columns to fetch at a time 208 | * @return 209 | */ 210 | public Iterator getRange(Date beginTime, Date endTime, int pageSize) { 211 | return getRange(beginTime.getTime(), endTime.getTime(), pageSize); 212 | } 213 | 214 | /** 215 | * Get the number of events for a given time range 216 | * 217 | * @return number of columns accross all keys 218 | */ 219 | public abstract long getNumEvents(long t1, long t2); 220 | 221 | public boolean isEventRecorded(long t1) { 222 | return getNumEvents(t1, t1) > 0; 223 | } 224 | 225 | /** 226 | * 227 | * @param d1 228 | * @param d2 229 | * @return the number of events between d1 and d2 230 | */ 231 | public long getNumEvents(Date d1, Date d2) { 232 | return getNumEvents(d1.getTime(), d2.getTime()); 233 | } 234 | 235 | /** 236 | * Delete the entire chronicle 237 | */ 238 | public abstract void delete(); 239 | 240 | /** 241 | * Delete a subset of the Chronicle 242 | * 243 | * @param beginTime 244 | * @param endTime 245 | */ 246 | public abstract void deleteRange(long t1, long t2); 247 | 248 | public void deleteRange(Date t1, Date t2) { 249 | deleteRange(t1.getTime(), t2.getTime()); 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/ChronicleBatch.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.util.ArrayList; 5 | import java.util.Date; 6 | import java.util.Iterator; 7 | import java.util.List; 8 | 9 | /** 10 | * Utility for batch updates of events 11 | * 12 | * @author Dan Simpson 13 | * 14 | */ 15 | public class ChronicleBatch { 16 | 17 | private List columns = new ArrayList(); 18 | 19 | public ChronicleBatch() { 20 | } 21 | 22 | /** 23 | * Batch add columns 24 | * 25 | * @param items 26 | */ 27 | public void add(ChronologicalRecord item) { 28 | columns.add(item); 29 | } 30 | 31 | /** 32 | * Add data with a given time stamp 33 | * 34 | * @param timestamp 35 | * @param data 36 | */ 37 | public void add(long timestamp, byte[] data) { 38 | add(new ChronologicalRecord(timestamp, data)); 39 | } 40 | 41 | /** 42 | * Add byte buffer with a given time stamp 43 | * 44 | * @param timestamp 45 | * @param data 46 | */ 47 | public void add(long timestamp, ByteBuffer data) { 48 | add(new ChronologicalRecord(timestamp, data.array())); 49 | } 50 | 51 | /** 52 | * Add string bytes with a given time stamp 53 | * 54 | * @param timestamp 55 | * @param data 56 | */ 57 | public void add(long timestamp, String data) { 58 | add(new ChronologicalRecord(timestamp, data.getBytes(Chronicle.CHARSET))); 59 | } 60 | 61 | /** 62 | * Add data with a given date time 63 | * 64 | * @param time 65 | * @param data 66 | */ 67 | public void add(Date time, byte[] data) { 68 | add(time.getTime(), data); 69 | } 70 | 71 | /** 72 | * Add data with a given date time 73 | * 74 | * @param time 75 | * @param data 76 | */ 77 | public void add(Date time, ByteBuffer data) { 78 | add(time.getTime(), data); 79 | } 80 | 81 | /** 82 | * Add data with a given date time 83 | * 84 | * @param time 85 | * @param data 86 | */ 87 | public void add(Date time, String data) { 88 | add(time.getTime(), data); 89 | } 90 | 91 | public Iterator iterator() { 92 | return columns.iterator(); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/ChronologicalRecord.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.nio.ByteBuffer; 4 | 5 | /** 6 | * A single Chronological Record 7 | * 8 | * @author Dan Simpson 9 | * 10 | */ 11 | public class ChronologicalRecord implements Comparable, Temporal { 12 | 13 | private long timestamp; 14 | private byte[] data; 15 | 16 | public ChronologicalRecord(long timestamp, byte[] data) { 17 | super(); 18 | this.timestamp = timestamp; 19 | this.data = data; 20 | } 21 | 22 | public ChronologicalRecord(long timestamp, ByteBuffer buffer) { 23 | super(); 24 | this.timestamp = timestamp; 25 | this.data = new byte[buffer.remaining()]; 26 | buffer.get(data); 27 | } 28 | 29 | /** 30 | * @return the timestamp 31 | */ 32 | public long getTimestamp() { 33 | return timestamp; 34 | } 35 | 36 | /** 37 | * @return the data 38 | */ 39 | public byte[] getData() { 40 | return data; 41 | } 42 | 43 | public ByteBuffer getValueBytes() { 44 | return ByteBuffer.wrap(data); 45 | } 46 | 47 | @Override 48 | public int compareTo(ChronologicalRecord o) { 49 | return Long.valueOf(timestamp).compareTo(o.timestamp); 50 | } 51 | 52 | /** 53 | * Get the number of bytes in the record 54 | * 55 | * @return 56 | */ 57 | public int getByteSize() { 58 | return data.length; 59 | } 60 | 61 | public String toString() { 62 | return String.format("%s - %d bytes", timestamp, getByteSize()); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/ChronosException.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | /** 4 | * 5 | * @author Dan Simpson 6 | * 7 | */ 8 | public class ChronosException extends Exception { 9 | 10 | private static final long serialVersionUID = 9037335065207393371L; 11 | 12 | public ChronosException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/PartitionIterator.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.Iterator; 4 | import java.util.LinkedList; 5 | 6 | 7 | /** 8 | * An iterator of iterators used for flattening blocks 9 | * 10 | * @author Dan Simpson 11 | * 12 | */ 13 | public class PartitionIterator implements Iterator { 14 | 15 | private LinkedList> iterators; 16 | 17 | public PartitionIterator(LinkedList> iterators) { 18 | this.iterators = iterators; 19 | } 20 | 21 | @Override 22 | public boolean hasNext() { 23 | while (!iterators.isEmpty()) { 24 | if (iterators.getFirst().hasNext()) { 25 | return true; 26 | } 27 | iterators.removeFirst(); 28 | } 29 | return false; 30 | } 31 | 32 | @Override 33 | public ChronologicalRecord next() { 34 | return iterators.getFirst().next(); 35 | } 36 | 37 | @Override 38 | public void remove() { 39 | iterators.getFirst().remove(); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/PartitionPeriod.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.Calendar; 4 | import java.util.Collections; 5 | import java.util.Date; 6 | import java.util.LinkedList; 7 | import java.util.List; 8 | import java.util.TimeZone; 9 | 10 | /** 11 | * A period of time for use with partitioning data over multiple keys 12 | * 13 | * @author Dan Simpson 14 | * 15 | */ 16 | public enum PartitionPeriod { 17 | 18 | HOUR(Calendar.HOUR), 19 | DAY(Calendar.DAY_OF_MONTH), 20 | MONTH(Calendar.MONTH), 21 | YEAR(Calendar.YEAR); 22 | 23 | private int calendarField; 24 | 25 | private PartitionPeriod(int calendarField) { 26 | this.calendarField = calendarField; 27 | } 28 | 29 | public final int getCalendarField() { 30 | return calendarField; 31 | } 32 | 33 | public final String getPeriodKey(String key, long timestamp) { 34 | return getPeriodKey(key, getCalendar(timestamp)); 35 | } 36 | 37 | public final String getPeriodKey(String key, Calendar date) { 38 | switch (this) { 39 | case HOUR: 40 | return String.format("%s-%04d-%02d-%02d-%02d", key, date.get(Calendar.YEAR), date.get(Calendar.MONTH) + 1, 41 | date.get(Calendar.DAY_OF_MONTH), date.get(Calendar.HOUR_OF_DAY)); 42 | case DAY: 43 | return String.format("%s-%04d-%02d-%02d", key, date.get(Calendar.YEAR), date.get(Calendar.MONTH) + 1, 44 | date.get(Calendar.DAY_OF_MONTH)); 45 | case MONTH: 46 | return String.format("%s-%04d-%02d", key, date.get(Calendar.YEAR), date.get(Calendar.MONTH) + 1); 47 | case YEAR: 48 | return String.format("%s-%04d", key, date.get(Calendar.YEAR)); 49 | default: 50 | return null; 51 | } 52 | } 53 | 54 | public final List getPeriodKeys(String prefix, Date d1, Date d2) { 55 | return getPeriodKeys(prefix, d1.getTime(), d2.getTime()); 56 | } 57 | 58 | private static final TimeZone TZ = TimeZone.getTimeZone("UTC"); 59 | 60 | public final List getPeriodKeys(String prefix, long t1, long t2) { 61 | if (t1 > t2) { 62 | List keys = getPeriodKeys(prefix, t2, t1); 63 | Collections.reverse(keys); 64 | return keys; 65 | } 66 | 67 | Calendar calendar = getCalendar(t1); 68 | 69 | LinkedList keys = new LinkedList(); 70 | while (calendar.getTimeInMillis() <= t2) { 71 | keys.add(getPeriodKey(prefix, calendar)); 72 | calendar.add(calendarField, 1); 73 | clearCal(calendar); 74 | } 75 | return keys; 76 | } 77 | 78 | private Calendar getCalendar(long timestamp) { 79 | Calendar calendar = Calendar.getInstance(); 80 | calendar.setTimeZone(TZ); 81 | calendar.setTimeInMillis(timestamp); 82 | clearCal(calendar); 83 | return calendar; 84 | } 85 | 86 | private void clearCal(Calendar calendar) { 87 | calendar.set(Calendar.MINUTE, 0); 88 | calendar.set(Calendar.SECOND, 0); 89 | calendar.set(Calendar.MILLISECOND, 0); 90 | 91 | if (this != HOUR) { 92 | calendar.set(Calendar.AM_PM, 0); 93 | calendar.set(Calendar.HOUR, 0); 94 | } 95 | 96 | // We need to fetch every key which could have data. If we 97 | // have a query that falls on the 8th, this will account for 98 | // that, and make sure we get all keys 99 | if (this == MONTH || this == YEAR) { 100 | calendar.set(Calendar.DAY_OF_MONTH, 1); 101 | } 102 | 103 | if (this == YEAR) { 104 | calendar.set(Calendar.MONTH, 0); 105 | } 106 | } 107 | } -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/PartitionedChronicle.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.Calendar; 4 | import java.util.HashMap; 5 | import java.util.Iterator; 6 | import java.util.LinkedList; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Map.Entry; 10 | 11 | import org.ds.chronos.timeline.SimpleTimeline; 12 | 13 | /** 14 | * 15 | * PartitionedChronicle 16 | *

17 | * A chronicle which stores data over many keys. This strategy is aimed at data with a predictable interval. 18 | *

19 | * Each event is mapped to a partition. A parition is a key which has Date information. For instance, if {@link PartitionPeriod} is YEAR, 20 | * then an event with date 2012-04-01 will partition onto the key: prefix-2012. 21 | *

22 | * 23 | * @see SimpleTimeline 24 | * @see PartitionPeriod 25 | * 26 | * TODO: Threaded prefetching? 27 | * 28 | * @author Dan Simpson 29 | * 30 | */ 31 | public abstract class PartitionedChronicle extends Chronicle { 32 | 33 | protected final String keyPrefix; 34 | protected final PartitionPeriod period; 35 | 36 | public PartitionedChronicle(String keyPrefix, PartitionPeriod period) { 37 | super(); 38 | this.keyPrefix = keyPrefix; 39 | this.period = period; 40 | } 41 | 42 | /** 43 | * Create a chronicle for a single partition 44 | * 45 | * @param key 46 | * the key of the partition 47 | * @return a chronicle which represents only the single partition 48 | */ 49 | protected abstract Chronicle getPartition(String key); 50 | 51 | @Override 52 | public void add(ChronologicalRecord item) { 53 | getPartition(getKeyName(item.getTimestamp())).add(item); 54 | } 55 | 56 | /** 57 | * Partition in memory and then batch write to each key 58 | * 59 | * @param columns 60 | */ 61 | @Override 62 | public void add(Iterator columns, int pageSize) { 63 | Map> groups = new HashMap>(); 64 | 65 | Calendar date = Calendar.getInstance(); 66 | while (columns.hasNext()) { 67 | ChronologicalRecord column = columns.next(); 68 | date.setTimeInMillis(column.getTimestamp()); 69 | String key = getKeyName(date); 70 | 71 | if (!groups.containsKey(key)) { 72 | groups.put(key, new LinkedList()); 73 | } 74 | groups.get(key).add(column); 75 | } 76 | 77 | for (Entry> entry : groups.entrySet()) { 78 | getPartition(entry.getKey()).add(entry.getValue()); 79 | } 80 | } 81 | 82 | @Override 83 | public Iterator getRange(long t1, long t2, int batchSize) { 84 | LinkedList> iterators = new LinkedList>(); 85 | for (Chronicle chronicle : getSubChronicles(t1, t2)) { 86 | iterators.add(chronicle.getRange(t1, t2, batchSize)); 87 | } 88 | return new PartitionIterator(iterators); 89 | } 90 | 91 | @Override 92 | public long getNumEvents(long t1, long t2) { 93 | long result = 0; 94 | for (Chronicle chronicle : getSubChronicles(t1, t2)) { 95 | result += chronicle.getNumEvents(t1, t2); 96 | } 97 | return result; 98 | } 99 | 100 | @Override 101 | public void delete() { 102 | } 103 | 104 | private String getKeyName(long timestamp) { 105 | return period.getPeriodKey(keyPrefix, timestamp); 106 | } 107 | 108 | private String getKeyName(Calendar date) { 109 | return period.getPeriodKey(keyPrefix, date); 110 | } 111 | 112 | @Override 113 | public void deleteRange(long t1, long t2) { 114 | for (Chronicle chronicle : getSubChronicles(t1, t2)) { 115 | chronicle.deleteRange(t1, t2); 116 | } 117 | } 118 | 119 | private List getSubChronicles(long t1, long t2) { 120 | List items = new LinkedList(); 121 | for (String key : period.getPeriodKeys(keyPrefix, t1, t2)) { 122 | items.add(getPartition(key)); 123 | } 124 | return items; 125 | } 126 | 127 | } 128 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/Temporal.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | /** 4 | * A contract for an object to have a temporal attribute 5 | * 6 | * @author Dan Simpson 7 | * 8 | */ 9 | public interface Temporal { 10 | 11 | /** 12 | * The timestamp for the temporal object 13 | * 14 | * @return millis since epoch 15 | */ 16 | public long getTimestamp(); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/Timeline.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.Collection; 4 | import java.util.Date; 5 | import java.util.Iterator; 6 | 7 | import org.ds.chronos.timeline.stream.DataStream; 8 | 9 | public abstract class Timeline { 10 | 11 | public static final int DEFAULT_BATCH_WRITE_SIZE = 512; 12 | public static final int DEFAULT_BATCH_READ_SIZE = 2048; 13 | 14 | protected int batchReadSize = DEFAULT_BATCH_READ_SIZE; 15 | protected int batchWriteSize = DEFAULT_BATCH_WRITE_SIZE; 16 | 17 | /** 18 | * Iterate over items and persist in batches of batchSize 19 | * 20 | * @param data 21 | * @param batchSize 22 | */ 23 | public abstract void add(Iterator data, int batchSize); 24 | 25 | /** 26 | * Add a collection of T items to the timeline 27 | * 28 | * @param data 29 | */ 30 | public void add(Collection data) { 31 | add(data.iterator(), batchWriteSize); 32 | } 33 | 34 | /** 35 | * Add a single T to the chronicle 36 | * 37 | * @param item 38 | * T object 39 | */ 40 | public abstract void add(T item); 41 | 42 | /** 43 | * Fetch a stream of metrics for a given timeframe 44 | * 45 | * @param d1 46 | * @param d2 47 | * @return iterable stream of metrics 48 | */ 49 | public abstract Iterator buildIterator(long t1, long t2, int batchSize); 50 | 51 | public Iterator buildIterator(long t1, long t2) { 52 | return buildIterator(t1, t2, batchReadSize); 53 | } 54 | 55 | /** 56 | * Iterable of timeline for a given range 57 | * 58 | * @param t1 59 | * @param t2 60 | * @return 61 | */ 62 | public Iterable iterable(final long t1, final long t2) { 63 | return new Iterable() { 64 | 65 | @Override 66 | public Iterator iterator() { 67 | return buildIterator(t1, t2); 68 | } 69 | 70 | }; 71 | } 72 | 73 | /** 74 | * Fetch a lazy data stream for the time range, of which has stages that ultimately produce objects of the output class 75 | * 76 | * @param t1 77 | * from time 78 | * @param t2 79 | * to time 80 | * @param klass 81 | * the type of the output (post transformation) 82 | * @return 83 | */ 84 | public DataStream query(long t1, long t2) { 85 | return new DataStream(iterable(t1, t2)); 86 | } 87 | 88 | /** 89 | * Fetch a lazy data stream for the time range, of which has stages that ultimately produce objects of the output class 90 | * 91 | * @param t1 92 | * @param t2 93 | * @return 94 | */ 95 | public DataStream query(Date t1, Date t2) { 96 | return query(t1.getTime(), t2.getTime()); 97 | } 98 | 99 | /** 100 | * @param t1 101 | * @param t2 102 | * @return 103 | * @see org.ds.chronos.api.Chronicle#getNumEvents(long, long) 104 | */ 105 | public abstract long getNumEvents(long t1, long t2); 106 | 107 | public long getNumEvents(Date d1, Date d2) { 108 | return getNumEvents(d1.getTime(), d2.getTime()); 109 | } 110 | 111 | /** 112 | * @param t1 113 | * @return 114 | * @see org.ds.chronos.api.Chronicle#isEventRecorded(long) 115 | */ 116 | public abstract boolean isEventRecorded(long time); 117 | 118 | public boolean isEventRecorded(Date date) { 119 | return isEventRecorded(date.getTime()); 120 | } 121 | 122 | /** 123 | * @param t1 124 | * @param t2 125 | * @see org.ds.chronos.api.Chronicle#deleteRange(long, long) 126 | */ 127 | public abstract void deleteRange(long t1, long t2); 128 | 129 | public void deleteRange(Date t1, Date t2) { 130 | deleteRange(t1.getTime(), t2.getTime()); 131 | } 132 | 133 | } 134 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/TimelineDecoder.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.Iterator; 4 | 5 | 6 | /** 7 | * 8 | * Implement this to allow streaming objects of your choosing, which are built 9 | * lazily for each page of data 10 | * 11 | * @author Dan Simpson 12 | * 13 | * @param 14 | */ 15 | public interface TimelineDecoder extends Iterator { 16 | 17 | /** 18 | * 19 | * @param input 20 | */ 21 | public void setInputStream(Iterator input); 22 | } 23 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/TimelineEncoder.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.Iterator; 4 | 5 | 6 | /** 7 | * 8 | * @author Dan Simpson 9 | * 10 | * @param 11 | */ 12 | public interface TimelineEncoder extends Iterator { 13 | 14 | /** 15 | * 16 | * @param input 17 | */ 18 | public void setInputStream(Iterator input); 19 | } 20 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/api/chronicle/MemoryChronicle.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api.chronicle; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Iterator; 5 | import java.util.LinkedList; 6 | import java.util.List; 7 | import java.util.TreeSet; 8 | 9 | import org.ds.chronos.api.Chronicle; 10 | import org.ds.chronos.api.ChronologicalRecord; 11 | 12 | /** 13 | * A memory based Chronicle for tests. 14 | * 15 | * @author Dan Simpson 16 | * 17 | */ 18 | public class MemoryChronicle extends Chronicle { 19 | 20 | protected TreeSet items = new TreeSet(); 21 | 22 | public MemoryChronicle() { 23 | } 24 | 25 | @Override 26 | public void add(ChronologicalRecord item) { 27 | items.add(item); 28 | } 29 | 30 | @Override 31 | public void add(Iterator itr, int pageSize) { 32 | while (itr.hasNext()) { 33 | items.add(itr.next()); 34 | } 35 | } 36 | 37 | @Override 38 | public Iterator getRange(long t1, long t2, int pageSize) { 39 | if (t1 > t2) { 40 | return getRangeReversed(t2, t1); 41 | } 42 | List result = new LinkedList(); 43 | for (ChronologicalRecord item : items.subSet(low(t1), true, high(t2), true)) { 44 | result.add(item); 45 | } 46 | return result.iterator(); 47 | } 48 | 49 | private Iterator getRangeReversed(long t1, long t2) { 50 | LinkedList result = new LinkedList(); 51 | for (ChronologicalRecord item : items.subSet(low(t1), true, high(t2), true)) { 52 | result.addFirst(item); 53 | } 54 | return result.iterator(); 55 | } 56 | 57 | @Override 58 | public long getNumEvents(long t1, long t2) { 59 | if (items.isEmpty()) { 60 | return 0; 61 | } 62 | try { 63 | return items.subSet(low(t1), true, high(t2), true).size(); 64 | } catch (Throwable t) { 65 | return 0; 66 | } 67 | } 68 | 69 | @Override 70 | public void delete() { 71 | items.clear(); 72 | } 73 | 74 | @Override 75 | public void deleteRange(long t1, long t2) { 76 | items.removeAll(toList(getRange(t1, t2))); 77 | } 78 | 79 | private ChronologicalRecord low(long time) { 80 | return items.ceiling(new ChronologicalRecord(time, new byte[0])); 81 | } 82 | 83 | private ChronologicalRecord high(long time) { 84 | return items.floor(new ChronologicalRecord(time, new byte[0])); 85 | } 86 | 87 | public static List toList(Iterator records) { 88 | List result = new ArrayList(); 89 | while (records.hasNext()) { 90 | result.add(records.next()); 91 | } 92 | return result; 93 | } 94 | 95 | public int size() { 96 | return items.size(); 97 | } 98 | 99 | public TreeSet all() { 100 | return items; 101 | } 102 | 103 | public boolean isEmpty() { 104 | return items.isEmpty(); 105 | } 106 | 107 | } -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/streams/TemporalIterator.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.streams; 2 | 3 | import java.util.Iterator; 4 | 5 | import org.ds.chronos.api.Temporal; 6 | 7 | public abstract class TemporalIterator implements Iterator, Temporal { 8 | 9 | private final long timestamp; 10 | 11 | public TemporalIterator(long timestamp) { 12 | this.timestamp = timestamp; 13 | } 14 | 15 | public long getTimestamp() { 16 | return timestamp; 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/streams/TemporalStream.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.streams; 2 | 3 | import java.util.stream.Stream; 4 | 5 | import org.ds.chronos.api.Temporal; 6 | 7 | public class TemporalStream implements Temporal { 8 | 9 | private final long timestamp; 10 | private final Stream stream; 11 | 12 | public TemporalStream(long timestamp, Stream stream) { 13 | super(); 14 | this.timestamp = timestamp; 15 | this.stream = stream; 16 | } 17 | 18 | /** 19 | * @return the timestamp 20 | */ 21 | public long getTimestamp() { 22 | return timestamp; 23 | } 24 | 25 | /** 26 | * @return the stream 27 | */ 28 | public Stream getStream() { 29 | return stream; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/streams/WindowIterator.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.streams; 2 | 3 | import java.time.Duration; 4 | import java.util.Iterator; 5 | import java.util.Spliterator; 6 | import java.util.Spliterators; 7 | import java.util.stream.Stream; 8 | import java.util.stream.StreamSupport; 9 | 10 | import org.ds.chronos.api.Temporal; 11 | 12 | /** 13 | * Window 14 | * 15 | * @author dan 16 | * 17 | * @param 18 | */ 19 | public class WindowIterator implements Iterator> { 20 | 21 | private long threshold = Long.MIN_VALUE; 22 | private T next; 23 | 24 | private final Iterator source; 25 | private final long window; 26 | 27 | public WindowIterator(Iterator source, Duration window) { 28 | this(source, window.toMillis()); 29 | } 30 | 31 | public WindowIterator(Iterator source, long window) { 32 | super(); 33 | this.source = source; 34 | this.window = window; 35 | } 36 | 37 | @Override 38 | public boolean hasNext() { 39 | return source.hasNext(); 40 | } 41 | 42 | private void advance() { 43 | if (hasNext()) { 44 | next = source.next(); 45 | if (next.getTimestamp() >= threshold) { 46 | if (threshold == Long.MIN_VALUE) { 47 | threshold = next.getTimestamp() + window; 48 | } else { 49 | threshold += window; 50 | } 51 | } 52 | } else { 53 | threshold = Long.MIN_VALUE; 54 | } 55 | } 56 | 57 | private T getAndAdvance() { 58 | T item = next; 59 | advance(); 60 | return item; 61 | } 62 | 63 | @Override 64 | public TemporalIterator next() { 65 | // Advance for the first check 66 | if (next == null) { 67 | advance(); 68 | } 69 | 70 | long track = threshold; 71 | 72 | // Handle a data gap by advancing and returning an empty iterator 73 | if (next.getTimestamp() >= threshold) { 74 | threshold += window; 75 | } 76 | 77 | return new TemporalIterator(track - window) { 78 | 79 | @Override 80 | public boolean hasNext() { 81 | return next.getTimestamp() < track && next.getTimestamp() < threshold; 82 | } 83 | 84 | @Override 85 | public T next() { 86 | return getAndAdvance(); 87 | } 88 | 89 | }; 90 | } 91 | 92 | private Stream stream(Iterator iterator) { 93 | return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false); 94 | } 95 | 96 | /** 97 | * @return Stream wrapping the iterators 98 | */ 99 | public Stream> stream() { 100 | return stream(this).map(i -> new TemporalStream(i.getTimestamp(), stream(i))); 101 | } 102 | 103 | } -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/timeline/ConcatenatedTimeline.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline; 2 | 3 | import java.util.Arrays; 4 | import java.util.Collection; 5 | import java.util.HashMap; 6 | import java.util.Iterator; 7 | import java.util.LinkedList; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Map.Entry; 11 | 12 | import org.ds.chronos.api.Temporal; 13 | import org.ds.chronos.api.Timeline; 14 | import org.ds.chronos.util.TimeFrame; 15 | 16 | import com.google.common.base.Function; 17 | import com.google.common.base.Predicate; 18 | import com.google.common.collect.Iterables; 19 | import com.google.common.collect.Iterators; 20 | import com.google.common.collect.Lists; 21 | 22 | /** 23 | * A Timeline composed of other Timelines, each of which hold events for a given TimeFrame or "scope" 24 | * 25 | * @author Dan Simpson 26 | * 27 | * @param 28 | */ 29 | public class ConcatenatedTimeline extends Timeline { 30 | 31 | public static final class ScopedTimeline { 32 | 33 | private TimeFrame scope; 34 | private Timeline timeline; 35 | 36 | public ScopedTimeline(Timeline timeline, TimeFrame scope) { 37 | this.timeline = timeline; 38 | this.scope = scope; 39 | } 40 | } 41 | 42 | final Collection> timelines; 43 | 44 | public ConcatenatedTimeline(ScopedTimeline... timelines) { 45 | this(Arrays.asList(timelines)); 46 | } 47 | 48 | public ConcatenatedTimeline(Collection> timelines) { 49 | this.timelines = timelines; 50 | } 51 | 52 | @Override 53 | public void add(Iterator data, final int batchSize) { 54 | Map, List> groups = new HashMap, List>(); 55 | 56 | while (data.hasNext()) { 57 | T item = data.next(); 58 | 59 | for (ScopedTimeline t : timelines) { 60 | if (t.scope.contains(item.getTimestamp())) { 61 | 62 | if (!groups.containsKey(t)) { 63 | groups.put(t, new LinkedList()); 64 | } 65 | groups.get(t).add(item); 66 | 67 | break; 68 | } 69 | } 70 | } 71 | 72 | for (Entry, List> entry : groups.entrySet()) { 73 | entry.getKey().timeline.add(entry.getValue()); 74 | } 75 | } 76 | 77 | @Override 78 | public void add(T item) { 79 | getTimeline(item.getTimestamp()).add(item); 80 | } 81 | 82 | @Override 83 | public Iterator buildIterator(long t1, long t2, final int batchSize) { 84 | 85 | List> items = Lists.newArrayList(Iterables.transform(fit(t1, t2), 86 | new Function, Iterator>() { 87 | 88 | public Iterator apply(ScopedTimeline t) { 89 | return t.timeline.buildIterator(t.scope.getStart(), t.scope.getEnd(), batchSize); 90 | } 91 | })); 92 | 93 | if (t2 < t1) { 94 | items = Lists.reverse(items); 95 | } 96 | 97 | return Iterators.concat(items.iterator()); 98 | } 99 | 100 | @Override 101 | public long getNumEvents(long t1, long t2) { 102 | long result = 0; 103 | for (ScopedTimeline t : fit(t1, t2)) { 104 | result += t.timeline.getNumEvents(t.scope.getStart(), t.scope.getEnd()); 105 | } 106 | return result; 107 | } 108 | 109 | @Override 110 | public boolean isEventRecorded(long time) { 111 | Timeline timeline = getTimeline(time); 112 | if (timeline == null) { 113 | return false; 114 | } 115 | return timeline.isEventRecorded(time); 116 | } 117 | 118 | @Override 119 | public void deleteRange(long t1, long t2) { 120 | for (ScopedTimeline t : fit(t1, t2)) { 121 | t.timeline.deleteRange(t.scope.getStart(), t.scope.getEnd()); 122 | } 123 | } 124 | 125 | private Timeline getTimeline(long time) { 126 | for (ScopedTimeline timeline : timelines) { 127 | if (timeline.scope.contains(time)) { 128 | return timeline.timeline; 129 | } 130 | } 131 | return null; 132 | } 133 | 134 | private Iterable> filter(final long start, final long end) { 135 | 136 | return Iterables.filter(timelines, new Predicate>() { 137 | 138 | public boolean apply(ScopedTimeline timeline) { 139 | return timeline.scope.intersects(start, end); 140 | } 141 | }); 142 | } 143 | 144 | private Iterable> fit(final long start, final long end) { 145 | return Iterables.transform(filter(start, end), new Function, ScopedTimeline>() { 146 | 147 | public ScopedTimeline apply(ScopedTimeline timeline) { 148 | return new ScopedTimeline(timeline.timeline, timeline.scope.fit(start, end)); 149 | } 150 | }); 151 | } 152 | 153 | } 154 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/timeline/SimpleTimeline.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline; 2 | 3 | import java.util.Arrays; 4 | import java.util.Iterator; 5 | 6 | import org.ds.chronos.api.Chronicle; 7 | import org.ds.chronos.api.Temporal; 8 | import org.ds.chronos.api.Timeline; 9 | import org.ds.chronos.api.TimelineDecoder; 10 | import org.ds.chronos.api.TimelineEncoder; 11 | 12 | /** 13 | * 14 | * A high level timeseries store with streaming encoding and decoding. 15 | *

16 | * Given a {@link Chronicle}, {@link TimelineEncoder}, and {@link TimelineDecoder}; we can produce a data layer for persisting and fetching 17 | * time stamped business objects. 18 | * 19 | * @author Dan Simpson 20 | * 21 | * @param 22 | * The type of object 23 | */ 24 | public class SimpleTimeline extends Timeline { 25 | 26 | protected Chronicle chronicle; 27 | 28 | protected TimelineDecoder decoder; 29 | protected TimelineEncoder encoder; 30 | 31 | /** 32 | * 33 | * @param chronicle 34 | * @param decoder 35 | * @param encoder 36 | */ 37 | public SimpleTimeline(Chronicle chronicle, TimelineDecoder decoder, TimelineEncoder encoder) { 38 | this.chronicle = chronicle; 39 | this.decoder = decoder; 40 | this.encoder = encoder; 41 | } 42 | 43 | /** 44 | * Add a batch of data 45 | * 46 | * @param data 47 | * @param batchSize 48 | */ 49 | public synchronized void add(Iterator data, int batchSize) { 50 | encoder.setInputStream(data); 51 | chronicle.add(encoder, batchSize); 52 | } 53 | 54 | /** 55 | * Add a single T to the chronicle 56 | * 57 | * @param item 58 | * T object 59 | */ 60 | @SuppressWarnings("unchecked") 61 | public void add(T item) { 62 | add(Arrays.asList(item).iterator(), 1); 63 | } 64 | 65 | /** 66 | * Fetch a stream of metrics for a given timeframe 67 | * 68 | * @param d1 69 | * @param d2 70 | * @return iterable stream of metrics 71 | */ 72 | public synchronized Iterator buildIterator(long t1, long t2, int batchSize) { 73 | decoder.setInputStream(chronicle.getRange(t1, t2, batchSize)); 74 | return decoder; 75 | } 76 | 77 | /** 78 | * @param t1 79 | * @param t2 80 | * @return 81 | * @see org.ds.chronos.api.Chronicle#getNumEvents(long, long) 82 | */ 83 | public long getNumEvents(long t1, long t2) { 84 | return chronicle.getNumEvents(t1, t2); 85 | } 86 | 87 | /** 88 | * @param t1 89 | * @return 90 | * @see org.ds.chronos.api.Chronicle#isEventRecorded(long) 91 | */ 92 | public boolean isEventRecorded(long t1) { 93 | return chronicle.isEventRecorded(t1); 94 | } 95 | 96 | /** 97 | * @param t1 98 | * @param t2 99 | * @see org.ds.chronos.api.Chronicle#deleteRange(long, long) 100 | */ 101 | public void deleteRange(long t1, long t2) { 102 | chronicle.deleteRange(t1, t2); 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/timeline/TimelineCopy.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline; 2 | 3 | import java.util.Iterator; 4 | 5 | import org.ds.chronos.api.Temporal; 6 | import org.ds.chronos.api.Timeline; 7 | 8 | 9 | /** 10 | * Copy utility for Timelines 11 | * 12 | * @author Dan Simpson 13 | * 14 | * @param 15 | */ 16 | public class TimelineCopy { 17 | 18 | private final Timeline source; 19 | private final Timeline destination; 20 | 21 | public TimelineCopy(Timeline source, Timeline destination) { 22 | this.source = source; 23 | this.destination = destination; 24 | } 25 | 26 | /** 27 | * Copy all events from source to destination 28 | * 29 | * @param t1 30 | * @param t2 31 | * @param batchSize 32 | */ 33 | public void copy(long t1, long t2, int batchSize) { 34 | copy(t1, t2, batchSize, batchSize); 35 | } 36 | 37 | /** 38 | * Copy all events from source to destination 39 | * 40 | * @param t1 41 | * - start time 42 | * @param t2 43 | * - end time 44 | * @param readBatch 45 | * - number of items to read at a time 46 | * @param writeBatch 47 | * - number of items to write at a time 48 | */ 49 | public void copy(long t1, long t2, int readBatch, int writeBatch) { 50 | Iterator iterator = source.buildIterator(t1, t2, readBatch); 51 | destination.add(iterator, writeBatch); 52 | } 53 | 54 | /** 55 | * Move items from source to destination, deleting source elements after write 56 | * 57 | * @param t1 58 | * @param t2 59 | * @param readBatch 60 | * @param writeBatch 61 | */ 62 | public void move(long t1, long t2, int readBatch, int writeBatch) { 63 | Iterator iterator = source.buildIterator(t1, t2, readBatch); 64 | destination.add(iterator, writeBatch); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/timeline/stream/DataStream.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.stream; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collection; 5 | import java.util.List; 6 | 7 | import org.ds.chronos.timeline.stream.partitioned.BucketSizePredicate; 8 | import org.ds.chronos.timeline.stream.partitioned.PartitionPredicate; 9 | import org.ds.chronos.timeline.stream.partitioned.PartitionedDataStream; 10 | 11 | import com.google.common.base.Function; 12 | import com.google.common.base.Optional; 13 | import com.google.common.base.Predicate; 14 | import com.google.common.collect.FluentIterable; 15 | import com.google.common.collect.ImmutableList; 16 | import com.google.common.collect.ImmutableListMultimap; 17 | import com.google.common.collect.ImmutableMap; 18 | import com.google.common.collect.ImmutableSet; 19 | 20 | /** 21 | * Lazy stream processing, with support for filters, transforms, partitioning, and reducing. 22 | * 23 | * @author Dan Simpson 24 | * 25 | * @param 26 | * The input type 27 | */ 28 | public class DataStream { 29 | 30 | private final FluentIterable source; 31 | 32 | public DataStream(Iterable source) { 33 | this.source = FluentIterable.from(source); 34 | } 35 | 36 | /** 37 | * Apply a filter condition 38 | * 39 | * @param condition 40 | * the condition, which returns true if the object should pass 41 | * @return new stream with filter applied 42 | */ 43 | public DataStream filter(final Predicate condition) { 44 | return new DataStream(source.filter(condition)); 45 | } 46 | 47 | /** 48 | * Apply a map function to each element and emit a stream of mapped objects 49 | * 50 | * @param fn 51 | * the map function 52 | * @return new stream 53 | */ 54 | public DataStream map(final Function fn) { 55 | return new DataStream(source.transform(fn)); 56 | } 57 | 58 | /** 59 | * Reduce the stream into a single object 60 | * 61 | * @param fn 62 | * the reduction function 63 | * @return resulting object 64 | */ 65 | public T reduce(final Function, T> fn) { 66 | return fn.apply(source); 67 | } 68 | 69 | /** 70 | * Partition the stream into sub streams based on a boundary predicate. 71 | * 72 | * @param predicate 73 | * the predicate which determines if a value belongs in a given parition 74 | * @return 75 | */ 76 | public PartitionedDataStream partition(final PartitionPredicate predicate) { 77 | return new PartitionedDataStream(this, predicate); 78 | } 79 | 80 | /** 81 | * Partition the stream into buckets of size bucketSize 82 | * 83 | * @param bucketSize 84 | * the number of elements to be returned per bucket 85 | * @return 86 | */ 87 | public PartitionedDataStream partition(int bucketSize) { 88 | return partition(new BucketSizePredicate(bucketSize)); 89 | } 90 | 91 | /** 92 | * 93 | * Joins this stream with another stream of the same type using a join function 94 | * 95 | * @param streams 96 | * @param transform 97 | * @return 98 | */ 99 | public DataStreamJoin join(Function, T> join, DataStream stream) { 100 | List> joining = new ArrayList>(); 101 | joining.add(this); 102 | joining.add(stream); 103 | return new DataStreamJoin(joining, join); 104 | } 105 | 106 | /** 107 | * Join this stream with one ore more other streams with a given join fn 108 | * 109 | * @param streams 110 | * @param transform 111 | * @return 112 | */ 113 | public DataStreamJoin join(Function, T> join, Collection> streams) { 114 | List> joining = new ArrayList>(); 115 | joining.add(this); 116 | joining.addAll(streams); 117 | return new DataStreamJoin(joining, join); 118 | } 119 | 120 | /** 121 | * 122 | * Create an interable for lazy streaming 123 | * 124 | * @return stream which lazy iterates 125 | */ 126 | public Iterable stream() { 127 | return source; 128 | } 129 | 130 | /** 131 | * Stream list downcasted 132 | * 133 | * @param klass 134 | * @return 135 | */ 136 | public Iterable streamAs(Class klass) { 137 | return map(new Function() { 138 | 139 | @SuppressWarnings("unchecked") 140 | public T apply(O o) { 141 | return (T) o; 142 | } 143 | }).stream(); 144 | } 145 | 146 | /** 147 | * @param predicate 148 | * @return 149 | * @see com.google.common.collect.FluentIterable#anyMatch(com.google.common.base.Predicate) 150 | */ 151 | public final boolean anyMatch(Predicate predicate) { 152 | return source.anyMatch(predicate); 153 | } 154 | 155 | /** 156 | * @param predicate 157 | * @return 158 | * @see com.google.common.collect.FluentIterable#allMatch(com.google.common.base.Predicate) 159 | */ 160 | public final boolean allMatch(Predicate predicate) { 161 | return source.allMatch(predicate); 162 | } 163 | 164 | /** 165 | * @param predicate 166 | * @return 167 | * @see com.google.common.collect.FluentIterable#firstMatch(com.google.common.base.Predicate) 168 | */ 169 | public final Optional firstMatch(Predicate predicate) { 170 | return source.firstMatch(predicate); 171 | } 172 | 173 | /** 174 | * @return 175 | * @see com.google.common.collect.FluentIterable#first() 176 | */ 177 | public final Optional first() { 178 | return source.first(); 179 | } 180 | 181 | /** 182 | * @return 183 | * @see com.google.common.collect.FluentIterable#last() 184 | */ 185 | public final Optional last() { 186 | return source.last(); 187 | } 188 | 189 | /** 190 | * @param size 191 | * @return 192 | * @see com.google.common.collect.FluentIterable#limit(int) 193 | */ 194 | public final FluentIterable limit(int size) { 195 | return source.limit(size); 196 | } 197 | 198 | /** 199 | * @return 200 | * @see com.google.common.collect.FluentIterable#toList() 201 | */ 202 | public final ImmutableList toList() { 203 | return source.toList(); 204 | } 205 | 206 | /** 207 | * @return 208 | * @see com.google.common.collect.FluentIterable#toSet() 209 | */ 210 | public final ImmutableSet toSet() { 211 | return source.toSet(); 212 | } 213 | 214 | /** 215 | * @param valueFunction 216 | * @return 217 | * @see com.google.common.collect.FluentIterable#toMap(com.google.common.base.Function) 218 | */ 219 | public final ImmutableMap toMap(Function valueFunction) { 220 | return source.toMap(valueFunction); 221 | } 222 | 223 | /** 224 | * @param keyFunction 225 | * @return 226 | * @see com.google.common.collect.FluentIterable#index(com.google.common.base.Function) 227 | */ 228 | public final ImmutableListMultimap index(Function keyFunction) { 229 | return source.index(keyFunction); 230 | } 231 | 232 | /** 233 | * @param keyFunction 234 | * @return 235 | * @see com.google.common.collect.FluentIterable#uniqueIndex(com.google.common.base.Function) 236 | */ 237 | public final ImmutableMap uniqueIndex(Function keyFunction) { 238 | return source.uniqueIndex(keyFunction); 239 | } 240 | 241 | /** 242 | * @param type 243 | * @return 244 | * @see com.google.common.collect.FluentIterable#toArray(java.lang.Class) 245 | */ 246 | public final O[] toArray(Class type) { 247 | return source.toArray(type); 248 | } 249 | 250 | /** 251 | * @param collection 252 | * @return 253 | * @see com.google.common.collect.FluentIterable#copyInto(java.util.Collection) 254 | */ 255 | public final > C copyInto(C collection) { 256 | return source.copyInto(collection); 257 | } 258 | 259 | /** 260 | * @return 261 | * @see com.google.common.collect.FluentIterable#size() 262 | */ 263 | public final int size() { 264 | return source.size(); 265 | } 266 | 267 | } 268 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/timeline/stream/DataStreamJoin.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.stream; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collection; 5 | import java.util.Iterator; 6 | import java.util.List; 7 | 8 | import org.ds.chronos.util.JoiningIterator; 9 | 10 | import com.google.common.base.Function; 11 | 12 | /** 13 | * A join of N data streams (of the same class, or contract). This allows joining of multiple streams with a transform function. 14 | * 15 | * @author Dan Simpson 16 | * 17 | * @param 18 | * the input type of eachs tream 19 | * @param 20 | * the resulting type of the join 21 | */ 22 | public class DataStreamJoin extends DataStream { 23 | 24 | public DataStreamJoin(Collection> streams, Function, O> fn) { 25 | super(wrap(streams, fn)); 26 | } 27 | 28 | private static Iterable wrap(final Collection> streams, final Function, O> fn) { 29 | return new Iterable() { 30 | 31 | @Override 32 | public Iterator iterator() { 33 | 34 | List> iterators = new ArrayList>(); 35 | for (DataStream stream : streams) { 36 | iterators.add(stream.stream().iterator()); 37 | } 38 | 39 | return new JoiningIterator(fn, iterators); 40 | } 41 | }; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/timeline/stream/partitioned/BucketSizePredicate.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.stream.partitioned; 2 | 3 | /** 4 | * A parition predicate which puts items in buckets based on count 5 | * 6 | * @author Dan Simpson 7 | * 8 | * @param 9 | */ 10 | public class BucketSizePredicate implements PartitionPredicate { 11 | 12 | private int count = 0; 13 | private final int bucketSize; 14 | 15 | public BucketSizePredicate(int bucketSize) { 16 | this.bucketSize = bucketSize; 17 | } 18 | 19 | public boolean apply(T t) { 20 | return count++ < bucketSize; 21 | } 22 | 23 | public void advance() { 24 | count = 0; 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/timeline/stream/partitioned/DurationPredicate.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.stream.partitioned; 2 | 3 | import org.ds.chronos.api.Temporal; 4 | import org.ds.chronos.util.Duration; 5 | 6 | /** 7 | * A partition predicate which leverage timestamps on temporal objects to lump them into time frame buckets 8 | * 9 | * @author Dan Simpson 10 | * 11 | * @param 12 | */ 13 | public class DurationPredicate implements PartitionPredicate { 14 | 15 | private Long start; 16 | private final long duration; 17 | private T last; 18 | 19 | /** 20 | * The size of the window, as a duration string see {@link Duration} 21 | * 22 | * @param duration 23 | */ 24 | public DurationPredicate(String duration) { 25 | this(new Duration(duration)); 26 | } 27 | 28 | /** 29 | * The size of the window, as a {@link Duration} 30 | * 31 | * @param duration 32 | */ 33 | public DurationPredicate(Duration duration) { 34 | this.duration = duration.getMillis(); 35 | } 36 | 37 | public boolean apply(T object) { 38 | last = object; 39 | if (start == null) { 40 | start = last.getTimestamp(); 41 | } 42 | 43 | return last.getTimestamp() >= start && last.getTimestamp() <= (start + duration); 44 | } 45 | 46 | public void advance() { 47 | start = last.getTimestamp(); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/timeline/stream/partitioned/PartitionPredicate.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.stream.partitioned; 2 | 3 | import com.google.common.base.Predicate; 4 | 5 | /** 6 | * A predicate designed specifically for determining if a DataStream object belongs in a given partition 7 | * 8 | * @author Dan Simpson 9 | * 10 | * @param 11 | */ 12 | public interface PartitionPredicate extends Predicate { 13 | 14 | /** 15 | * When the predicate fails, the caller will want to advance the predicate for the rest of the sequence 16 | */ 17 | public void advance(); 18 | } 19 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/timeline/stream/partitioned/PartitionedDataStream.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.stream.partitioned; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Iterator; 5 | import java.util.List; 6 | 7 | import org.ds.chronos.timeline.stream.DataStream; 8 | 9 | import com.google.common.base.Function; 10 | 11 | /** 12 | * A partitioned data stream which leverages a predicate to determine if a given item in the stream belongs to the "active" partition. The 13 | * goal is to convert a large data stream into a stream of streams. This gives us the ability to partition, then mapReduce the results in a 14 | * streaming fashion 15 | * 16 | * TODO: We can do this without buffering the objects in an array 17 | * 18 | * @author Dan Simpson 19 | * 20 | * @param 21 | */ 22 | public class PartitionedDataStream extends DataStream> { 23 | 24 | /** 25 | * 26 | * @param stream 27 | * the stream to partition 28 | * @param boundary 29 | * the boundary predicate. When returning false, a new parition will start and the active partition will be passed downstream 30 | */ 31 | public PartitionedDataStream(DataStream stream, final PartitionPredicate boundary) { 32 | super(wrap(stream.stream().iterator(), boundary)); 33 | } 34 | 35 | private static Iterable> wrap(final Iterator source, final PartitionPredicate boundary) { 36 | return new Iterable>() { 37 | 38 | private T last; 39 | 40 | @Override 41 | public Iterator> iterator() { 42 | return new Iterator>() { 43 | 44 | @Override 45 | public boolean hasNext() { 46 | return source.hasNext(); 47 | } 48 | 49 | @Override 50 | public DataStream next() { 51 | 52 | List buffer = new ArrayList(); 53 | 54 | if (last == null) { 55 | last = source.next(); 56 | } 57 | 58 | while (boundary.apply(last)) { 59 | buffer.add(last); 60 | 61 | if (!hasNext()) { 62 | break; 63 | } 64 | last = source.next(); 65 | } 66 | 67 | boundary.advance(); 68 | 69 | return new DataStream(buffer); 70 | } 71 | 72 | @Override 73 | public void remove() { 74 | } 75 | 76 | }; 77 | } 78 | 79 | }; 80 | } 81 | 82 | /** 83 | * Reduce each stream with a given reduce function, emitting a stream of reduced objects 84 | * 85 | * @param fn 86 | * reduce function 87 | * @return 88 | */ 89 | public DataStream reduceAll(final Function, K> fn) { 90 | 91 | final Iterator> source = stream().iterator(); 92 | 93 | return new DataStream(new Iterable() { 94 | 95 | @Override 96 | public Iterator iterator() { 97 | return new Iterator() { 98 | 99 | @Override 100 | public boolean hasNext() { 101 | return source.hasNext(); 102 | } 103 | 104 | @Override 105 | public K next() { 106 | return source.next().reduce(fn); 107 | } 108 | 109 | @Override 110 | public void remove() { 111 | } 112 | }; 113 | } 114 | }); 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/util/Duration.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.util; 2 | 3 | import java.util.Date; 4 | import java.util.Map; 5 | import java.util.concurrent.ConcurrentSkipListMap; 6 | import java.util.regex.Matcher; 7 | import java.util.regex.Pattern; 8 | 9 | /** 10 | * 11 | * Utility class for representing a duration of time. Supports short hand codes for durations such as 1d, 1h, or 1h30m. 12 | * 13 | * @author Dan Simpson 14 | */ 15 | public class Duration { 16 | 17 | private static final long MILLIS_IN_SECOND = 1000l; 18 | private static final long MILLIS_IN_MINUTE = MILLIS_IN_SECOND * 60; 19 | private static final long MILLIS_IN_HOUR = MILLIS_IN_MINUTE * 60; 20 | private static final long MILLIS_IN_DAY = MILLIS_IN_HOUR * 24; 21 | private static final long MILLIS_IN_WEEK = MILLIS_IN_DAY * 7; 22 | private static final long MILLIS_IN_YEAR = MILLIS_IN_DAY * 365; 23 | 24 | private static final Map codes = new ConcurrentSkipListMap(); 25 | static { 26 | codes.put("y", MILLIS_IN_YEAR); 27 | codes.put("w", MILLIS_IN_WEEK); 28 | codes.put("d", MILLIS_IN_DAY); 29 | codes.put("h", MILLIS_IN_HOUR); 30 | codes.put("m", MILLIS_IN_MINUTE); 31 | codes.put("s", MILLIS_IN_SECOND); 32 | codes.put("ms", 1l); 33 | } 34 | 35 | /** 36 | * Regex represents a number followed by a time code from the abbreviations map above. Eg: 1d, 1d2h, 1d 3h, 1d 1d 1d... odd 37 | */ 38 | private static final Pattern pattern = Pattern.compile("(\\d+)\\s*([a-z]{1,2})"); 39 | 40 | private long millis; 41 | 42 | /** 43 | * 44 | * @param number 45 | * of milliseconds for duration 46 | */ 47 | public Duration(long millis) { 48 | assert (millis > 0); 49 | this.millis = millis; 50 | } 51 | 52 | /** 53 | * 54 | * Parse a duration string into milliseconds. 55 | *

56 | * Examples: 1d = 1 day 3d4h = 3 days 4 hours 57 | *

58 | * units: y = year, w = week, d = day, h = hour, m = minute, s = second, ms = milliseconds. 59 | * 60 | * @param duration 61 | * abbreviation 62 | */ 63 | public Duration(String shorthand) { 64 | this(parse(shorthand)); 65 | } 66 | 67 | /** 68 | * @return The number of milliseconds in this duration 69 | */ 70 | public long getMillis() { 71 | return millis; 72 | } 73 | 74 | /** 75 | * Reduce a number into a sub unit and return the rest 76 | * 77 | * @param value 78 | * @param target 79 | * @param code 80 | * @param b 81 | * @return 82 | */ 83 | private long reduce(long value, long target, String code, StringBuilder b) { 84 | if (value >= target) { 85 | b.append(value / target); 86 | b.append(code); 87 | return value % target; 88 | } 89 | return value; 90 | } 91 | 92 | /** 93 | * Generate a string representation for duration 94 | */ 95 | public String toString() { 96 | if (millis <= 0) { 97 | return "0ms"; 98 | } 99 | 100 | long acc = millis; 101 | 102 | StringBuilder b = new StringBuilder(); 103 | acc = reduce(acc, MILLIS_IN_YEAR, "y", b); 104 | acc = reduce(acc, MILLIS_IN_WEEK, "w", b); 105 | acc = reduce(acc, MILLIS_IN_DAY, "d", b); 106 | acc = reduce(acc, MILLIS_IN_HOUR, "h", b); 107 | acc = reduce(acc, MILLIS_IN_MINUTE, "m", b); 108 | acc = reduce(acc, MILLIS_IN_SECOND, "s", b); 109 | acc = reduce(acc, 1l, "ms", b); 110 | return b.toString(); 111 | } 112 | 113 | /** 114 | * Use the regex to match the pattern and sum the output 115 | * 116 | * @param duration 117 | * @return 118 | */ 119 | private static long parse(String duration) { 120 | long result = 0; 121 | 122 | Matcher matcher = pattern.matcher(duration.toLowerCase()); 123 | while (matcher.find()) { 124 | if (matcher.groupCount() >= 2) { 125 | result += Long.parseLong(matcher.group(1)) * codes.get(matcher.group(2)); 126 | } 127 | } 128 | 129 | return result; 130 | } 131 | 132 | /** 133 | * Justify date to the previous (past) interval for this duration 134 | * 135 | * @param input 136 | * the date 137 | * @return justified date 138 | */ 139 | public final Date justifyPast(Date input) { 140 | return new Date(justifyPast(input.getTime())); 141 | } 142 | 143 | /** 144 | * @param input 145 | * timetamp 146 | * @return 147 | */ 148 | public final long justifyPast(long input) { 149 | return input - (input % millis); 150 | } 151 | 152 | /** 153 | * Justify date to the previous or current interval if the current time happens to fall on an interval line. 154 | * 155 | * @param input 156 | * @return 157 | */ 158 | public final Date justifyPastOrNow(Date input) { 159 | return new Date(justifyPastOrNow(input.getTime())); 160 | } 161 | 162 | /** 163 | * @param input 164 | * timetamp 165 | * @return 166 | */ 167 | public final long justifyPastOrNow(long input) { 168 | long diff = (input % millis); 169 | if (diff == millis) { 170 | return input; 171 | } 172 | return input - diff; 173 | } 174 | 175 | /** 176 | * Justify date to the next (future) interval for this duration 177 | * 178 | * @param input 179 | * the date 180 | * @return justified date 181 | */ 182 | public Date justifyFuture(Date input) { 183 | return new Date(justifyFuture(input.getTime())); 184 | } 185 | 186 | public final long justifyFuture(long input) { 187 | long remainder = input % millis; 188 | if (remainder == 0) { 189 | return input; 190 | } 191 | return input + millis - (input % millis); 192 | } 193 | 194 | /** 195 | * Increment date by the given duration 196 | * 197 | * @param input 198 | * @return new date 199 | */ 200 | public Date add(Date input) { 201 | return new Date(input.getTime() + millis); 202 | } 203 | 204 | /** 205 | * Decrement date by given duration 206 | * 207 | * @param input 208 | * @return new date 209 | */ 210 | public Date subtract(Date input) { 211 | return new Date(input.getTime() - millis); 212 | } 213 | } -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/util/JoiningIterator.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.util; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Iterator; 5 | import java.util.List; 6 | 7 | import com.google.common.base.Function; 8 | 9 | 10 | /** 11 | * Joining Iterator (AKA Joinerator). Takes a series of iterators, and joins them, and allows iterating on that joined result. 12 | * 13 | * If there are 2 input iterators A, and B, the sequence of iteration is: A, B, A, B... A, B etc. This is not a concatenating iterator 14 | * 15 | * @author Dan Simpson 16 | * 17 | * @param 18 | * The input type of each iterator 19 | * @param 20 | * The output type of the Join 21 | */ 22 | public class JoiningIterator implements Iterator { 23 | 24 | final Iterable> streams; 25 | final Function, K> fn; 26 | 27 | /** 28 | * 29 | * @param joinFn 30 | * A function which takes a List of T objects, and should produce a K object 31 | * @param streams 32 | * the streams which we join 33 | */ 34 | public JoiningIterator(Function, K> joinFn, Iterable> streams) { 35 | super(); 36 | this.fn = joinFn; 37 | this.streams = streams; 38 | } 39 | 40 | @Override 41 | public boolean hasNext() { 42 | boolean result = true; 43 | for (Iterator stream : streams) { 44 | result &= stream.hasNext(); 45 | } 46 | return result; 47 | } 48 | 49 | @Override 50 | public K next() { 51 | List input = new ArrayList(); 52 | for (Iterator stream : streams) { 53 | input.add(stream.next()); 54 | } 55 | return fn.apply(input); 56 | } 57 | 58 | @Override 59 | public void remove() { 60 | } 61 | 62 | } -------------------------------------------------------------------------------- /chronos-api/src/main/java/org/ds/chronos/util/TimeFrame.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.util; 2 | 3 | import java.util.Date; 4 | 5 | /** 6 | * Representation of a time frame. 7 | * 8 | * @author Dan Simpson 9 | * 10 | */ 11 | public class TimeFrame { 12 | 13 | private final long t1; 14 | private final long t2; 15 | 16 | public TimeFrame(long start, long end) { 17 | this.t1 = start; 18 | this.t2 = end; 19 | } 20 | 21 | public TimeFrame(long start, Duration duration) { 22 | this(start, start + duration.getMillis()); 23 | } 24 | 25 | public TimeFrame(Date start, Date end) { 26 | this(start.getTime(), end.getTime()); 27 | } 28 | 29 | public Duration getDuration() { 30 | return new Duration(t2 - t1); 31 | } 32 | 33 | /** 34 | * @return the t1 35 | */ 36 | public long getStart() { 37 | return t1; 38 | } 39 | 40 | /** 41 | * @return the t2 42 | */ 43 | public long getEnd() { 44 | return t2; 45 | } 46 | 47 | public boolean contains(long time) { 48 | return time >= t1 && time <= t2; 49 | } 50 | 51 | /** 52 | * Check if an entire time frame is contained in this time frame 53 | * 54 | * @param start 55 | * @param end 56 | * @return 57 | */ 58 | public boolean contains(long start, long end) { 59 | return contains(start) && contains(end); 60 | } 61 | 62 | /** 63 | * Check if this time frame intersects another time frame 64 | * 65 | * @param start 66 | * @param end 67 | * @return 68 | */ 69 | public boolean intersects(long start, long end) { 70 | return contains(start) || contains(end) || (t1 > start && t2 < end); 71 | } 72 | 73 | /** 74 | * Fit this time frame within another time frame, or return null if impossible. 75 | * 76 | * @param start 77 | * @param end 78 | * @return 79 | */ 80 | public TimeFrame fit(long start, long end) { 81 | if (!intersects(start, end)) { 82 | return null; 83 | } 84 | return new TimeFrame(Math.max(start, t1), Math.min(end, t2)); 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/api/ConcatenatedTimelineTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import junit.framework.Assert; 7 | 8 | import org.ds.chronos.api.chronicle.MemoryChronicle; 9 | import org.ds.chronos.support.TestBase; 10 | import org.ds.chronos.support.TestData; 11 | import org.ds.chronos.support.TestDecoder; 12 | import org.ds.chronos.support.TestEncoder; 13 | import org.ds.chronos.timeline.ConcatenatedTimeline; 14 | import org.ds.chronos.timeline.ConcatenatedTimeline.ScopedTimeline; 15 | import org.ds.chronos.timeline.SimpleTimeline; 16 | import org.ds.chronos.util.TimeFrame; 17 | import org.junit.Before; 18 | import org.junit.Test; 19 | 20 | import com.google.common.collect.Iterables; 21 | 22 | public class ConcatenatedTimelineTest extends TestBase { 23 | 24 | private MemoryChronicle chronicle1; 25 | private MemoryChronicle chronicle2; 26 | 27 | private Timeline t1; 28 | private Timeline t2; 29 | private ConcatenatedTimeline store; 30 | 31 | @SuppressWarnings("unchecked") 32 | @Before 33 | public void createChronicle() { 34 | chronicle1 = new MemoryChronicle(); 35 | chronicle2 = new MemoryChronicle(); 36 | t1 = new SimpleTimeline(chronicle1, new TestDecoder(), new TestEncoder()); 37 | t2 = new SimpleTimeline(chronicle2, new TestDecoder(), new TestEncoder()); 38 | 39 | store = new ConcatenatedTimeline(new ScopedTimeline(t1, new TimeFrame(0, 1000)), 40 | new ScopedTimeline(t2, new TimeFrame(1001, 2000))); 41 | } 42 | 43 | @Test 44 | public void distributeItem() throws ChronosException { 45 | store.add(new TestData(100, (byte) 0, 20d)); 46 | store.add(new TestData(1100, (byte) 0, 20d)); 47 | Assert.assertEquals(2, store.getNumEvents(0, 5000)); 48 | Assert.assertEquals(1, t1.getNumEvents(0, 5000)); 49 | Assert.assertEquals(1, t2.getNumEvents(0, 5000)); 50 | } 51 | 52 | @Test 53 | public void exists() throws ChronosException { 54 | store.add(new TestData(100, (byte) 0, 20d)); 55 | Assert.assertTrue(store.isEventRecorded(100)); 56 | Assert.assertTrue(t1.isEventRecorded(100)); 57 | Assert.assertFalse(t2.isEventRecorded(100)); 58 | } 59 | 60 | @Test 61 | public void remove() throws ChronosException { 62 | store.add(new TestData(100, (byte) 0, 20d)); 63 | store.add(new TestData(1100, (byte) 0, 20d)); 64 | store.deleteRange(0, 5500); 65 | Assert.assertEquals(0, store.getNumEvents(0, 5000)); 66 | Assert.assertEquals(0, t1.getNumEvents(0, 5000)); 67 | Assert.assertEquals(0, t2.getNumEvents(0, 5000)); 68 | } 69 | 70 | @Test 71 | public void iterate() throws ChronosException { 72 | store.add(new TestData(100, (byte) 0, 20d)); 73 | store.add(new TestData(1100, (byte) 0, 20d)); 74 | Assert.assertEquals(2, Iterables.size(store.iterable(0, 2000))); 75 | Assert.assertEquals(100, store.query(0, 2000).first().get().getTimestamp()); 76 | } 77 | 78 | @Test 79 | public void iterateReverse() throws ChronosException { 80 | store.add(new TestData(100, (byte) 0, 20d)); 81 | store.add(new TestData(1100, (byte) 0, 20d)); 82 | Assert.assertEquals(2, Iterables.size(store.iterable(2000, 0))); 83 | Assert.assertEquals(1100, store.query(2000, 0).first().get().getTimestamp()); 84 | } 85 | 86 | @Test 87 | public void addBatch() throws ChronosException { 88 | int count = 2000; 89 | List list = new ArrayList(); 90 | for (int i = 0; i < count; i++) { 91 | TestData data = new TestData(); 92 | data.time = i; 93 | data.type = 0x05; 94 | data.value = i <= 1000 ? 1d : 2d; 95 | list.add(data); 96 | } 97 | store.add(list); 98 | 99 | Assert.assertEquals(count, store.getNumEvents(0, 5000)); 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/api/DataStreamTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Iterator; 5 | import java.util.List; 6 | 7 | import org.ds.chronos.support.TestData; 8 | import org.ds.chronos.timeline.stream.DataStream; 9 | import org.ds.chronos.timeline.stream.partitioned.PartitionedDataStream; 10 | import org.ds.chronos.timeline.stream.partitioned.DurationPredicate; 11 | import org.junit.Assert; 12 | import org.junit.Before; 13 | import org.junit.Test; 14 | 15 | import com.google.common.base.Function; 16 | import com.google.common.base.Optional; 17 | import com.google.common.base.Predicate; 18 | 19 | public class DataStreamTest { 20 | 21 | public static Predicate range(final double low, final double high) { 22 | return new Predicate() { 23 | 24 | public boolean apply(TestData item) { 25 | return item.value >= low && item.value < high; 26 | } 27 | }; 28 | } 29 | 30 | public static Function, Double> sum() { 31 | return new Function, Double>() { 32 | 33 | public Double apply(Iterable items) { 34 | double sum = 0d; 35 | for (TestData data : items) { 36 | sum += data.value; 37 | } 38 | return sum; 39 | } 40 | }; 41 | } 42 | 43 | public static Predicate gte(final double v) { 44 | return new Predicate() { 45 | 46 | public boolean apply(Double item) { 47 | return item >= v; 48 | } 49 | }; 50 | } 51 | 52 | public static Function doublize() { 53 | return new Function() { 54 | 55 | public Double apply(TestData item) { 56 | return item.value; 57 | } 58 | }; 59 | } 60 | 61 | public static Function multiply(final double factor) { 62 | return new Function() { 63 | 64 | public Double apply(Double d) { 65 | return d * factor; 66 | } 67 | }; 68 | } 69 | 70 | private List items; 71 | 72 | private static final int STORE_SIZE = 1000; 73 | 74 | private ArrayList getData(int count) { 75 | ArrayList items = new ArrayList(); 76 | for (int i = 0; i < count; i++) { 77 | TestData data = new TestData(); 78 | data.time = 1000 * i; 79 | data.type = 0x05; 80 | data.value = i; 81 | items.add(data); 82 | } 83 | return items; 84 | } 85 | 86 | @Before 87 | public void createChronicle() { 88 | items = getData(STORE_SIZE); 89 | } 90 | 91 | @Test 92 | public void testPipeline() { 93 | DataStream pipeline = new DataStream(items); 94 | int count = 0; 95 | for (@SuppressWarnings("unused") 96 | TestData data : pipeline.stream()) { 97 | count++; 98 | } 99 | Assert.assertEquals(STORE_SIZE, count); 100 | Assert.assertEquals(STORE_SIZE, pipeline.size()); 101 | } 102 | 103 | @Test 104 | public void testResultSet() { 105 | List result = new DataStream(items).toList(); 106 | Assert.assertEquals(STORE_SIZE, result.size()); 107 | } 108 | 109 | @Test 110 | @SuppressWarnings("unused") 111 | public void testStream() { 112 | int count = 0; 113 | for (TestData data : new DataStream(items).stream()) { 114 | count++; 115 | } 116 | Assert.assertEquals(STORE_SIZE, count); 117 | } 118 | 119 | @Test 120 | public void testPipelineFilter() { 121 | List result = new DataStream(items).filter(range(100, 200)).toList(); 122 | Assert.assertEquals(200 - 100, result.size()); 123 | } 124 | 125 | @Test 126 | public void testMultifilter() { 127 | List result = new DataStream(items).filter(range(100, 200)).filter(range(110, 120)).toList(); 128 | 129 | Assert.assertEquals(120 - 110, result.size()); 130 | } 131 | 132 | @Test 133 | public void testMap() { 134 | List result = new DataStream(items).map(doublize()).toList(); 135 | Assert.assertEquals(STORE_SIZE, result.size()); 136 | } 137 | 138 | @Test 139 | public void testReduce() { 140 | Double result = new DataStream(items).reduce(sum()); 141 | Assert.assertEquals(linearSum(items.size()), result, 0.0); 142 | } 143 | 144 | @Test 145 | public void testPartition() { 146 | PartitionedDataStream result = new DataStream(items).partition(20); 147 | Assert.assertEquals(STORE_SIZE / 20, result.size()); 148 | } 149 | 150 | @Test 151 | public void testTimePartition() { 152 | PartitionedDataStream result = new DataStream(items) 153 | .partition(new DurationPredicate("44s")); 154 | Assert.assertEquals(Math.ceil(STORE_SIZE / 44.0), result.size(), 0.0); 155 | } 156 | 157 | @Test 158 | public void testParitionMapReduce() { 159 | DataStream result = new DataStream(items).partition(20).reduceAll(sum()); 160 | 161 | Optional first = result.first(); 162 | Assert.assertTrue(first.isPresent()); 163 | Assert.assertEquals(linearSum(20), first.get(), 0.0); 164 | } 165 | 166 | @Test 167 | public void testTransform() { 168 | List result = new DataStream(items).map(doublize()).filter(gte(5)).map(multiply(-1d)) 169 | .map(multiply(2d)).toList(); 170 | 171 | Assert.assertEquals(STORE_SIZE - 5, result.size()); 172 | Assert.assertEquals(-10d, result.get(0), 0.1); 173 | } 174 | 175 | private static class JoinSum implements Function, Double> { 176 | 177 | public Double apply(Iterable input) { 178 | double sum = 0; 179 | for (Double d : input) { 180 | sum += d; 181 | } 182 | return sum; 183 | } 184 | 185 | } 186 | 187 | @Test 188 | public void testJoin() { 189 | DataStream s1 = new DataStream(items).map(doublize()); 190 | DataStream s2 = new DataStream(items).map(doublize()); 191 | DataStream sj = s1.join(new JoinSum(), s2); 192 | List result = sj.toList(); 193 | Assert.assertEquals(STORE_SIZE, sj.size()); 194 | Assert.assertEquals(2, result.get(1), 0.1); 195 | } 196 | 197 | @Test 198 | public void testJoin3() { 199 | DataStream s1 = new DataStream(items).map(doublize()); 200 | 201 | List> others = new ArrayList>(); 202 | others.add(new DataStream(items).map(doublize())); 203 | others.add(new DataStream(items).map(doublize())); 204 | 205 | DataStream sj = s1.join(new JoinSum(), others); 206 | List result = sj.toList(); 207 | Assert.assertEquals(STORE_SIZE, sj.size()); 208 | Assert.assertEquals(3, result.get(1), 0.1); 209 | } 210 | 211 | /** 212 | * @Test 213 | * 214 | * 10BB items ready 11 MB, 10BB items staged 11 MB, 10BB items processed 29 MB ~5mins 215 | */ 216 | public void testMemory() { 217 | Runtime r = Runtime.getRuntime(); 218 | 219 | // increase to 10B 220 | final long n = 10000000000l; 221 | 222 | final TestData data = new TestData(); 223 | data.time = 1000; 224 | data.type = 0x05; 225 | data.value = 50; 226 | 227 | Iterable itr = new Iterable() { 228 | 229 | @Override 230 | public Iterator iterator() { 231 | return new Iterator() { 232 | 233 | long x = 0; 234 | 235 | public boolean hasNext() { 236 | return x++ < n; 237 | } 238 | 239 | public TestData next() { 240 | return data; 241 | } 242 | 243 | public void remove() { 244 | } 245 | }; 246 | } 247 | 248 | }; 249 | 250 | System.out.printf("%d items ready %d MB\n", n, (r.totalMemory() - r.freeMemory()) / (1024 * 1024)); 251 | 252 | long t1 = System.currentTimeMillis(); 253 | Iterator result = new DataStream(itr).map(doublize()).filter(gte(5)).map(multiply(-1d)) 254 | .map(multiply(2d)).stream().iterator(); 255 | 256 | System.out.printf("%d items staged %d MB\n", n, (r.totalMemory() - r.freeMemory()) / (1024 * 1024)); 257 | 258 | while (result.hasNext()) { 259 | result.next(); 260 | } 261 | 262 | System.out.printf("%d items processed %d MB %d ms\n", n, (r.totalMemory() - r.freeMemory()) / (1024 * 1024), 263 | System.currentTimeMillis() - t1); 264 | } 265 | 266 | private static double linearSum(int count) { 267 | double result = 0d; 268 | for (int i = 0; i < count; i++) { 269 | result += i; 270 | } 271 | return result; 272 | } 273 | 274 | } 275 | -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/api/DurationTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.Calendar; 4 | import java.util.Date; 5 | 6 | import org.ds.chronos.api.ChronosException; 7 | import org.ds.chronos.support.TestBase; 8 | import org.ds.chronos.util.Duration; 9 | import org.junit.Assert; 10 | import org.junit.Test; 11 | 12 | public class DurationTest extends TestBase { 13 | 14 | @Test 15 | public void simpleTest() throws ChronosException { 16 | Duration duration = new Duration(1000); 17 | Assert.assertEquals(1000, duration.getMillis()); 18 | } 19 | 20 | @Test 21 | public void testYears() { 22 | Duration duration = new Duration("1y"); 23 | Assert.assertEquals(1000l * 60 * 60 * 24 * 365, duration.getMillis()); 24 | } 25 | 26 | @Test 27 | public void test2Years() { 28 | Duration duration = new Duration("2y"); 29 | Assert.assertEquals(1000l * 60 * 60 * 24 * 365 * 2, duration.getMillis()); 30 | } 31 | 32 | @Test 33 | public void testWeek() { 34 | Duration duration = new Duration("1w"); 35 | Assert.assertEquals(1000 * 60 * 60 * 24 * 7, duration.getMillis()); 36 | } 37 | 38 | @Test 39 | public void testDay() { 40 | Duration duration = new Duration("1d"); 41 | Assert.assertEquals(1000 * 60 * 60 * 24, duration.getMillis()); 42 | } 43 | 44 | @Test 45 | public void testHour() { 46 | Duration duration = new Duration("1h"); 47 | Assert.assertEquals(1000 * 60 * 60, duration.getMillis()); 48 | } 49 | 50 | @Test 51 | public void testMinute() { 52 | Duration duration = new Duration("1m"); 53 | Assert.assertEquals(1000 * 60, duration.getMillis()); 54 | } 55 | 56 | @Test 57 | public void testSecond() { 58 | Duration duration = new Duration("1s"); 59 | Assert.assertEquals(1000, duration.getMillis()); 60 | } 61 | 62 | @Test 63 | public void testMillis() { 64 | Duration duration = new Duration("10ms"); 65 | Assert.assertEquals(10, duration.getMillis()); 66 | } 67 | 68 | @Test 69 | public void testSpaced() { 70 | Duration duration = new Duration("10 ms"); 71 | Assert.assertEquals(10, duration.getMillis()); 72 | } 73 | 74 | @Test 75 | public void testCaps() { 76 | Duration duration = new Duration("10MS"); 77 | Assert.assertEquals(10, duration.getMillis()); 78 | } 79 | 80 | @Test 81 | public void testComplex() { 82 | Duration duration = new Duration("1h30m3ms"); 83 | Assert.assertEquals(1000 * 60 * 90 + 3, duration.getMillis()); 84 | } 85 | 86 | @Test 87 | public void testComplexSpaced() { 88 | Duration duration = new Duration("1h 30m 3ms"); 89 | Assert.assertEquals(1000 * 60 * 90 + 3, duration.getMillis()); 90 | } 91 | 92 | @Test 93 | public void testStrings() { 94 | Assert.assertEquals("999ms", new Duration("999ms").toString()); 95 | Assert.assertEquals("1s", new Duration("1000ms").toString()); 96 | Assert.assertEquals("59s", new Duration("59s").toString()); 97 | Assert.assertEquals("1m", new Duration("60s").toString()); 98 | Assert.assertEquals("59m", new Duration("59m").toString()); 99 | Assert.assertEquals("1h", new Duration("60m").toString()); 100 | Assert.assertEquals("6d", new Duration("6d").toString()); 101 | Assert.assertEquals("1w", new Duration("7d").toString()); 102 | Assert.assertEquals("1w", new Duration("1w").toString()); 103 | Assert.assertEquals("1y", new Duration("1y").toString()); 104 | Assert.assertEquals("1y1w1d1h1m1s15ms", new Duration( 105 | "1y 1w 1d 1h 1m 1s 15ms").toString()); 106 | } 107 | 108 | @Test 109 | public void testPastJustify() { 110 | Calendar c = getCal(); 111 | Date expected = c.getTime(); 112 | c.set(2012, 1, 5, 15, 13, 22); 113 | Date test = c.getTime(); 114 | 115 | Duration duration = new Duration("1h"); 116 | Assert.assertEquals(expected, duration.justifyPast(test)); 117 | Assert.assertEquals(expected, 118 | duration.justifyPast(duration.justifyPast(test))); 119 | } 120 | 121 | @Test 122 | public void testFutureJustify() { 123 | Calendar c = getCal(); 124 | c.add(Calendar.HOUR, 1); 125 | Date expected = c.getTime(); 126 | c.add(Calendar.MINUTE, -25); 127 | Date test = c.getTime(); 128 | 129 | Duration duration = new Duration("1h"); 130 | Assert.assertEquals(expected, duration.justifyFuture(test)); 131 | Assert.assertEquals(expected, 132 | duration.justifyFuture(duration.justifyFuture(test))); 133 | } 134 | 135 | @Test 136 | public void testAdd() { 137 | Calendar c = getCal(); 138 | Date expected = c.getTime(); 139 | c.add(Calendar.HOUR, -1); 140 | Date test = c.getTime(); 141 | Assert.assertEquals(expected, new Duration("1h").add(test)); 142 | } 143 | 144 | @Test 145 | public void testSubtract() { 146 | Calendar c = getCal(); 147 | Date expected = c.getTime(); 148 | c.add(Calendar.HOUR, 1); 149 | Date test = c.getTime(); 150 | Assert.assertEquals(expected, new Duration("1h").subtract(test)); 151 | } 152 | 153 | private Calendar getCal() { 154 | Calendar c = Calendar.getInstance(); 155 | c.set(2012, 1, 5, 15, 0, 0); 156 | c.set(Calendar.MILLISECOND, 0); 157 | return c; 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/api/MemoryChronicleTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import junit.framework.Assert; 7 | 8 | import org.ds.chronos.api.chronicle.MemoryChronicle; 9 | import org.ds.chronos.support.TestBase; 10 | import org.junit.Test; 11 | 12 | public class MemoryChronicleTest extends TestBase { 13 | 14 | protected ChronologicalRecord getTestItem(long time) { 15 | return new ChronologicalRecord(time, "Hello".getBytes()); 16 | } 17 | 18 | protected List getTestItemList(long startTime, 19 | long periodInMillis, int count) { 20 | List result = new ArrayList(); 21 | for (int i = 0; i < count; i++) { 22 | result.add(getTestItem(startTime + (periodInMillis * i))); 23 | } 24 | return result; 25 | } 26 | 27 | @Test 28 | public void testAdd() { 29 | Chronicle chronicle = new MemoryChronicle(); 30 | chronicle.add(getTestItem(0)); 31 | Assert.assertEquals(1, 32 | chronicle.getNumEvents(0, System.currentTimeMillis())); 33 | } 34 | 35 | @Test 36 | public void testAddBatch() { 37 | Chronicle chronicle = new MemoryChronicle(); 38 | chronicle.add(getTestItemList(0, 1000, 100)); 39 | Assert.assertEquals(100, 40 | chronicle.getNumEvents(0, System.currentTimeMillis())); 41 | } 42 | 43 | @Test 44 | public void testRange() { 45 | Chronicle chronicle = new MemoryChronicle(); 46 | chronicle.add(getTestItemList(0, 1000, 100)); 47 | 48 | List items = MemoryChronicle.toList(chronicle.getRange(1000, 5000)); 49 | 50 | Assert.assertEquals(5, items.size()); 51 | Assert.assertEquals(1000, items.get(0).getTimestamp()); 52 | Assert.assertEquals(5000, items.get(4).getTimestamp()); 53 | } 54 | 55 | @Test 56 | public void testReverseRange() { 57 | Chronicle chronicle = new MemoryChronicle(); 58 | chronicle.add(getTestItemList(0, 1000, 100)); 59 | 60 | List items = MemoryChronicle.toList(chronicle.getRange(5000, 1000)); 61 | 62 | Assert.assertEquals(5, items.size()); 63 | Assert.assertEquals(5000, items.get(0).getTimestamp()); 64 | Assert.assertEquals(1000, items.get(4).getTimestamp()); 65 | } 66 | 67 | @Test 68 | public void testCount() { 69 | Chronicle chronicle = new MemoryChronicle(); 70 | chronicle.add(getTestItemList(0, 1000, 100)); 71 | Assert.assertEquals(50, chronicle.getNumEvents(1, 50000)); 72 | } 73 | 74 | } -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/api/PartitionPeriodTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.Calendar; 4 | import java.util.Date; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import junit.framework.Assert; 10 | 11 | import org.ds.chronos.api.chronicle.MemoryChronicle; 12 | import org.ds.chronos.support.TestBase; 13 | import org.ds.chronos.util.Duration; 14 | import org.junit.Test; 15 | 16 | import com.google.common.collect.Iterators; 17 | 18 | public class PartitionPeriodTest extends TestBase { 19 | 20 | @Test 21 | public void shouldGenerateHourKey() throws ChronosException { 22 | Assert.assertEquals("key-2018-10-08-18", PartitionPeriod.HOUR.getPeriodKey("key", 1539023421354l)); 23 | } 24 | 25 | @Test 26 | public void shouldGenerateRowKeysForHours() throws ChronosException { 27 | PartitionPeriod period = PartitionPeriod.HOUR; 28 | 29 | Calendar calendar = Calendar.getInstance(); 30 | calendar.set(2012, 0, 1, 0, 0, 0); 31 | Date d1 = calendar.getTime(); 32 | calendar.set(2012, 0, 2, 23, 59, 59); 33 | Date d2 = calendar.getTime(); 34 | 35 | List keys = period.getPeriodKeys("key", d1, d2); 36 | Assert.assertEquals("key-2012-01-01-00", keys.get(0)); 37 | Assert.assertEquals("key-2012-01-02-23", keys.get(keys.size() - 1)); 38 | } 39 | 40 | @Test 41 | public void shouldGenerateRowKeysForDays() throws ChronosException { 42 | PartitionPeriod period = PartitionPeriod.DAY; 43 | 44 | Calendar calendar = Calendar.getInstance(); 45 | calendar.set(2012, 0, 1); 46 | Date d1 = calendar.getTime(); 47 | calendar.set(2012, 8, 1); 48 | Date d2 = calendar.getTime(); 49 | 50 | List keys = period.getPeriodKeys("key", d1, d2); 51 | 52 | Assert.assertEquals(245, keys.size()); 53 | Assert.assertEquals("key-2012-01-01", keys.get(0)); 54 | Assert.assertEquals("key-2012-09-01", keys.get(keys.size() - 1)); 55 | } 56 | 57 | @Test 58 | public void shouldWorkOnTightBoundaries() throws ChronosException { 59 | PartitionPeriod period = PartitionPeriod.DAY; 60 | 61 | Calendar calendar = Calendar.getInstance(); 62 | calendar.set(2012, 0, 1, 23, 0, 0); 63 | Date d1 = calendar.getTime(); 64 | calendar.set(2012, 0, 2, 1, 0, 0); 65 | Date d2 = calendar.getTime(); 66 | 67 | List keys = period.getPeriodKeys("key", d1, d2); 68 | 69 | Assert.assertEquals(2, keys.size()); 70 | Assert.assertEquals("key-2012-01-01", keys.get(0)); 71 | Assert.assertEquals("key-2012-01-02", keys.get(keys.size() - 1)); 72 | } 73 | 74 | @Test 75 | public void shouldWorkInReverse() throws ChronosException { 76 | PartitionPeriod period = PartitionPeriod.DAY; 77 | 78 | Calendar calendar = Calendar.getInstance(); 79 | calendar.set(2012, 0, 1, 23, 0, 0); 80 | Date d1 = calendar.getTime(); 81 | calendar.set(2012, 0, 2, 1, 0, 0); 82 | Date d2 = calendar.getTime(); 83 | 84 | List keys = period.getPeriodKeys("key", d2, d1); 85 | 86 | Assert.assertEquals(2, keys.size()); 87 | Assert.assertEquals("key-2012-01-02", keys.get(0)); 88 | Assert.assertEquals("key-2012-01-01", keys.get(keys.size() - 1)); 89 | } 90 | 91 | @Test 92 | public void shouldGenerateRowKeysForMonths() throws ChronosException { 93 | PartitionPeriod period = PartitionPeriod.MONTH; 94 | 95 | Calendar calendar = Calendar.getInstance(); 96 | calendar.set(2012, 0, 1); 97 | Date d1 = calendar.getTime(); 98 | calendar.set(2012, 8, 1); 99 | Date d2 = calendar.getTime(); 100 | 101 | List keys = period.getPeriodKeys("key", d1, d2); 102 | 103 | Assert.assertEquals(9, keys.size()); 104 | Assert.assertEquals("key-2012-01", keys.get(0)); 105 | Assert.assertEquals("key-2012-09", keys.get(keys.size() - 1)); 106 | } 107 | 108 | @Test 109 | public void shouldGenerateRowKeysForYears() throws ChronosException { 110 | PartitionPeriod period = PartitionPeriod.YEAR; 111 | 112 | Calendar calendar = Calendar.getInstance(); 113 | calendar.set(2012, 0, 1); 114 | Date d1 = calendar.getTime(); 115 | calendar.set(2013, 8, 1); 116 | Date d2 = calendar.getTime(); 117 | 118 | List keys = period.getPeriodKeys("key", d1, d2); 119 | 120 | Assert.assertEquals(2, keys.size()); 121 | Assert.assertEquals("key-2012", keys.get(0)); 122 | Assert.assertEquals("key-2013", keys.get(1)); 123 | } 124 | 125 | @Test 126 | public void shouldGenerateRowKeyForYear() throws ChronosException { 127 | PartitionPeriod period = PartitionPeriod.YEAR; 128 | 129 | Calendar calendar = Calendar.getInstance(); 130 | calendar.set(2012, 0, 1); 131 | Date d1 = calendar.getTime(); 132 | calendar.set(2012, 8, 1); 133 | Date d2 = calendar.getTime(); 134 | List keys = period.getPeriodKeys("key", d1, d2); 135 | 136 | Assert.assertEquals(1, keys.size()); 137 | Assert.assertEquals("key-2012", keys.get(0)); 138 | } 139 | 140 | @Test 141 | public void shouldIncludeBothYears() throws ChronosException { 142 | PartitionPeriod period = PartitionPeriod.YEAR; 143 | 144 | Calendar calendar = Calendar.getInstance(); 145 | calendar.set(2012, 11, 29, 0, 0, 0); 146 | calendar.set(Calendar.MILLISECOND, 0); 147 | Date d1 = calendar.getTime(); 148 | calendar.add(Calendar.DATE, 15); 149 | Date d2 = calendar.getTime(); 150 | List keys = period.getPeriodKeys("key", d1, d2); 151 | 152 | Assert.assertEquals(2, keys.size()); 153 | Assert.assertEquals("key-2012", keys.get(0)); 154 | Assert.assertEquals("key-2013", keys.get(1)); 155 | } 156 | 157 | @Test 158 | public void shouldInclude3Years() throws ChronosException { 159 | PartitionPeriod period = PartitionPeriod.YEAR; 160 | 161 | Calendar calendar = Calendar.getInstance(); 162 | calendar.set(2012, 11, 29, 0, 0, 0); 163 | calendar.set(Calendar.MILLISECOND, 0); 164 | Date d1 = calendar.getTime(); 165 | calendar.add(Calendar.DATE, 15 + 365); 166 | Date d2 = calendar.getTime(); 167 | List keys = period.getPeriodKeys("key", d1, d2); 168 | 169 | Assert.assertEquals(3, keys.size()); 170 | Assert.assertEquals("key-2012", keys.get(0)); 171 | Assert.assertEquals("key-2013", keys.get(1)); 172 | Assert.assertEquals("key-2014", keys.get(2)); 173 | } 174 | 175 | @Test 176 | public void shouldInclude3Months() throws ChronosException { 177 | PartitionPeriod period = PartitionPeriod.MONTH; 178 | 179 | Calendar calendar = Calendar.getInstance(); 180 | calendar.set(2012, 11, 29, 0, 0, 0); 181 | calendar.set(Calendar.MILLISECOND, 0); 182 | Date d1 = calendar.getTime(); 183 | calendar.add(Calendar.DATE, 45); 184 | Date d2 = calendar.getTime(); 185 | List keys = period.getPeriodKeys("key", d1, d2); 186 | 187 | Assert.assertEquals(3, keys.size()); 188 | Assert.assertEquals("key-2012-12", keys.get(0)); 189 | Assert.assertEquals("key-2013-01", keys.get(1)); 190 | Assert.assertEquals("key-2013-02", keys.get(2)); 191 | } 192 | 193 | @Test 194 | public void shouldHitProperChronicle() throws ChronosException { 195 | PartitionPeriod period = PartitionPeriod.YEAR; 196 | 197 | Calendar calendar = Calendar.getInstance(); 198 | calendar.set(2012, 11, 29, 0, 0, 0); 199 | calendar.set(Calendar.MILLISECOND, 0); 200 | long start = calendar.getTimeInMillis(); 201 | calendar.add(Calendar.DATE, 15); 202 | long end = calendar.getTimeInMillis(); 203 | 204 | TestChronicle chron = new TestChronicle("test", period); 205 | 206 | long time = start; 207 | while (time < end) { 208 | chron.add(new ChronologicalRecord(time, "x".getBytes())); 209 | time += new Duration("1m").getMillis(); 210 | } 211 | 212 | Assert.assertEquals(2, chron.chronicles.size()); 213 | Assert.assertNotNull(chron.chronicles.get("test-2012")); 214 | Assert.assertNotNull(chron.chronicles.get("test-2013")); 215 | Assert.assertEquals(1440 * 3, chron.chronicles.get("test-2012").size()); 216 | Assert.assertEquals(1440 * 12, chron.chronicles.get("test-2013").size()); 217 | Assert.assertEquals(1440 * 3, chron.chronicles.get("test-2012").getNumEvents(start, end)); 218 | Assert.assertEquals(1440 * 12, chron.chronicles.get("test-2013").getNumEvents(start, end)); 219 | Assert.assertEquals(1440 * 15, chron.getNumEvents(start, end)); 220 | Assert.assertEquals(1440 * 15, Iterators.size(chron.getRange(start, end))); 221 | } 222 | 223 | private static class TestChronicle extends PartitionedChronicle { 224 | 225 | public final Map chronicles; 226 | 227 | public TestChronicle(String keyPrefix, PartitionPeriod period) { 228 | super(keyPrefix, period); 229 | chronicles = new HashMap(); 230 | } 231 | 232 | @Override 233 | public Chronicle getPartition(String key) { 234 | if (!chronicles.containsKey(key)) { 235 | chronicles.put(key, new MemoryChronicle()); 236 | } 237 | return chronicles.get(key); 238 | } 239 | 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/api/TimeFrameTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import org.ds.chronos.support.TestBase; 4 | import org.ds.chronos.util.TimeFrame; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | public class TimeFrameTest extends TestBase { 9 | 10 | @Test 11 | public void giveDuration() throws ChronosException { 12 | Assert.assertEquals("100ms", new TimeFrame(0, 100).getDuration().toString()); 13 | } 14 | 15 | @Test 16 | public void contains() throws ChronosException { 17 | TimeFrame t = new TimeFrame(10, 100); 18 | 19 | Assert.assertTrue(t.contains(55)); 20 | Assert.assertFalse(t.contains(101)); 21 | Assert.assertFalse(t.contains(1)); 22 | 23 | 24 | Assert.assertFalse(t.contains(1, 20)); 25 | Assert.assertTrue(t.intersects(1, 20)); 26 | Assert.assertTrue(t.contains(11, 20)); 27 | Assert.assertFalse(t.contains(90, 110)); 28 | Assert.assertTrue(t.intersects(90, 110)); 29 | } 30 | 31 | @Test 32 | public void narrowTest() throws ChronosException { 33 | TimeFrame t = new TimeFrame(0, 100); 34 | 35 | Assert.assertEquals(0, t.fit(-1, 101).getStart()); 36 | Assert.assertEquals(100, t.fit(-1, 101).getEnd()); 37 | Assert.assertEquals(10, t.fit(10, 99).getStart()); 38 | Assert.assertEquals(99, t.fit(10, 99).getEnd()); 39 | Assert.assertEquals(null, t.fit(101, 999)); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/api/TimelineTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Date; 5 | import java.util.List; 6 | 7 | import junit.framework.Assert; 8 | 9 | import org.ds.chronos.api.chronicle.MemoryChronicle; 10 | import org.ds.chronos.support.TestData; 11 | import org.ds.chronos.support.TestDecoder; 12 | import org.ds.chronos.support.TestEncoder; 13 | import org.ds.chronos.timeline.SimpleTimeline; 14 | import org.junit.Before; 15 | import org.junit.Test; 16 | 17 | import com.google.common.base.Function; 18 | import com.google.common.base.Optional; 19 | import com.google.common.base.Predicate; 20 | 21 | public class TimelineTest { 22 | 23 | private MemoryChronicle chronicle; 24 | private SimpleTimeline store; 25 | 26 | @Before 27 | public void createChronicle() { 28 | chronicle = new MemoryChronicle(); 29 | store = new SimpleTimeline(chronicle, new TestDecoder(), new TestEncoder()); 30 | } 31 | 32 | @Test 33 | public void testSingleItem() { 34 | TestData data = new TestData(); 35 | data.time = 1000; 36 | data.type = 0x05; 37 | data.value = 1337.1337d; 38 | store.add(data); 39 | 40 | Assert.assertEquals(1, chronicle.getNumEvents(0, System.currentTimeMillis())); 41 | 42 | Optional option = store.query(new Date(0), new Date()).first(); 43 | Assert.assertTrue(option.isPresent()); 44 | 45 | TestData compare = option.get(); 46 | Assert.assertEquals(data.time, compare.time); 47 | Assert.assertEquals(data.type, compare.type); 48 | Assert.assertEquals(data.value, compare.value, 0.0); 49 | } 50 | 51 | @Test(timeout = 3000) 52 | public void testManyItems() { 53 | int count = 500000; 54 | List list = new ArrayList(); 55 | for (int i = 0; i < count; i++) { 56 | TestData data = new TestData(); 57 | data.time = i * 1000; 58 | data.type = 0x05; 59 | data.value = 1337.1337d; 60 | list.add(data); 61 | } 62 | store.add(list); 63 | 64 | Assert.assertEquals(count, chronicle.getNumEvents(0, System.currentTimeMillis())); 65 | Iterable data = store.query(new Date(0), new Date()).stream(); 66 | 67 | long t1 = System.currentTimeMillis(); 68 | count = 0; 69 | for (TestData item : data) { 70 | Assert.assertEquals(count++ * 1000, item.time); 71 | } 72 | System.out.printf("%d items decoded in %d", count, System.currentTimeMillis() - t1); 73 | } 74 | 75 | @Test(timeout = 3000) 76 | public void testQuery() { 77 | int count = 5000; 78 | List list = new ArrayList(); 79 | for (int i = 0; i < count; i++) { 80 | TestData data = new TestData(); 81 | data.time = i * 1000; 82 | data.type = 0x05; 83 | data.value = 1337.1337d; 84 | list.add(data); 85 | } 86 | store.add(list); 87 | 88 | Assert.assertEquals(count, store.query(0, count * 1000).toList().size()); 89 | } 90 | 91 | @Test(timeout = 3000) 92 | public void testFluent() { 93 | int count = 5000; 94 | List list = new ArrayList(); 95 | for (int i = 0; i < count; i++) { 96 | TestData data = new TestData(); 97 | data.time = i * 1000; 98 | data.type = 0x05; 99 | data.value = i; 100 | list.add(data); 101 | } 102 | store.add(list); 103 | 104 | Optional value = store.query(0, count * 1000).map(new Function() { 105 | 106 | public Double apply(TestData test) { 107 | return test.value; 108 | } 109 | }).firstMatch(new Predicate() { 110 | 111 | public boolean apply(Double value) { 112 | return value > 1000; 113 | } 114 | }); 115 | 116 | Assert.assertTrue(value.isPresent()); 117 | Assert.assertEquals(1001, value.get(), 0.0); 118 | 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/api/WindowTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.api; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.stream.Stream; 6 | 7 | import org.ds.chronos.api.chronicle.MemoryChronicle; 8 | import org.ds.chronos.streams.TemporalIterator; 9 | import org.ds.chronos.streams.TemporalStream; 10 | import org.ds.chronos.streams.WindowIterator; 11 | import org.junit.Assert; 12 | import org.junit.Test; 13 | 14 | public class WindowTest { 15 | 16 | protected ChronologicalRecord getTestItem(long time) { 17 | return new ChronologicalRecord(time, new byte[0]); 18 | } 19 | 20 | protected List getTestItemList(long startTime, long periodInMillis, int count) { 21 | List result = new ArrayList(); 22 | for (int i = 0; i < count; i++) { 23 | result.add(getTestItem(startTime + (periodInMillis * i))); 24 | } 25 | return result; 26 | } 27 | 28 | @Test(timeout = 1000) 29 | public void testIterator() { 30 | Chronicle chronicle = new MemoryChronicle(); 31 | chronicle.add(getTestItemList(0, 1000, 1000)); 32 | WindowIterator itr = new WindowIterator(chronicle.getRange(0, 1000000), 33 | 10000); 34 | 35 | int count = 0; 36 | 37 | while (itr.hasNext()) { 38 | count++; 39 | TemporalIterator window = itr.next(); 40 | int wcount = 0; 41 | while (window.hasNext()) { 42 | wcount++; 43 | window.next(); 44 | } 45 | Assert.assertEquals(10, wcount); 46 | } 47 | 48 | Assert.assertEquals(100, count); 49 | } 50 | 51 | @Test(timeout = 1000) 52 | public void testGap() { 53 | Chronicle chronicle = new MemoryChronicle(); 54 | chronicle.add(getTestItemList(0, 1000, 1000)); 55 | chronicle.deleteRange(20000, 49999); 56 | WindowIterator itr = new WindowIterator(chronicle.getRange(0, 1000000), 57 | 10000); 58 | 59 | int count = 0; 60 | 61 | while (itr.hasNext()) { 62 | count++; 63 | TemporalIterator window = itr.next(); 64 | int wcount = 0; 65 | while (window.hasNext()) { 66 | wcount++; 67 | window.next(); 68 | } 69 | if (window.getTimestamp() >= 20000 && window.getTimestamp() < 50000) { 70 | Assert.assertEquals(0, wcount); 71 | } else { 72 | Assert.assertEquals(10, wcount); 73 | } 74 | } 75 | 76 | Assert.assertEquals(100, count); 77 | } 78 | 79 | @Test(timeout = 1000) 80 | public void testStream() { 81 | Chronicle chronicle = new MemoryChronicle(); 82 | chronicle.add(getTestItemList(0, 1000, 1000)); 83 | Stream> stream = new WindowIterator( 84 | chronicle.getRange(0, 1000000), 10000).stream(); 85 | Assert.assertEquals(1000, stream.mapToLong(s -> s.getStream().count()).sum()); 86 | } 87 | 88 | @Test(timeout = 1000) 89 | public void testReduce() { 90 | Chronicle chronicle = new MemoryChronicle(); 91 | chronicle.add(getTestItemList(0, 1000, 1000)); 92 | Stream> stream = new WindowIterator( 93 | chronicle.getRange(0, 1000000), 10000).stream(); 94 | Assert.assertEquals(100, stream.map(t -> t.getStream().reduce((x, y) -> x).get()).count()); 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/support/TestBase.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.support; 2 | 3 | import java.util.TimeZone; 4 | 5 | import org.junit.BeforeClass; 6 | 7 | public class TestBase { 8 | 9 | @BeforeClass 10 | public static void setupBasic() { 11 | TimeZone.setDefault(TimeZone.getTimeZone("UTC")); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/support/TestData.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.support; 2 | 3 | import org.ds.chronos.api.Temporal; 4 | 5 | public class TestData implements Temporal, Comparable { 6 | 7 | public long time; 8 | public byte type; 9 | public double value; 10 | 11 | public TestData() { 12 | } 13 | 14 | public TestData(long time, byte type, double value) { 15 | super(); 16 | this.time = time; 17 | this.type = type; 18 | this.value = value; 19 | } 20 | 21 | @Override 22 | public long getTimestamp() { 23 | return time; 24 | } 25 | 26 | @Override 27 | public int compareTo(Temporal o) { 28 | return Long.valueOf(time).compareTo(o.getTimestamp()); 29 | } 30 | 31 | } -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/support/TestDecoder.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.support; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.util.Iterator; 5 | 6 | import org.ds.chronos.api.ChronologicalRecord; 7 | import org.ds.chronos.api.TimelineDecoder; 8 | 9 | public class TestDecoder implements TimelineDecoder { 10 | 11 | private Iterator input; 12 | 13 | @Override 14 | public void setInputStream(Iterator input) { 15 | this.input = input; 16 | } 17 | 18 | @Override 19 | public boolean hasNext() { 20 | return input.hasNext(); 21 | } 22 | 23 | @Override 24 | public TestData next() { 25 | ChronologicalRecord column = input.next(); 26 | ByteBuffer buffer = column.getValueBytes(); 27 | 28 | TestData data = new TestData(); 29 | data.time = column.getTimestamp(); 30 | data.type = buffer.get(); 31 | data.value = buffer.getDouble(); 32 | return data; 33 | } 34 | 35 | @Override 36 | public void remove() { 37 | } 38 | 39 | } -------------------------------------------------------------------------------- /chronos-api/src/test/java/org/ds/chronos/support/TestEncoder.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.support; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.util.Iterator; 5 | 6 | import org.ds.chronos.api.ChronologicalRecord; 7 | import org.ds.chronos.api.TimelineEncoder; 8 | 9 | public class TestEncoder implements TimelineEncoder { 10 | 11 | private Iterator input; 12 | 13 | @Override 14 | public boolean hasNext() { 15 | return input.hasNext(); 16 | } 17 | 18 | @Override 19 | public ChronologicalRecord next() { 20 | TestData data = input.next(); 21 | ByteBuffer buffer = ByteBuffer.allocate(9); 22 | buffer.put(data.type); 23 | buffer.putDouble(data.value); 24 | buffer.rewind(); 25 | return new ChronologicalRecord(data.time, buffer.array()); 26 | } 27 | 28 | @Override 29 | public void remove() { 30 | } 31 | 32 | @Override 33 | public void setInputStream(Iterator input) { 34 | this.input = input; 35 | } 36 | } -------------------------------------------------------------------------------- /chronos-aws/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | description = 'chronos aws tools' 3 | dependencies { 4 | compile project(':chronos-api') 5 | compile group: 'com.amazonaws', name: 'aws-java-sdk', version:'1.4.2.1' 6 | compile group: 'org.slf4j', name: 'slf4j-api', version:'1.6.6' 7 | compile group: 'com.google.guava', name: 'guava', version:'19.0' 8 | testCompile group: 'com.amazonaws', name: 'aws-java-sdk', version:'1.4.2.1', classifier:'sources' 9 | testCompile group: 'org.mockito', name: 'mockito-all', version:'1.9.5' 10 | } 11 | 12 | task packageTests(type: Jar) { 13 | from sourceSets.test.output 14 | classifier = 'tests' 15 | } 16 | artifacts.archives packageTests 17 | -------------------------------------------------------------------------------- /chronos-aws/src/main/java/org/ds/chronos/aws/S3Chronicle.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.aws; 2 | 3 | import java.io.ByteArrayInputStream; 4 | import java.io.ByteArrayOutputStream; 5 | import java.io.IOException; 6 | import java.nio.ByteBuffer; 7 | import java.util.Iterator; 8 | import java.util.zip.GZIPInputStream; 9 | import java.util.zip.GZIPOutputStream; 10 | 11 | import org.ds.chronos.api.Chronicle; 12 | import org.ds.chronos.api.ChronologicalRecord; 13 | import org.ds.chronos.api.chronicle.MemoryChronicle; 14 | 15 | import com.amazonaws.services.s3.AmazonS3Client; 16 | import com.amazonaws.services.s3.model.ObjectMetadata; 17 | import com.amazonaws.services.s3.model.S3Object; 18 | import com.amazonaws.services.s3.model.S3ObjectInputStream; 19 | import com.google.common.io.ByteArrayDataOutput; 20 | import com.google.common.io.ByteStreams; 21 | 22 | /** 23 | * 24 | * A chronicle backed by an S3 Object. 25 | *

26 | * The motivation for this implementation is archive related. S3Chronicle should not be used for write heavy workloads, as the entire object 27 | * needs to be fetched in order to write a single value. See {@link S3PartitionedChronicle} for a more robust implementation, which can 28 | * partition data by date, into multiple keys. 29 | *

30 | * 31 | * TODO: Optimize getRange with a streaming decoder. 32 | * 33 | * @author Dan Simpson 34 | * 35 | */ 36 | public class S3Chronicle extends Chronicle { 37 | 38 | private static final int BUFSIZE = 1024; 39 | 40 | final AmazonS3Client client; 41 | final String bucket; 42 | final String key; 43 | 44 | final MemoryChronicle records; 45 | final boolean gzip; 46 | 47 | public S3Chronicle(AmazonS3Client client, String bucket, String key) { 48 | this(client, bucket, key, false); 49 | } 50 | 51 | public S3Chronicle(AmazonS3Client client, String bucket, String key, boolean gzip) { 52 | this.client = client; 53 | this.bucket = bucket; 54 | this.key = key; 55 | this.gzip = gzip; 56 | this.records = new MemoryChronicle(); 57 | } 58 | 59 | @Override 60 | public void add(ChronologicalRecord item) { 61 | load(); 62 | records.add(item); 63 | save(); 64 | } 65 | 66 | @Override 67 | public void add(Iterator items, int pageSize) { 68 | load(); 69 | records.add(items, pageSize); 70 | save(); 71 | } 72 | 73 | @Override 74 | public Iterator getRange(long t1, long t2, int pageSize) { 75 | load(); 76 | return records.getRange(t1, t2, pageSize); 77 | } 78 | 79 | @Override 80 | public long getNumEvents(long t1, long t2) { 81 | load(); 82 | return records.getNumEvents(t1, t2); 83 | } 84 | 85 | @Override 86 | public void delete() { 87 | client.deleteObject(bucket, key); 88 | } 89 | 90 | @Override 91 | public void deleteRange(long t1, long t2) { 92 | load(); 93 | records.deleteRange(t1, t2); 94 | save(); 95 | } 96 | 97 | /** 98 | * Load the data from S3, decode it, and store it in the memory chronicle 99 | */ 100 | private synchronized void load() { 101 | if (!records.isEmpty()) { 102 | return; 103 | } 104 | 105 | ByteBuffer buffer = getBytes(); 106 | 107 | if (buffer.remaining() < 4) { 108 | return; 109 | } 110 | 111 | long count = buffer.getInt(); 112 | for (int i = 0; i < count; i++) { 113 | long time = buffer.getLong(); 114 | byte[] data = new byte[buffer.getInt()]; 115 | buffer.get(data); 116 | records.add(new ChronologicalRecord(time, data)); 117 | } 118 | } 119 | 120 | private synchronized void save() { 121 | byte[] result = encode(); 122 | 123 | ObjectMetadata meta = new ObjectMetadata(); 124 | meta.setContentLength(result.length); 125 | 126 | if (gzip) { 127 | meta.setContentType("application/gzip"); 128 | } else { 129 | meta.setContentType("application/octet-stream"); 130 | } 131 | 132 | client.putObject(bucket, key, new ByteArrayInputStream(result), meta); 133 | } 134 | 135 | protected byte[] encode() { 136 | ByteArrayDataOutput out = ByteStreams.newDataOutput(); 137 | out.writeInt(records.size()); 138 | 139 | for (ChronologicalRecord record : records.all()) { 140 | out.writeLong(record.getTimestamp()); 141 | out.writeInt(record.getByteSize()); 142 | out.write(record.getData()); 143 | } 144 | 145 | if (!gzip) { 146 | return out.toByteArray(); 147 | } 148 | 149 | ByteArrayOutputStream output = new ByteArrayOutputStream(); 150 | try { 151 | GZIPOutputStream gzipOutputStream = new GZIPOutputStream(output); 152 | gzipOutputStream.write(out.toByteArray()); 153 | gzipOutputStream.close(); 154 | } catch (IOException e) { 155 | throw new RuntimeException(e); 156 | } 157 | 158 | return output.toByteArray(); 159 | } 160 | 161 | /** 162 | * Convert an S3 object to a ByteBuffer 163 | * 164 | * @return 165 | */ 166 | private ByteBuffer getBytes() { 167 | S3Object object; 168 | try { 169 | object = client.getObject(bucket, key); 170 | } catch (Throwable t) { 171 | return ByteBuffer.allocate(0); 172 | } 173 | 174 | S3ObjectInputStream input = object.getObjectContent(); 175 | ByteArrayOutputStream output = new ByteArrayOutputStream(); 176 | 177 | try { 178 | if (gzip) { 179 | readGzipped(input, output); 180 | } else { 181 | read(input, output); 182 | } 183 | input.close(); 184 | } catch (IOException e) { 185 | throw new RuntimeException(e); 186 | } 187 | 188 | return ByteBuffer.wrap(output.toByteArray()); 189 | } 190 | 191 | private void read(S3ObjectInputStream input, ByteArrayOutputStream output) throws IOException { 192 | int size; 193 | byte[] tmp = new byte[BUFSIZE]; 194 | while ((size = input.read(tmp)) != -1) { 195 | output.write(tmp, 0, size); 196 | } 197 | } 198 | 199 | private void readGzipped(S3ObjectInputStream input, ByteArrayOutputStream output) throws IOException { 200 | GZIPInputStream gzip = new GZIPInputStream(input); 201 | int size; 202 | byte[] tmp = new byte[BUFSIZE]; 203 | while ((size = gzip.read(tmp)) != -1) { 204 | output.write(tmp, 0, size); 205 | } 206 | gzip.close(); 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /chronos-aws/src/main/java/org/ds/chronos/aws/S3ParitionedChronicle.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.aws; 2 | 3 | import org.ds.chronos.api.Chronicle; 4 | import org.ds.chronos.api.PartitionPeriod; 5 | import org.ds.chronos.api.PartitionedChronicle; 6 | 7 | import com.amazonaws.services.s3.AmazonS3Client; 8 | 9 | public class S3ParitionedChronicle extends PartitionedChronicle { 10 | 11 | final AmazonS3Client client; 12 | final String bucket; 13 | final boolean gzip; 14 | 15 | public S3ParitionedChronicle(AmazonS3Client client, String bucket, String keyPrefix, PartitionPeriod period) { 16 | this(client, bucket, keyPrefix, period, false); 17 | } 18 | 19 | public S3ParitionedChronicle(AmazonS3Client client, String bucket, String keyPrefix, PartitionPeriod period, 20 | boolean gzip) { 21 | super(keyPrefix, period); 22 | this.client = client; 23 | this.bucket = bucket; 24 | this.gzip = gzip; 25 | } 26 | 27 | @Override 28 | protected Chronicle getPartition(String key) { 29 | return new S3Chronicle(client, bucket, key, gzip); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /chronos-aws/src/main/java/org/ds/chronos/aws/SimpleDBChronicle.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.aws; 2 | 3 | import java.util.Collection; 4 | import java.util.Collections; 5 | import java.util.Iterator; 6 | import java.util.LinkedList; 7 | 8 | import org.ds.chronos.api.Chronicle; 9 | import org.ds.chronos.api.ChronologicalRecord; 10 | 11 | import com.amazonaws.services.simpledb.AmazonSimpleDBClient; 12 | import com.amazonaws.services.simpledb.model.BatchPutAttributesRequest; 13 | import com.amazonaws.services.simpledb.model.CreateDomainRequest; 14 | import com.amazonaws.services.simpledb.model.DeleteDomainRequest; 15 | import com.amazonaws.services.simpledb.model.ReplaceableAttribute; 16 | import com.amazonaws.services.simpledb.model.ReplaceableItem; 17 | import com.amazonaws.services.simpledb.model.SelectRequest; 18 | import com.amazonaws.services.simpledb.model.SelectResult; 19 | 20 | public class SimpleDBChronicle extends Chronicle { 21 | 22 | final AmazonSimpleDBClient client; 23 | final String key; 24 | 25 | public SimpleDBChronicle(AmazonSimpleDBClient client, String key) { 26 | super(); 27 | this.client = client; 28 | this.key = key; 29 | } 30 | 31 | @Override 32 | public void add(ChronologicalRecord item) { 33 | add(Collections.singleton(item).iterator(), 0); 34 | } 35 | 36 | private ReplaceableItem getItem(ChronologicalRecord record) { 37 | ReplaceableItem item = new ReplaceableItem(); 38 | item.setName(String.format("%013d", record.getTimestamp())); 39 | item.setAttributes(Collections.singleton(new ReplaceableAttribute("v", new String(record.getData()), true))); 40 | return item; 41 | } 42 | 43 | private final int PAGE_SIZE = 25; 44 | 45 | @Override 46 | public void add(Iterator items, int pageSize) { 47 | BatchPutAttributesRequest request = new BatchPutAttributesRequest(); 48 | request.setDomainName(key); 49 | 50 | int count = 0; 51 | Collection mapped = new LinkedList(); 52 | while (items.hasNext()) { 53 | mapped.add(getItem(items.next())); 54 | 55 | if (++count % PAGE_SIZE == 0) { 56 | request.setItems(mapped); 57 | client.batchPutAttributes(request); 58 | mapped.clear(); 59 | } 60 | } 61 | 62 | if (mapped.size() > 0) { 63 | request.setItems(mapped); 64 | client.batchPutAttributes(request); 65 | } 66 | } 67 | 68 | @Override 69 | public Iterator getRange(long t1, long t2, int pageSize) { 70 | SelectRequest request = new SelectRequest(); 71 | request.setConsistentRead(true); 72 | request.setSelectExpression(getQuery("*", t1, t2)); 73 | return new SimpleDBIterator(client, request); 74 | } 75 | 76 | private String getQuery(String projection, long t1, long t2) { 77 | if (t1 > t2) { 78 | return String.format( 79 | "SELECT %s FROM `%s` WHERE itemName() >= '%013d' AND itemName() <= '%013d' ORDER BY itemName() DESC", 80 | projection, key, t2, t1); 81 | } 82 | 83 | return String.format("SELECT %s FROM `%s` WHERE itemName() >= '%013d' AND itemName() <= '%013d'", projection, key, 84 | t1, t2); 85 | } 86 | 87 | @Override 88 | public long getNumEvents(long t1, long t2) { 89 | SelectRequest request = new SelectRequest(); 90 | request.setConsistentRead(true); 91 | request.setSelectExpression(getQuery("count(*)", t1, t2)); 92 | 93 | long count = 0; 94 | String token = null; 95 | 96 | do { 97 | if (token != null) { 98 | request.setNextToken(token); 99 | } 100 | 101 | SelectResult result = client.select(request); 102 | token = result.getNextToken(); 103 | count += Integer.parseInt(result.getItems().get(0).getAttributes().get(0).getValue()); 104 | } while (token != null); 105 | 106 | return count; 107 | } 108 | 109 | @Override 110 | public void delete() { 111 | client.deleteDomain(new DeleteDomainRequest().withDomainName(key)); 112 | } 113 | 114 | @Override 115 | public void deleteRange(long t1, long t2) { 116 | // DeleteAttributesRequest request = new DeleteAttributesRequest(); 117 | // request.setDomainName(key); 118 | // request.setItemName(itemName); 119 | // client.deleteAttributes(request); 120 | } 121 | 122 | public void createDomain() { 123 | client.createDomain(new CreateDomainRequest().withDomainName(key)); 124 | } 125 | 126 | } 127 | -------------------------------------------------------------------------------- /chronos-aws/src/main/java/org/ds/chronos/aws/SimpleDBIterator.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.aws; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Iterator; 5 | import java.util.List; 6 | 7 | import org.ds.chronos.api.ChronologicalRecord; 8 | 9 | import com.amazonaws.services.simpledb.AmazonSimpleDBClient; 10 | import com.amazonaws.services.simpledb.model.Item; 11 | import com.amazonaws.services.simpledb.model.SelectRequest; 12 | import com.amazonaws.services.simpledb.model.SelectResult; 13 | 14 | public class SimpleDBIterator implements Iterator { 15 | 16 | private final AmazonSimpleDBClient client; 17 | private final SelectRequest request; 18 | 19 | private List buffer; 20 | private int index = 0; 21 | private boolean finished = false; 22 | 23 | public SimpleDBIterator(AmazonSimpleDBClient client, SelectRequest request) { 24 | this.client = client; 25 | this.request = request; 26 | this.buffer = new ArrayList(); 27 | } 28 | 29 | private void load() { 30 | if (finished) { 31 | return; 32 | } 33 | 34 | SelectResult result = client.select(request); 35 | if (result.getNextToken() == null) { 36 | finished = true; 37 | } else { 38 | request.setNextToken(result.getNextToken()); 39 | } 40 | 41 | index = 0; 42 | buffer = result.getItems(); 43 | } 44 | 45 | @Override 46 | public boolean hasNext() { 47 | if (index < buffer.size()) { 48 | return true; 49 | } 50 | 51 | load(); 52 | 53 | return index < buffer.size(); 54 | } 55 | 56 | @Override 57 | public ChronologicalRecord next() { 58 | Item item = buffer.get(index++); 59 | 60 | long time = Long.parseLong(item.getName()); 61 | byte[] data = item.getAttributes().get(0).getValue().getBytes(); 62 | 63 | return new ChronologicalRecord(time, data); 64 | } 65 | 66 | @Override 67 | public void remove() { 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /chronos-aws/src/test/java/org/ds/chronos/aws/S3ChronicleTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.aws; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import junit.framework.Assert; 7 | 8 | import org.ds.chronos.api.ChronologicalRecord; 9 | import org.ds.chronos.api.chronicle.MemoryChronicle; 10 | import org.junit.Assume; 11 | import org.junit.Before; 12 | import org.junit.Test; 13 | 14 | import com.amazonaws.auth.BasicAWSCredentials; 15 | import com.amazonaws.regions.Region; 16 | import com.amazonaws.regions.Regions; 17 | import com.amazonaws.services.s3.AmazonS3Client; 18 | 19 | public class S3ChronicleTest { 20 | 21 | protected ChronologicalRecord getTestItem(long time) { 22 | return new ChronologicalRecord(time, "Hello".getBytes()); 23 | } 24 | 25 | protected List getTestItemList(long startTime, long periodInMillis, int count) { 26 | List result = new ArrayList(); 27 | for (int i = 0; i < count; i++) { 28 | result.add(getTestItem(startTime + (periodInMillis * i))); 29 | } 30 | return result; 31 | } 32 | 33 | AmazonS3Client client; 34 | S3Chronicle chronicle; 35 | String bucket = "chronos-tests"; 36 | 37 | public AmazonS3Client getClient() { 38 | String ak = System.getenv("AWS_AK"); 39 | String sk = System.getenv("AWS_SK"); 40 | 41 | if (ak != null && sk != null) { 42 | AmazonS3Client client = new AmazonS3Client(new BasicAWSCredentials(ak, sk)); 43 | client.setRegion(Region.getRegion(Regions.US_WEST_1)); 44 | return client; 45 | } else { 46 | System.err.println("AWS Credentials not set in env"); 47 | } 48 | 49 | return null; 50 | } 51 | 52 | @Before 53 | public void setup() { 54 | client = getClient(); 55 | 56 | Assume.assumeTrue(client != null); 57 | 58 | chronicle = getChronicle(); 59 | chronicle.delete(); 60 | } 61 | 62 | public S3Chronicle getChronicle() { 63 | return new S3Chronicle(client, bucket, "test"); 64 | } 65 | 66 | @Test 67 | public void testAdd() { 68 | chronicle.add(getTestItem(0)); 69 | Assert.assertEquals(1, chronicle.getNumEvents(0, System.currentTimeMillis())); 70 | Assert.assertEquals(1, getChronicle().getNumEvents(0, System.currentTimeMillis())); 71 | } 72 | 73 | @Test 74 | public void testAddBatch() { 75 | int items = 1440 * 30; 76 | int iters = 1; 77 | 78 | chronicle.add(getTestItemList(0, 1000, items)); 79 | Assert.assertEquals(items, chronicle.getNumEvents(0, System.currentTimeMillis())); 80 | 81 | long t1 = System.currentTimeMillis(); 82 | for (int i = 0; i < iters; i++) { 83 | Assert.assertEquals(items, getChronicle().getNumEvents(0, System.currentTimeMillis())); 84 | } 85 | long t2 = System.currentTimeMillis(); 86 | 87 | System.out.printf("%d items loaded and decoded %d times in %d ms - %d bytes\n", items, iters, t2 - t1, 88 | chronicle.encode().length); 89 | } 90 | 91 | @Test 92 | public void testRange() { 93 | chronicle.add(getTestItemList(0, 1000, 100)); 94 | 95 | List items = MemoryChronicle.toList(getChronicle().getRange(1000, 5000)); 96 | 97 | Assert.assertEquals(5, items.size()); 98 | Assert.assertEquals(1000, items.get(0).getTimestamp()); 99 | Assert.assertEquals(5000, items.get(4).getTimestamp()); 100 | } 101 | 102 | @Test 103 | public void testReverseRange() { 104 | chronicle.add(getTestItemList(0, 1000, 100)); 105 | 106 | List items = MemoryChronicle.toList(getChronicle().getRange(5000, 1000)); 107 | 108 | Assert.assertEquals(5, items.size()); 109 | Assert.assertEquals(5000, items.get(0).getTimestamp()); 110 | Assert.assertEquals(1000, items.get(4).getTimestamp()); 111 | } 112 | 113 | @Test 114 | public void testCount() { 115 | chronicle.add(getTestItemList(0, 1000, 100)); 116 | Assert.assertEquals(50, getChronicle().getNumEvents(1, 50000)); 117 | } 118 | 119 | @Test 120 | public void testGzip() { 121 | int items = 1440 * 30; 122 | int iters = 1; 123 | 124 | S3Chronicle chronicle = new S3Chronicle(client, bucket, "test", true); 125 | 126 | chronicle.add(getTestItemList(0, 1000, items)); 127 | byte[] data = chronicle.encode(); 128 | 129 | Assert.assertTrue(data.length > 0); 130 | 131 | long t1 = System.currentTimeMillis(); 132 | for (int i = 0; i < iters; i++) { 133 | Assert.assertEquals(items, 134 | new S3Chronicle(client, bucket, "test", true).getNumEvents(0, System.currentTimeMillis())); 135 | } 136 | long t2 = System.currentTimeMillis(); 137 | 138 | System.out.printf("%d items loaded and decoded %d times in %d ms - %d bytes\n", items, iters, t2 - t1, data.length); 139 | } 140 | 141 | } 142 | -------------------------------------------------------------------------------- /chronos-aws/src/test/java/org/ds/chronos/aws/SimpleDBChronicleTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.aws; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import junit.framework.Assert; 7 | 8 | import org.ds.chronos.api.ChronologicalRecord; 9 | import org.ds.chronos.api.chronicle.MemoryChronicle; 10 | import org.junit.After; 11 | import org.junit.Assume; 12 | import org.junit.Before; 13 | import org.junit.Test; 14 | 15 | import com.amazonaws.auth.BasicAWSCredentials; 16 | import com.amazonaws.regions.Region; 17 | import com.amazonaws.regions.Regions; 18 | import com.amazonaws.services.simpledb.AmazonSimpleDBClient; 19 | 20 | public class SimpleDBChronicleTest { 21 | 22 | protected ChronologicalRecord getTestItem(long time) { 23 | return new ChronologicalRecord(time, "Hello".getBytes()); 24 | } 25 | 26 | protected List getTestItemList(long startTime, long periodInMillis, int count) { 27 | List result = new ArrayList(); 28 | for (int i = 0; i < count; i++) { 29 | result.add(getTestItem(startTime + (periodInMillis * i))); 30 | } 31 | return result; 32 | } 33 | 34 | AmazonSimpleDBClient client; 35 | SimpleDBChronicle chronicle; 36 | String bucket = "chronos-tests"; 37 | 38 | public AmazonSimpleDBClient getClient() { 39 | String ak = System.getenv("AWS_AK"); 40 | String sk = System.getenv("AWS_SK"); 41 | 42 | if (ak != null && sk != null) { 43 | AmazonSimpleDBClient client = new AmazonSimpleDBClient(new BasicAWSCredentials(ak, sk)); 44 | client.setRegion(Region.getRegion(Regions.US_WEST_1)); 45 | return client; 46 | } else { 47 | System.err.println("AWS Credentials not set in env"); 48 | } 49 | 50 | return null; 51 | } 52 | 53 | @Before 54 | public void setup() { 55 | client = getClient(); 56 | 57 | Assume.assumeTrue(client != null); 58 | 59 | chronicle = getChronicle(); 60 | 61 | try { 62 | chronicle.delete(); 63 | chronicle.createDomain(); 64 | } catch (Throwable t) { 65 | System.err.println("Can't manipulate domain"); 66 | } 67 | 68 | } 69 | 70 | @After 71 | public void cleanup() { 72 | try { 73 | chronicle.delete(); 74 | } catch (Throwable t) { 75 | System.err.println("Can't delete domain"); 76 | } 77 | } 78 | 79 | public SimpleDBChronicle getChronicle() { 80 | return new SimpleDBChronicle(client, "chronos-test"); 81 | } 82 | 83 | @Test 84 | public void testAdd() { 85 | chronicle.add(getTestItem(0)); 86 | Assert.assertEquals(1, chronicle.getNumEvents(0, System.currentTimeMillis())); 87 | Assert.assertEquals(1, getChronicle().getNumEvents(0, System.currentTimeMillis())); 88 | } 89 | 90 | @Test 91 | public void testRange() { 92 | chronicle.add(getTestItemList(0, 1000, 100)); 93 | 94 | List items = MemoryChronicle.toList(getChronicle().getRange(1000, 5000)); 95 | 96 | Assert.assertEquals(5, items.size()); 97 | Assert.assertEquals(1000, items.get(0).getTimestamp()); 98 | Assert.assertEquals(5000, items.get(4).getTimestamp()); 99 | } 100 | 101 | @Test 102 | public void testReverseRange() { 103 | chronicle.add(getTestItemList(0, 1000, 100)); 104 | 105 | List items = MemoryChronicle.toList(getChronicle().getRange(5000, 1000)); 106 | 107 | Assert.assertEquals(5, items.size()); 108 | Assert.assertEquals(5000, items.get(0).getTimestamp()); 109 | Assert.assertEquals(1000, items.get(4).getTimestamp()); 110 | } 111 | 112 | @Test 113 | public void testCount() { 114 | chronicle.add(getTestItemList(0, 1000, 100)); 115 | Assert.assertEquals(50, getChronicle().getNumEvents(1, 50000)); 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /chronos-datastax/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | description = 'chronos datastax' 3 | dependencies { 4 | compile project(':chronos-api') 5 | compile group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version:'3.0.2' 6 | compile group: 'org.slf4j', name: 'slf4j-api', version:'1.6.6' 7 | testCompile group: 'com.datastax.cassandra', name: 'cassandra-driver-core', version:'3.0.2', classifier:'sources' 8 | testCompile group: 'org.slf4j', name: 'slf4j-simple', version:'1.6.6' 9 | } 10 | 11 | task packageTests(type: Jar) { 12 | from sourceSets.test.output 13 | classifier = 'tests' 14 | } 15 | artifacts.archives packageTests 16 | -------------------------------------------------------------------------------- /chronos-datastax/src/main/java/org/ds/chronos/chronicle/DatastaxChronicle.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.chronicle; 2 | 3 | import static com.datastax.driver.core.querybuilder.QueryBuilder.eq; 4 | import static com.datastax.driver.core.querybuilder.QueryBuilder.gte; 5 | import static com.datastax.driver.core.querybuilder.QueryBuilder.lte; 6 | import static com.datastax.driver.core.querybuilder.QueryBuilder.select; 7 | 8 | import java.util.Iterator; 9 | 10 | import org.ds.chronos.api.Chronicle; 11 | import org.ds.chronos.api.ChronologicalRecord; 12 | 13 | import com.datastax.driver.core.BatchStatement; 14 | import com.datastax.driver.core.PreparedStatement; 15 | import com.datastax.driver.core.ResultSet; 16 | import com.datastax.driver.core.Row; 17 | import com.datastax.driver.core.Session; 18 | import com.datastax.driver.core.querybuilder.Delete; 19 | import com.datastax.driver.core.querybuilder.QueryBuilder; 20 | import com.datastax.driver.core.querybuilder.Select.Where; 21 | import com.google.common.base.Function; 22 | import com.google.common.collect.Iterators; 23 | 24 | /** 25 | * 26 | * A chronicle backed by cassandra and assisted by the Datastax java client. 27 | * 28 | * @author Dan Simpson 29 | * 30 | */ 31 | public class DatastaxChronicle extends Chronicle { 32 | 33 | /** 34 | * Compact storage tends to be faster for this type of data 35 | */ 36 | private static final String CREATE_STATEMENT = "CREATE TABLE %s (%s varchar, %s bigint, %s blob, PRIMARY KEY(%2$s, %3$s)) WITH COMPACT STORAGE"; 37 | 38 | /** 39 | * Structure to define Table, Key, Clustering Column, Data names 40 | * 41 | * @author Dan Simpson 42 | * 43 | */ 44 | public static final class Settings { 45 | 46 | protected final String table; 47 | protected final String key; 48 | protected final String cluster; 49 | protected final String data; 50 | 51 | private PreparedStatement insert; 52 | 53 | public Settings(String table, String key, String cluster, String data) { 54 | super(); 55 | this.table = table; 56 | this.key = key; // series 57 | this.cluster = cluster; // timestamp 58 | this.data = data; // value 59 | 60 | } 61 | 62 | protected PreparedStatement insert(Session session) { 63 | if (insert == null) { 64 | synchronized (this) { 65 | insert = session.prepare( 66 | String.format("INSERT INTO %s (%s, %s, %s) VALUES (?, ?, ?) USING TTL ?", table, key, cluster, data)); 67 | } 68 | } 69 | return insert; 70 | } 71 | 72 | public static Settings legacy(String table) { 73 | return new Settings(table, "key", "column1", "value"); 74 | } 75 | 76 | public static Settings modern(String table) { 77 | return new Settings(table, "name", "time", "data"); 78 | } 79 | 80 | } 81 | 82 | protected final String name; 83 | protected final Settings table; 84 | protected final Session session; 85 | protected final int ttl; 86 | 87 | public DatastaxChronicle(Session session, Settings table, String name) { 88 | this(session, table, name, 0); 89 | } 90 | 91 | public DatastaxChronicle(Session session, Settings table, String name, int ttl) { 92 | this.session = session; 93 | this.table = table; 94 | this.name = name; 95 | this.ttl = ttl; 96 | 97 | // Increasing this can decrease throughput 98 | WRITE_PAGE_SIZE = 2048; 99 | } 100 | 101 | @Override 102 | public void add(ChronologicalRecord column) { 103 | session.execute(table.insert(session).bind(name, column.getTimestamp(), column.getValueBytes(), ttl)); 104 | } 105 | 106 | @Override 107 | public void add(Iterator items, int pageSize) { 108 | PreparedStatement insert = table.insert(session); 109 | 110 | BatchStatement statement = new BatchStatement(); 111 | while (items.hasNext()) { 112 | ChronologicalRecord item = items.next(); 113 | statement.add(insert.bind(name, item.getTimestamp(), item.getValueBytes(), ttl)); 114 | if (statement.size() >= pageSize) { 115 | session.execute(statement); 116 | statement.clear(); 117 | } 118 | } 119 | 120 | if (statement.size() > 0) { 121 | session.execute(statement); 122 | } 123 | } 124 | 125 | @Override 126 | public long getNumEvents(long t1, long t2) { 127 | assert (t1 <= t2); 128 | 129 | Where query = select().countAll().from(table.table).where(eq(table.key, name)).and(gte(table.cluster, t1)) 130 | .and(lte(table.cluster, t2)); 131 | ResultSet result = session.execute(query); 132 | if (result.isExhausted()) { 133 | return 0; 134 | } 135 | return result.iterator().next().getLong(0); 136 | } 137 | 138 | @Override 139 | public boolean isEventRecorded(long time) { 140 | Where query = QueryBuilder.select().from(table.table).where(eq(table.key, name)).and(eq(table.cluster, time)); 141 | return !session.execute(query).isExhausted(); 142 | } 143 | 144 | @Override 145 | public void delete() { 146 | Delete.Where query = QueryBuilder.delete().from(table.table).where(eq(table.key, name)); 147 | session.execute(query); 148 | } 149 | 150 | @Override 151 | public Iterator getRange(long t1, long t2, int pageSize) { 152 | 153 | long begin = Math.min(t1, t2); 154 | long end = Math.max(t1, t2); 155 | 156 | Where query = select().column(table.cluster).column(table.data).from(table.table).where(eq(table.key, name)) 157 | .and(gte(table.cluster, begin)).and(lte(table.cluster, end)); 158 | 159 | if (t1 > t2) { 160 | query.orderBy(QueryBuilder.desc(table.cluster)); 161 | } 162 | 163 | ResultSet set = session.execute(query); 164 | return Iterators.transform(set.iterator(), new Function() { 165 | 166 | public ChronologicalRecord apply(Row row) { 167 | return new ChronologicalRecord(row.getLong(0), row.getBytes(1)); 168 | } 169 | }); 170 | } 171 | 172 | @Override 173 | public void deleteRange(long t1, long t2) { 174 | assert (t1 <= t2); 175 | Iterator range = getRange(t1, t2); 176 | while (range.hasNext()) { 177 | ChronologicalRecord record = range.next(); 178 | session.execute(QueryBuilder.delete().from(table.table).where(eq(table.key, name)) 179 | .and(eq(table.cluster, record.getTimestamp()))); 180 | } 181 | } 182 | 183 | /** 184 | * Create the schema for the current session and table name, using compact storage and defaults 185 | */ 186 | public static void createTable(Session session, Settings settings) { 187 | session.execute(String.format(CREATE_STATEMENT, settings.table, settings.key, settings.cluster, settings.data)); 188 | } 189 | 190 | } 191 | -------------------------------------------------------------------------------- /chronos-datastax/src/main/java/org/ds/chronos/chronicle/DatastaxPartitionedChronicle.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.chronicle; 2 | 3 | import org.ds.chronos.api.Chronicle; 4 | import org.ds.chronos.api.PartitionPeriod; 5 | import org.ds.chronos.api.PartitionedChronicle; 6 | import org.ds.chronos.chronicle.DatastaxChronicle.Settings; 7 | 8 | import com.datastax.driver.core.Session; 9 | 10 | public class DatastaxPartitionedChronicle extends PartitionedChronicle { 11 | 12 | protected Session session; 13 | protected Settings settings; 14 | 15 | public DatastaxPartitionedChronicle(Session session, Settings settings, String keyPrefix, PartitionPeriod period) { 16 | super(keyPrefix, period); 17 | this.session = session; 18 | this.settings = settings; 19 | } 20 | 21 | @Override 22 | protected Chronicle getPartition(String key) { 23 | return new DatastaxChronicle(session, settings, key); 24 | } 25 | 26 | } -------------------------------------------------------------------------------- /chronos-datastax/src/test/java/org/ds/chronos/chronicle/ChronicleTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.chronicle; 2 | 3 | import java.util.Date; 4 | import java.util.Iterator; 5 | 6 | import junit.framework.Assert; 7 | 8 | import org.ds.chronos.api.Chronicle; 9 | import org.ds.chronos.api.ChronicleBatch; 10 | import org.ds.chronos.api.ChronologicalRecord; 11 | import org.ds.chronos.api.ChronosException; 12 | import org.ds.chronos.api.chronicle.MemoryChronicle; 13 | import org.junit.Before; 14 | import org.junit.Test; 15 | 16 | import com.google.common.base.Charsets; 17 | import com.google.common.collect.Iterators; 18 | 19 | public class ChronicleTest extends TestBase { 20 | 21 | private static int count = 1; 22 | private Chronicle chronicle; 23 | 24 | @Before 25 | public void create() throws ChronosException { 26 | chronicle = getChronicle("basicTest" + count++); 27 | } 28 | 29 | @Test 30 | public void shouldBeEmpty() throws ChronosException { 31 | Assert.assertEquals(0, chronicle.getNumEvents(0, System.currentTimeMillis())); 32 | } 33 | 34 | @Test 35 | public void shouldAddEventAndCount() throws ChronosException { 36 | chronicle.add(1000, "test"); 37 | Assert.assertEquals(1, chronicle.getNumEvents(999, 1001)); 38 | } 39 | 40 | @Test 41 | public void shouldBatch() throws ChronosException { 42 | long count = 100; 43 | long period = 1000 * 60; 44 | long time = 0; 45 | 46 | ChronicleBatch batch = new ChronicleBatch(); 47 | for (int i = 0; i < count; i++) { 48 | batch.add(time + i * period, "test"); 49 | } 50 | chronicle.add(batch); 51 | 52 | Assert.assertEquals(count, chronicle.getNumEvents(0, count * period)); 53 | } 54 | 55 | @Test 56 | public void shouldGetSingleRange() throws ChronosException { 57 | chronicle.add(1000, "test"); 58 | Iterator slice = chronicle.getRange(0, 2000); 59 | 60 | Assert.assertTrue(slice.hasNext()); 61 | ChronologicalRecord event = slice.next(); 62 | Assert.assertEquals(1000, event.getTimestamp()); 63 | Assert.assertEquals("test", new String(event.getData(), Charsets.UTF_8)); 64 | } 65 | 66 | @Test 67 | public void shouldGetRange() throws ChronosException { 68 | chronicle.add(1000, "test"); 69 | chronicle.add(2000, "test"); 70 | Iterator slice = chronicle.getRange(1000, 3000); 71 | 72 | Assert.assertTrue(slice.hasNext()); 73 | ChronologicalRecord event = slice.next(); 74 | 75 | Assert.assertEquals(1000, event.getTimestamp()); 76 | Assert.assertEquals("test", new String(event.getData(), Charsets.UTF_8)); 77 | } 78 | 79 | @Test 80 | public void shouldGetRangeReversed() throws ChronosException { 81 | chronicle.add(1000, "test"); 82 | chronicle.add(2000, "test"); 83 | Iterator slice = chronicle.getRange(3000, 1000); 84 | 85 | Assert.assertTrue(slice.hasNext()); 86 | ChronologicalRecord event = slice.next(); 87 | 88 | Assert.assertEquals(2000, event.getTimestamp()); 89 | Assert.assertEquals("test", new String(event.getData(), Charsets.UTF_8)); 90 | } 91 | 92 | @Test 93 | public void shouldDelete() throws ChronosException { 94 | Date time = new Date(); 95 | 96 | chronicle.add(time, "test"); 97 | chronicle.deleteRange(time.getTime() - 1000, time.getTime() + 1000); 98 | 99 | Assert.assertEquals(0, chronicle.getNumEvents(0, System.currentTimeMillis())); 100 | } 101 | 102 | @Test 103 | public void shouldDeleteFragment() throws ChronosException { 104 | chronicle.add(1000, "test"); 105 | chronicle.add(2000, "test"); 106 | chronicle.add(3000, "test"); 107 | chronicle.add(4000, "test"); 108 | 109 | chronicle.deleteRange(0, 2000); 110 | 111 | Assert.assertEquals(2, chronicle.getNumEvents(0, 4000)); 112 | Assert.assertEquals(3000, MemoryChronicle.toList(chronicle.getRange(0, 4000)).get(0).getTimestamp()); 113 | } 114 | 115 | @Test 116 | public void shouldDeleteRow() throws ChronosException { 117 | 118 | Chronicle other = getChronicle("nottest"); 119 | other.add(1000, "ok"); 120 | 121 | chronicle.add(1000, "test"); 122 | chronicle.delete(); 123 | 124 | Assert.assertEquals(0, chronicle.getNumEvents(0, 2000)); 125 | Assert.assertEquals(1, other.getNumEvents(0, 2000)); 126 | } 127 | 128 | @Test 129 | public void shouldDetectExistence() throws ChronosException { 130 | chronicle.add(1000, "test"); 131 | Assert.assertTrue(chronicle.isEventRecorded(1000)); 132 | Assert.assertFalse(chronicle.isEventRecorded(999)); 133 | Assert.assertFalse(chronicle.isEventRecorded(1001)); 134 | } 135 | 136 | @Test(timeout = 5000) 137 | public void testPerf() { 138 | long count = 2800; 139 | long period = 1000 * 60; 140 | long time = System.currentTimeMillis(); 141 | long end = time + (count * period); 142 | 143 | ChronicleBatch batch = new ChronicleBatch(); 144 | for (int i = 0; i < count; i++) { 145 | batch.add(time + i * period, "test"); 146 | } 147 | long t1 = System.currentTimeMillis(); 148 | chronicle.add(batch); 149 | long t2 = System.currentTimeMillis(); 150 | System.out.printf("%d ms for %d items\n", t2 - t1, count); 151 | 152 | t1 = System.currentTimeMillis(); 153 | chronicle.add(new ChronologicalRecord(time - period, "test".getBytes())); 154 | t2 = System.currentTimeMillis(); 155 | 156 | System.out.printf("%d ms for appending one more\n", t2 - t1); 157 | 158 | int numRead = 0; 159 | 160 | t1 = System.currentTimeMillis(); 161 | for (int i = 0; i < 100; i++) { 162 | numRead += Iterators.toArray(chronicle.getRange(time, end), ChronologicalRecord.class).length; 163 | } 164 | t2 = System.currentTimeMillis(); 165 | 166 | System.out.printf("%d ms for ranging 100 times, reading %d samples\n", t2 - t1, numRead); 167 | } 168 | 169 | } 170 | -------------------------------------------------------------------------------- /chronos-datastax/src/test/java/org/ds/chronos/chronicle/TestBase.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.chronicle; 2 | 3 | import org.ds.chronos.api.ChronosException; 4 | import org.ds.chronos.chronicle.DatastaxChronicle.Settings; 5 | import org.junit.BeforeClass; 6 | 7 | import com.datastax.driver.core.Cluster; 8 | import com.datastax.driver.core.Session; 9 | 10 | public class TestBase { 11 | 12 | protected static Cluster cluster; 13 | protected static Session session; 14 | 15 | protected static final String keyspace = "staxtests"; 16 | protected static Settings settings; 17 | 18 | @BeforeClass 19 | public static void setup() throws ChronosException { 20 | if (cluster == null) { 21 | cluster = new Cluster.Builder().addContactPoints("127.0.0.1").build(); 22 | session = cluster.connect(); 23 | 24 | try { 25 | session.execute("DROP KEYSPACE " + keyspace); 26 | } catch (Throwable t) { 27 | } 28 | 29 | session.execute(String.format( 30 | "CREATE KEYSPACE %s WITH replication= {'class':'SimpleStrategy', 'replication_factor':1};", keyspace)); 31 | session.close(); 32 | } 33 | 34 | session = cluster.connect(keyspace); 35 | 36 | settings = Settings.modern("testable"); 37 | 38 | DatastaxChronicle.createTable(session, settings); 39 | } 40 | 41 | public static DatastaxChronicle getChronicle(String name) throws ChronosException { 42 | return new DatastaxChronicle(session, settings, name); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /chronos-jackson/README.md: -------------------------------------------------------------------------------- 1 | chronos-jackson 2 | --------------- 3 | 4 | chronos + jackson = storage timeseries objects that 5 | can map to json and back 6 | 7 | ### Setup and usage 8 | 9 | Create factory: 10 | 11 | ```java 12 | JsonTimelineFactory factory = new JsonTimelineFactory(chronos); 13 | ``` 14 | 15 | Create a timeline: 16 | 17 | ```java 18 | TypeReference typeRef = new TypeReference() {}; 19 | Timeline timeline = factory.createTimeline("events-xyz", typeRef); 20 | ``` 21 | A type ref must be passed in to avoid erasure :( 22 | 23 | Add/Fetch/Delete: 24 | 25 | ```java 26 | timeline.add(new MyObject(...)); 27 | timeline.add(myListOfObjects()); 28 | 29 | Iterable stream 30 | = timeline.query(begin, end, MyObject.class) 31 | .filter(tagfilter("good", "ok")) // custom FilterFn 32 | .stream(); 33 | 34 | for(MyObject object: stream) { 35 | //... 36 | } 37 | 38 | timeline.deleteRange(begin, end); 39 | timeline.getNumEvents(begin, end); 40 | timeline.isEventRecorded(timestamp); 41 | ``` 42 | See chronos Timeline for all options and other details. 43 | 44 | 45 | Maven 46 | ----- 47 | 48 | ```xml 49 | 50 | org.ds 51 | chronos-jackson 52 | 1.0.0 53 | 54 | ``` -------------------------------------------------------------------------------- /chronos-jackson/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | description = 'chronos jackson' 3 | dependencies { 4 | compile project(':chronos-api') 5 | compile group: 'org.slf4j', name: 'slf4j-api', version:'1.6.6' 6 | compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version:'2.1.4' 7 | compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version:'2.1.4' 8 | testCompile group: 'org.mockito', name: 'mockito-all', version:'1.9.5' 9 | } 10 | 11 | task packageTests(type: Jar) { 12 | from sourceSets.test.output 13 | classifier = 'tests' 14 | } 15 | artifacts.archives packageTests 16 | -------------------------------------------------------------------------------- /chronos-jackson/src/main/java/org/ds/chronos/timeline/json/JsonTimeline.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.json; 2 | 3 | import org.ds.chronos.api.Chronicle; 4 | import org.ds.chronos.timeline.SimpleTimeline; 5 | 6 | import com.fasterxml.jackson.core.type.TypeReference; 7 | 8 | /** 9 | * JSON Timeline 10 | * 11 | * @author Dan Simpson 12 | * 13 | * @param type which can be serialized to and from json 14 | */ 15 | public class JsonTimeline extends SimpleTimeline { 16 | 17 | /** 18 | * Create a new JSON Timeline 19 | * @param chronicle the underlying storage 20 | * @param ref the typeref for decoding 21 | */ 22 | public JsonTimeline(Chronicle chronicle, TypeReference ref) { 23 | super(chronicle, new JsonTimelineDecoder(ref), new JsonTimelineEncoder()); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /chronos-jackson/src/main/java/org/ds/chronos/timeline/json/JsonTimelineDecoder.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.json; 2 | 3 | import java.util.Iterator; 4 | 5 | import org.ds.chronos.api.ChronologicalRecord; 6 | import org.ds.chronos.api.TimelineDecoder; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import com.fasterxml.jackson.core.type.TypeReference; 11 | import com.fasterxml.jackson.databind.DeserializationFeature; 12 | import com.fasterxml.jackson.databind.ObjectMapper; 13 | 14 | /** 15 | * A Timeline decoder which converts stored JSON into objects of generic type. 16 | * 17 | * @author Dan Simpson 18 | * 19 | * @param 20 | */ 21 | public class JsonTimelineDecoder implements TimelineDecoder { 22 | 23 | private static final Logger log = LoggerFactory.getLogger(JsonTimelineEncoder.class); 24 | private static final ObjectMapper mapper = new ObjectMapper(); 25 | static { 26 | mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 27 | } 28 | 29 | private TypeReference typeRef; 30 | 31 | public JsonTimelineDecoder(TypeReference typeRef) { 32 | this.typeRef = typeRef; 33 | } 34 | 35 | private Iterator upstream; 36 | 37 | @Override 38 | public boolean hasNext() { 39 | return upstream.hasNext(); 40 | } 41 | 42 | @Override 43 | public T next() { 44 | ChronologicalRecord column = upstream.next(); 45 | T item = null; 46 | try { 47 | item = mapper.readValue(column.getData(), typeRef); 48 | item.setTimestamp(column.getTimestamp()); 49 | } catch (Throwable t) { 50 | log.error("Error decoding column", t); 51 | } 52 | return item; 53 | } 54 | 55 | @Override 56 | public void remove() { 57 | } 58 | 59 | @Override 60 | public void setInputStream(Iterator input) { 61 | upstream = input; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /chronos-jackson/src/main/java/org/ds/chronos/timeline/json/JsonTimelineEncoder.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.json; 2 | 3 | import java.util.Iterator; 4 | 5 | import org.ds.chronos.api.ChronologicalRecord; 6 | import org.ds.chronos.api.TimelineEncoder; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import com.fasterxml.jackson.databind.DeserializationFeature; 11 | import com.fasterxml.jackson.databind.ObjectMapper; 12 | 13 | /** 14 | * Encoder of objects to JSON for a timeline. 15 | * 16 | * @author Dan Simpson 17 | * 18 | * @param 19 | */ 20 | public class JsonTimelineEncoder implements TimelineEncoder { 21 | 22 | private static final Logger log = LoggerFactory.getLogger(JsonTimelineEncoder.class); 23 | 24 | private static final ObjectMapper mapper = new ObjectMapper(); 25 | static { 26 | mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 27 | } 28 | 29 | private Iterator upstream; 30 | 31 | public JsonTimelineEncoder() { 32 | } 33 | 34 | @Override 35 | public boolean hasNext() { 36 | return upstream.hasNext(); 37 | } 38 | 39 | @Override 40 | public ChronologicalRecord next() { 41 | T item = upstream.next(); 42 | try { 43 | return new ChronologicalRecord(item.getTimestamp(), mapper.writeValueAsBytes(item)); 44 | } catch (Throwable t) { 45 | log.error("Error encoding object", t); 46 | } 47 | return null; 48 | } 49 | 50 | @Override 51 | public void remove() { 52 | } 53 | 54 | @Override 55 | public void setInputStream(Iterator input) { 56 | upstream = input; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /chronos-jackson/src/main/java/org/ds/chronos/timeline/json/Timestamped.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.json; 2 | 3 | import org.ds.chronos.api.Temporal; 4 | 5 | /** 6 | * An interface which exposes a timestamp for a given object 7 | * 8 | * @author Dan Simpson 9 | * 10 | */ 11 | public interface Timestamped extends Temporal { 12 | 13 | /** 14 | * Set the timestamp 15 | * 16 | * @param timestamp 17 | * unix timestamp in millis 18 | */ 19 | public void setTimestamp(long timestamp); 20 | } 21 | -------------------------------------------------------------------------------- /chronos-jackson/src/test/java/org/ds/chronos/timeline/json/JsonTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.json; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import org.ds.chronos.api.ChronosException; 7 | import org.ds.chronos.api.chronicle.MemoryChronicle; 8 | import org.ds.chronos.timeline.SimpleTimeline; 9 | import org.junit.Assert; 10 | import org.junit.Before; 11 | import org.junit.Test; 12 | 13 | import com.fasterxml.jackson.core.type.TypeReference; 14 | 15 | public class JsonTest { 16 | 17 | SimpleTimeline timeline; 18 | 19 | @Before 20 | public void setup() throws ChronosException { 21 | timeline = new JsonTimeline(new MemoryChronicle(), new TypeReference() { 22 | }); 23 | } 24 | 25 | public JsonTestObject buildObject(long time) { 26 | JsonTestObject object = new JsonTestObject(); 27 | object.setId(4); 28 | object.setName("Dan"); 29 | object.setAge(28); 30 | object.setWeight(171.0); 31 | object.setTimestamp(time); 32 | return object; 33 | } 34 | 35 | @Test 36 | public void testCodec() { 37 | JsonTestObject object = buildObject(1000); 38 | timeline.add(buildObject(1000)); 39 | JsonTestObject other = timeline.query(0, 1000).first().get(); 40 | Assert.assertNotNull(other); 41 | Assert.assertEquals(object.getId(), other.getId()); 42 | Assert.assertEquals(object.getName(), other.getName()); 43 | Assert.assertEquals(object.getAge(), other.getAge()); 44 | Assert.assertEquals(object.getWeight(), other.getWeight(), 0.1); 45 | Assert.assertEquals(object.getTimestamp(), other.getTimestamp()); 46 | } 47 | 48 | @Test 49 | public void testBatch() { 50 | List list = new ArrayList(); 51 | for (int i = 0; i < 100; i++) { 52 | list.add(buildObject(i * 1000)); 53 | } 54 | timeline.add(list); 55 | 56 | List other = timeline.query(0, 1000 * 100).toList(); 57 | Assert.assertEquals(list.size(), other.size()); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /chronos-jackson/src/test/java/org/ds/chronos/timeline/json/JsonTestObject.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.json; 2 | 3 | class JsonTestObject implements Timestamped { 4 | 5 | private int id; 6 | private String name; 7 | private int age; 8 | private double weight; 9 | private long birthdate; 10 | 11 | /** 12 | * @return the id 13 | */ 14 | public int getId() { 15 | return id; 16 | } 17 | 18 | /** 19 | * @param id 20 | * the id to set 21 | */ 22 | public void setId(int id) { 23 | this.id = id; 24 | } 25 | 26 | /** 27 | * @return the name 28 | */ 29 | public String getName() { 30 | return name; 31 | } 32 | 33 | /** 34 | * @param name 35 | * the name to set 36 | */ 37 | public void setName(String name) { 38 | this.name = name; 39 | } 40 | 41 | /** 42 | * @return the age 43 | */ 44 | public int getAge() { 45 | return age; 46 | } 47 | 48 | /** 49 | * @param age 50 | * the age to set 51 | */ 52 | public void setAge(int age) { 53 | this.age = age; 54 | } 55 | 56 | /** 57 | * @return the weight 58 | */ 59 | public double getWeight() { 60 | return weight; 61 | } 62 | 63 | /** 64 | * @param weight 65 | * the weight to set 66 | */ 67 | public void setWeight(double weight) { 68 | this.weight = weight; 69 | } 70 | 71 | @Override 72 | public long getTimestamp() { 73 | return birthdate; 74 | } 75 | 76 | @Override 77 | public void setTimestamp(long timestamp) { 78 | birthdate = timestamp; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /chronos-msgpack/README.md: -------------------------------------------------------------------------------- 1 | chronos-msgpack 2 | --------------- 3 | 4 | ### Setup and usage 5 | 6 | Create a timeline: 7 | 8 | ```java 9 | Timeline timeline = new MsgPackTimeline(chronicle, MyObject.class); 10 | ``` 11 | 12 | Add/Fetch/Delete: 13 | 14 | ```java 15 | timeline.add(new MyObject(...)); 16 | timeline.add(myListOfObjects()); 17 | 18 | Iterable stream 19 | = timeline.query(begin, end).streamAs(MyObject.class); 20 | 21 | for(MyObject object: stream) { 22 | //... 23 | } 24 | 25 | timeline.deleteRange(begin, end); 26 | timeline.getNumEvents(begin, end); 27 | timeline.isEventRecorded(timestamp); 28 | ``` 29 | See chronos Timeline for all options and other details. 30 | 31 | 32 | Maven 33 | ----- 34 | 35 | ```xml 36 | 37 | org.ds 38 | chronos-msgpack 39 | ... 40 | 41 | ``` -------------------------------------------------------------------------------- /chronos-msgpack/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | description = 'chronos msgpack' 3 | dependencies { 4 | compile project(':chronos-api') 5 | compile group: 'org.slf4j', name: 'slf4j-api', version:'1.6.6' 6 | compile group: 'org.msgpack', name: 'msgpack', version:'0.6.7' 7 | } 8 | 9 | task packageTests(type: Jar) { 10 | from sourceSets.test.output 11 | classifier = 'tests' 12 | } 13 | artifacts.archives packageTests 14 | -------------------------------------------------------------------------------- /chronos-msgpack/src/main/java/org/ds/chronos/timeline/msgpack/MsgPackTimeline.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.msgpack; 2 | 3 | import org.ds.chronos.api.Chronicle; 4 | import org.ds.chronos.timeline.SimpleTimeline; 5 | import org.msgpack.MessagePack; 6 | 7 | /** 8 | * JSON Timeline 9 | * 10 | * @author Dan Simpson 11 | * 12 | * @param 13 | * type which can be serialized to and from json 14 | */ 15 | public class MsgPackTimeline extends SimpleTimeline { 16 | 17 | public static final MessagePack msgpack = new MessagePack(); 18 | 19 | /** 20 | * Create a new JSON Timeline 21 | * 22 | * @param chronicle 23 | * the underlying storage 24 | * @param ref 25 | * the typeref for decoding 26 | */ 27 | public MsgPackTimeline(Chronicle chronicle, Class klass) { 28 | super(chronicle, new MsgPackTimelineDecoder(klass), new MsgPackTimelineEncoder()); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /chronos-msgpack/src/main/java/org/ds/chronos/timeline/msgpack/MsgPackTimelineDecoder.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.msgpack; 2 | 3 | import java.util.Iterator; 4 | 5 | import org.ds.chronos.api.ChronologicalRecord; 6 | import org.ds.chronos.api.TimelineDecoder; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * A Timeline decoder which converts stored JSON into objects of generic type. 12 | * 13 | * @author Dan Simpson 14 | * 15 | * @param 16 | */ 17 | public class MsgPackTimelineDecoder implements TimelineDecoder { 18 | 19 | private static final Logger log = LoggerFactory.getLogger(MsgPackTimelineEncoder.class); 20 | 21 | private Class klass; 22 | 23 | public MsgPackTimelineDecoder(Class klass) { 24 | this.klass = klass; 25 | } 26 | 27 | private Iterator upstream; 28 | 29 | @Override 30 | public boolean hasNext() { 31 | return upstream.hasNext(); 32 | } 33 | 34 | @Override 35 | public T next() { 36 | ChronologicalRecord column = upstream.next(); 37 | T item = null; 38 | try { 39 | item = MsgPackTimeline.msgpack.read(column.getData(), klass); 40 | item.setTimestamp(column.getTimestamp()); 41 | } catch (Throwable t) { 42 | log.error("Error decoding column", t); 43 | } 44 | return item; 45 | } 46 | 47 | @Override 48 | public void remove() { 49 | } 50 | 51 | @Override 52 | public void setInputStream(Iterator input) { 53 | upstream = input; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /chronos-msgpack/src/main/java/org/ds/chronos/timeline/msgpack/MsgPackTimelineEncoder.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.msgpack; 2 | 3 | import java.util.Iterator; 4 | 5 | import org.ds.chronos.api.ChronologicalRecord; 6 | import org.ds.chronos.api.TimelineEncoder; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * Encoder of objects to JSON for a timeline. 12 | * 13 | * @author Dan Simpson 14 | * 15 | * @param 16 | */ 17 | public class MsgPackTimelineEncoder implements TimelineEncoder { 18 | 19 | private static final Logger log = LoggerFactory.getLogger(MsgPackTimelineEncoder.class); 20 | 21 | private Iterator upstream; 22 | 23 | public MsgPackTimelineEncoder() { 24 | } 25 | 26 | @Override 27 | public boolean hasNext() { 28 | return upstream.hasNext(); 29 | } 30 | 31 | @Override 32 | public ChronologicalRecord next() { 33 | T item = upstream.next(); 34 | try { 35 | return new ChronologicalRecord(item.getTimestamp(), MsgPackTimeline.msgpack.write(item)); 36 | } catch (Throwable t) { 37 | log.error("Error encoding object", t); 38 | } 39 | return null; 40 | } 41 | 42 | @Override 43 | public void remove() { 44 | } 45 | 46 | @Override 47 | public void setInputStream(Iterator input) { 48 | upstream = input; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /chronos-msgpack/src/main/java/org/ds/chronos/timeline/msgpack/Timestamped.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.msgpack; 2 | 3 | import org.ds.chronos.api.Temporal; 4 | 5 | /** 6 | * An interface which exposes a timestamp for a given object 7 | * 8 | * @author Dan Simpson 9 | * 10 | */ 11 | public interface Timestamped extends Temporal { 12 | 13 | /** 14 | * Set the timestamp 15 | * 16 | * @param timestamp 17 | * unix timestamp in millis 18 | */ 19 | public void setTimestamp(long timestamp); 20 | } 21 | -------------------------------------------------------------------------------- /chronos-msgpack/src/test/java/org/ds/chronos/timeline/msgpack/MsgPackTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.msgpack; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import org.ds.chronos.api.ChronosException; 7 | import org.ds.chronos.api.chronicle.MemoryChronicle; 8 | import org.ds.chronos.timeline.SimpleTimeline; 9 | import org.ds.chronos.timeline.msgpack.MsgPackTimeline; 10 | import org.junit.Assert; 11 | import org.junit.Before; 12 | import org.junit.Test; 13 | 14 | public class MsgPackTest { 15 | 16 | SimpleTimeline timeline; 17 | 18 | @Before 19 | public void setup() throws ChronosException { 20 | timeline = new MsgPackTimeline(new MemoryChronicle(), TestObject.class); 21 | } 22 | 23 | public TestObject buildObject(long time) { 24 | TestObject object = new TestObject(); 25 | object.setId(4); 26 | object.setName("Dan"); 27 | object.setAge(28); 28 | object.setWeight(171.0); 29 | object.setTimestamp(time); 30 | return object; 31 | } 32 | 33 | @Test 34 | public void testCodec() { 35 | TestObject object = buildObject(1000); 36 | timeline.add(buildObject(1000)); 37 | TestObject other = timeline.query(0, 1000).first().get(); 38 | Assert.assertNotNull(other); 39 | Assert.assertEquals(object.getId(), other.getId()); 40 | Assert.assertEquals(object.getName(), other.getName()); 41 | Assert.assertEquals(object.getAge(), other.getAge()); 42 | Assert.assertEquals(object.getWeight(), other.getWeight(), 0.1); 43 | Assert.assertEquals(object.getTimestamp(), other.getTimestamp()); 44 | } 45 | 46 | @Test 47 | public void testBatch() { 48 | List list = new ArrayList(); 49 | for (int i = 0; i < 100; i++) { 50 | list.add(buildObject(i * 1000)); 51 | } 52 | timeline.add(list); 53 | 54 | List other = timeline.query(0, 1000 * 100).toList(); 55 | Assert.assertEquals(list.size(), other.size()); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /chronos-msgpack/src/test/java/org/ds/chronos/timeline/msgpack/TestObject.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.timeline.msgpack; 2 | 3 | import org.ds.chronos.timeline.msgpack.Timestamped; 4 | import org.msgpack.annotation.Message; 5 | 6 | @Message 7 | class TestObject implements Timestamped { 8 | 9 | private int id; 10 | private String name; 11 | private int age; 12 | private double weight; 13 | private long birthdate; 14 | 15 | /** 16 | * @return the id 17 | */ 18 | public int getId() { 19 | return id; 20 | } 21 | 22 | /** 23 | * @param id 24 | * the id to set 25 | */ 26 | public void setId(int id) { 27 | this.id = id; 28 | } 29 | 30 | /** 31 | * @return the name 32 | */ 33 | public String getName() { 34 | return name; 35 | } 36 | 37 | /** 38 | * @param name 39 | * the name to set 40 | */ 41 | public void setName(String name) { 42 | this.name = name; 43 | } 44 | 45 | /** 46 | * @return the age 47 | */ 48 | public int getAge() { 49 | return age; 50 | } 51 | 52 | /** 53 | * @param age 54 | * the age to set 55 | */ 56 | public void setAge(int age) { 57 | this.age = age; 58 | } 59 | 60 | /** 61 | * @return the weight 62 | */ 63 | public double getWeight() { 64 | return weight; 65 | } 66 | 67 | /** 68 | * @param weight 69 | * the weight to set 70 | */ 71 | public void setWeight(double weight) { 72 | this.weight = weight; 73 | } 74 | 75 | @Override 76 | public long getTimestamp() { 77 | return birthdate; 78 | } 79 | 80 | @Override 81 | public void setTimestamp(long timestamp) { 82 | birthdate = timestamp; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /chronos-redis/README.md: -------------------------------------------------------------------------------- 1 | # chronos-redis 2 | 3 | Timeseries storage using redis sorted sets. 4 | 5 | ### Setup and usage 6 | 7 | Create a Chronicle: 8 | 9 | ```java 10 | Chronicle chronicle = new RedisChronicle(JedisPool pool, String key); 11 | ``` 12 | 13 | See chronos-api for abstractions. 14 | 15 | Maven 16 | ----- 17 | 18 | ```xml 19 | 20 | org.ds.chronos 21 | chronos-redis 22 | version 23 | 24 | ``` -------------------------------------------------------------------------------- /chronos-redis/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | description = 'chronos redis' 3 | dependencies { 4 | compile project(':chronos-api') 5 | compile group: 'org.slf4j', name: 'slf4j-api', version:'1.6.6' 6 | compile group: 'redis.clients', name: 'jedis', version:'2.1.0' 7 | testCompile group: 'redis.clients', name: 'jedis', version:'2.1.0', classifier:'sources' 8 | } 9 | 10 | task packageTests(type: Jar) { 11 | from sourceSets.test.output 12 | classifier = 'tests' 13 | } 14 | artifacts.archives packageTests 15 | -------------------------------------------------------------------------------- /chronos-redis/src/main/java/org/ds/chronos/chronicle/RedisChronicle.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos.chronicle; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.util.Iterator; 5 | import java.util.Set; 6 | 7 | import org.ds.chronos.api.Chronicle; 8 | import org.ds.chronos.api.ChronologicalRecord; 9 | 10 | import redis.clients.jedis.Jedis; 11 | import redis.clients.jedis.JedisPool; 12 | import redis.clients.jedis.Pipeline; 13 | 14 | import com.google.common.base.Function; 15 | import com.google.common.collect.Iterators; 16 | 17 | public class RedisChronicle extends Chronicle { 18 | 19 | private final JedisPool pool; 20 | private final byte[] key; 21 | 22 | public RedisChronicle(JedisPool pool, String key) { 23 | this.pool = pool; 24 | this.key = key.getBytes(); 25 | } 26 | 27 | private abstract class RedisCommand { 28 | 29 | public T execute() { 30 | 31 | Jedis redis = null; 32 | try { 33 | redis = pool.getResource(); 34 | return execute(redis); 35 | } finally { 36 | if (redis != null) { 37 | pool.returnResource(redis); 38 | } 39 | 40 | } 41 | } 42 | 43 | protected abstract T execute(Jedis redis); 44 | } 45 | 46 | private final byte[] encodeRecord(ChronologicalRecord record) { 47 | ByteBuffer buffer = ByteBuffer.allocate(8 + record.getByteSize()); 48 | buffer.putLong(record.getTimestamp()); 49 | buffer.put(record.getData()); 50 | return buffer.array(); 51 | } 52 | 53 | private final ChronologicalRecord decodeRecord(byte[] raw) { 54 | ByteBuffer buffer = ByteBuffer.wrap(raw); 55 | long time = buffer.getLong(); 56 | byte[] data = new byte[buffer.remaining()]; 57 | buffer.get(data); 58 | return new ChronologicalRecord(time, data); 59 | } 60 | 61 | @Override 62 | public void add(final ChronologicalRecord item) { 63 | new RedisCommand() { 64 | 65 | @Override 66 | protected Void execute(Jedis redis) { 67 | redis.zadd(key, item.getTimestamp(), encodeRecord(item)); 68 | return null; 69 | } 70 | }.execute(); 71 | } 72 | 73 | @Override 74 | public void add(final Iterator items, int pageSize) { 75 | new RedisCommand() { 76 | 77 | @Override 78 | protected Void execute(Jedis redis) { 79 | Pipeline pipeline = redis.pipelined(); 80 | while (items.hasNext()) { 81 | ChronologicalRecord item = items.next(); 82 | pipeline.zadd(key, item.getTimestamp(), encodeRecord(item)); 83 | } 84 | pipeline.exec(); 85 | return null; 86 | } 87 | }.execute(); 88 | } 89 | 90 | @Override 91 | public Iterator getRange(final long t1, final long t2, int pageSize) { 92 | 93 | Set raw = new RedisCommand>() { 94 | 95 | @Override 96 | protected Set execute(Jedis redis) { 97 | if (t1 <= t2) { 98 | return redis.zrangeByScore(key, t1, t2); 99 | } 100 | return redis.zrevrangeByScore(key, t1, t2); 101 | } 102 | }.execute(); 103 | 104 | return Iterators.transform(raw.iterator(), new Function() { 105 | 106 | @Override 107 | public ChronologicalRecord apply(byte[] raw) { 108 | return decodeRecord(raw); 109 | } 110 | }); 111 | } 112 | 113 | @Override 114 | public long getNumEvents(final long t1, final long t2) { 115 | return new RedisCommand() { 116 | 117 | @Override 118 | protected Long execute(Jedis redis) { 119 | return redis.zcount(key, t1, t2); 120 | } 121 | }.execute(); 122 | } 123 | 124 | @Override 125 | public void delete() { 126 | new RedisCommand() { 127 | 128 | @Override 129 | protected Void execute(Jedis redis) { 130 | redis.del(key); 131 | return null; 132 | } 133 | }.execute(); 134 | } 135 | 136 | @Override 137 | public void deleteRange(final long t1, final long t2) { 138 | new RedisCommand() { 139 | 140 | @Override 141 | protected Void execute(Jedis redis) { 142 | redis.zremrangeByScore(key, t1, t2); 143 | return null; 144 | } 145 | }.execute(); 146 | } 147 | 148 | } 149 | -------------------------------------------------------------------------------- /chronos-redis/src/test/java/org/ds/chronos/RedisChronicleTest.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos; 2 | 3 | import java.util.Date; 4 | import java.util.Iterator; 5 | 6 | import junit.framework.Assert; 7 | 8 | import org.ds.chronos.api.Chronicle; 9 | import org.ds.chronos.api.ChronicleBatch; 10 | import org.ds.chronos.api.ChronologicalRecord; 11 | import org.ds.chronos.api.ChronosException; 12 | import org.ds.chronos.api.chronicle.MemoryChronicle; 13 | import org.ds.chronos.chronicle.RedisChronicle; 14 | import org.junit.After; 15 | import org.junit.Before; 16 | import org.junit.Test; 17 | 18 | import com.google.common.base.Charsets; 19 | import com.google.common.collect.Iterators; 20 | 21 | public class RedisChronicleTest extends TestBase { 22 | 23 | private Chronicle chronicle; 24 | 25 | @Before 26 | public void create() throws ChronosException { 27 | chronicle = new RedisChronicle(getPool(), "chronos-test"); 28 | chronicle.delete(); 29 | } 30 | 31 | @After 32 | public void destroy() throws ChronosException { 33 | chronicle.delete(); 34 | } 35 | 36 | @Test 37 | public void shouldBeEmpty() throws ChronosException { 38 | Assert.assertEquals(0, chronicle.getNumEvents(0, System.currentTimeMillis())); 39 | } 40 | 41 | @Test 42 | public void shouldAddEventAndCount() throws ChronosException { 43 | chronicle.add(1000, "test"); 44 | Assert.assertEquals(1, chronicle.getNumEvents(999, 1001)); 45 | } 46 | 47 | @Test 48 | public void shouldBatch() throws ChronosException { 49 | long count = 100; 50 | long period = 1000 * 60; 51 | long time = 0; 52 | 53 | ChronicleBatch batch = new ChronicleBatch(); 54 | for (int i = 0; i < count; i++) { 55 | batch.add(time + i * period, "test"); 56 | } 57 | chronicle.add(batch); 58 | 59 | Assert.assertEquals(count, chronicle.getNumEvents(0, count * period)); 60 | } 61 | 62 | @Test 63 | public void shouldGetSingleRange() throws ChronosException { 64 | chronicle.add(1000, "test"); 65 | Iterator slice = chronicle.getRange(0, 2000); 66 | 67 | Assert.assertTrue(slice.hasNext()); 68 | ChronologicalRecord event = slice.next(); 69 | Assert.assertEquals(1000, event.getTimestamp()); 70 | Assert.assertEquals("test", new String(event.getData(), Charsets.UTF_8)); 71 | } 72 | 73 | @Test 74 | public void shouldGetRange() throws ChronosException { 75 | chronicle.add(1000, "test"); 76 | chronicle.add(2000, "test"); 77 | Iterator slice = chronicle.getRange(1000, 3000); 78 | 79 | Assert.assertTrue(slice.hasNext()); 80 | ChronologicalRecord event = slice.next(); 81 | 82 | Assert.assertEquals(1000, event.getTimestamp()); 83 | Assert.assertEquals("test", new String(event.getData(), Charsets.UTF_8)); 84 | } 85 | 86 | @Test 87 | public void shouldGetRangeReversed() throws ChronosException { 88 | chronicle.add(1000, "test"); 89 | chronicle.add(2000, "test"); 90 | Iterator slice = chronicle.getRange(3000, 1000); 91 | 92 | Assert.assertTrue(slice.hasNext()); 93 | ChronologicalRecord event = slice.next(); 94 | 95 | Assert.assertEquals(2000, event.getTimestamp()); 96 | Assert.assertEquals("test", new String(event.getData(), Charsets.UTF_8)); 97 | } 98 | 99 | @Test 100 | public void shouldDelete() throws ChronosException { 101 | Date time = new Date(); 102 | 103 | chronicle.add(time, "test"); 104 | chronicle.deleteRange(time.getTime() - 1000, time.getTime() + 1000); 105 | 106 | Assert.assertEquals(0, chronicle.getNumEvents(0, System.currentTimeMillis())); 107 | } 108 | 109 | @Test 110 | public void shouldDeleteFragment() throws ChronosException { 111 | chronicle.add(1000, "test"); 112 | chronicle.add(2000, "test"); 113 | chronicle.add(3000, "test"); 114 | chronicle.add(4000, "test"); 115 | 116 | chronicle.deleteRange(0, 2000); 117 | 118 | Assert.assertEquals(2, chronicle.getNumEvents(0, 4000)); 119 | Assert.assertEquals(3000, MemoryChronicle.toList(chronicle.getRange(0, 4000)).get(0).getTimestamp()); 120 | } 121 | 122 | @Test 123 | public void shouldDeleteRow() throws ChronosException { 124 | chronicle.add(1000, "test"); 125 | chronicle.delete(); 126 | 127 | Assert.assertEquals(0, chronicle.getNumEvents(0, 2000)); 128 | } 129 | 130 | @Test 131 | public void shouldDetectExistence() throws ChronosException { 132 | chronicle.add(1000, "test"); 133 | Assert.assertTrue(chronicle.isEventRecorded(1000)); 134 | Assert.assertFalse(chronicle.isEventRecorded(999)); 135 | Assert.assertFalse(chronicle.isEventRecorded(1001)); 136 | } 137 | 138 | @Test(timeout = 5000) 139 | public void testPerf() { 140 | long count = 2800; 141 | long period = 1000 * 60; 142 | long time = System.currentTimeMillis(); 143 | 144 | ChronicleBatch batch = new ChronicleBatch(); 145 | for (int i = 0; i < count; i++) { 146 | batch.add(time + i * period, "test"); 147 | } 148 | long t1 = System.currentTimeMillis(); 149 | chronicle.add(batch); 150 | long t2 = System.currentTimeMillis(); 151 | System.out.printf("%d ms for %d items\n", t2 - t1, count); 152 | 153 | t1 = System.currentTimeMillis(); 154 | chronicle.add(new ChronologicalRecord(time - period, "test".getBytes())); 155 | t2 = System.currentTimeMillis(); 156 | 157 | System.out.printf("%d ms for appending one more\n", t2 - t1); 158 | 159 | int numRead = 0; 160 | 161 | t1 = System.currentTimeMillis(); 162 | for (int i = 0; i < 100; i++) { 163 | long start = time; 164 | long end = time + Math.round(Math.random() * (count * period)); 165 | numRead += Iterators.toArray(chronicle.getRange(start, end), ChronologicalRecord.class).length; 166 | } 167 | t2 = System.currentTimeMillis(); 168 | 169 | System.out.printf("%d ms for ranging 100 times, reading %d samples\n", t2 - t1, numRead); 170 | } 171 | 172 | } -------------------------------------------------------------------------------- /chronos-redis/src/test/java/org/ds/chronos/TestBase.java: -------------------------------------------------------------------------------- 1 | package org.ds.chronos; 2 | 3 | import java.util.TimeZone; 4 | 5 | import org.apache.commons.pool.impl.GenericObjectPool.Config; 6 | import org.junit.BeforeClass; 7 | 8 | import redis.clients.jedis.JedisPool; 9 | 10 | public class TestBase { 11 | 12 | private static JedisPool pool; 13 | 14 | @BeforeClass 15 | public static void setupBasic() { 16 | TimeZone.setDefault(TimeZone.getTimeZone("UTC")); 17 | } 18 | 19 | public static JedisPool getPool() { 20 | if (pool == null) { 21 | Config config = new Config(); 22 | config.testOnBorrow = true; 23 | config.maxActive = 4; 24 | config.maxWait = 2000; 25 | pool = new JedisPool(config, "localhost"); 26 | } 27 | return pool; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dansimpson/chronos/9cd0b92a36355536838cd3ae571338149a2dda70/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'chronos-parent' 2 | include ':chronos-api' 3 | include ':chronos-jackson' 4 | include ':chronos-aws' 5 | include ':chronos-redis' 6 | include ':chronos-msgpack' 7 | include ':chronos-datastax' 8 | 9 | project(':chronos-api').projectDir = "$rootDir/chronos-api" as File 10 | project(':chronos-jackson').projectDir = "$rootDir/chronos-jackson" as File 11 | project(':chronos-aws').projectDir = "$rootDir/chronos-aws" as File 12 | project(':chronos-redis').projectDir = "$rootDir/chronos-redis" as File 13 | project(':chronos-msgpack').projectDir = "$rootDir/chronos-msgpack" as File 14 | project(':chronos-datastax').projectDir = "$rootDir/chronos-datastax" as File 15 | --------------------------------------------------------------------------------