├── .gitignore ├── LICENSE ├── README.md ├── pom.xml └── src ├── main └── java │ └── net │ └── openhft │ └── chronicle │ └── timeseries │ ├── AbstractColumn.java │ ├── BytesDoubleLookup.java │ ├── BytesDoubleLookups.java │ ├── BytesLongLookup.java │ ├── BytesLongLookups.java │ ├── Column.java │ ├── ColumnCommon.java │ ├── Columns.java │ ├── DoubleColumn.java │ ├── InMemoryColumn.java │ ├── InMemoryDoubleColumn.java │ ├── InMemoryLongColumn.java │ ├── InMemoryTimeSeries.java │ ├── LongColumn.java │ ├── LongColumnIndexObjectConsumer.java │ └── TimeSeries.java └── test └── java └── net └── openhft └── chronicle └── timeseries └── InMemoryTimeSeriesTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | .idea/* 3 | target/* 4 | *.iml 5 | 6 | # Mobile Tools for Java (J2ME) 7 | .mtj.tmp/ 8 | 9 | # Package Files # 10 | *.jar 11 | *.war 12 | *.ear 13 | 14 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 15 | hs_err_pid* 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Chronicle-TimeSeries 2 | Multi-Threaded Time Series library 3 | 4 | # Purpose 5 | This library has two efficiency objectives 6 | 7 | - efficient storage on long sequences of data in a column based database. 8 | - multi-threaded processing where possible 9 | - integration with engine for lookup and management of the TimeSeries. 10 | - perform calculation on time series where the timings are in micro-seconds and each time series has it's own timestamps i.e. they don't have to be in sync or vectorized. 11 | 12 | # Enterprise edition 13 | The enterprise version 14 | 15 | - has more multi-threaded implementations. 16 | - peristsed timeseries (via memory mapped files) 17 | - remote access to time series (no need to have the time series locally) 18 | - distributed times series data where data is processed locally. e.g. if you have N servers, each server can process 1/N of the work. 19 | 20 | # Sample program 21 | This program creates two series for the mid of an instrumentent and attempts to see if there is any correlation. 22 | 23 | Note: the two time series have different times. 24 | 25 | ```java 26 | long size = 600_000_000; 27 | 28 | // generate series 1 29 | TimeSeries ts = new InMemoryTimeSeries(null); 30 | ts.setLength(size); 31 | 32 | LongColumn time = ts.getTimestamp(); 33 | time.setAll(Random::new, (c, i, r) -> c.set(i, 9 + (int) Math.pow(1e6, sqr(r.nextFloat())))); 34 | long sum = time.integrate(); // sum all the intervals 35 | 36 | System.out.printf("%.1f days%n", sum/86400e6); 37 | 38 | DoubleColumn mid = ts.acquireDoubleColumn("mid", BytesDoubleLookups.INT16_4); 39 | mid.generateBrownian(1, 2, 0.0005); 40 | 41 | // generate series 2 42 | TimeSeries ts2 = new InMemoryTimeSeries(null); 43 | ts2.setLength(size); 44 | 45 | LongColumn time2 = ts.getTimestamp(); 46 | time2.setAll(Random::new, (c, i, r) -> c.set(i, 9 + (int) Math.pow(1e6, sqr(r.nextFloat())))); // 47 | long sum2 = time2.integrate(); // sum all the intervals 48 | 49 | System.out.printf("%.1f days%n", sum2/86400e6); 50 | 51 | DoubleColumn mid2 = ts2.acquireDoubleColumn("mid", BytesDoubleLookups.INT16_4); 52 | mid2.generateBrownian(1, 2, 0.0005); 53 | 54 | 55 | // compare the correlation 56 | CorralationStatistic stats = PearsonsCorrelation.calcCorrelation(mid, mid2, Mode.AFTER_BOTH_CHANGE); 57 | ``` 58 | 59 | This takes around 30 seconds on a 16 core machine for two sets of 600 million data points generated and compared (Notionally >250 business days, ie a year) 60 | 61 | When comparing correlations there is many different ways you might do this when the spacing between events varies. 62 | 63 | You can look to see when either changes by a minimum amounts, or when one changes, or when both have changed. 64 | You might also prefer to sub-sample the data before performing correlation to reduce noise. 65 | 66 | # Predictive testing. 67 | The ultimate purpose of the library is to find patterns which might have predictive power. To do this 68 | you need to estimate a forward movement in a metric you would want to predict and find inputs which 69 | would can help estimate this forward. 70 | 71 | The steps to do this are 72 | 73 | - estimate the time horizon you need e.g. 5 mins, an hour, 5 days. THZ 74 | - generate a forward by comparing the difference between the current and current + THZ. 75 | - find a correlation between the inputs and this forward. 76 | 77 | If you look at enough inputs you will find spurious correlations so you need to take a view on which 78 | correlations are predictive or not. [http://tylervigen.com/spurious-correlations] 79 | 80 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 19 | 20 | 21 | net.openhft 22 | java-parent-pom 23 | 1.1.10 24 | 25 | 26 | 27 | 4.0.0 28 | chronicle-timeseries 29 | 0.0.1-alpha-SNAPSHOT 30 | OpenHFT/Chronicle-TimeSeries 31 | Chronicle-TimeSeries 32 | bundle 33 | 34 | 35 | 36 | 37 | net.openhft 38 | third-party-bom 39 | pom 40 | 3.6.0 41 | import 42 | 43 | 44 | 45 | net.openhft 46 | chronicle-bom 47 | 1.16.58 48 | pom 49 | import 50 | 51 | 52 | 53 | 54 | 55 | 56 | net.openhft 57 | chronicle-wire 58 | 59 | 60 | 61 | org.apache.commons 62 | commons-math3 63 | 3.6 64 | 65 | 66 | 67 | 68 | 69 | junit 70 | junit 71 | test 72 | 73 | 74 | 75 | org.slf4j 76 | slf4j-simple 77 | test 78 | 79 | 80 | 81 | 82 | 83 | 84 | org.apache.maven.plugins 85 | maven-scm-publish-plugin 86 | 87 | ${project.build.directory}/scmpublish/javadoc 88 | 89 | Publishing javadoc for ${project.artifactId}:${project.version} 90 | 91 | ${project.reporting.outputDirectory} 92 | true 93 | scm:git:git@github.com:OpenHFT/Chronicle-TimeSeries 94 | gh-pages 95 | 96 | 97 | 98 | 99 | org.apache.maven.plugins 100 | maven-javadoc-plugin 101 | 102 | 103 | http://openhft.github.io/Chronicle-TimeSeries/apidocs/ 104 | 105 | 106 | 107 | 108 | 109 | org.apache.maven.plugins 110 | maven-compiler-plugin 111 | 112 | -Xlint:deprecation 113 | 1.8 114 | 1.8 115 | UTF-8 116 | 117 | 118 | 126 | 127 | org.apache.servicemix.tooling 128 | depends-maven-plugin 129 | 130 | 131 | generate-depends-file 132 | 133 | generate-depends-file 134 | 135 | 136 | 137 | 138 | 139 | org.apache.felix 140 | maven-bundle-plugin 141 | true 142 | 143 | 144 | ${project.groupId}.${project.artifactId} 145 | 146 | OpenHFT :: ${project.artifactId} 147 | ${project.version} 148 | 149 | net.openhft.chronicle.wire.*;-noimport:=true 150 | 151 | 152 | org.xerial.snappy;resolution:=optional, 153 | * 154 | 155 | 156 | 157 | 158 | 162 | 163 | 164 | manifest 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | scm:git:git@github.com:OpenHFT/Chronicle-TimeSeries.git 174 | scm:git:git@github.com:OpenHFT/Chronicle-TimeSeries.git 175 | scm:git:git@github.com:OpenHFT/Chronicle-TimeSeries.git 176 | master 177 | 178 | 179 | 180 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/AbstractColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | /** 22 | * Created by peter on 20/02/16. 23 | */ 24 | public abstract class AbstractColumn implements ColumnCommon { 25 | private final TimeSeries timeSeries; 26 | private final String name; 27 | 28 | protected AbstractColumn(TimeSeries timeSeries, String name) { 29 | this.timeSeries = timeSeries; 30 | this.name = name; 31 | } 32 | 33 | @Override 34 | public TimeSeries timeSeries() { 35 | return timeSeries; 36 | } 37 | 38 | @Override 39 | public String name() { 40 | return name; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/BytesDoubleLookup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import net.openhft.chronicle.bytes.BytesStore; 22 | 23 | /** 24 | * Created by peter on 19/02/16. 25 | */ 26 | public interface BytesDoubleLookup { 27 | double get(BytesStore bytes, long index); 28 | 29 | void set(BytesStore bytes, long index, double value); 30 | 31 | long sizeFor(long capacity); 32 | 33 | default double add(BytesStore bytes, long index, double value) { 34 | double x = get(bytes, index); 35 | x += value; 36 | set(bytes, index, x); 37 | return x; 38 | } 39 | 40 | default boolean supportNaN() { 41 | return false; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/BytesDoubleLookups.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import net.openhft.chronicle.bytes.BytesStore; 22 | import net.openhft.chronicle.core.Maths; 23 | 24 | /** 25 | * Created by peter on 19/02/16. 26 | */ 27 | public enum BytesDoubleLookups implements BytesDoubleLookup { 28 | FLOAT64 { 29 | @Override 30 | public double get(BytesStore bytes, long index) { 31 | return bytes.readDouble(index << 3); 32 | } 33 | 34 | @Override 35 | public void set(BytesStore bytes, long index, double value) { 36 | bytes.writeDouble(index << 3, value); 37 | } 38 | 39 | @Override 40 | public long sizeFor(long capacity) { 41 | return capacity << 3; 42 | } 43 | 44 | @Override 45 | public boolean supportNaN() { 46 | return true; 47 | } 48 | }, 49 | FLOAT32 { 50 | @Override 51 | public double get(BytesStore bytes, long index) { 52 | return bytes.readFloat(index << 2); 53 | } 54 | 55 | @Override 56 | public void set(BytesStore bytes, long index, double value) { 57 | bytes.writeFloat(index << 2, (float) value); 58 | } 59 | 60 | @Override 61 | public long sizeFor(long capacity) { 62 | return capacity << 2; 63 | } 64 | 65 | @Override 66 | public boolean supportNaN() { 67 | return true; 68 | } 69 | }, 70 | INT16_1 { 71 | final double factor = 1e1; 72 | 73 | @Override 74 | public double get(BytesStore bytes, long index) { 75 | short i = bytes.readShort(index << 1); 76 | return i == Short.MIN_VALUE ? Double.NaN : i / factor; 77 | } 78 | 79 | @Override 80 | public void set(BytesStore bytes, long index, double value) { 81 | short i = Double.isNaN(value) ? Short.MIN_VALUE : Maths.toInt16(Math.round(value * factor)); 82 | bytes.writeShort(index << 1, i); 83 | } 84 | 85 | @Override 86 | public long sizeFor(long capacity) { 87 | return capacity << 1; 88 | } 89 | 90 | @Override 91 | public boolean supportNaN() { 92 | return true; 93 | } 94 | }, 95 | INT16_2 { 96 | final double factor = 1e2; 97 | 98 | @Override 99 | public double get(BytesStore bytes, long index) { 100 | short i = bytes.readShort(index << 1); 101 | return i == Short.MIN_VALUE ? Double.NaN : i / factor; 102 | } 103 | 104 | @Override 105 | public void set(BytesStore bytes, long index, double value) { 106 | short i = Double.isNaN(value) ? Short.MIN_VALUE : Maths.toInt16(Math.round(value * factor)); 107 | bytes.writeShort(index << 1, i); 108 | } 109 | 110 | @Override 111 | public long sizeFor(long capacity) { 112 | return capacity << 1; 113 | } 114 | 115 | @Override 116 | public boolean supportNaN() { 117 | return true; 118 | } 119 | }, 120 | INT16_3 { 121 | final double factor = 1e3; 122 | 123 | @Override 124 | public double get(BytesStore bytes, long index) { 125 | short i = bytes.readShort(index << 1); 126 | return i == Short.MIN_VALUE ? Double.NaN : i / factor; 127 | } 128 | 129 | @Override 130 | public void set(BytesStore bytes, long index, double value) { 131 | short i = Double.isNaN(value) ? Short.MIN_VALUE : Maths.toInt16(Math.round(value * factor)); 132 | bytes.writeShort(index << 1, i); 133 | } 134 | 135 | @Override 136 | public long sizeFor(long capacity) { 137 | return capacity << 1; 138 | } 139 | 140 | @Override 141 | public boolean supportNaN() { 142 | return true; 143 | } 144 | }, 145 | INT16_4 { 146 | final double factor = 1e4; 147 | 148 | @Override 149 | public double get(BytesStore bytes, long index) { 150 | short i = bytes.readShort(index << 1); 151 | return i == Short.MIN_VALUE ? Double.NaN : i / factor; 152 | } 153 | 154 | @Override 155 | public void set(BytesStore bytes, long index, double value) { 156 | short i = Double.isNaN(value) ? Short.MIN_VALUE : Maths.toInt16(Math.round(value * factor)); 157 | bytes.writeShort(index << 1, i); 158 | } 159 | 160 | @Override 161 | public long sizeFor(long capacity) { 162 | return capacity << 1; 163 | } 164 | 165 | @Override 166 | public boolean supportNaN() { 167 | return true; 168 | } 169 | }, 170 | INT32_1 { 171 | final double factor = 1e1; 172 | 173 | @Override 174 | public double get(BytesStore bytes, long index) { 175 | int i = bytes.readInt(index << 2); 176 | return i == Integer.MIN_VALUE ? Double.NaN : i / factor; 177 | } 178 | 179 | @Override 180 | public void set(BytesStore bytes, long index, double value) { 181 | int i = Double.isNaN(value) ? Integer.MIN_VALUE : Math.toIntExact(Math.round(value * factor)); 182 | bytes.writeInt(index << 2, i); 183 | } 184 | 185 | @Override 186 | public long sizeFor(long capacity) { 187 | return capacity << 2; 188 | } 189 | 190 | @Override 191 | public boolean supportNaN() { 192 | return true; 193 | } 194 | }, 195 | INT32_2 { 196 | final double factor = 1e2; 197 | 198 | @Override 199 | public double get(BytesStore bytes, long index) { 200 | int i = bytes.readInt(index << 2); 201 | return i == Integer.MIN_VALUE ? Double.NaN : i / factor; 202 | } 203 | 204 | @Override 205 | public void set(BytesStore bytes, long index, double value) { 206 | int i = Double.isNaN(value) ? Integer.MIN_VALUE : Math.toIntExact(Math.round(value * factor)); 207 | bytes.writeInt(index << 2, i); 208 | } 209 | 210 | @Override 211 | public long sizeFor(long capacity) { 212 | return capacity << 2; 213 | } 214 | 215 | @Override 216 | public boolean supportNaN() { 217 | return true; 218 | } 219 | }, 220 | INT32_3 { 221 | final double factor = 1e3; 222 | 223 | @Override 224 | public double get(BytesStore bytes, long index) { 225 | int i = bytes.readInt(index << 2); 226 | return i == Integer.MIN_VALUE ? Double.NaN : i / factor; 227 | } 228 | 229 | @Override 230 | public void set(BytesStore bytes, long index, double value) { 231 | int i = Double.isNaN(value) ? Integer.MIN_VALUE : Math.toIntExact(Math.round(value * factor)); 232 | bytes.writeInt(index << 2, i); 233 | } 234 | 235 | @Override 236 | public long sizeFor(long capacity) { 237 | return capacity << 2; 238 | } 239 | 240 | @Override 241 | public boolean supportNaN() { 242 | return true; 243 | } 244 | }, 245 | INT32_4 { 246 | final double factor = 1e4; 247 | 248 | @Override 249 | public double get(BytesStore bytes, long index) { 250 | int i = bytes.readInt(index << 2); 251 | return i == Integer.MIN_VALUE ? Double.NaN : i / factor; 252 | } 253 | 254 | @Override 255 | public void set(BytesStore bytes, long index, double value) { 256 | int i = Double.isNaN(value) ? Integer.MIN_VALUE : Math.toIntExact(Math.round(value * factor)); 257 | bytes.writeInt(index << 2, i); 258 | } 259 | 260 | @Override 261 | public long sizeFor(long capacity) { 262 | return capacity << 2; 263 | } 264 | 265 | @Override 266 | public boolean supportNaN() { 267 | return true; 268 | } 269 | }, 270 | INT32_6 { 271 | final double factor = 1e6; 272 | 273 | @Override 274 | public double get(BytesStore bytes, long index) { 275 | int i = bytes.readInt(index << 2); 276 | return i == Integer.MIN_VALUE ? Double.NaN : i / factor; 277 | } 278 | 279 | @Override 280 | public void set(BytesStore bytes, long index, double value) { 281 | int i = Double.isNaN(value) ? Integer.MIN_VALUE : Math.toIntExact(Math.round(value * factor)); 282 | bytes.writeInt(index << 2, i); 283 | } 284 | 285 | @Override 286 | public long sizeFor(long capacity) { 287 | return capacity << 2; 288 | } 289 | 290 | @Override 291 | public boolean supportNaN() { 292 | return true; 293 | } 294 | } 295 | } 296 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/BytesLongLookup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import net.openhft.chronicle.bytes.BytesStore; 22 | 23 | /** 24 | * Created by peter on 19/02/16. 25 | */ 26 | public interface BytesLongLookup { 27 | long get(BytesStore bytes, long index); 28 | 29 | void set(BytesStore bytes, long index, long value); 30 | 31 | long sizeFor(long capacity); 32 | 33 | default boolean supportsNaN() { 34 | return false; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/BytesLongLookups.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import net.openhft.chronicle.bytes.BytesStore; 22 | import net.openhft.chronicle.core.Maths; 23 | 24 | /** 25 | * Created by peter on 19/02/16. 26 | */ 27 | public enum BytesLongLookups implements BytesLongLookup { 28 | INT64 { 29 | @Override 30 | public long get(BytesStore bytes, long index) { 31 | return bytes.readLong(index << 3); 32 | } 33 | 34 | @Override 35 | public void set(BytesStore bytes, long index, long value) { 36 | bytes.writeLong(index << 3, value); 37 | } 38 | 39 | @Override 40 | public long sizeFor(long capacity) { 41 | return capacity << 3; 42 | } 43 | 44 | @Override 45 | public boolean supportsNaN() { 46 | return true; 47 | } 48 | }, 49 | INT32 { 50 | @Override 51 | public long get(BytesStore bytes, long index) { 52 | int i = bytes.readInt(index << 2); 53 | return i == Integer.MIN_VALUE ? TimeSeries.LONG_NAN : i; 54 | } 55 | 56 | @Override 57 | public void set(BytesStore bytes, long index, long value) { 58 | int i = value == TimeSeries.LONG_NAN ? Integer.MIN_VALUE : Math.toIntExact(value); 59 | bytes.writeInt(index << 2, i); 60 | } 61 | 62 | @Override 63 | public long sizeFor(long capacity) { 64 | return capacity << 2; 65 | } 66 | 67 | @Override 68 | public boolean supportsNaN() { 69 | return true; 70 | } 71 | }, 72 | INT16 { 73 | @Override 74 | public long get(BytesStore bytes, long index) { 75 | short i = bytes.readShort(index << 1); 76 | return i == Short.MIN_VALUE ? Long.MIN_VALUE : i; 77 | } 78 | 79 | @Override 80 | public void set(BytesStore bytes, long index, long value) { 81 | short i = value == Long.MIN_VALUE ? Short.MIN_VALUE : Maths.toInt16(value); 82 | bytes.writeShort(index << 1, i); 83 | } 84 | 85 | @Override 86 | public long sizeFor(long capacity) { 87 | return capacity << 1; 88 | } 89 | 90 | @Override 91 | public boolean supportsNaN() { 92 | return true; 93 | } 94 | }, 95 | INT8 { 96 | @Override 97 | public long get(BytesStore bytes, long index) { 98 | return bytes.readByte(index); 99 | } 100 | 101 | @Override 102 | public void set(BytesStore bytes, long index, long value) { 103 | bytes.writeByte(index, Maths.toInt8(value)); 104 | } 105 | 106 | @Override 107 | public long sizeFor(long capacity) { 108 | return capacity; 109 | } 110 | }, 111 | UINT8 { 112 | @Override 113 | public long get(BytesStore bytes, long index) { 114 | return bytes.readUnsignedByte(index); 115 | } 116 | 117 | @Override 118 | public void set(BytesStore bytes, long index, long value) { 119 | bytes.writeByte(index, Maths.toUInt8(value)); 120 | } 121 | 122 | @Override 123 | public long sizeFor(long capacity) { 124 | return capacity; 125 | } 126 | }, 127 | UINT4 { 128 | @Override 129 | public long get(BytesStore bytes, long index) { 130 | int i = bytes.readUnsignedByte(index >> 1); 131 | return (index & 1) != 0 ? (i >> 4) : (i & 0xF); 132 | } 133 | 134 | @Override 135 | public void set(BytesStore bytes, long index, long value) { 136 | int i = bytes.readUnsignedByte(index >> 1); 137 | int i2 = (int) ((index & 1) != 0 ? (i & 0xF0) | (value & 0xF) : (i & 0xF) | ((value & 0xf) << 4)); 138 | bytes.writeByte(index, Maths.toUInt8(value)); 139 | } 140 | 141 | @Override 142 | public long sizeFor(long capacity) { 143 | return (capacity + 1) >> 1; 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/Column.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | /** 22 | * Created by peter on 19/02/16. 23 | */ 24 | public interface Column extends ColumnCommon { 25 | void set(long index, T t); 26 | 27 | T get(long index); 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/ColumnCommon.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | /** 22 | * Created by peter on 19/02/16. 23 | */ 24 | public interface ColumnCommon { 25 | void ensureCapacity(long capacity); 26 | 27 | TimeSeries timeSeries(); 28 | 29 | String name(); 30 | 31 | default long length() { 32 | return timeSeries().length(); 33 | } 34 | 35 | boolean supportsNaN(); 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/Columns.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import net.openhft.chronicle.bytes.BytesStore; 22 | import net.openhft.chronicle.bytes.NativeBytesStore; 23 | import org.apache.commons.math3.distribution.NormalDistribution; 24 | import org.apache.commons.math3.random.MersenneTwister; 25 | import org.apache.commons.math3.random.RandomGenerator; 26 | 27 | import java.util.ArrayList; 28 | import java.util.List; 29 | import java.util.concurrent.ForkJoinPool; 30 | import java.util.concurrent.ForkJoinTask; 31 | import java.util.function.Supplier; 32 | 33 | /** 34 | * Created by peter on 20/02/16. 35 | */ 36 | public enum Columns { 37 | ; 38 | 39 | static int CHUNK_SIZE_SQRT = 64; 40 | static int CHUNK_SIZE = CHUNK_SIZE_SQRT * CHUNK_SIZE_SQRT; 41 | 42 | public static void generateBrownian(DoubleColumn col, double start, double end, double sd) { 43 | long length = col.length(); 44 | double sd2 = sd / Math.sqrt(length); 45 | NormalDistribution nd = new NormalDistribution(0, sd2 * CHUNK_SIZE); 46 | int trendLength = Math.toIntExact((length - 1) / CHUNK_SIZE + 2); 47 | BytesStore trend = NativeBytesStore.lazyNativeBytesStoreWithFixedCapacity(trendLength * 8L); 48 | double x = start; 49 | RandomGenerator rand = new MersenneTwister(); 50 | for (int i = 0; i < trendLength - 1; i++) { 51 | float f = rand.nextFloat(); 52 | trend.writeDouble((long) i << 3, x); 53 | x += nd.inverseCumulativeProbability(f); 54 | } 55 | trend.writeDouble((long) (trendLength - 1) << 3, x); 56 | double diff = end - x; 57 | double gradient = diff / (trendLength - 1); 58 | for (int i = 0; i < trendLength; i++) { 59 | double y = trend.addAndGetDoubleNotAtomic((long) i << 3, i * gradient); 60 | // System.out.println(i + ": "+y); 61 | } 62 | int procs = Runtime.getRuntime().availableProcessors(); 63 | int chunksPerTask = (trendLength - 1) / procs + 1; 64 | ForkJoinPool fjp = ForkJoinPool.commonPool(); 65 | List tasks = new ArrayList<>(procs); 66 | for (int i = 0; i < procs; i++) { 67 | int si = i * chunksPerTask; 68 | int ei = Math.min(trendLength, si + chunksPerTask); 69 | tasks.add(fjp.submit(() -> { 70 | NormalDistribution nd2 = new NormalDistribution(0, sd2); 71 | RandomGenerator rand2 = new MersenneTwister(); 72 | for (int j = si; j < ei; j++) { 73 | generateBrownian(col, 74 | (long) j * CHUNK_SIZE, 75 | trend.readDouble((long) j << 3), 76 | trend.readDouble((long) (j + 1) << 3), nd2, rand2); 77 | } 78 | })); 79 | } 80 | for (ForkJoinTask task : tasks) { 81 | task.join(); 82 | } 83 | trend.release(); 84 | } 85 | 86 | private static void generateBrownian(DoubleColumn col, long first, double start, double end, NormalDistribution nd, RandomGenerator rand) { 87 | double x = start; 88 | int chunkSize = (int) Math.min(col.length() - first, CHUNK_SIZE); 89 | for (int i = 0; i < chunkSize; i++) { 90 | col.set(first + i, x); 91 | double p = rand.nextFloat() + 0.5 / (1 << 24); 92 | double v = nd.inverseCumulativeProbability(p); 93 | x += v; 94 | assert !Double.isInfinite(x); 95 | } 96 | double diff = end - x; 97 | double gradient = diff / chunkSize; 98 | for (int i = 0; i < chunkSize; i++) { 99 | col.add(first + i, i * gradient); 100 | } 101 | } 102 | 103 | public static void setAll(LongColumn col, Supplier perThread, LongColumnIndexObjectConsumer consumer) { 104 | long length = col.length(); 105 | int chunks = Math.toIntExact((length - 1) / CHUNK_SIZE + 1); 106 | ForkJoinPool fjp = ForkJoinPool.commonPool(); 107 | int procs = Runtime.getRuntime().availableProcessors(); 108 | List tasks = new ArrayList<>(procs); 109 | int chunksPerTask = (chunks - 1) / procs + 1; 110 | for (int i = 0; i < procs; i++) { 111 | int si = i * chunksPerTask; 112 | int ei = Math.min(chunks, si + chunksPerTask); 113 | tasks.add(fjp.submit(() -> { 114 | T t = perThread.get(); 115 | long first = (long) si * CHUNK_SIZE; 116 | int max = (int) Math.min((ei - si) * CHUNK_SIZE, length - first); 117 | for (int j = 0; j < max; j++) { 118 | consumer.apply(col, first + j, t); 119 | } 120 | })); 121 | } 122 | for (ForkJoinTask task : tasks) { 123 | task.join(); 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/DoubleColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import java.util.DoubleSummaryStatistics; 22 | 23 | /** 24 | * Created by peter on 19/02/16. 25 | */ 26 | public interface DoubleColumn extends ColumnCommon { 27 | void set(long index, double value); 28 | 29 | double get(long index); 30 | 31 | double add(long index, double v); 32 | 33 | void generateBrownian(double start, double end, double sd); 34 | 35 | DoubleSummaryStatistics summaryStatistics(); 36 | 37 | BytesDoubleLookup lookup(); 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/InMemoryColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import java.util.ArrayList; 22 | 23 | /** 24 | * Created by peter on 19/02/16. 25 | */ 26 | public class InMemoryColumn extends AbstractColumn implements Column { 27 | private final ArrayList list = new ArrayList<>(); 28 | 29 | public InMemoryColumn(TimeSeries timeSeries, String name, long capacity) { 30 | super(timeSeries, name); 31 | 32 | ensureCapacity(capacity); 33 | } 34 | 35 | @Override 36 | public void set(long index, T t) { 37 | int index0 = Math.toIntExact(index); 38 | while (list.size() <= index0) 39 | list.add(null); 40 | list.set(index0, t); 41 | } 42 | 43 | @Override 44 | public T get(long index) { 45 | return list.get(Math.toIntExact(index)); 46 | } 47 | 48 | @Override 49 | public void ensureCapacity(long capacity) { 50 | list.ensureCapacity(Math.toIntExact(capacity)); 51 | while (list.size() <= capacity) 52 | list.add(null); 53 | } 54 | 55 | @Override 56 | public boolean supportsNaN() { 57 | return false; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/InMemoryDoubleColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import net.openhft.chronicle.bytes.Bytes; 22 | import net.openhft.chronicle.bytes.BytesStore; 23 | 24 | import java.nio.ByteBuffer; 25 | import java.util.DoubleSummaryStatistics; 26 | 27 | /** 28 | * Created by peter on 19/02/16. 29 | */ 30 | public class InMemoryDoubleColumn extends AbstractColumn implements DoubleColumn { 31 | private final BytesDoubleLookup lookup; 32 | private BytesStore bytes; 33 | 34 | public InMemoryDoubleColumn(TimeSeries timeSeries, String name, BytesDoubleLookup lookup, long capacity) { 35 | super(timeSeries, name); 36 | this.lookup = lookup; 37 | // this.bytes = NativeBytesStore.lazyNativeBytesStoreWithFixedCapacity(lookup.sizeFor(capacity)); 38 | this.bytes = Bytes.wrapForRead(ByteBuffer.allocateDirect(Math.toIntExact(lookup.sizeFor(capacity)))); 39 | } 40 | 41 | @Override 42 | public void ensureCapacity(long capacity) { 43 | long cap = lookup.sizeFor(capacity); 44 | if (cap > bytes.realCapacity()) { 45 | // BytesStore bytes2 = NativeBytesStore.lazyNativeBytesStoreWithFixedCapacity(Maths.divideRoundUp(cap, OS.pageSize())); 46 | BytesStore bytes2 = Bytes.wrapForRead(ByteBuffer.allocateDirect(Math.toIntExact(lookup.sizeFor(capacity)))); 47 | bytes2.write(0, bytes); 48 | bytes.release(); 49 | bytes = bytes2; 50 | } 51 | } 52 | 53 | @Override 54 | public void set(long index, double value) { 55 | if (index < 0 || index > bytes.realCapacity()) 56 | throw new AssertionError("index: " + index); 57 | lookup.set(bytes, index, value); 58 | } 59 | 60 | @Override 61 | public double get(long index) { 62 | return lookup.get(bytes, index); 63 | } 64 | 65 | public double add(long index, double value) { 66 | return lookup.add(bytes, index, value); 67 | } 68 | 69 | @Override 70 | public void generateBrownian(double start, double end, double sd) { 71 | Columns.generateBrownian(this, start, end, sd); 72 | } 73 | 74 | @Override 75 | public DoubleSummaryStatistics summaryStatistics() { 76 | throw new UnsupportedOperationException(); 77 | } 78 | 79 | @Override 80 | public boolean supportsNaN() { 81 | return lookup.supportNaN(); 82 | } 83 | 84 | @Override 85 | public BytesDoubleLookup lookup() { 86 | return lookup; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/InMemoryLongColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import net.openhft.chronicle.bytes.Bytes; 22 | import net.openhft.chronicle.bytes.BytesStore; 23 | import net.openhft.chronicle.bytes.NativeBytesStore; 24 | import net.openhft.chronicle.core.Jvm; 25 | 26 | import java.nio.ByteBuffer; 27 | import java.util.function.Supplier; 28 | 29 | /** 30 | * Created by peter on 19/02/16. 31 | */ 32 | public class InMemoryLongColumn extends AbstractColumn implements LongColumn { 33 | private final BytesLongLookup lookup; 34 | private BytesStore bytes; 35 | 36 | public InMemoryLongColumn(TimeSeries timeSeries, String name, BytesLongLookup lookup, long capacity) { 37 | super(timeSeries, name); 38 | this.lookup = lookup; 39 | long value = lookup.sizeFor(capacity); 40 | this.bytes = Jvm.isDebug() 41 | ? Bytes.wrapForRead(ByteBuffer.allocateDirect(Math.toIntExact(value))) 42 | : NativeBytesStore.lazyNativeBytesStoreWithFixedCapacity(value); 43 | } 44 | 45 | @Override 46 | public void ensureCapacity(long capacity) { 47 | long cap = lookup.sizeFor(capacity); 48 | if (cap > bytes.realCapacity()) { 49 | long value = lookup.sizeFor(capacity); 50 | BytesStore bytes2 = Jvm.isDebug() 51 | ? Bytes.wrapForRead(ByteBuffer.allocateDirect(Math.toIntExact(value))) 52 | : NativeBytesStore.lazyNativeBytesStoreWithFixedCapacity(value); 53 | bytes2.write(0, bytes); 54 | bytes.release(); 55 | bytes = bytes2; 56 | } 57 | } 58 | 59 | @Override 60 | public void set(long index, long value) { 61 | lookup.set(bytes, index, value); 62 | } 63 | 64 | @Override 65 | public long get(long index) { 66 | return lookup.get(bytes, index); 67 | } 68 | 69 | @Override 70 | public boolean supportsNaN() { 71 | return lookup.supportsNaN(); 72 | } 73 | 74 | @Override 75 | public void setAll(Supplier perThread, LongColumnIndexObjectConsumer consumer) { 76 | Columns.setAll(this, perThread, consumer); 77 | } 78 | 79 | @Override 80 | public long integrate() { 81 | long sum = 0; 82 | for (long i = 0; i < length(); i++) { 83 | long v = get(i); 84 | sum += v; 85 | set(i, v); 86 | } 87 | return sum; 88 | } 89 | 90 | @Override 91 | public BytesLongLookup lookup() { 92 | return lookup; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/InMemoryTimeSeries.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import java.util.ArrayList; 22 | import java.util.LinkedHashMap; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | /** 27 | * Created by peter on 19/02/16. 28 | */ 29 | public class InMemoryTimeSeries implements TimeSeries { 30 | private final Map columnMap = new LinkedHashMap<>(); 31 | private final TimeSeries parent; 32 | private long length = 0; 33 | private long capacity = 1 << 20; 34 | 35 | public InMemoryTimeSeries(TimeSeries parent) { 36 | this.parent = parent; 37 | } 38 | 39 | @Override 40 | public void setLength(long size) { 41 | ensureCapacity(size); 42 | this.length = size; 43 | } 44 | 45 | @Override 46 | public void ensureCapacity(long capacity) { 47 | if (capacity > this.capacity) { 48 | for (ColumnCommon c : columnMap.values()) { 49 | c.ensureCapacity(capacity); 50 | } 51 | } 52 | this.capacity = capacity; 53 | } 54 | 55 | @Override 56 | public long addIndex(long timeStampMicros) { 57 | LongColumn ts = getTimestamp(); 58 | ts.set(length, timeStampMicros); 59 | if (length + 1 >= capacity) 60 | ensureCapacity(length + (1 << 20)); 61 | return length++; 62 | } 63 | 64 | @Override 65 | public List getColumns() { 66 | List columns = new ArrayList<>(); 67 | if (parent != null) 68 | columns.addAll(parent.getColumns()); 69 | columns.addAll(columnMap.keySet()); 70 | return columns; 71 | } 72 | 73 | @Override 74 | public LongColumn getTimestamp() { 75 | return acquireLongColumn(TIMESTAMP); 76 | } 77 | 78 | @Override 79 | public LongColumn acquireLongColumn(String name, BytesLongLookup lookup) { 80 | return (LongColumn) columnMap.computeIfAbsent(name, (n) -> new InMemoryLongColumn(this, n, lookup, capacity)); 81 | } 82 | 83 | @Override 84 | public LongColumn getLongColumn(String name) { 85 | return (LongColumn) columnMap.get(name); 86 | } 87 | 88 | @Override 89 | public DoubleColumn acquireDoubleColumn(String name, BytesDoubleLookup lookup) { 90 | return (DoubleColumn) columnMap.computeIfAbsent(name, (n) -> new InMemoryDoubleColumn(this, n, lookup, capacity)); 91 | } 92 | 93 | @Override 94 | public DoubleColumn getDoubleColumn(String name) { 95 | return (DoubleColumn) columnMap.get(name); 96 | } 97 | 98 | @Override 99 | public Column acquireColumn(String name, Class tClass) { 100 | return (Column) columnMap.computeIfAbsent(name, (n) -> new InMemoryColumn<>(this, n, capacity)); 101 | } 102 | 103 | @Override 104 | public Column getColumn(String name, Class tClass) { 105 | return (Column) columnMap.get(name); 106 | } 107 | 108 | @Override 109 | public long length() { 110 | return length; 111 | } 112 | 113 | @Override 114 | public DoubleColumn projectAs(String name, DoubleColumn source) { 115 | LongColumn ts = getTimestamp(); 116 | LongColumn ts2 = source.timeSeries().getTimestamp(); 117 | DoubleColumn result = acquireDoubleColumn(name, source.lookup()); 118 | long i2 = 0, time2 = ts2.get(0); 119 | double v2 = source.get(0); 120 | for (long i = 0; i < length(); i++) { 121 | long time = ts.get(i); 122 | OUTER: 123 | if (time > time2) { 124 | do { 125 | if (i2 + 1 >= ts2.length()) { 126 | v2 = Double.NaN; 127 | time2 = Long.MAX_VALUE; 128 | break OUTER; 129 | } 130 | time2 = ts2.get(++i2); 131 | } while (time > time2); 132 | v2 = source.get(i2); 133 | } 134 | result.set(i, v2); 135 | } 136 | return result; 137 | } 138 | 139 | @Override 140 | public LongColumn projectAs(String name, LongColumn source) { 141 | LongColumn ts = getTimestamp(); 142 | LongColumn ts2 = source.timeSeries().getTimestamp(); 143 | LongColumn result = acquireLongColumn(name, source.lookup()); 144 | long i2 = 0, time2 = ts2.get(0); 145 | long v2 = source.get(0); 146 | for (long i = 0; i < length(); i++) { 147 | long time = ts.get(i); 148 | OUTER: 149 | if (time > time2) { 150 | do { 151 | if (i2 + 1 >= ts2.length()) { 152 | v2 = Long.MIN_VALUE; 153 | time2 = Long.MAX_VALUE; 154 | break OUTER; 155 | } 156 | time2 = ts2.get(++i2); 157 | } while (time > time2); 158 | v2 = source.get(i2); 159 | } 160 | result.set(i, v2); 161 | } 162 | return result; 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/LongColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import java.util.function.Supplier; 22 | 23 | /** 24 | * Created by peter on 19/02/16. 25 | */ 26 | public interface LongColumn extends ColumnCommon { 27 | void set(long index, long value); 28 | 29 | long get(long index); 30 | 31 | BytesLongLookup lookup(); 32 | 33 | void setAll(Supplier perThread, LongColumnIndexObjectConsumer consumer); 34 | 35 | long integrate(); 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/LongColumnIndexObjectConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | /** 22 | * Created by peter on 21/02/16. 23 | */ 24 | @FunctionalInterface 25 | public interface LongColumnIndexObjectConsumer { 26 | void apply(LongColumn col, long index, T t); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/net/openhft/chronicle/timeseries/TimeSeries.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import java.util.List; 22 | 23 | /** 24 | * Created by peter on 19/02/16. 25 | */ 26 | public interface TimeSeries { 27 | long LONG_NAN = Long.MIN_VALUE; 28 | String TIMESTAMP = "time"; 29 | 30 | void setLength(long size); 31 | 32 | void ensureCapacity(long capacity); 33 | 34 | long addIndex(long timeStampMicros); 35 | 36 | List getColumns(); 37 | 38 | default LongColumn acquireLongColumn(String name) { 39 | return acquireLongColumn(name, BytesLongLookups.INT64); 40 | } 41 | 42 | LongColumn acquireLongColumn(String name, BytesLongLookup lookup); 43 | 44 | LongColumn getLongColumn(String name); 45 | 46 | default DoubleColumn acquireDoubleColumn(String name) { 47 | return acquireDoubleColumn(name, BytesDoubleLookups.FLOAT64); 48 | } 49 | 50 | DoubleColumn acquireDoubleColumn(String name, BytesDoubleLookup lookup); 51 | 52 | DoubleColumn getDoubleColumn(String name); 53 | 54 | Column acquireColumn(String name, Class tClass); 55 | 56 | Column getColumn(String name, Class tClass); 57 | 58 | long length(); 59 | 60 | DoubleColumn projectAs(String name, DoubleColumn source); 61 | 62 | LongColumn projectAs(String name, LongColumn source); 63 | 64 | LongColumn getTimestamp(); 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/net/openhft/chronicle/timeseries/InMemoryTimeSeriesTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * * Copyright (C) 2016 higherfrequencytrading.com 4 | * * 5 | * * This program is free software: you can redistribute it and/or modify 6 | * * it under the terms of the GNU Lesser General Public License as published by 7 | * * the Free Software Foundation, either version 3 of the License. 8 | * * 9 | * * This program is distributed in the hope that it will be useful, 10 | * * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * * GNU Lesser General Public License for more details. 13 | * * 14 | * * You should have received a copy of the GNU Lesser General Public License 15 | * * along with this program. If not, see . 16 | * 17 | */ 18 | 19 | package net.openhft.chronicle.timeseries; 20 | 21 | import org.junit.Ignore; 22 | import org.junit.Test; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | import java.util.Random; 27 | import java.util.concurrent.ExecutionException; 28 | import java.util.concurrent.ForkJoinPool; 29 | import java.util.concurrent.ForkJoinTask; 30 | 31 | /** 32 | * Created by peter on 19/02/16. 33 | */ 34 | public class InMemoryTimeSeriesTest { 35 | static float sqr(float f) { 36 | return f * f; 37 | } 38 | 39 | @Test 40 | public void testBidAsk() throws ExecutionException, InterruptedException { 41 | TimeSeries ts = new InMemoryTimeSeries(null); 42 | long size = 10_000_000_000L; 43 | ts.ensureCapacity(size); 44 | LongColumn time = ts.acquireLongColumn("ts", BytesLongLookups.INT64); 45 | DoubleColumn bid = ts.acquireDoubleColumn("bid", BytesDoubleLookups.INT16_4); 46 | DoubleColumn ask = ts.acquireDoubleColumn("ask", BytesDoubleLookups.INT16_4); 47 | 48 | int threads = Runtime.getRuntime().availableProcessors() * 2; 49 | long start = System.currentTimeMillis(); 50 | long block = (((((size + threads - 1) / threads) - 1) | 63) + 1); 51 | 52 | List> tasks = new ArrayList<>(); 53 | for (int i = 0; i < threads; i++) { 54 | final int finalI = i; 55 | tasks.add(ForkJoinPool.commonPool().submit(() -> { 56 | Random rand1 = new Random(); 57 | Random rand2 = new Random(); 58 | long first = finalI * block; 59 | for (int j = 0, max = (int) Math.min(size - first, block); j < max; j++) { 60 | long v = first + j; 61 | time.set(v, v); 62 | int r1 = rand1.nextInt(1000); 63 | int r2 = rand2.nextInt(1000); 64 | bid.set(v, Math.min(r1, r2) / 1e3); 65 | ask.set(v, Math.max(r1, r2) / 1e3); 66 | } 67 | })); 68 | } 69 | for (ForkJoinTask task : tasks) { 70 | task.get(); 71 | } 72 | long took = System.currentTimeMillis() - start; 73 | System.out.printf("%d threads took %.3f secs%n", threads, took / 1e3); 74 | } 75 | 76 | @Test 77 | public void testGenerateBrownian() { 78 | TimeSeries ts = new InMemoryTimeSeries(null); 79 | long size = 4L << 30; 80 | ts.setLength(size); 81 | 82 | long start = System.currentTimeMillis(); 83 | DoubleColumn mid = ts.acquireDoubleColumn("mid", BytesDoubleLookups.INT16_4); 84 | mid.generateBrownian(1, 2, 0.001); 85 | 86 | DoubleColumn spread = ts.acquireDoubleColumn("spread", BytesDoubleLookups.INT16_4); 87 | spread.generateBrownian(0.001, 0.001, 0.0001); 88 | long took = System.currentTimeMillis() - start; 89 | 90 | System.out.printf("generateBrownian took %.3f secs%n", took / 1e3); 91 | } 92 | 93 | @Test 94 | @Ignore("TODO FIX") 95 | public void testGenerateRandomSequence() { 96 | long size = 600_000_000L; 97 | 98 | // generate series 1 99 | TimeSeries ts = new InMemoryTimeSeries(null); 100 | ts.setLength(size); 101 | 102 | LongColumn time = ts.getTimestamp(); 103 | time.setAll(Random::new, (c, i, r) -> c.set(i, 9 + (int) Math.pow(1e6, sqr(r.nextFloat())))); 104 | long sum = time.integrate(); // sum all the intervals 105 | 106 | System.out.printf("%.1f days%n", sum / 86400e6); 107 | 108 | DoubleColumn mid = ts.acquireDoubleColumn("mid", BytesDoubleLookups.INT16_4); 109 | mid.generateBrownian(1, 2, 0.0005); 110 | 111 | // generate series 2 112 | TimeSeries ts2 = new InMemoryTimeSeries(null); 113 | ts2.setLength(size); 114 | 115 | LongColumn time2 = ts.getTimestamp(); 116 | time2.setAll(Random::new, (c, i, r) -> c.set(i, 9 + (int) Math.pow(1e6, sqr(r.nextFloat())))); 117 | long sum2 = time2.integrate(); // sum all the intervals 118 | 119 | System.out.printf("%.1f days%n", sum2 / 86400e6); 120 | 121 | DoubleColumn mid2 = ts2.acquireDoubleColumn("mid", BytesDoubleLookups.INT16_4); 122 | mid2.generateBrownian(1, 2, 0.0005); 123 | 124 | // compare the correlation 125 | // CorrelationStatistic stats = PearsonsCorrelation.calcCorrelation(mid, mid2, Mode.AFTER_BOTH_CHANGE); 126 | } 127 | } 128 | --------------------------------------------------------------------------------