30 | * Computing Extremely Accurate Quantiles Using t-Digests, 31 | * Ted Dunning and Otmar Ertl, 32 | * https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf 33 | *34 | * 35 | *
36 | * import org.isarnproject.sketches.java.TDigest; 37 | * double[] data = // data that you would like to sketch 38 | * TDigest sketch = TDigest.sketch(data) 39 | * // the cumulative distribution function of the sketch; cdf(x) at x = 0 40 | * double cdf = sketch.cdf(0.0) 41 | * // inverse of the CDF, evaluated at q = 0.5 42 | * double cdfi = sketch.cdfInverse(0.5) 43 | *44 | */ 45 | public class TDigest implements Serializable { 46 | /** compression setting (delta in original paper) */ 47 | protected final double C; 48 | /** maximum number of unique discrete values to track */ 49 | protected final int maxDiscrete; 50 | /** current number of clusters */ 51 | protected int nclusters = 0; 52 | /** total mass of data sampled so far */ 53 | protected double M = 0.0; 54 | /** cluster centers */ 55 | protected double[] cent = null; 56 | /** cluster masses */ 57 | protected double[] mass = null; 58 | /** cumulative cluster masses, represented as a Fenwick Tree */ 59 | protected double[] ftre = null; 60 | 61 | /** A new t-digest sketching structure with default compression and maximum discrete tracking. */ 62 | public TDigest() { 63 | this(COMPRESSION_DEFAULT, 0, INIT_SIZE_DEFAULT); 64 | } 65 | 66 | /** Construct a t-digest with the given compression. 67 | * Maximum discrete tracking defaults to zero. 68 | * @param compression sketching compression setting. Higher = more compression. 69 | * Must be > 0. 70 | */ 71 | public TDigest(double compression) { 72 | this(compression, 0, INIT_SIZE_DEFAULT); 73 | } 74 | 75 | /** Construct a t-digest with the given compression and maximum discrete tracking. 76 | * @param compression sketching compression setting. Higher = more compression. 77 | * Must be > 0. 78 | * @param maxDiscrete maximum number of unique discrete values to track. Must be ≥ 0. 79 | * If this number of values is exceeded, the sketch will begin to operate in 80 | * normal continuous mode. 81 | */ 82 | public TDigest(double compression, int maxDiscrete) { 83 | this(compression, maxDiscrete, INIT_SIZE_DEFAULT); 84 | } 85 | 86 | /** Construct a t-digest with the given compression and maximum discrete tracking. 87 | * @param compression sketching compression setting. Higher = more compression. 88 | * Must be > 0. 89 | * @param maxDiscrete maximum number of unique discrete values to track. Must be ≥ 0. 90 | * If this number of values is exceeded, the sketch will begin to operate in 91 | * normal continuous mode. 92 | * @param sz initial capacity to use for internal arrays. Must be > 0. 93 | */ 94 | public TDigest(double compression, int maxDiscrete, int sz) { 95 | assert compression > 0.0; 96 | assert maxDiscrete >= 0; 97 | assert sz > 0; 98 | C = compression; 99 | this.maxDiscrete = maxDiscrete; 100 | cent = new double[sz]; 101 | mass = new double[sz]; 102 | ftre = new double[1 + sz]; 103 | // ftre is 1-based. set ftre[0] to zero just to be tidy 104 | ftre[0] = 0.0; 105 | } 106 | 107 | /** 108 | * Construct a t-digest from a list of cluster centers and masses. 109 | * Object deserialization is one of the intended use cases for this constructor. 110 | * NOTE: This constructor assumes the 'cent' and 'mass' arrays will be owned 111 | * by the new t-digest object. If 'cent' and 'mass' are both null then an empty cluster 112 | * will be created. 113 | * @param compression sketching compression setting. Higher = more compression. 114 | * Must be > 0. 115 | * @param maxDiscrete maximum number of unique discrete values to track. Must be ≥ 0. 116 | * If this number of values is exceeded, the sketch will begin to operate in 117 | * @param cent the list of cluster centers. Assumed to be in sorted order. 118 | * This array is assumed to be owned by the t-digest object after construction. 119 | * @param mass a list of cluster masses. Assumed to be parallel to centers. 120 | * This array is assumed to be owned by the t-digest object after construction. 121 | */ 122 | public TDigest(double compression, int maxDiscrete, double cent[], double mass[]) { 123 | assert compression > 0.0; 124 | assert maxDiscrete >= 0; 125 | this.C = compression; 126 | this.maxDiscrete = maxDiscrete; 127 | assert (cent != null && mass != null) || (cent == null && mass == null); 128 | this.nclusters = (cent != null) ? cent.length : 0; 129 | int sz = nclusters; 130 | if (sz == 0) { 131 | // cent, mass and ftre cannot be zero length 132 | sz = INIT_SIZE_DEFAULT; 133 | this.cent = new double[sz]; 134 | this.mass = new double[sz]; 135 | } else { 136 | this.cent = cent; 137 | this.mass = mass; 138 | } 139 | assert cent != null && mass != null; 140 | assert cent.length == sz; 141 | assert cent.length == mass.length; 142 | assert cent.length > 0; 143 | this.ftre = new double[1 + sz]; 144 | Arrays.fill(ftre, 0, 1 + nclusters, 0.0); 145 | this.M = 0.0; 146 | for (int j = 0; j < nclusters; ++j) { 147 | M += mass[j]; 148 | ftInc(j, mass[j]); 149 | } 150 | } 151 | 152 | /** Construct a deep copy of another t-digest */ 153 | public TDigest(TDigest that) { 154 | C = that.C; 155 | maxDiscrete = that.maxDiscrete; 156 | nclusters = that.nclusters; 157 | M = that.M; 158 | cent = Arrays.copyOf(that.cent, nclusters); 159 | mass = Arrays.copyOf(that.mass, nclusters); 160 | ftre = Arrays.copyOf(that.ftre, nclusters); 161 | } 162 | 163 | /** Update the sketch with a new sampled value 164 | * @param x the new sampled value 165 | */ 166 | public final void update(double x) { 167 | update(x, 1.0); 168 | } 169 | 170 | /** Update the sketch with a new sampled value 171 | * @param x the new sampled value 172 | * @param w the weight (aka mass) associated with x 173 | */ 174 | public final void update(double x, double w) { 175 | updateLogic(x, w); 176 | if ((nclusters > maxDiscrete) && (nclusters > R())) recluster(); 177 | } 178 | 179 | private final void updateLogic(double x, double w) { 180 | if (nclusters == 0) { 181 | // clusters are empty, so (x,w) becomes the first cluster 182 | cent[0] = x; 183 | M = w; 184 | mass[0] = w; 185 | ftre[1] = w; 186 | nclusters += 1; 187 | return; 188 | } 189 | if (nclusters <= maxDiscrete) { 190 | // we are under the limit for discrete values to track 191 | int j = Arrays.binarySearch(cent, 0, nclusters, x); 192 | if (j >= 0) { 193 | // landed on existing cluster: add its mass and we're done 194 | M += w; 195 | mass[j] += w; 196 | ftInc(j, w); 197 | } else { 198 | // a new x value: insert as a new discrete cluster 199 | newCluster(-(j + 1), x, w); 200 | } 201 | return; 202 | } 203 | // get the index of the cluster closest to x 204 | int j = closest(x); 205 | if (x == cent[j]) { 206 | // landed on existing cluster: add its mass and we're done 207 | M += w; 208 | mass[j] += w; 209 | ftInc(j, w); 210 | return; 211 | } 212 | double m = mass[j]; 213 | // q is the quantile of the closest cluster to x 214 | // (ftSum does the right thing (return 0) for j = 0) 215 | double q = (ftSum(j - 1) + (m / 2.0)) / M; 216 | // this is the upper-bound for the mass of closest cluster 217 | double ub = C * M * q * (1.0 - q); 218 | // dm is how much mass we're allowed to add to closest cluster 219 | double dm = Math.min(w, Math.max(0.0, ub - m)); 220 | // rm is the remainder of the mass 221 | double rm = w - dm; 222 | if (dm > 0.0) { 223 | // Add any allowable mass to closest cluster and update its center. 224 | // It is safe to update center this way because it will remain 225 | // between x and original center, and so cannot move out of its original 226 | // ordering relative to its neighbors, because x is by previous logic 227 | // closer to cent[j] than any other cluster. 228 | double dc = dm * (x - cent[j]) / (m + dm); 229 | cent[j] += dc; 230 | M += dm; 231 | mass[j] += dm; 232 | ftInc(j, dm); 233 | } 234 | // if there is remaining mass, it becomes a new cluster 235 | if (rm > 0.0) newCluster((x < cent[j]) ? j : j + 1, x, rm); 236 | } 237 | 238 | /** Merge another t-digest into this one. 239 | * @param that the t-digest to merge. This t-digest is unaltered. 240 | */ 241 | public final void merge(TDigest that) { 242 | Integer[] indexes = new Integer[that.nclusters]; 243 | for (int j = 0; j < that.nclusters; ++j) indexes[j] = j; 244 | // sort so that largest clusters are first. 245 | // inserting large to small yields stable distribution estimations 246 | Comparator
ltd.merge(rtd), 748 | * otherwise
rtd.merge(ltd)749 | */ 750 | public static TDigest merge(TDigest ltd, TDigest rtd) { 751 | if (ltd.size() < rtd.size()) return merge(rtd, ltd); 752 | if (rtd.size() == 0) return ltd; 753 | if (rtd.size() == 1) { 754 | ltd.update(rtd.cent[0], rtd.mass[0]); 755 | return ltd; 756 | } 757 | if (rtd.mass() < ltd.mass()) { 758 | ltd.merge(rtd); 759 | return ltd; 760 | } else { 761 | rtd.merge(ltd); 762 | return rtd; 763 | } 764 | } 765 | 766 | /** 767 | * Sketch data using a t-digest with default compression and maximum discrete tracking. 768 | * @param data the data to sketch 769 | * @return a t-digest sketch of the data 770 | */ 771 | public static TDigest sketch(double[] data) { 772 | return sketch(data, COMPRESSION_DEFAULT, 0, INIT_SIZE_DEFAULT); 773 | } 774 | 775 | /** 776 | * Sketch data using a t-digest. 777 | * maxDiscrete defaults to zero. 778 | * @param data the data to sketch 779 | * @param compression sketching compression setting. Higher = more compression. 780 | * Must be > 0. 781 | * @return a t-digest sketch of the data 782 | */ 783 | public static TDigest sketch(double[] data, double compression) { 784 | return sketch(data, compression, 0, INIT_SIZE_DEFAULT); 785 | } 786 | 787 | /** 788 | * Sketch data using a t-digest. 789 | * @param data the data to sketch 790 | * @param compression sketching compression setting. Higher = more compression. 791 | * Must be > 0. 792 | * @param maxDiscrete maximum number of unique discrete values to track. Must be ≥ 0. 793 | * If this number of values is exceeded, the sketch will begin to operate in 794 | * normal continuous mode. 795 | * @return a t-digest sketch of the data 796 | */ 797 | public static TDigest sketch(double[] data, double compression, int maxDiscrete) { 798 | return sketch(data, compression, maxDiscrete, INIT_SIZE_DEFAULT); 799 | } 800 | 801 | /** 802 | * Sketch data using a t-digest. 803 | * @param data the data to sketch 804 | * @param compression sketching compression setting. Higher = more compression. 805 | * Must be > 0. 806 | * @param maxDiscrete maximum number of unique discrete values to track. Must be ≥ 0. 807 | * If this number of values is exceeded, the sketch will begin to operate in 808 | * normal continuous mode. 809 | * @param sz initial capacity to use for internal arrays. Must be > 0. 810 | * @return a t-digest sketch of the data 811 | */ 812 | public static TDigest sketch(double[] data, double compression, int maxDiscrete, int sz) { 813 | TDigest td = empty(compression, maxDiscrete, sz); 814 | for (double x: data) td.update(x, 1.0); 815 | if (td.size() > maxDiscrete) td.recluster(); 816 | return td; 817 | } 818 | 819 | static void intShuffle(int[] data) { 820 | intShuffle(data, 0, data.length); 821 | } 822 | 823 | static void intShuffle(int[] data, int end) { 824 | intShuffle(data, 0, end); 825 | } 826 | 827 | static void intShuffle(int[] data, int beg, int end) { 828 | ThreadLocalRandom rnd = ThreadLocalRandom.current(); 829 | end -= 1; 830 | while (end > beg) { 831 | int r = rnd.nextInt(beg, end); 832 | int d = data[end]; 833 | data[end] = data[r]; 834 | data[r] = d; 835 | end -= 1; 836 | } 837 | } 838 | } 839 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.3.12 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | resolvers += Resolver.url( 2 | "bintray-sbt-plugin-releases", 3 | url("http://dl.bintray.com/content/sbt/sbt-plugin-releases"))( 4 | Resolver.ivyStylePatterns) 5 | 6 | resolvers += "sonatype-releases" at "https://oss.sonatype.org/content/repositories/releases/" 7 | 8 | resolvers += "jgit-repo" at "http://download.eclipse.org/jgit/maven" 9 | 10 | addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.3") 11 | 12 | addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3") 13 | 14 | addSbtPlugin("io.crashbox" % "sbt-gpg" % "0.2.1") 15 | 16 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.2") 17 | 18 | // scoverage and coveralls deps are at old versions to avoid a bug in the current versions 19 | // update these when this fix is released: https://github.com/scoverage/sbt-coveralls/issues/73 20 | //addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4") 21 | 22 | //addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0") 23 | 24 | //addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.6.0") 25 | -------------------------------------------------------------------------------- /src/main/scala/org/isarnproject/sketches/TDigest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 Erik Erlandson 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package org.isarnproject.sketches 18 | 19 | import scala.util.Random 20 | 21 | import tdmap.TDigestMap 22 | 23 | /** 24 | * A t-digest sketch of sampled numeric data, as described in: 25 | * Computing Extremely Accurate Quantiles Using t-Digests, 26 | * Ted Dunning and Otmar Ertl, 27 | * https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf 28 | * 29 | * {{{ 30 | * import org.isarnproject.sketches.TDigest 31 | * val data = Vector.fill(10000) { scala.util.Random.nextGaussian() } 32 | * // sketch of some Gaussian data 33 | * val sketch = TDigest.sketch(data) 34 | * // the cumulative distribution function of the sketch; cdf(x) at x = 0 35 | * val cdf = sketch.cdf(0.0) 36 | * // inverse of the CDF, evaluated at q = 0.5 37 | * val cdfi = sketch.cdfInverse(0.5) 38 | * }}} 39 | */ 40 | case class TDigest( 41 | delta: Double, 42 | maxDiscrete: Int, 43 | nclusters: Int, 44 | clusters: TDigestMap) extends Serializable { 45 | 46 | // re-cluster when number of clusters exceeds this threshold 47 | @inline private def R = (TDigest.K / delta).toInt 48 | 49 | /** 50 | * Returns a new t-digest with value x included in its sketch; td + x is equivalent to 51 | * td + (x, 1). 52 | * @param x The numeric data value to include in the sketch 53 | * @return the updated sketch 54 | */ 55 | def +[N](x: N)(implicit num: Numeric[N]): TDigest = this.plus(num.toDouble(x), 1.0) 56 | 57 | /** 58 | * Returns a new t-digest with new pair (x, w) included in its sketch. 59 | * @param xw A pair (x, w) where x is the numeric value and w is its weight 60 | * @return the updated sketch 61 | * @note This implements 'algorithm 1' from: 62 | * Computing Extremely Accurate Quantiles Using t-Digests, 63 | * Ted Dunning and Otmar Ertl, 64 | * https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf 65 | */ 66 | def +[N1, N2](xw: (N1, N2))(implicit num1: Numeric[N1], num2: Numeric[N2]): TDigest = 67 | this.plus(num1.toDouble(xw._1), num2.toDouble(xw._2)) 68 | 69 | private def plus(x: Double, w: Double): TDigest = { 70 | if (nclusters <= maxDiscrete) { 71 | clusters.getNode(x).fold { 72 | TDigest(delta, maxDiscrete, nclusters + 1, clusters + (x -> w)) 73 | } { xnode => 74 | TDigest(delta, maxDiscrete, nclusters, clusters.update(x, x, xnode.data.value + w)) 75 | } 76 | } else { 77 | val s = this.update(x, w) 78 | if (s.nclusters <= R) s 79 | else { 80 | // too many clusters: attempt to compress it by re-clustering 81 | val ds = TDigest.shuffle(s.clusters.toVector) 82 | ds.foldLeft(TDigest.empty(delta, maxDiscrete)) { case (d, (x, w)) => d.update(x, w) } 83 | } 84 | } 85 | } 86 | 87 | /** 88 | * Add this digest to another 89 | * @param that The right-hand t-digest operand 90 | * @return the result of combining left and right digests 91 | */ 92 | def ++(that: TDigest): TDigest = TDigest.combine(this, that, this.delta, this.maxDiscrete) 93 | 94 | // This is most of 'algorithm 1', except for re-clustering which is factored out to avoid 95 | // recursive calls during a reclustering phase 96 | private def update(x: Double, w: Double) = { 97 | require(w > 0.0, "data weight must be > 0") 98 | 99 | if (nclusters == 0) { 100 | // our map is empty, so insert this pair as the first cluster 101 | TDigest(delta, maxDiscrete, nclusters + 1, clusters + (x -> w)) 102 | } else { 103 | // Get the current cluster nearest to incoming (x) 104 | val (c, m, psum) = clusters.nearTD(x) 105 | if (x == c) { 106 | // data landed on an existing cluster: increment that cluster's mass directly 107 | TDigest(delta, maxDiscrete, nclusters, clusters.update(c, c, m + w)) 108 | } else { 109 | val M = clusters.sum 110 | val q = (psum + m / 2.0) / M 111 | val ub = M * delta * q * (1.0 - q) 112 | 113 | val dm = math.min(w, math.max(0.0, ub - m)) 114 | val rm = w - dm 115 | 116 | val tClust = if (dm > 0.0) { 117 | val nm = m + dm 118 | val dc = dm * (x - c) / nm 119 | clusters.update(c, c + dc, nm) 120 | } else clusters 121 | 122 | val uClust = if (rm > 0.0) tClust + (x -> rm) else tClust 123 | 124 | // return the updated t-digest 125 | TDigest(delta, maxDiscrete, nclusters + (if (rm > 0.0) 1 else 0), uClust) 126 | } 127 | } 128 | } 129 | 130 | /** 131 | * Compute a cumulative probability (CDF) for a numeric value, from the estimated probability 132 | * distribution represented by this t-digest sketch. 133 | * @param x a numeric value 134 | * @return the cumulative probability that a random sample from the distribution is <= x 135 | */ 136 | def cdf[N](x: N)(implicit num: Numeric[N]): Double = clusters.cdf(x) 137 | 138 | /** 139 | * Compute the inverse cumulative probability (inverse-CDF) for a quantile value, from the 140 | * estimated probability distribution represented by this t-digest sketch. 141 | * @param q a quantile value. The value of q is expected to be on interval [0, 1] 142 | * @return the value x such that cdf(x) = q 143 | */ 144 | def cdfInverse[N](q: N)(implicit num: Numeric[N]): Double = clusters.cdfInverse(q) 145 | 146 | /** 147 | * Compute a cumulative probability (CDF) for a numeric value, from the estimated probability 148 | * distribution represented by this t-digest sketch, assuming sketch is "discrete" 149 | * (e.g. if number of clusters <= maxDiscrete setting) 150 | * @param x a numeric value 151 | * @return the cumulative probability that a random sample from the distribution is <= x 152 | */ 153 | def cdfDiscrete[N](x: N)(implicit num: Numeric[N]): Double = 154 | clusters.cdfDiscrete(x) 155 | 156 | /** 157 | * Compute the inverse cumulative probability (inverse-CDF) for a quantile value, from the 158 | * estimated probability distribution represented by this t-digest sketch, 159 | * assuming the sketch is "discrete" (e.g. if number of clusters <= maxDiscrete setting) 160 | * @param q a quantile value. The value of q is expected to be on interval [0, 1] 161 | * @return the smallest value x such that q <= cdf(x) 162 | */ 163 | def cdfDiscreteInverse[N](q: N)(implicit num: Numeric[N]): Double = 164 | clusters.cdfDiscreteInverse(q) 165 | 166 | /** 167 | * Perform a random sampling from the distribution as sketched by this t-digest, in 168 | * "probability density" mode. 169 | * @return A random number sampled from the sketched distribution 170 | * @note uses the inverse transform sampling method 171 | */ 172 | def samplePDF: Double = clusters.cdfInverse(Random.nextDouble) 173 | 174 | /** 175 | * Perform a random sampling from the distribution as sketched by this t-digest, in 176 | * "probability mass" (i.e. discrete) mode. 177 | * @return A random number sampled from the sketched distribution 178 | * @note uses the inverse transform sampling method 179 | */ 180 | def samplePMF: Double = clusters.cdfDiscreteInverse(Random.nextDouble) 181 | 182 | /** 183 | * Perform a random sampling from the distribution as sketched by this t-digest, 184 | * using "discrete" (PMF) mode if the number of clusters <= maxDiscrete setting, 185 | * and "density" (PDF) mode otherwise. 186 | * @return A random number sampled from the sketched distribution 187 | * @note uses the inverse transform sampling method 188 | */ 189 | def sample: Double = if (nclusters <= maxDiscrete) samplePMF else samplePDF 190 | } 191 | 192 | /** Factory functions for TDigest */ 193 | object TDigest { 194 | import scala.language.higherKinds 195 | import scala.collection.SeqLike 196 | import scala.collection.generic.CanBuildFrom 197 | 198 | /** 199 | * Default value for a t-digest delta parameter. The number of clusters varies, roughly, as 200 | * about (50/delta), when data are presented in random order 201 | * (it may grow larger if data are not presented randomly). The default corresponds to 202 | * an expected number of clusters of about 100. 203 | */ 204 | val deltaDefault: Double = (50.0 / 100.0) // delta * E[clusters] ~ 50 205 | 206 | /** 207 | * The t-digest algorithm will re-cluster itself whenever its number of clusters exceeds 208 | * (K/delta). This value is set such that the threshold is about 10x the heuristically 209 | * expected number of clusters for the user-specified delta value. Generally the number of 210 | * clusters will only trigger the corresponding re-clustering threshold when data are being 211 | * presented in a non-random order. 212 | */ 213 | val K: Double = 10.0 * 50.0 214 | 215 | /** 216 | * Obtain an empty t-digest 217 | * @param delta a sketch resolution parameter. 218 | * @param maxDiscrete sketch in discrete distribution mode up to this number of 219 | * unique values. Defaults to zero; normal continuous mode. 220 | * @note Smaller values of delta yield sketches with more clusters, and higher resolution 221 | * @note The expected number of clusters will vary (roughly) as (50/delta) 222 | */ 223 | def empty(delta: Double = deltaDefault, maxDiscrete: Int = 0): TDigest = { 224 | require(delta > 0.0, s"delta was not > 0") 225 | require(maxDiscrete >= 0, s"maxDiscrete was not >= 0") 226 | TDigest(delta, maxDiscrete, 0, TDigestMap.empty) 227 | } 228 | 229 | /** 230 | * Sketch some data with a t-digest 231 | * @param data The data elements to sketch 232 | * @param delta The sketch resolution parameter. 233 | * @param maxDiscrete sketch in discrete distribution mode up to this number of 234 | * unique values. Defaults to zero; normal continuous mode. 235 | * @return A t-digest sketch of the input data 236 | * @note Smaller values of delta yield sketches with more clusters, and higher resolution 237 | * @note The expected number of clusters will vary (roughly) as (50/delta) 238 | */ 239 | def sketch[N]( 240 | data: TraversableOnce[N], 241 | delta: Double = deltaDefault, 242 | maxDiscrete: Int = 0)(implicit num: Numeric[N]): TDigest = { 243 | require(delta > 0.0, s"delta was not > 0") 244 | require(maxDiscrete >= 0, s"maxDiscrete was not >= 0") 245 | val td = data.foldLeft(empty(delta, maxDiscrete))((c, e) => c + e) 246 | TDigest.shuffle(td.clusters.toVector).foldLeft(empty(delta, maxDiscrete))((c, e) => c + e) 247 | } 248 | 249 | /** 250 | * Combine two t-digests to yield a new digest 251 | * @param ltd the left-hand t-digest operand 252 | * @param rtd the right hand t-digest 253 | * @param delta a sketch resolution parameter. 254 | * @param maxDiscrete sketch in discrete distribution mode up to this number of 255 | * unique values. Defaults to zero; normal continuous mode. 256 | * @return the sum of left and right digests, defined as their aggregation 257 | * @note This operation satisfies a Semigroup law, with the caveat 258 | * that it is only "statistically" associative: d1++(d2++d3) will be statistically 259 | * similar to (d1++d2)++d3, but rarely identical. 260 | */ 261 | def combine(ltd: TDigest, rtd: TDigest, 262 | delta: Double = deltaDefault, 263 | maxDiscrete: Int = 0): TDigest = { 264 | if (ltd.nclusters <= 1 && rtd.nclusters > 1) combine(rtd, ltd, delta, maxDiscrete) 265 | else if (rtd.nclusters == 0) ltd 266 | else if (rtd.nclusters == 1) { 267 | // handle the singleton RHS case specially to prevent quadratic catastrophe when 268 | // it is being used in the Aggregator use case 269 | val d = rtd.clusters.asInstanceOf[tdmap.tree.INodeTD].data 270 | ltd + ((d.key, d.value)) 271 | } else { 272 | // insert clusters from largest to smallest 273 | (ltd.clusters.toVector ++ rtd.clusters.toVector).sortWith((a, b) => a._2 > b._2) 274 | .foldLeft(empty(delta, maxDiscrete))((d, e) => d + e) 275 | } 276 | } 277 | 278 | // Shuffle a sequence in a referentially-transparent way: pseudo-randomly, but with a random 279 | // seed that is a function of the sequence argument. 280 | private[sketches] def shuffle[E, S[X] <: SeqLike[X, S[X]]]( 281 | seq: S[E])(implicit cbf: CanBuildFrom[S[E], E, S[E]]) = 282 | if (seq.length <= 1) seq 283 | else { 284 | val seed = scala.util.hashing.MurmurHash3.productHash((seq(0), seq(1), seq.length)) 285 | (new scala.util.Random(seed)).shuffle(seq) 286 | } 287 | } 288 | -------------------------------------------------------------------------------- /src/main/scala/org/isarnproject/sketches/tdmap/TDigestMap.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016-2018 Erik Erlandson 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package org.isarnproject.sketches.tdmap 18 | 19 | import math.Numeric 20 | 21 | import scala.collection.SortedMap 22 | 23 | import org.isarnproject.algebraAPI.{ MonoidAPI => Monoid, AggregatorAPI => Aggregator } 24 | 25 | import org.isarnproject.collections.mixmaps.increment._ 26 | import org.isarnproject.collections.mixmaps.prefixsum._ 27 | import org.isarnproject.collections.mixmaps.nearest._ 28 | 29 | object tree { 30 | import org.isarnproject.collections.mixmaps.redblack.tree._ 31 | import org.isarnproject.collections.mixmaps.ordered._ 32 | import org.isarnproject.collections.mixmaps.ordered.tree.DataMap 33 | import org.isarnproject.collections.mixmaps.increment.tree._ 34 | import org.isarnproject.collections.mixmaps.prefixsum.tree._ 35 | import org.isarnproject.collections.mixmaps.nearest.tree._ 36 | 37 | /** Base class of tree node for a TDigestMap */ 38 | trait NodeTD extends NodePS[Double, Double, Double] 39 | with NodeInc[Double, Double] with NodeNearMap[Double, Double] { 40 | 41 | /** 42 | * Obtain a "mass cover": two adjacent nodes in the tree such that the cumulative mass 43 | * of the left node is <= (m) and the cumulative mass of the right node is > (m) 44 | * @param m The target mass to cover between two adjacent nodes 45 | * @return a Cover instance with the left and right covering tree nodes. If the (m) 46 | * was < the mass of the left-most tree node, the left cover value will be None. Similarly 47 | * if the mass was >= the cumulative mass of the right-most node (equivalent to sum of all 48 | * node masses in the tree), then the right cover value will be None. 49 | */ 50 | final def mCover(m: Double) = mcov(m, 0.0, Cover[INodeTD](None, None)) 51 | 52 | private[tree] def mcov(m: Double, psum: Double, cov: Cover[INodeTD]): Cover[INodeTD] 53 | 54 | // Find the cluster whose prefix sum is the least upper bound of mass 'm' 55 | // domain specific to t-digest algorithms 56 | private[sketches] final def keyPFSLUB(m: Double) = this match { 57 | case _: LNodeTD => Double.NaN 58 | case _ if (m < 0.0 || m > this.pfs) => Double.NaN 59 | case _ if (m == 0.0) => this.nodeMin.get.asInstanceOf[INodeTD].data.key 60 | case _ => this.kpl(m, 0.0) 61 | } 62 | 63 | // recursive implementation of keyPFSLUB 64 | private[tree] def kpl(m: Double, psum: Double): Double 65 | 66 | // obtains the nearest cluster to 'x'. Returns the cluster (location, mass, prefix-sum) 67 | private[sketches] final def nearTD(x: Double): (Double, Double, Double) = ntd(x, 0.0) 68 | 69 | // recursive implementation for nearTD 70 | private[tree] def ntd(x: Double, psum: Double): (Double, Double, Double) 71 | 72 | // recursive implementation of 'update' method 73 | private[tdmap] def upd(x0: Double, x: Double, m: Double): Node[Double] 74 | } 75 | 76 | trait LNodeTD extends NodeTD 77 | with LNodePS[Double, Double, Double] with LNodeInc[Double, Double] 78 | with LNodeNearMap[Double, Double] { 79 | final def mcov(m: Double, psum: Double, cov: Cover[INodeTD]) = cov 80 | final def kpl(m: Double, psum: Double) = Double.NaN 81 | final def ntd(x: Double, psum: Double) = (Double.NaN, Double.NaN, Double.NaN) 82 | final def upd(x0: Double, x: Double, m: Double) = 83 | throw new Exception("If this exception threw, there is a bug in this code") 84 | } 85 | 86 | trait INodeTD extends NodeTD 87 | with INodePS[Double, Double, Double] with INodeInc[Double, Double] 88 | with INodeNearMap[Double, Double] { 89 | val lsub: NodeTD 90 | val rsub: NodeTD 91 | 92 | final def mcov(m: Double, psum: Double, cov: Cover[INodeTD]) = { 93 | if (m < psum + lsub.pfs) { 94 | lsub match { 95 | case n: INodeTD => 96 | lsub.mcov(m, psum, cov.copy(r = Some(n.nodeMax.get.asInstanceOf[INodeTD]))) 97 | case _ => cov.copy(r = Some(this)) 98 | } 99 | } else { 100 | val t = psum + lsub.pfs + data.value 101 | if (m >= t) rsub.mcov(m, t, cov.copy(l = Some(this))) 102 | else { 103 | lsub match { 104 | case n: INodeTD => Cover(Some(lsub.nodeMax.get.asInstanceOf[INodeTD]), Some(this)) 105 | case _ => cov.copy(r = Some(this)) 106 | } 107 | } 108 | } 109 | } 110 | 111 | final def kpl(m: Double, psum: Double) = { 112 | val lb = psum + lsub.pfs 113 | val ub = lb + data.value 114 | if (m > ub) { 115 | rsub.kpl(m, ub) 116 | } else if (m > lb) { 117 | data.key 118 | } else { 119 | lsub.kpl(m, psum) 120 | } 121 | } 122 | 123 | final def ntd(x: Double, psum: Double) = { 124 | if (x < data.key) { 125 | lsub match { 126 | case ls: INodeTD => { 127 | if (x <= ls.kmax) ls.ntd(x, psum) 128 | else { 129 | val (dk, ldk) = (math.abs(x - data.key), math.abs(x - ls.kmax)) 130 | if (dk <= ldk) (data.key, data.value, psum + lsub.pfs) 131 | else { 132 | val n = ls.node(ls.kmax).get.asInstanceOf[INodeTD] 133 | (n.data.key, n.data.value, psum + lsub.pfs - n.data.value) 134 | } 135 | } 136 | } 137 | case _ => (data.key, data.value, psum + lsub.pfs) 138 | } 139 | } else if (x > data.key) { 140 | rsub match { 141 | case rs: INodeTD => { 142 | if (x >= rs.kmin) rs.ntd(x, psum + lsub.pfs + data.value) 143 | else { 144 | val (dk, rdk) = (math.abs(x - data.key), math.abs(x - rs.kmin)) 145 | if (dk <= rdk) (data.key, data.value, psum + lsub.pfs) 146 | else { 147 | val n = rs.node(rs.kmin).get.asInstanceOf[INodeTD] 148 | (n.data.key, n.data.value, psum + lsub.pfs + data.value) 149 | } 150 | } 151 | } 152 | case _ => (data.key, data.value, psum + lsub.pfs) 153 | } 154 | } else (data.key, data.value, psum + lsub.pfs) 155 | } 156 | 157 | final def upd(x0: Double, x: Double, m: Double) = 158 | if (color == R) { 159 | if (x0 < data.key) rNode(data, lsub.upd(x0, x, m), rsub) 160 | else if (x0 > data.key) rNode(data, lsub, rsub.upd(x0, x, m)) 161 | else { 162 | val d = new DataMap[Double, Double] { 163 | val key = x 164 | val value = m 165 | } 166 | rNode(d, lsub, rsub) 167 | } 168 | } else { 169 | // We know we are directly replacing a node, so no need to call balance() 170 | // in the case of black nodes. This is quite a bit faster. \o/ 171 | if (x0 < data.key) bNode(data, lsub.upd(x0, x, m), rsub) 172 | else if (x0 > data.key) bNode(data, lsub, rsub.upd(x0, x, m)) 173 | else { 174 | val d = new DataMap[Double, Double] { 175 | val key = x 176 | val value = m 177 | } 178 | bNode(d, lsub, rsub) 179 | } 180 | } 181 | } 182 | } 183 | 184 | import tree._ 185 | 186 | object infra { 187 | import org.isarnproject.collections.mixmaps.redblack.tree._ 188 | import org.isarnproject.collections.mixmaps.ordered.tree.DataMap 189 | 190 | object tdmapMonoid extends Monoid[Double] { 191 | def empty = 0.0 192 | def combine(x: Double, y: Double) = x + y 193 | def combineAll(as: TraversableOnce[Double]) = as.fold(0.0)(_ + _) 194 | def combineAllOption(as: TraversableOnce[Double]) = 195 | if (as.isEmpty) None else Some(combineAll(as)) 196 | } 197 | 198 | object tdmapAggregator extends Aggregator[Double, Double] { 199 | def monoid = tdmapMonoid 200 | def lff = (m: Double, d: Double) => m + d 201 | def mf = (d: Double) => d 202 | def aggregate(as: TraversableOnce[Double]) = as.fold(0.0)(_ + _) 203 | } 204 | 205 | /** Dependency injection class for TDigestMap */ 206 | class Inject extends Serializable { 207 | // Typeclasses corresponding to "regular real numbers": 208 | val keyOrdering = implicitly[Numeric[Double]] 209 | 210 | val valueMonoid = tdmapMonoid 211 | 212 | val prefixAggregator = tdmapAggregator 213 | 214 | def iNode(clr: Color, dat: Data[Double], ls: Node[Double], rs: Node[Double]) = 215 | new Inject with INodeTD with TDigestMap { 216 | // INode 217 | val color = clr 218 | val lsub = ls.asInstanceOf[NodeTD] 219 | val rsub = rs.asInstanceOf[NodeTD] 220 | val data = dat.asInstanceOf[DataMap[Double, Double]] 221 | // INodePS 222 | val prefix = prefixAggregator.lff( 223 | prefixAggregator.monoid.combine(lsub.pfs, rsub.pfs), data.value) 224 | // INodeNear 225 | val kmin = lsub match { 226 | case n: INodeTD => n.kmin 227 | case _ => data.key 228 | } 229 | val kmax = rsub match { 230 | case n: INodeTD => n.kmax 231 | case _ => data.key 232 | } 233 | } 234 | } 235 | 236 | } 237 | 238 | import infra._ 239 | 240 | /** 241 | * The tree-backed map object a TDigest uses to store and update its clusters. TDigestMap 242 | * inherits functionality for value increment, prefix-sum and nearest-neighbor queries. 243 | */ 244 | sealed trait TDigestMap extends SortedMap[Double, Double] with NodeTD 245 | with IncrementMapLike[Double, Double, INodeTD, TDigestMap] 246 | with PrefixSumMapLike[Double, Double, Double, INodeTD, TDigestMap] 247 | with NearestMapLike[Double, Double, INodeTD, TDigestMap] { 248 | 249 | override def empty = TDigestMap.empty 250 | 251 | private def m1m2(c1: Double, tm1: Double, c2: Double, tm2: Double) = { 252 | val s = this.prefixSum(c1, open = true) 253 | val d1 = if (c1 == this.keyMin.get) 0.0 else tm1 / 2.0 254 | val m1 = s + d1 255 | val m2 = m1 + (tm1 - d1) + (if (c2 == this.keyMax.get) tm2 else tm2 / 2.0) 256 | (m1, m2) 257 | } 258 | 259 | // This updates an existing cluster with a new location and mass. It does this 260 | // efficiently by taking advantage of the knowledge that (a) this kind of update 261 | // never changes the key ordering, and therefore that (b) this operation can 262 | // always directly replace an existing node, without otherwise changing the topology 263 | // of the tree. Clearly, this is a domain-dependent method, and not exposed to the 264 | // public API 265 | private[sketches] def update(x0: Double, x: Double, m: Double): TDigestMap = 266 | this.upd(x0, x, m).asInstanceOf[TDigestMap] 267 | 268 | /** Compute the CDF for a value, using piece-wise linear between clusters */ 269 | def cdf[N](xx: N)(implicit num: Numeric[N]) = { 270 | val x = num.toDouble(xx) 271 | this.coverR(x) match { 272 | case Cover(Some((c1, tm1)), Some((c2, tm2))) => { 273 | val (m1, m2) = m1m2(c1, tm1, c2, tm2) 274 | val m = m1 + (x - c1) * (m2 - m1) / (c2 - c1) 275 | // Clipping to [m1,m2] corrects numeric precision errors that can cause non-monotonic quirks 276 | math.min(m2, math.max(m1, m)) / this.sum 277 | } 278 | case Cover(Some(_), None) => 1.0 279 | case _ => 0.0 280 | } 281 | } 282 | 283 | def cdfDiscrete[N](xx: N)(implicit num: Numeric[N]) = { 284 | if (this.isEmpty) 0.0 else { 285 | val x = num.toDouble(xx) 286 | this.prefixSum(x) / this.sum 287 | } 288 | } 289 | 290 | def cdfDiscreteInverse[N](qq: N)(implicit num: Numeric[N]) = { 291 | val q = num.toDouble(qq) 292 | keyPFSLUB(q * this.sum) 293 | } 294 | 295 | /** 296 | * Compute the inverse-CDF from a given quantile on interval [0, 1], using piecewise linear 297 | * interpolation between clusters 298 | */ 299 | def cdfInverse[N](qq: N)(implicit num: Numeric[N]) = { 300 | def cdfI(m: Double, c1: Double, tm1: Double, c2: Double, tm2: Double) = { 301 | val (m1, m2) = m1m2(c1, tm1, c2, tm2) 302 | val x = c1 + (m - m1) * (c2 - c1) / (m2 - m1) 303 | // Clipping to [c1,c2] corrects numeric precision errors that can cause non-monotonic quirks 304 | math.min(c2, math.max(c1, x)) 305 | } 306 | 307 | val q = num.toDouble(qq) 308 | if (q < 0.0 || q > 1.0) Double.NaN 309 | else { 310 | val m = q * this.sum 311 | this.mCover(m).map(n => (n.data.key, n.data.value)) match { 312 | case Cover(Some((c1, tm1)), Some((c2, tm2))) => cdfI(m, c1, tm1, c2, tm2) 313 | case Cover(None, Some((c, _))) => this.coverR(c) match { 314 | case Cover(Some((c1, tm1)), Some((c2, tm2))) => cdfI(m, c1, tm1, c2, tm2) 315 | case _ => Double.NaN 316 | } 317 | case Cover(Some((c, _)), None) => c 318 | case _ => Double.NaN 319 | } 320 | } 321 | } 322 | 323 | override def toString = 324 | "TDigestMap(" + 325 | iterator.zip(prefixSumsIterator()) 326 | .map(x => s"${x._1._1} -> (${x._1._2}, ${x._2})").mkString(", ") + 327 | ")" 328 | } 329 | 330 | /** factory functions for TDigestMap */ 331 | object TDigestMap { 332 | /** Obtain an empty TDigestMap instance */ 333 | def empty: TDigestMap = new Inject with LNodeTD with TDigestMap 334 | } 335 | -------------------------------------------------------------------------------- /src/site/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |
10 | Scala API documentation 11 |
12 | Java API documentation 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/test/scala/org/isarnproject/sketches/TDigestTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016-2018 Erik Erlandson 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package org.isarnproject.sketches 18 | 19 | import org.scalatest._ 20 | 21 | import org.isarnproject.scalatest.matchers.seq._ 22 | 23 | class TDigestTest extends FlatSpec with Matchers { 24 | import org.apache.commons.math3.distribution.RealDistribution 25 | import org.apache.commons.math3.distribution.IntegerDistribution 26 | 27 | val seed = 235711L 28 | scala.util.Random.setSeed(seed) 29 | 30 | val ss = 100000 31 | val delta = 50.0 / 1000 32 | 33 | val maxD = 0.05 34 | val maxDI = 0.1 35 | 36 | def testTDvsDist(td: TDigest, dist: RealDistribution, stdv: Double): Boolean = { 37 | val xmin = td.clusters.keyMin.get 38 | val xmax = td.clusters.keyMax.get 39 | val step = (xmax - xmin) / 1000 40 | val d = (xmin to xmax by step).iterator 41 | .map(x => math.abs(td.cdf(x) - dist.cumulativeProbability(x))).max 42 | 43 | val dInv = (0.01 to 0.99 by 0.01).iterator 44 | .map(x => math.abs(td.cdfInverse(x) - dist.inverseCumulativeProbability(x))).max / stdv 45 | 46 | val pass = d <= maxD && dInv <= maxDI 47 | if (!pass) Console.err.println(s"testTDvsDist failure: d= $d dInv= $dInv") 48 | pass 49 | } 50 | 51 | def testSamplingPDF(td: TDigest, dist: RealDistribution): Boolean = { 52 | val tdSamples = Array.fill(10000) { td.samplePDF } 53 | val distSamples = Array.fill(10000) { dist.sample } 54 | val kst = new org.apache.commons.math3.stat.inference.KolmogorovSmirnovTest() 55 | val d = kst.kolmogorovSmirnovStatistic(tdSamples, distSamples) 56 | val pass = d <= maxD 57 | if (!pass) Console.err.println(s"testSamplingPDF failure: d= $d") 58 | pass 59 | } 60 | 61 | def testSamplingPMF(td: TDigest, dist: IntegerDistribution): Boolean = { 62 | td.nclusters should be <=(td.maxDiscrete) 63 | val tdSamples = Array.fill(10000) { td.samplePMF } 64 | val distSamples = Array.fill(10000) { dist.sample.toDouble } 65 | val kst = new org.apache.commons.math3.stat.inference.KolmogorovSmirnovTest() 66 | val d = kst.kolmogorovSmirnovStatistic(tdSamples, distSamples) 67 | val pass = d <= maxD 68 | if (!pass) Console.err.println(s"testSamplingPDF failure: d= $d") 69 | pass 70 | } 71 | 72 | def testDistribution(dist: RealDistribution, stdv: Double): Boolean = { 73 | dist.reseedRandomGenerator(seed) 74 | 75 | val td = TDigest.sketch(Iterator.fill(ss) { dist.sample }, delta = delta) 76 | 77 | testTDvsDist(td, dist, stdv) && testSamplingPDF(td, dist) 78 | } 79 | 80 | def testMonotoneCDF(dist: RealDistribution): Boolean = { 81 | dist.reseedRandomGenerator(seed) 82 | val td = TDigest.sketch(Iterator.fill(ss) { dist.sample }, delta = delta) 83 | val (xmin, xmax) = (td.clusters.keyMin.get, td.clusters.keyMax.get) 84 | val step = (xmax - xmin) / 100000 85 | val t = (xmin to xmax by step).iterator.map(x => td.cdf(x)).sliding(2).map(w => w(1) - w(0)).min 86 | val pass = t >= 0.0 87 | if (!pass) Console.err.println(s"testMonotoneCDF failure: t= $t") 88 | pass 89 | } 90 | 91 | def testMonotoneCDFI(dist: RealDistribution): Boolean = { 92 | dist.reseedRandomGenerator(seed) 93 | val td = TDigest.sketch(Iterator.fill(ss) { dist.sample }, delta = delta) 94 | val (xmin, xmax) = (0.0, 1.0) 95 | val step = (xmax - xmin) / 100000 96 | val t = (xmin to xmax by step).iterator.map(q => td.cdfInverse(q)).sliding(2).map(w => w(1) - w(0)).min 97 | val pass = t >= 0.0 98 | if (!pass) Console.err.println(s"testMonotoneCDFI failure: t= $t") 99 | pass 100 | } 101 | 102 | def testMonotone(dist: RealDistribution): Boolean = { 103 | testMonotoneCDF(dist) && testMonotoneCDFI(dist) 104 | } 105 | 106 | it should "sketch a uniform distribution" in { 107 | import org.apache.commons.math3.distribution.UniformRealDistribution 108 | val dist = new UniformRealDistribution() 109 | testDistribution(dist, math.sqrt(dist.getNumericalVariance())) should be (true) 110 | } 111 | 112 | it should "sketch a normal distribution" in { 113 | import org.apache.commons.math3.distribution.NormalDistribution 114 | val dist = new NormalDistribution() 115 | testDistribution(dist, math.sqrt(dist.getNumericalVariance())) should be (true) 116 | } 117 | 118 | it should "sketch an exponential distribution" in { 119 | import org.apache.commons.math3.distribution.ExponentialDistribution 120 | val dist = new ExponentialDistribution(1.0) 121 | testDistribution(dist, math.sqrt(dist.getNumericalVariance())) should be (true) 122 | } 123 | 124 | it should "aggregate with another t-digest using ++" in { 125 | import org.apache.commons.math3.distribution.NormalDistribution 126 | val dist = new NormalDistribution() 127 | dist.reseedRandomGenerator(seed) 128 | 129 | val td1 = TDigest.sketch(Iterator.fill(ss) { dist.sample }, delta = delta) 130 | val td2 = TDigest.sketch(Iterator.fill(ss) { dist.sample }, delta = delta) 131 | 132 | testTDvsDist(td1 ++ td2, dist, math.sqrt(dist.getNumericalVariance())) should be (true) 133 | } 134 | 135 | it should "respect monotonic cdf and inverse" in { 136 | import org.apache.commons.math3.distribution.ExponentialDistribution 137 | import org.apache.commons.math3.distribution.NormalDistribution 138 | import org.apache.commons.math3.distribution.UniformRealDistribution 139 | 140 | testMonotone(new UniformRealDistribution()) should be (true) 141 | testMonotone(new ExponentialDistribution(1.0)) should be (true) 142 | testMonotone(new NormalDistribution(0.0, 0.1)) should be (true) 143 | } 144 | 145 | it should "respect maxDiscrete parameter" in { 146 | import org.apache.commons.math3.distribution.GeometricDistribution 147 | val gd = new GeometricDistribution(0.33) 148 | val data = gd.sample(1000000) 149 | val dataUniq = data.distinct.sorted 150 | val kt = dataUniq.map(_.toDouble).toSet 151 | val td = TDigest.sketch(data, maxDiscrete = 50) 152 | val clust = td.clusters 153 | clust.keys.toSet should be (kt) 154 | val D = clust.keys.map { x => td.cdfDiscrete(x) } 155 | .zip(dataUniq.map { k => gd.cumulativeProbability(k) }) 156 | .map { case (p1, p2) => math.abs(p1 - p2) } 157 | .max 158 | (D <= 0.01) should be (true) 159 | testSamplingPMF(td, gd) should be (true) 160 | } 161 | 162 | it should "respect maxDiscrete parameter over ++" in { 163 | import org.apache.commons.math3.distribution.GeometricDistribution 164 | val gd = new GeometricDistribution(0.33) 165 | val tdvec = Vector.fill(10) { TDigest.sketch(gd.sample(100000), maxDiscrete = 50) } 166 | val td = tdvec.reduce(_ ++ _) 167 | val clust = td.clusters 168 | clust.keys.map(_.toInt).map(_.toDouble) should beEqSeq(clust.keys) 169 | val D = clust.keys.map { x => td.cdfDiscrete(x) } 170 | .zip(clust.keys.map(_.toInt).map { k => gd.cumulativeProbability(k) }) 171 | .map { case (p1, p2) => math.abs(p1 - p2) } 172 | .max 173 | (D <= 0.01) should be (true) 174 | testSamplingPMF(td, gd) should be (true) 175 | } 176 | 177 | it should "serialize and deserialize" in { 178 | import org.apache.commons.math3.distribution.NormalDistribution 179 | 180 | import org.isarnproject.scalatest.serde.roundTripSerDe 181 | 182 | val dist = new NormalDistribution() 183 | dist.reseedRandomGenerator(seed) 184 | 185 | val tdo = TDigest.sketch(Iterator.fill(ss) { dist.sample }, delta = delta) 186 | 187 | val tdi = roundTripSerDe(tdo) 188 | 189 | (tdi == tdo) should be (true) 190 | 191 | testTDvsDist(tdi, dist, math.sqrt(dist.getNumericalVariance())) should be (true) 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /src/test/scala/org/isarnproject/sketches/java/JavaTDigestTest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016-2018 Erik Erlandson 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package org.isarnproject.sketches.java 18 | 19 | import org.scalatest._ 20 | 21 | import org.isarnproject.scalatest.matchers.seq._ 22 | 23 | class JavaTDigestTest extends FlatSpec with Matchers { 24 | import org.apache.commons.math3.distribution.RealDistribution 25 | import org.apache.commons.math3.distribution.IntegerDistribution 26 | 27 | val seed = 235711L 28 | scala.util.Random.setSeed(seed) 29 | 30 | val ss = 100000 31 | val delta = 50.0 / 1000 32 | 33 | val maxD = 0.05 34 | val maxDI = 0.1 35 | 36 | def testTDvsDist(td: TDigest, dist: RealDistribution, stdv: Double): Boolean = { 37 | val xmin = td.cent(0) 38 | val xmax = td.cent(td.nclusters - 1) 39 | val step = (xmax - xmin) / 1000 40 | val d = (xmin to xmax by step).iterator 41 | .map(x => math.abs(td.cdf(x) - dist.cumulativeProbability(x))).max 42 | 43 | val dInv = (0.01 to 0.99 by 0.01).iterator 44 | .map(x => math.abs(td.cdfInverse(x) - dist.inverseCumulativeProbability(x))).max / stdv 45 | 46 | val pass = d <= maxD && dInv <= maxDI 47 | if (!pass) Console.err.println(s"testTDvsDist failure: d= $d dInv= $dInv") 48 | pass 49 | } 50 | 51 | def testSamplingPDF(td: TDigest, dist: RealDistribution): Boolean = { 52 | val tdSamples = Array.fill(10000) { td.samplePDF } 53 | val distSamples = Array.fill(10000) { dist.sample } 54 | val kst = new org.apache.commons.math3.stat.inference.KolmogorovSmirnovTest() 55 | val d = kst.kolmogorovSmirnovStatistic(tdSamples, distSamples) 56 | val pass = d <= maxD 57 | if (!pass) Console.err.println(s"testSamplingPDF failure: d= $d") 58 | pass 59 | } 60 | 61 | def testSamplingPMF(td: TDigest, dist: IntegerDistribution): Boolean = { 62 | td.nclusters should be <=(td.maxDiscrete) 63 | val tdSamples = Array.fill(10000) { td.samplePMF } 64 | val distSamples = Array.fill(10000) { dist.sample.toDouble } 65 | val kst = new org.apache.commons.math3.stat.inference.KolmogorovSmirnovTest() 66 | val d = kst.kolmogorovSmirnovStatistic(tdSamples, distSamples) 67 | val pass = d <= maxD 68 | if (!pass) Console.err.println(s"testSamplingPDF failure: d= $d") 69 | pass 70 | } 71 | 72 | def testDistribution(dist: RealDistribution, stdv: Double): Boolean = { 73 | dist.reseedRandomGenerator(seed) 74 | 75 | val td = TDigest.sketch(Array.fill(ss) { dist.sample }, delta) 76 | 77 | testTDvsDist(td, dist, stdv) && testSamplingPDF(td, dist) 78 | } 79 | 80 | def testMonotoneCDF(dist: RealDistribution): Boolean = { 81 | dist.reseedRandomGenerator(seed) 82 | val td = TDigest.sketch(Array.fill(ss) { dist.sample }, delta) 83 | val (xmin, xmax) = (td.cent(0), td.cent(td.nclusters - 1)) 84 | val step = (xmax - xmin) / 100000 85 | val t = (xmin to xmax by step).iterator.map(x => td.cdf(x)).sliding(2).map(w => w(1) - w(0)).min 86 | val pass = t >= 0.0 87 | if (!pass) Console.err.println(s"testMonotoneCDF failure: t= $t") 88 | pass 89 | } 90 | 91 | def testMonotoneCDFI(dist: RealDistribution): Boolean = { 92 | dist.reseedRandomGenerator(seed) 93 | val td = TDigest.sketch(Array.fill(ss) { dist.sample }, delta) 94 | val (xmin, xmax) = (0.0, 1.0) 95 | val step = (xmax - xmin) / 100000 96 | val t = (xmin to xmax by step).iterator.map(q => td.cdfInverse(q)).sliding(2).map(w => w(1) - w(0)).min 97 | val pass = t >= 0.0 98 | if (!pass) Console.err.println(s"testMonotoneCDFI failure: t= $t") 99 | pass 100 | } 101 | 102 | def testMonotone(dist: RealDistribution): Boolean = { 103 | testMonotoneCDF(dist) && testMonotoneCDFI(dist) 104 | } 105 | 106 | it should "sketch a uniform distribution" in { 107 | import org.apache.commons.math3.distribution.UniformRealDistribution 108 | val dist = new UniformRealDistribution() 109 | testDistribution(dist, math.sqrt(dist.getNumericalVariance())) should be (true) 110 | } 111 | 112 | it should "sketch a normal distribution" in { 113 | import org.apache.commons.math3.distribution.NormalDistribution 114 | val dist = new NormalDistribution() 115 | testDistribution(dist, math.sqrt(dist.getNumericalVariance())) should be (true) 116 | } 117 | 118 | it should "sketch an exponential distribution" in { 119 | import org.apache.commons.math3.distribution.ExponentialDistribution 120 | val dist = new ExponentialDistribution(1.0) 121 | testDistribution(dist, math.sqrt(dist.getNumericalVariance())) should be (true) 122 | } 123 | 124 | it should "aggregate with another t-digest using merge method" in { 125 | import org.apache.commons.math3.distribution.NormalDistribution 126 | val dist = new NormalDistribution() 127 | dist.reseedRandomGenerator(seed) 128 | 129 | val td1 = TDigest.sketch(Array.fill(ss) { dist.sample }, delta) 130 | val td2 = TDigest.sketch(Array.fill(ss) { dist.sample }, delta) 131 | 132 | testTDvsDist(TDigest.merge(td1, td2), dist, math.sqrt(dist.getNumericalVariance())) should be (true) 133 | } 134 | 135 | it should "respect monotonic cdf and inverse" in { 136 | import org.apache.commons.math3.distribution.ExponentialDistribution 137 | import org.apache.commons.math3.distribution.NormalDistribution 138 | import org.apache.commons.math3.distribution.UniformRealDistribution 139 | 140 | testMonotone(new UniformRealDistribution()) should be (true) 141 | testMonotone(new ExponentialDistribution(1.0)) should be (true) 142 | testMonotone(new NormalDistribution(0.0, 0.1)) should be (true) 143 | } 144 | 145 | it should "respect maxDiscrete parameter" in { 146 | import org.apache.commons.math3.distribution.GeometricDistribution 147 | val gd = new GeometricDistribution(0.33) 148 | val data = gd.sample(1000000).map(_.toDouble) 149 | val dataUniq = data.distinct.sorted 150 | val kt = dataUniq.map(_.toDouble).toSet 151 | val td = TDigest.sketch(data, delta, 50) 152 | val clust = td.cent 153 | clust.toSet should be (kt) 154 | val D = clust.map { x => td.cdfDiscrete(x) } 155 | .zip(dataUniq.map { k => gd.cumulativeProbability(k.toInt) }) 156 | .map { case (p1, p2) => math.abs(p1 - p2) } 157 | .max 158 | (D <= 0.01) should be (true) 159 | testSamplingPMF(td, gd) should be (true) 160 | } 161 | 162 | it should "respect maxDiscrete parameter over merge" in { 163 | import org.apache.commons.math3.distribution.GeometricDistribution 164 | val gd = new GeometricDistribution(0.33) 165 | val tdvec = Vector.fill(10) { TDigest.sketch(gd.sample(100000).map(_.toDouble), delta, 50) } 166 | val td = tdvec.reduce((a, b) => TDigest.merge(a, b)) 167 | val clust = td.cent 168 | clust.map(_.toInt).map(_.toDouble).toVector should beEqSeq(clust.toVector) 169 | val D = clust.map { x => td.cdfDiscrete(x) } 170 | .zip(clust.map(_.toInt).map { k => gd.cumulativeProbability(k) }) 171 | .map { case (p1, p2) => math.abs(p1 - p2) } 172 | .max 173 | (D <= 0.01) should be (true) 174 | testSamplingPMF(td, gd) should be (true) 175 | } 176 | 177 | it should "support copy constructor" in { 178 | import org.apache.commons.math3.distribution.NormalDistribution 179 | 180 | val dist = new NormalDistribution() 181 | dist.reseedRandomGenerator(seed) 182 | val data = Array.fill(ss) { dist.sample } 183 | val td1 = TDigest.sketch(data, delta) 184 | val td2 = new TDigest(td1) 185 | (td2.equals(td1)) should be (true) 186 | (td1.equals(td2)) should be (true) 187 | 188 | // add more data and re-check equality to ensure 189 | // that all state for future updates was correctly copied 190 | for { x <- data } { 191 | td1.update(x) 192 | td2.update(x) 193 | } 194 | (td2.equals(td1)) should be (true) 195 | (td1.equals(td2)) should be (true) 196 | } 197 | 198 | def testTDClose(td1: TDigest, td2: TDigest, eps: Double = 1e-6): Unit = { 199 | td1.getCompression() should be (td2.getCompression()) 200 | td1.getMaxDiscrete() should be (td2.getMaxDiscrete()) 201 | td1.size() should be (td2.size()) 202 | td1.mass() should be (td2.mass() +- eps) 203 | for { j <- 0 until td1.size() } { 204 | td1.getCentUnsafe()(j) should be (td2.getCentUnsafe()(j) +- eps) 205 | td1.getMassUnsafe()(j) should be (td2.getMassUnsafe()(j) +- eps) 206 | td1.getFTUnsafe()(1 + j) should be (td2.getFTUnsafe()(1 + j) +- eps) 207 | } 208 | } 209 | 210 | it should "support dser constructor" in { 211 | import java.util.Arrays; 212 | import org.apache.commons.math3.distribution.NormalDistribution 213 | 214 | val eps = 1e-9 215 | 216 | val dist = new NormalDistribution() 217 | dist.reseedRandomGenerator(seed) 218 | val data = Array.fill(ss) { dist.sample } 219 | 220 | // test constructing empty t-digests 221 | val td1 = new TDigest(0.5, 0, Array.empty[Double], Array.empty[Double]) 222 | val td2 = new TDigest( 223 | td1.getCompression(), 224 | td1.getMaxDiscrete(), 225 | Arrays.copyOf(td1.getCentUnsafe(), td1.size()), 226 | Arrays.copyOf(td1.getMassUnsafe(), td1.size()) 227 | ) 228 | testTDClose(td1, td2, eps) 229 | 230 | // test sketching from empty state 231 | for { x <- data } { 232 | td1.update(x) 233 | td2.update(x) 234 | } 235 | testTDClose(td1, td2, eps) 236 | 237 | // copy from non-empty state 238 | val td3 = new TDigest( 239 | td1.getCompression(), 240 | td1.getMaxDiscrete(), 241 | Arrays.copyOf(td1.getCentUnsafe(), td1.size()), 242 | Arrays.copyOf(td1.getMassUnsafe(), td1.size()) 243 | ) 244 | testTDClose(td1, td3, eps) 245 | 246 | // test from non-empty state 247 | for { x <- data } { 248 | td1.update(x) 249 | td3.update(x) 250 | } 251 | testTDClose(td1, td3, eps) 252 | } 253 | 254 | it should "serialize and deserialize" in { 255 | import org.apache.commons.math3.distribution.NormalDistribution 256 | 257 | import org.isarnproject.scalatest.serde.roundTripSerDe 258 | 259 | val dist = new NormalDistribution() 260 | dist.reseedRandomGenerator(seed) 261 | 262 | val tdo = TDigest.sketch(Array.fill(ss) { dist.sample }, delta) 263 | 264 | val tdi = roundTripSerDe(tdo) 265 | 266 | (tdi.equals(tdo)) should be (true) 267 | 268 | testTDvsDist(tdi, dist, math.sqrt(dist.getNumericalVariance())) should be (true) 269 | } 270 | } 271 | --------------------------------------------------------------------------------