├── .docker └── config.json ├── .gitignore ├── AUTHORS.md ├── LICENSE ├── README.md ├── build.gradle ├── docker ├── Dockerfile └── version.sh ├── docs └── media │ ├── perf-comparison.png │ └── splicer-arch.png ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── libs └── tsdb-2.3.0-SNAPSHOT.jar ├── pom.xml ├── settings.gradle └── src ├── main ├── java │ └── com │ │ └── turn │ │ └── splicer │ │ ├── Config.java │ │ ├── ConfigServlet.java │ │ ├── Const.java │ │ ├── HttpWorker.java │ │ ├── Splicer.java │ │ ├── SplicerMain.java │ │ ├── SplicerServlet.java │ │ ├── SuggestHttpWorker.java │ │ ├── SuggestServlet.java │ │ ├── cache │ │ └── JedisClient.java │ │ ├── hbase │ │ ├── MetricLookupException.java │ │ ├── MetricsCache.java │ │ ├── RegionCheckException.java │ │ ├── RegionChecker.java │ │ └── RegionUtil.java │ │ ├── merge │ │ ├── MergeException.java │ │ ├── QueryAwareResultsMerger.java │ │ ├── ResultsMerger.java │ │ └── TsdbResult.java │ │ └── tsdbutils │ │ ├── DateTime.java │ │ ├── Functions.java │ │ ├── JSON.java │ │ ├── RateOptions.java │ │ ├── SplicerQueryRunner.java │ │ ├── SplicerUtils.java │ │ ├── TSSubQuery.java │ │ ├── TsQuery.java │ │ ├── TsQuerySerializer.java │ │ └── expression │ │ ├── BadNumberException.java │ │ ├── EndpointAligningAggregationIterator.java │ │ ├── ExprReader.java │ │ ├── Expression.java │ │ ├── ExpressionFactory.java │ │ ├── ExpressionTree.java │ │ ├── ExpressionTreeWorker.java │ │ ├── Expressions.java │ │ ├── QueryRunnerWorker.java │ │ ├── SeekableViewDataPointImpl.java │ │ └── parser │ │ ├── ParseException.java │ │ ├── SimpleCharStream.java │ │ ├── SyntaxChecker.java │ │ ├── SyntaxCheckerConstants.java │ │ ├── SyntaxCheckerTest.java │ │ ├── SyntaxCheckerTokenManager.java │ │ ├── Tester.java │ │ ├── Token.java │ │ ├── TokenMgrError.java │ │ ├── parser.jj │ │ └── run_javacc.sh └── resources │ ├── logback.xml │ └── splicer.conf └── test └── java └── com └── turn └── splicer ├── merge ├── QueryAwareResultsMergerTest.java └── TsdbResultTest.java └── tsdbutils ├── TsQueryDeserializeTest.java ├── TsdbResultMergerTests.java └── expression ├── ExprReaderTest.java ├── ExpressionTreeTest.java ├── FunctionsTest.java ├── InterpolationTest.java └── MovingAverageTest.java /.docker/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auths": { 3 | "docker.turn.com": { 4 | "auth": "amVua2luc190c2RiOkw1KT1wOVNRVTUkcyRRRmU=", 5 | "email": "foo@bar.com" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | /target/ 3 | 4 | # Temporary dependency on turn codebase 5 | /lib/turn.jar 6 | 7 | # No need to check in VERSION parameters to git 8 | docker/VERSION 9 | 10 | # Eclipse files we don't want to track 11 | /.metadata 12 | /.pydevproject* 13 | /.settings/ 14 | /.checkstyle 15 | 16 | # Intellij files we don't want to track 17 | /.idea/ 18 | *.iml 19 | *.eml 20 | 21 | # tmp folder not to track 22 | /tmp/ 23 | 24 | # Ignore various temp files 25 | *.swp 26 | *~ 27 | .DS_Store 28 | 29 | ## Gradle directories 30 | .gradle/ 31 | build/ 32 | 33 | ## Docker Directory 34 | docker/*.jar 35 | docker/resources 36 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | h1. Corporate Authors 2 | 3 | - Turn Inc 4 | - Arjun Satish 5 | - Brian Peltz 6 | - Jonathan Creasy 7 | 8 | h1. Individual Authors 9 | - __none yet__ *your name here* 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Splicer: Faster Queries on TSDB 2 | 3 | Splicer is a tool built at [Turn Inc](http://www.turn.com/) to quickly retrieve metrics from [TSDB](http://opentsdb.net/). TSDB is generally efficient at executing queries, but in our installation we found that for certain queries, especially those which contain many tag combinations or for very long time ranges, the average time to return results can be in order of few minutes or even tens of minutes. When you want metrics to diagnose issues quickly, this can be a long wait. 4 | 5 | Splicer attempts to solve this problem by slicing long range time intervals in queries into smaller chunks and parallely executing them on an array of TSDBs (as opposed to one). The results from these slices are cached in a Redis instance. Future queries on the same metrics are looked up from this cache, and hence can be much faster. Here is a graph showing the time taken by TSDB and Splicer (with and without caching) to retrieve results for a query with varying time ranges (4 hours to 48 hours). 6 | 7 | ![Performance Comparison](docs/media/perf-comparison.png) 8 | 9 | ## Design 10 | 11 | TSDB stores its metrics in HBase. HBase is a persistent replicated distributed sorted map, where data is physically stored in [Region Servers](http://hbase.apache.org/0.94/book/regions.arch.html). TSDB has a [clever way](http://opentsdb.net/docs/build/html/user_guide/backends/hbase.html) to construct fixed size keys for the metrics it ingests. All data from the same metric are co-located with each other. More often than not, a query will find all its data points in a region. 12 | 13 | Typically, the TSDB which executes a query is not co-located with the region server. This causes all the metric data to be sent over the network to the TSDB process. If this data set is large (we have noticed metric data of over 5 GB being transferred over the network per query), it will incur significant network resources. Moreover, downsampling and aggregating this stream will also be significant. If multiple queries are sent to this TSD, they will all compete for the same network and CPU resources. 14 | 15 | ## Splicer 16 | 17 | Splicer is used to overcome these performance issues with TSDB. Each TSDB query is sliced up into multiple pieces by time, and executed independently on different TSDs. We recommend co-locating multiple TSDBs with each region server in your HBase cluster (At Turn, we have around 10-12 TSDs per HBase node). For each slice, splicer can determine which region server holds the metrics, and execute that slice on a TSD available on that server. This reduces the network dependency, and also makes your query faster as data has to pass from one process to another across system memory. The returned results are stored in Redis (which acts as an LRU cache). This enables us to serve future queries faster. Also, the external cache gives us some redundancy across Splicer nodes. The returned results are _spliced_ together to form a final result before returning to the user. 18 | 19 | The figure below shows the main architectural pieces of Splicer. 20 | 21 | ![Splicer Architecture](docs/media/splicer-arch.png) 22 | 23 | In order to over come these problems, we recommend using Splicer in the following manner: 24 | 25 | 1. Co-locate an array of TSDBs with each region server in your HBase cluster. 26 | 2. Use one or more splicer nodes (atleast two is recommended) to receive your original TSDB queries. 27 | 3. Use a **shared** Redis cluster to cache the intermediate results so future queries are faster. 28 | 4. Use Splicer on always-running dashboards ([Grafana](http://grafana.org/), for instance) which lookup multi-day or week long plots. 29 | 30 | ## Configuration 31 | 32 | You can configure splicer with the following options. An example config can be found at src/main/resources/splicer.conf. 33 | 34 | ``` 35 | ## the port on which the splicer listens for HTTP requests 36 | splicer.port = 4245 37 | ``` 38 | 39 | Use the following config to tell Splicer which hosts are running the TSDs, and which ports they are listening on. In this example, there are 3x3 = 9 TSDs available for splicer to delegate queries to. 40 | 41 | ``` 42 | ## hosts on which the TSDs are running 43 | tsd.hosts = data001.turn.com,data2.atl2.turn.com,data3.atl2.turn.com 44 | 45 | ## start and end port for TSDs on data nodes. Start is inclusive, End is exclusive 46 | tsd.start.port = 4242 47 | tsd.end.port = 4245 48 | ``` 49 | 50 | By default, each query slice is sent to one host:port combination. To increase this number, use the following config. This is also useful if you have a load balancer between Splicer and the query nodes. Saves you the trouble of changing the config each time a TSD goes down. 51 | 52 | ``` 53 | tsd.queries.per.port = 10 54 | ``` 55 | 56 | Enable overflow for slices. In order to query our splices, we read more data than what is needed for that time interval (so we get correct results on the boundaries of that slice). In order to grab a bigger chunk of data (1 hour before and 1 hour after), enable this setting: 57 | ``` 58 | ## enable overflow for slices 59 | slice.overflow.enable = false 60 | ``` 61 | 62 | To enable redis caching, and provide it the host and port details for the redis server: 63 | ``` 64 | ## is redis caching enabled (disable this flag if Redis is not available) 65 | caching.enabled = true 66 | 67 | ## redis host:port setups for 68 | caching.hosts = 172.17.42.1:6379 69 | ``` 70 | 71 | Tell Splice where the HBase Zookeeper is: 72 | ``` 73 | ## hbase configuration 74 | hbase.zookeeper.quorum = hbase-zk.turn.com,hbase-zk.turn.com,hbase-zk.turn.com 75 | hbase.znode.parent = /hbase 76 | ``` 77 | 78 | ## Running Splicer 79 | 80 | After setting up the different TSDs along with the Hbase Region Server, run the main class to run Splicer fom (SplicerMain)[https://stash.turn.com/projects/REPORT/repos/tsdb-splicer/browse/src/main/java/com/turn/splicer/SplicerMain.java]. 81 | 82 | You also can use the docker scripts provided in the `docker` directory to build and run containers. 83 | 84 | ## Applying Functions on Time Series 85 | 86 | Besides executing regular TSDB functions, Splicer also gives you the ability to run different functions on TSDB results. For example, you can add two time series, create a new time series which is the sum of two time series divided by a third one. We provide the following functions as of now. 87 | 88 | ### SUM 89 | 90 | Sum two time series 91 | 92 | ``` 93 | sum(sum:1m-avg:proc.stat.cpu.percpu{cpu=1},sum:1m-avg:proc.stat.cpu.percpu{cpu=2}) 94 | ``` 95 | 96 | ### DIFFERENCE 97 | 98 | Substract the second series from the first 99 | 100 | ``` 101 | difference(sum:1m-avg:proc.stat.cpu.percpu{cpu=1},sum:1m-avg:proc.stat.cpu.percpu{cpu=2}) 102 | ``` 103 | 104 | ### MULTIPLY 105 | 106 | Multiply two time series 107 | 108 | ``` 109 | multiply(sum:1m-avg:proc.stat.cpu.percpu{cpu=1},sum:1m-avg:proc.stat.cpu.percpu{cpu=2}) 110 | 111 | ``` 112 | 113 | ### DIVIDE 114 | 115 | Takes two series and divide the first one with the second. 116 | 117 | ``` 118 | divide(sum:1m-avg:proc.stat.cpu.percpu{cpu=1},sum:1m-avg:proc.stat.cpu.percpu{cpu=2}) 119 | ``` 120 | 121 | ### MOVING AVERAGE 122 | 123 | The moving average of a metric over a fixed number of past points, or a time interval. 124 | 125 | ``` 126 | // average over last 10 points 127 | movingAverage(sum:1m-avg:proc.stat.cpu.percpu{cpu=1}, 10) 128 | 129 | // average over last 1 min 130 | movingAverage(sum:1m-avg:proc.stat.cpu.percpu{cpu=1}, '1min') 131 | ``` 132 | 133 | ### HIGHEST CURRENT 134 | 135 | Takes one metric or a wildcard seriesList followed by an integer N. Out of all metrics passed, return only the N metrics with the highest value at the end of the time period specified. 136 | 137 | ``` 138 | // find the top two metrics which have the highest values at the end of the time period specified. 139 | highestCurrent(sum:1m-avg:proc.stat.cpu.percpu{cpu=1|2|3|4}, 2) 140 | 141 | ``` 142 | 143 | ### HIGHEST MAX 144 | 145 | Takes one metric or a wildcard seriesList followed by an integer N. Out of all metrics passed, return only the N metrics with the highest maximum value in the time period specified. 146 | 147 | 148 | ``` 149 | // find the top two metrics which have the highest maximum anywhere in the end of the time period specified. 150 | highestMax(sum:1m-avg:proc.stat.cpu.percpu{cpu=1|2|3|4}, 2) 151 | 152 | ``` 153 | 154 | ### SCALE 155 | 156 | Takes one metric or a wildcard seriesList followed by a constant, and multiplies the datapoint by the constant provided at each point. 157 | 158 | ``` 159 | // multiply each datapoint value with 10 160 | scale(avg:1m-avg:proc.stat.cpu.percpu{cpu=1},10) 161 | 162 | // multiply each datapoint value with 0.0001 163 | scale(avg:1m-avg:proc.stat.cpu.percpu{cpu=1},0.0001) 164 | ``` 165 | 166 | ### ALIAS 167 | 168 | Takes one metric or a wildcard seriesList and a string. Returns the string instead of the metric name in the legend. 169 | 170 | ``` 171 | // quotes not needed around cpu_1 172 | alias(avg:1m-avg:proc.stat.cpu.percpu{cpu=1},cpu_1) 173 | 174 | // every instance of @cpu will be replaced by the value of the tag "cpu". 175 | alias(avg:1m-avg:proc.stat.cpu.percpu{cpu=1},@cpu) 176 | ``` 177 | 178 | ### ABSOLUTE 179 | 180 | Take one or metrics and apply the absolute function to every data point in it. 181 | 182 | ``` 183 | absolute(avg:1m-avg:proc.stat.cpu.percpu{cpu=1}). 184 | 185 | // scale each value by -1 and then return their absolute value 186 | absolute(scale(sum:1m-avg:rate:proc.stat.cpu.percpu{cpu=*},-1)) 187 | ``` 188 | 189 | An example curl command to run such a function on Splicer: 190 | 191 | 192 | ## Running commands: 193 | 194 | Splicer does its best to mimic TSDB's 1.x Query API. For example, you can run a curl command against Splicer: 195 | 196 | ``` 197 | curl 'http://splicer.host.com/splicer/api/query' --data-binary '{"start":1473715620025,"queries":[{"metric":"proc.stat.cpu.percpu","aggregator":"sum","rate":true,"rateOptions":{"counter":false},"downsample":"10m-max","tags":{}}]}' 198 | ``` 199 | 200 | In order to run function expressions, pass in the entire expression in the HTTP Query string (with param name 'x') 201 | 202 | ``` 203 | curl 'http://splicer.host.com/api/query/query?start=1473715143790&x=absolute(scale(sum:1m-avg:rate:proc.stat.cpu.percpu{cpu=*},-1))' 204 | ``` -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'maven' 3 | 4 | group = 'com.turn' 5 | version = '0.1.5-2016011901' 6 | ext.image = rootProject.name 7 | ext.urlDev = 'docker-dev.turn.com:5000' 8 | ext.urlProd = 'docker.turn.com' 9 | ext.namespace = '/turn-tsdb/' 10 | ext.tagDev = urlDev + namespace + image + ':' + version 11 | ext.tagProd = urlProd + namespace + image + ':' + version 12 | 13 | description = "This project implements a TSDB Splicer, which will " + 14 | "partition long TSDB queries into smaller chunks, execute them in parallel, and splice the " + 15 | "individual results" 16 | 17 | sourceCompatibility = 1.7 18 | targetCompatibility = 1.7 19 | 20 | repositories { 21 | flatDir { 22 | dirs 'libs' 23 | } 24 | maven { url "http://repo.maven.apache.org/maven2" } 25 | mavenCentral() 26 | } 27 | 28 | configurations{ 29 | providedCompile 30 | compile.extendsFrom providedCompile 31 | } 32 | 33 | dependencies { 34 | compile group: 'com.google.guava', name: 'guava', version:'18.0' 35 | compile group: 'org.eclipse.jetty', name: 'jetty-server', version:'8.1.16.v20140903' 36 | compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version:'8.1.16.v20140903' 37 | compile(group: 'org.apache.hbase', name: 'hbase-client', version:'0.98.0-hadoop1') { 38 | exclude(module: 'slf4j-log4j12') 39 | exclude(module: 'org.codehaus.jackson') 40 | } 41 | compile group: 'org.apache.httpcomponents', name: 'httpclient', version:'4.5' 42 | compile group: 'com.google.code.findbugs', name: 'jsr305', version:'3.0.0' 43 | compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version:'2.4.3' 44 | compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version:'2.4.3' 45 | compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version:'2.4.3' 46 | compile group: 'org.slf4j', name: 'slf4j-api', version:'1.7.7' 47 | compile group: 'org.slf4j', name: 'log4j-over-slf4j', version:'1.7.7' 48 | compile group: 'ch.qos.logback', name: 'logback-core', version:'1.0.13' 49 | compile group: 'ch.qos.logback', name: 'logback-classic', version:'1.0.13' 50 | compile group: 'redis.clients', name: 'jedis', version:'2.7.2' 51 | compile group: 'com.stumbleupon', name: 'async', version:'1.4.0' 52 | compile group: 'org.apache.commons', name: 'commons-math3', version:'3.4.1' 53 | providedCompile files('libs/tsdb-2.3.0-SNAPSHOT.jar') 54 | testCompile group: 'org.hbase', name: 'asynchbase', version:'1.7.1' 55 | testCompile group: 'org.testng', name: 'testng', version:'6.8' 56 | testCompile group: 'junit', name: 'junit', version:'4.11' 57 | testCompile group: 'org.mockito', name: 'mockito-core', version:'1.9.5' 58 | } 59 | 60 | task fatJar(type: Jar, dependsOn: build) { 61 | baseName = project.name + '-all' 62 | from { 63 | configurations.compile.collect { 64 | it.isDirectory() ? it : zipTree(it).matching { 65 | exclude { 66 | it.path.contains('META-INF') 67 | } 68 | } 69 | } 70 | } 71 | with jar 72 | } 73 | 74 | task generateVersion(type: Exec) { 75 | workingDir 'docker' 76 | commandLine './version.sh' 77 | } 78 | 79 | task copyJar(type: Copy, dependsOn: fatJar) { 80 | from('build/libs') { 81 | include fatJar.archiveName 82 | } 83 | into('docker') 84 | } 85 | 86 | task copyConfig(type: Copy) { 87 | from('src/main/resources') { 88 | include '*' 89 | } 90 | into('docker/resources') 91 | } 92 | 93 | task buildDockerImage(type: Exec, dependsOn: ['copyJar', 'copyConfig', 'generateVersion']) { 94 | commandLine 'docker', 'build', '-t', image, './docker' 95 | } 96 | 97 | task tagDockerDevImage(type: Exec, dependsOn: buildDockerImage) { 98 | commandLine 'docker', 'tag', '-f', image, tagDev 99 | } 100 | 101 | task tagDockerProdImage(type: Exec, dependsOn: buildDockerImage) { 102 | commandLine 'docker', 'tag', '-f', image, tagProd 103 | } 104 | 105 | task pushDockerDevImage(type: Exec, dependsOn: tagDockerDevImage) { 106 | commandLine 'docker', 'push', tagDev 107 | } 108 | 109 | task pushDockerProdImage(type: Exec, dependsOn: tagDockerProdImage) { 110 | environment 'HOME', '.' 111 | commandLine 'docker', 'push', tagProd 112 | } 113 | 114 | task getDeps(type: Copy) { 115 | from sourceSets.main.runtimeClasspath 116 | into 'runtime/' 117 | } 118 | 119 | clean.doLast { 120 | def cmd = "rm -f docker/${fatJar.archiveName}" 121 | println cmd 122 | cmd.execute() 123 | cmd = "rm -rf docker/resources" 124 | println cmd 125 | cmd.execute() 126 | } 127 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.turn.com/tops-base/jdk7:latest 2 | MAINTAINER instrumentation@turn.com 3 | 4 | ENV VERSION 0.1.5-2016011901 5 | ENV WORKDIR /usr/share/tsdb-splicer 6 | ENV DATADIR /data/tsdb-splicer 7 | ENV LOGDIR /var/log/tsdb-splicer 8 | RUN mkdir -p $WORKDIR/resources 9 | RUN mkdir -p $DATADIR/cache 10 | RUN mkdir -p $LOGDIR 11 | 12 | ENV CLASSPATH $WORKDIR 13 | ENV BASE $WORKDIR 14 | ENV SPLICER_PORT 4245 15 | EXPOSE $SPLICER_PORT 16 | 17 | ADD tsdb-splicer-all-$VERSION.jar $WORKDIR/ 18 | 19 | ADD VERSION $WORKDIR/resources/ 20 | ADD resources $WORKDIR/resources/ 21 | 22 | WORKDIR $WORKDIR 23 | 24 | ENTRYPOINT /bin/bash -c "java -server -classpath $WORKDIR/resources:$WORKDIR:$WORKDIR/tsdb-splicer-all-$VERSION.jar com.turn.splicer.SplicerMain --port $SPLICER_PORT --config=$WORKDIR/resources/splicer.conf" 25 | -------------------------------------------------------------------------------- /docker/version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [[ -z "$(command -v git)" ]]; then 3 | echo "Could not find git. Exiting." 4 | exit 5 | fi 6 | 7 | echo "# VERSION generated by $(whoami) on $(date) from $(hostname)" > VERSION 8 | 9 | echo "git.branch = $(git symbolic-ref HEAD)" >> VERSION 10 | echo "git.commit = $(git rev-parse HEAD)" >> VERSION 11 | 12 | echo "build.hostname = $(hostname)" >> VERSION 13 | echo "build.date = $(date)" >> VERSION 14 | echo "build.user = $(whoami)" >> VERSION 15 | 16 | # mkdir -p resources 17 | # mv VERSION resources/ 18 | -------------------------------------------------------------------------------- /docs/media/perf-comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turn/splicer/1e01538c939c254742cd3d5543eb0e4618f94b57/docs/media/perf-comparison.png -------------------------------------------------------------------------------- /docs/media/splicer-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turn/splicer/1e01538c939c254742cd3d5543eb0e4618f94b57/docs/media/splicer-arch.png -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turn/splicer/1e01538c939c254742cd3d5543eb0e4618f94b57/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Wed Sep 02 10:39:05 PDT 2015 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # For Cygwin, ensure paths are in UNIX format before anything is touched. 46 | if $cygwin ; then 47 | [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 48 | fi 49 | 50 | # Attempt to set APP_HOME 51 | # Resolve links: $0 may be a link 52 | PRG="$0" 53 | # Need this for relative symlinks. 54 | while [ -h "$PRG" ] ; do 55 | ls=`ls -ld "$PRG"` 56 | link=`expr "$ls" : '.*-> \(.*\)$'` 57 | if expr "$link" : '/.*' > /dev/null; then 58 | PRG="$link" 59 | else 60 | PRG=`dirname "$PRG"`"/$link" 61 | fi 62 | done 63 | SAVED="`pwd`" 64 | cd "`dirname \"$PRG\"`/" >&- 65 | APP_HOME="`pwd -P`" 66 | cd "$SAVED" >&- 67 | 68 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 69 | 70 | # Determine the Java command to use to start the JVM. 71 | if [ -n "$JAVA_HOME" ] ; then 72 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 73 | # IBM's JDK on AIX uses strange locations for the executables 74 | JAVACMD="$JAVA_HOME/jre/sh/java" 75 | else 76 | JAVACMD="$JAVA_HOME/bin/java" 77 | fi 78 | if [ ! -x "$JAVACMD" ] ; then 79 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 80 | 81 | Please set the JAVA_HOME variable in your environment to match the 82 | location of your Java installation." 83 | fi 84 | else 85 | JAVACMD="java" 86 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 87 | 88 | Please set the JAVA_HOME variable in your environment to match the 89 | location of your Java installation." 90 | fi 91 | 92 | # Increase the maximum file descriptors if we can. 93 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 94 | MAX_FD_LIMIT=`ulimit -H -n` 95 | if [ $? -eq 0 ] ; then 96 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 97 | MAX_FD="$MAX_FD_LIMIT" 98 | fi 99 | ulimit -n $MAX_FD 100 | if [ $? -ne 0 ] ; then 101 | warn "Could not set maximum file descriptor limit: $MAX_FD" 102 | fi 103 | else 104 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 105 | fi 106 | fi 107 | 108 | # For Darwin, add options to specify how the application appears in the dock 109 | if $darwin; then 110 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 111 | fi 112 | 113 | # For Cygwin, switch paths to Windows format before running java 114 | if $cygwin ; then 115 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 116 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 158 | function splitJvmOpts() { 159 | JVM_OPTS=("$@") 160 | } 161 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 162 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 163 | 164 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 165 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /libs/tsdb-2.3.0-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/turn/splicer/1e01538c939c254742cd3d5543eb0e4618f94b57/libs/tsdb-2.3.0-SNAPSHOT.jar -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.turn 8 | tsdbsplicer 9 | 0.1 10 | 11 | 12 | 13 | 14 | 15 | maven-compiler-plugin 16 | 2.3.2 17 | 18 | 1.7 19 | 1.7 20 | 21 | 22 | 23 | org.apache.maven.plugins 24 | maven-surefire-plugin 25 | 2.16 26 | 27 | 28 | 29 | 30 | 31 | 32 | UTF-8 33 | 8.1.16.v20140903 34 | 2.4.2 35 | 18.0 36 | 37 | 38 | 39 | 40 | com.google.guava 41 | guava 42 | ${GUAVA_VERSION} 43 | 44 | 45 | org.eclipse.jetty 46 | jetty-server 47 | ${JETTY_VERSION} 48 | 49 | 50 | org.eclipse.jetty 51 | jetty-servlet 52 | ${JETTY_VERSION} 53 | 54 | 55 | org.apache.hbase 56 | hbase-client 57 | 0.98.0-hadoop1 58 | 59 | 60 | slf4j-log4j12 61 | org.slf4j 62 | 63 | 64 | org.codehaus.jackson 65 | jackson-mapper-asl 66 | 67 | 68 | 69 | 70 | org.apache.httpcomponents 71 | httpclient 72 | 4.5 73 | 74 | 75 | com.google.code.findbugs 76 | jsr305 77 | 3.0.0 78 | 79 | 80 | 81 | 82 | com.fasterxml.jackson.core 83 | jackson-core 84 | 2.4.3 85 | 86 | 87 | com.fasterxml.jackson.core 88 | jackson-databind 89 | 2.4.3 90 | 91 | 92 | com.fasterxml.jackson.core 93 | jackson-annotations 94 | 2.4.3 95 | 96 | 97 | 98 | org.slf4j 99 | slf4j-api 100 | 1.7.7 101 | 102 | 103 | 104 | org.slf4j 105 | log4j-over-slf4j 106 | 1.7.7 107 | 108 | 109 | 110 | ch.qos.logback 111 | logback-core 112 | 1.0.13 113 | 114 | 115 | 116 | ch.qos.logback 117 | logback-classic 118 | 1.0.13 119 | 120 | 121 | 122 | org.testng 123 | testng 124 | 6.8 125 | test 126 | 127 | 128 | junit 129 | junit 130 | 4.11 131 | test 132 | 133 | 134 | 135 | org.mockito 136 | mockito-core 137 | 1.9.5 138 | 139 | 140 | 141 | redis.clients 142 | jedis 143 | 2.7.2 144 | jar 145 | compile 146 | 147 | 148 | 149 | com.stumbleupon 150 | async 151 | 1.4.0 152 | 153 | 154 | 155 | org.apache.commons 156 | commons-math3 157 | 3.4.1 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'tsdb-splicer' 2 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/Config.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer; 18 | 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | import java.util.Map; 22 | import java.util.Properties; 23 | import java.util.TreeMap; 24 | 25 | import com.fasterxml.jackson.core.JsonGenerator; 26 | import com.google.common.base.Splitter; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | public class Config { 31 | 32 | private static final Logger LOG = LoggerFactory.getLogger(Config.class); 33 | 34 | private static final String CONFIG_FILE = "splicer.conf"; 35 | 36 | private static final String VERSION_FILE = "VERSION"; 37 | 38 | private static Splitter COMMA_SPLITTER = Splitter.on(',').omitEmptyStrings().trimResults(); 39 | 40 | private static final Config INSTANCE = new Config(); 41 | 42 | protected Properties properties = new Properties(); 43 | 44 | private Config() { 45 | try { 46 | InputStream is = getClass().getClassLoader().getResourceAsStream(CONFIG_FILE); 47 | if (is != null) { 48 | LOG.info("Loaded {} bytes of configuration", is.available()); 49 | } 50 | 51 | properties.load(is); 52 | } catch (IOException e) { 53 | LOG.error("Could not load " + CONFIG_FILE, e); 54 | } 55 | } 56 | 57 | public static Config get() { 58 | return INSTANCE; 59 | } 60 | 61 | /** 62 | * Read an integer from the config file 63 | * @param field property name 64 | * @return integer or throws NumberFormatException if property doesn't 65 | * exist or is not an integer 66 | */ 67 | public int getInt(String field) { 68 | return Integer.parseInt(properties.getProperty(field)); 69 | } 70 | 71 | /** 72 | * Read an integer from the config file 73 | * @param field property name 74 | * @param defaultVal default to return if no property 75 | * @return integer or defaultVal if property doesn't exist or is not an 76 | * integer 77 | */ 78 | public int getInt(String field, int defaultVal) { 79 | try { 80 | return Integer.parseInt(properties.getProperty(field)); 81 | } catch (NumberFormatException e) { 82 | return defaultVal; 83 | } 84 | } 85 | 86 | public boolean getBoolean(String field) { 87 | return Boolean.parseBoolean(properties.getProperty(field)); 88 | } 89 | 90 | /** 91 | * Read a string from the config file 92 | * @param field property name 93 | * @return string or null if property doesn't exist 94 | */ 95 | public String getString(String field) { 96 | return properties.getProperty(field); 97 | } 98 | 99 | /** 100 | * For properties of the form: 101 | * 102 | * key = value1,value2,value3....valuek 103 | * this will return an iterable of the values. 104 | * 105 | * @param field property name 106 | * @return list of (value1, value2... valuek) 107 | */ 108 | public Iterable getStrings(String field) { 109 | String all = getString(field); 110 | return COMMA_SPLITTER.split(all); 111 | } 112 | 113 | public void writeAsJson(JsonGenerator jgen) throws IOException 114 | { 115 | if (properties == null) { 116 | jgen.writeStartObject(); 117 | jgen.writeEndObject(); 118 | return; 119 | } 120 | 121 | TreeMap map = new TreeMap<>(); 122 | for (Map.Entry e: properties.entrySet()) { 123 | map.put(String.valueOf(e.getKey()), String.valueOf(e.getValue())); 124 | } 125 | 126 | InputStream is = getClass().getClassLoader().getResourceAsStream(VERSION_FILE); 127 | if (is != null) { 128 | LOG.debug("Loaded {} bytes of version file configuration", is.available()); 129 | Properties versionProps = new Properties(); 130 | versionProps.load(is); 131 | for (Map.Entry e: versionProps.entrySet()) { 132 | map.put(String.valueOf(e.getKey()), String.valueOf(e.getValue())); 133 | } 134 | } else { 135 | LOG.error("No version file found on classpath. VERSION_FILE={}", VERSION_FILE); 136 | } 137 | 138 | jgen.writeStartObject(); 139 | for (Map.Entry e: map.entrySet()) { 140 | if (e.getValue().indexOf(',') > 0) { 141 | splitList(e.getKey(), e.getValue(), jgen); 142 | } else { 143 | jgen.writeStringField(e.getKey(), e.getValue()); 144 | } 145 | } 146 | jgen.writeEndObject(); 147 | } 148 | 149 | private void splitList(String key, String value, JsonGenerator jgen) throws IOException { 150 | jgen.writeArrayFieldStart(key); 151 | for (String o: value.split(",")) { 152 | jgen.writeString(o); 153 | } 154 | jgen.writeEndArray(); 155 | } 156 | 157 | } 158 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/ConfigServlet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer; 18 | 19 | import com.turn.splicer.hbase.RegionUtil; 20 | 21 | import javax.servlet.http.HttpServlet; 22 | import javax.servlet.http.HttpServletRequest; 23 | import javax.servlet.http.HttpServletResponse; 24 | import java.io.IOException; 25 | 26 | import com.fasterxml.jackson.core.JsonEncoding; 27 | import com.fasterxml.jackson.core.JsonFactory; 28 | import com.fasterxml.jackson.core.JsonGenerator; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | public class ConfigServlet extends HttpServlet { 33 | 34 | private static final Logger LOG = LoggerFactory.getLogger(ConfigServlet.class); 35 | 36 | public static RegionUtil REGION_UTIL = new RegionUtil(); 37 | 38 | @Override 39 | protected void doGet(HttpServletRequest request, HttpServletResponse response) 40 | throws IOException 41 | { 42 | try { 43 | doGetWork(request, response); 44 | } catch (IOException e) { 45 | LOG.error("IOException which processing GET request", e); 46 | } catch (Exception e) { 47 | LOG.error("Exception which processing GET request", e); 48 | } 49 | } 50 | 51 | private void doGetWork(HttpServletRequest request, HttpServletResponse response) 52 | throws IOException 53 | { 54 | response.setContentType("application/json"); 55 | JsonGenerator generator = new JsonFactory() 56 | .createGenerator(response.getOutputStream(), JsonEncoding.UTF8); 57 | Config.get().writeAsJson(generator); 58 | generator.close(); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/Const.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer; 18 | 19 | import java.text.SimpleDateFormat; 20 | import java.util.Date; 21 | 22 | public class Const { 23 | 24 | public static String tsFormat(long val) { 25 | SimpleDateFormat SDF = new SimpleDateFormat("yyyy/MM/dd-HH:mm:ss"); 26 | return SDF.format(new Date(val)); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/HttpWorker.java: -------------------------------------------------------------------------------- 1 | package com.turn.splicer; 2 | 3 | import com.turn.splicer.cache.JedisClient; 4 | import com.turn.splicer.hbase.RegionChecker; 5 | import com.turn.splicer.tsdbutils.JSON; 6 | import com.turn.splicer.tsdbutils.TSSubQuery; 7 | import com.turn.splicer.tsdbutils.TsQuery; 8 | 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.Random; 13 | import java.util.Set; 14 | import java.util.concurrent.Callable; 15 | import java.util.concurrent.LinkedBlockingQueue; 16 | 17 | import org.apache.commons.io.IOUtils; 18 | import org.apache.commons.lang.StringUtils; 19 | import org.apache.http.HttpResponse; 20 | import org.apache.http.client.methods.HttpPost; 21 | import org.apache.http.entity.StringEntity; 22 | import org.apache.http.impl.client.CloseableHttpClient; 23 | import org.apache.http.impl.client.HttpClientBuilder; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | public class HttpWorker implements Callable { 28 | 29 | private static final Logger LOG = LoggerFactory.getLogger(HttpWorker.class); 30 | 31 | private static final Random RANDOM_GENERATOR = new Random(); 32 | 33 | public static final Map> TSDMap = new HashMap<>(); 34 | 35 | private final TsQuery query; 36 | private final RegionChecker checker; 37 | 38 | private final String[] hosts; 39 | 40 | public HttpWorker(TsQuery query, RegionChecker checker) { 41 | this.query = query; 42 | this.checker = checker; 43 | 44 | Set hosts = TSDMap.keySet(); 45 | if (hosts.size() == 0) { 46 | throw new NullPointerException("No Query Hosts. TSDMap.size = 0"); 47 | } 48 | 49 | this.hosts = new String[hosts.size()]; 50 | int i=0; 51 | for (String h: hosts) { 52 | this.hosts[i] = h; 53 | i++; 54 | } 55 | } 56 | 57 | @Override 58 | public String call() throws Exception 59 | { 60 | LOG.debug("Start time={}, End time={}", Const.tsFormat(query.startTime()), 61 | Const.tsFormat(query.endTime())); 62 | 63 | String metricName = query.getQueries().get(0).getMetric(); 64 | String cacheResult = JedisClient.get().get(this.query.toString()); 65 | if (cacheResult != null) { 66 | LOG.debug("Cache hit for start=" + query.startTime() 67 | + ", end=" + query.endTime() + ", metric=" + metricName); 68 | return cacheResult; 69 | } 70 | 71 | String hostname = checker.getBestRegionHost(metricName, 72 | query.startTime() / 1000, query.endTime() / 1000); 73 | LOG.debug("Found region server hostname={} for metric={}", hostname, metricName); 74 | 75 | LinkedBlockingQueue TSDs; 76 | if (hostname == null) { 77 | LOG.error("Could not find region server for metric={}", metricName); 78 | return "{'error': 'Could not find region server for metric=" + metricName + "'}"; 79 | } 80 | 81 | TSDs = TSDMap.get(hostname); 82 | if (TSDs == null) { 83 | String host = select(); // randomly select a host (basic load balancing) 84 | TSDs = TSDMap.get(host); 85 | if (TSDs == null) { 86 | LOG.error("We are not running TSDs on regionserver={}. Fallback failed. Returning error", hostname); 87 | return "{'error': 'Fallback to hostname=" + hostname + " failed.'}"; 88 | } else { 89 | LOG.info("Falling back to " + host + " for queries"); 90 | } 91 | } 92 | 93 | String server = TSDs.take(); 94 | String uri = "http://" + server + "/api/query/qexp/"; 95 | 96 | CloseableHttpClient postman = HttpClientBuilder.create().build(); 97 | try { 98 | 99 | HttpPost postRequest = new HttpPost(uri); 100 | 101 | StringEntity input = new StringEntity(JSON.serializeToString(query)); 102 | input.setContentType("application/json"); 103 | postRequest.setEntity(input); 104 | LOG.debug("Sending request to: {} for query {} ", uri, query); 105 | 106 | HttpResponse response = postman.execute(postRequest); 107 | 108 | if (response.getStatusLine().getStatusCode() != 200) { 109 | throw new RuntimeException("Failed : HTTP error code : " 110 | + response.getStatusLine().getStatusCode()); 111 | } 112 | 113 | List dl = IOUtils.readLines(response.getEntity().getContent()); 114 | String result = StringUtils.join(dl, ""); 115 | LOG.debug("Result={}", result); 116 | if (isCacheable(query)) { 117 | JedisClient.get().put(this.query.toString(), result); 118 | } 119 | return result; 120 | } finally { 121 | IOUtils.closeQuietly(postman); 122 | 123 | TSDs.put(server); 124 | LOG.debug("Returned {} into the available queue", server); 125 | } 126 | } 127 | 128 | private boolean isCacheable(TsQuery query) { 129 | long interval = query.endTime() - query.startTime(); 130 | 131 | if (Config.get().getBoolean("slice.overflow.enable")) { 132 | return interval == (Splicer.SLICE_SIZE + Splicer.OVERFLOW) * 1000; 133 | } else { 134 | return interval == (Splicer.SLICE_SIZE) * 1000; 135 | } 136 | } 137 | 138 | private String select() { 139 | int index = RANDOM_GENERATOR.nextInt(hosts.length); 140 | return hosts[index]; 141 | } 142 | 143 | private String stringify(TsQuery query) 144 | { 145 | String subs = ""; 146 | for (TSSubQuery sub: query.getQueries()) { 147 | if (subs.length() > 0) subs += ","; 148 | subs += "m=[" + sub.getMetric() + sub.getTags() 149 | + ", downsample=" + sub.getDownsample() 150 | + ", rate=" + sub.getRate() + "]"; 151 | } 152 | return "{" + Const.tsFormat(query.startTime()) + " to " + Const.tsFormat(query.endTime()) + ", " + subs + "}"; 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/Splicer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer; 18 | 19 | import com.turn.splicer.tsdbutils.TsQuery; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | public class Splicer { 28 | 29 | private static final Logger LOG = LoggerFactory.getLogger(Splicer.class); 30 | 31 | public static final int SLICE_SIZE = 3600; 32 | public static final int OVERFLOW = 300; 33 | 34 | private final TsQuery tsQuery; 35 | 36 | public Splicer(TsQuery tsQuery) { 37 | this.tsQuery = tsQuery; 38 | } 39 | 40 | /** 41 | * Slices a query into pieces with each with time frame atmost of {@link #SLICE_SIZE}. 42 | * 43 | * NOTE: It must be noted that this method does not take into consideration the downsample. 44 | * 45 | * What does this mean? If we have a downsample of 30m, and each slice is divided into one hour 46 | * blocks, then we will only get one point per hour slice. This will reduce the output of the 47 | * final query by 50%. Thus, if {@link #SLICE_SIZE} is one hour, we must add some overflow 48 | * time to accomodate for these points. By default, this is set to {@link #OVERFLOW}. 49 | * 50 | * This can be better tuned, than hardcoding to a fixed {@link #OVERFLOW} value. As most queries 51 | * use a downsample of 1-5 minutes, this would mean dropping 1 out of 60-12 points respectively. 52 | * 53 | * @return list of queries 54 | */ 55 | public List sliceQuery() 56 | { 57 | final long bucket_size = SLICE_SIZE * 1000; 58 | 59 | final long overflow_in_millis; 60 | if (Config.get().getBoolean("slice.overflow.enable")) { 61 | overflow_in_millis = OVERFLOW * 1000; 62 | } else { 63 | overflow_in_millis = 0; 64 | } 65 | 66 | long startTime = tsQuery.startTime(); 67 | long endTime = tsQuery.endTime(); 68 | 69 | List slices = new ArrayList<>(); 70 | 71 | long end = startTime - (startTime % bucket_size) + bucket_size; 72 | TsQuery first = TsQuery.sliceOf(tsQuery, startTime - (startTime % bucket_size), end + overflow_in_millis); 73 | slices.add(first); 74 | LOG.debug("First interval is {} to {}", Const.tsFormat(first.startTime()), Const.tsFormat(first.endTime())); 75 | 76 | while (end + bucket_size < endTime) { 77 | TsQuery slice = TsQuery.sliceOf(tsQuery, end, end + bucket_size + overflow_in_millis); 78 | slices.add(slice); 79 | end = end + bucket_size; 80 | LOG.debug("Add interval# {} from {} to {}", slices.size(), 81 | Const.tsFormat(slice.startTime()), 82 | Const.tsFormat(slice.endTime())); 83 | } 84 | 85 | slices.add(TsQuery.sliceOf(tsQuery, end, endTime)); 86 | LOG.debug("Last interval is {} to {}", Const.tsFormat(end), Const.tsFormat(endTime)); 87 | 88 | return slices; 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/SplicerMain.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer; 18 | 19 | import com.turn.splicer.cache.JedisClient; 20 | 21 | import java.util.concurrent.LinkedBlockingQueue; 22 | 23 | import org.eclipse.jetty.server.Connector; 24 | import org.eclipse.jetty.server.Handler; 25 | import org.eclipse.jetty.server.Server; 26 | import org.eclipse.jetty.server.handler.DefaultHandler; 27 | import org.eclipse.jetty.server.handler.HandlerList; 28 | import org.eclipse.jetty.server.nio.SelectChannelConnector; 29 | import org.eclipse.jetty.servlet.ServletHandler; 30 | import org.eclipse.jetty.util.thread.QueuedThreadPool; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | public class SplicerMain { 35 | 36 | private static final Logger LOG = LoggerFactory.getLogger(SplicerMain.class); 37 | 38 | // port on which the splicer is listening 39 | private static final int PORT = Config.get().getInt("splicer.port"); 40 | 41 | // comma separated hosts on which the TSDs are running 42 | private static final String TSD_HOSTS = Config.get().getString("tsd.hosts"); 43 | 44 | // start and end port for TSDs on data nodes. Start is inclusive, End is exclusive. 45 | private static final int TSD_START_PORT = Config.get().getInt("tsd.start.port"); 46 | private static final int TSD_END_PORT = Config.get().getInt("tsd.end.port"); 47 | 48 | private static final int TSD_QUERIES_PER_PORT = Config.get().getInt("tsd.queries.per.port", 1); 49 | 50 | public static void main(String[] args) throws InterruptedException { 51 | 52 | if (Config.get().getBoolean("tsd.connect.enable")) { 53 | String[] TSDs = TSD_HOSTS.split(","); 54 | for (int i = 0; i < TSDs.length; i++) { 55 | TSDs[i] = TSDs[i].trim(); 56 | } 57 | 58 | for (String TSD : TSDs) { 59 | for (int port = TSD_START_PORT; port < TSD_END_PORT; port++) { 60 | String r = TSD + ":" + port; 61 | if (HttpWorker.TSDMap.get(TSD) == null) { 62 | HttpWorker.TSDMap.put(TSD, new LinkedBlockingQueue()); 63 | } 64 | 65 | //with load balancer we actually have 10 tsdb instances behind each port 66 | for(int j = 0; j < TSD_QUERIES_PER_PORT; j++) { 67 | HttpWorker.TSDMap.get(TSD).put(r); 68 | } 69 | LOG.info("Registering {}", r); 70 | } 71 | } 72 | } 73 | 74 | LOG.info("JedisClient Status: " + JedisClient.get().config()); 75 | 76 | final Server server = new Server(); 77 | 78 | Connector connector = new SelectChannelConnector(); 79 | connector.setPort(PORT); 80 | server.addConnector(connector); 81 | 82 | QueuedThreadPool qtp = new QueuedThreadPool(); 83 | qtp.setName("jetty-qt-pool"); 84 | server.setThreadPool(qtp); 85 | 86 | ServletHandler servletHandler = new ServletHandler(); 87 | servletHandler.addServletWithMapping(SplicerServlet.class.getName(), "/api/query"); 88 | servletHandler.addServletWithMapping(SplicerServlet.class.getName(), "/api/query/query"); 89 | servletHandler.addServletWithMapping(SplicerServlet.class.getName(), "/api/query/qexp"); 90 | servletHandler.addServletWithMapping(ConfigServlet.class.getName(), "/api/config"); 91 | servletHandler.addServletWithMapping(SuggestServlet.class.getName(), "/api/suggest"); 92 | 93 | HandlerList handlers = new HandlerList(); 94 | handlers.setHandlers(new Handler[]{ 95 | servletHandler, 96 | new DefaultHandler() 97 | }); 98 | server.setHandler(handlers); 99 | 100 | // register shutdown hook 101 | Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { 102 | @Override 103 | public void run() { 104 | try { 105 | server.stop(); 106 | } catch (Exception e) { 107 | LOG.error("Jetty Shutdown Error", e); 108 | } finally { 109 | LOG.info("Shutdown Server..."); 110 | } 111 | } 112 | }, "jetty-shutdown-hook")); 113 | 114 | // and start the server 115 | try { 116 | LOG.info("Starting web server at port=" + PORT); 117 | server.start(); 118 | server.join(); 119 | } catch (Exception e) { 120 | throw new RuntimeException("Could not start web server", e); 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/SplicerServlet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer; 18 | 19 | import com.google.common.collect.Lists; 20 | import com.turn.splicer.hbase.RegionChecker; 21 | import com.turn.splicer.hbase.RegionUtil; 22 | import com.turn.splicer.merge.ResultsMerger; 23 | import com.turn.splicer.merge.TsdbResult; 24 | import com.turn.splicer.tsdbutils.*; 25 | import com.turn.splicer.tsdbutils.expression.Expression; 26 | import com.turn.splicer.tsdbutils.expression.ExpressionTree; 27 | 28 | import javax.servlet.http.HttpServlet; 29 | import javax.servlet.http.HttpServletRequest; 30 | import javax.servlet.http.HttpServletResponse; 31 | import java.io.BufferedReader; 32 | import java.io.IOException; 33 | import java.util.*; 34 | import java.util.concurrent.*; 35 | import java.util.concurrent.atomic.AtomicInteger; 36 | 37 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 38 | 39 | import com.turn.splicer.tsdbutils.expression.ExpressionTreeWorker; 40 | import org.apache.commons.lang.StringUtils; 41 | import org.slf4j.Logger; 42 | import org.slf4j.LoggerFactory; 43 | 44 | public class SplicerServlet extends HttpServlet { 45 | 46 | private static final Logger LOG = LoggerFactory.getLogger(SplicerServlet.class); 47 | 48 | public static RegionUtil REGION_UTIL = new RegionUtil(); 49 | 50 | private static ExecutorService pool = Executors.newCachedThreadPool(); 51 | 52 | @Override 53 | protected void doGet(HttpServletRequest request, HttpServletResponse response) 54 | throws IOException 55 | { 56 | try { 57 | doGetWork(request, response); 58 | } catch (IOException e) { 59 | LOG.error("IOException which processing GET request", e); 60 | } catch (Exception e) { 61 | LOG.error("Exception which processing GET request", e); 62 | } 63 | } 64 | 65 | @Override 66 | protected void doPost(HttpServletRequest request, HttpServletResponse response) 67 | throws IOException 68 | { 69 | try { 70 | doPostWork(request, response); 71 | } catch (Exception e) { 72 | LOG.error("Exception which processing POST request", e); 73 | 74 | String stackTrace = ""; 75 | if (e.getStackTrace() != null && e.getStackTrace().length > 2) { 76 | stackTrace += e.getStackTrace()[0] + ", " + e.getStackTrace()[1]; 77 | } else { 78 | stackTrace = ""; 79 | } 80 | 81 | response.getWriter().write("{\"error\": \"" 82 | + e.getMessage() + ", stacktrace=" + stackTrace + "\"}\n"); 83 | } 84 | } 85 | 86 | /** 87 | * Parses a TsQuery out of request, divides into subqueries, and writes result 88 | * from openTsdb 89 | * 90 | * Format for GET request: 91 | * start - required start time of query 92 | * end - optional end time of query, if not provided will use current time as end 93 | * x - expression (functions + metrics) 94 | * m - metrics only - no functions, [aggregator]:[optional_downsampling]:metric{optional tags} 95 | * either x or m must be provided, otherwise nothing to query! 96 | * ms - optional for millisecond resolution 97 | * padding - optional pad front of value's with 0's 98 | * 99 | *example: 100 | * /api/query?start=1436910725795&x=abs(sum:1m-avg:tcollector.collector.lines_received)" 101 | * @param request 102 | * @param response 103 | * @throws IOException 104 | */ 105 | private void doGetWork(HttpServletRequest request, HttpServletResponse response) 106 | throws IOException 107 | { 108 | LOG.info(request.getQueryString()); 109 | 110 | final TsQuery dataQuery = new TsQuery(); 111 | 112 | dataQuery.setStart(request.getParameter("start")); 113 | dataQuery.setEnd(request.getParameter("end")); 114 | 115 | dataQuery.setPadding(Boolean.valueOf(request.getParameter("padding"))); 116 | 117 | if(request.getParameter("ms") != null) { 118 | dataQuery.setMsResolution(true); 119 | } 120 | 121 | List expressionTrees = null; 122 | 123 | final String[] expressions = request.getParameterValues("x"); 124 | 125 | if(expressions != null) { 126 | expressionTrees = new ArrayList(); 127 | List metricQueries = new ArrayList(); 128 | 129 | SplicerUtils.syntaxCheck(expressions, dataQuery, metricQueries, expressionTrees); 130 | } 131 | 132 | //not supporting metric queries from GET yet... 133 | //TODO: fix this or decide if we need to support "m" type queries 134 | if(request.getParameter("m") != null) { 135 | final List legacy_queries = Arrays.asList(request.getParameterValues("m")); 136 | for(String q: legacy_queries) { 137 | SplicerUtils.parseMTypeSubQuery(q, dataQuery); 138 | } 139 | } 140 | 141 | LOG.info("Serving query={}", dataQuery); 142 | 143 | LOG.info("Original TsQuery Start time={}, End time={}", 144 | Const.tsFormat(dataQuery.startTime()), 145 | Const.tsFormat(dataQuery.endTime())); 146 | 147 | response.setCharacterEncoding("UTF-8"); 148 | response.setContentType("application/json"); 149 | 150 | try (RegionChecker checker = REGION_UTIL.getRegionChecker()) { 151 | List exprResults = Lists.newArrayList(); 152 | if(expressionTrees == null || expressionTrees.size() == 0) { 153 | System.out.println("expression trees == null...figure this out later"); 154 | response.getWriter().write("No expression or error parsing expression"); 155 | } 156 | else { 157 | try { 158 | List> futureList = new ArrayList>(expressionTrees.size()); 159 | 160 | TsQuery prev = null; 161 | 162 | for (ExpressionTree expressionTree : expressionTrees) { 163 | futureList.add(pool.submit(new ExpressionTreeWorker(expressionTree))); 164 | } 165 | 166 | for (Future future : futureList) { 167 | exprResults.add(future.get()); 168 | } 169 | response.getWriter().write(TsdbResult.toJson(SplicerUtils.flatten( 170 | exprResults))); 171 | } catch (Exception e) { 172 | LOG.error("Could not evaluate expression tree", e); 173 | e.printStackTrace(); 174 | } 175 | } 176 | } 177 | } 178 | 179 | private void doPostWork(HttpServletRequest request, HttpServletResponse response) 180 | throws IOException 181 | { 182 | BufferedReader reader = request.getReader(); 183 | String line; 184 | 185 | StringBuilder builder = new StringBuilder(); 186 | while ((line = reader.readLine()) != null) { 187 | builder.append(line); 188 | } 189 | 190 | String jsonPostRequest = builder.toString(); 191 | 192 | TsQuery tsQuery = TsQuerySerializer.deserializeFromJson(jsonPostRequest); 193 | tsQuery.validateAndSetQuery(); 194 | 195 | LOG.info("Serving query={}", tsQuery); 196 | 197 | LOG.info("Original TsQuery Start time={}, End time={}", 198 | Const.tsFormat(tsQuery.startTime()), 199 | Const.tsFormat(tsQuery.endTime())); 200 | 201 | response.setCharacterEncoding("UTF-8"); 202 | response.setContentType("application/json"); 203 | 204 | try (RegionChecker checker = REGION_UTIL.getRegionChecker()) { 205 | 206 | List subQueries = new ArrayList<>(tsQuery.getQueries()); 207 | SplicerQueryRunner queryRunner = new SplicerQueryRunner(); 208 | 209 | if (subQueries.size() == 1) { 210 | TsdbResult[] results = queryRunner.sliceAndRunQuery(tsQuery, checker); 211 | if (results == null || results.length == 0) { 212 | response.getWriter().write("[]"); 213 | } else { 214 | response.getWriter().write(TsdbResult.toJson(results)); 215 | } 216 | response.getWriter().flush(); 217 | } else { 218 | List resultsFromAllSubQueries = new ArrayList<>(); 219 | for (TSSubQuery subQuery: subQueries) { 220 | TsQuery tsQueryCopy = TsQuery.validCopyOf(tsQuery); 221 | tsQueryCopy.addSubQuery(subQuery); 222 | TsdbResult[] results = queryRunner.sliceAndRunQuery(tsQueryCopy, checker); 223 | resultsFromAllSubQueries.add(results); 224 | } 225 | response.getWriter().write(TsdbResult.toJson(SplicerUtils.flatten( 226 | resultsFromAllSubQueries))); 227 | } 228 | } 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/SuggestHttpWorker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer; 18 | 19 | import org.apache.commons.io.IOUtils; 20 | import org.apache.commons.lang.StringUtils; 21 | import org.apache.http.HttpResponse; 22 | import org.apache.http.client.methods.HttpGet; 23 | import org.apache.http.impl.client.CloseableHttpClient; 24 | import org.apache.http.impl.client.HttpClientBuilder; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import java.util.List; 29 | import java.util.Random; 30 | import java.util.Set; 31 | import java.util.concurrent.Callable; 32 | import java.util.concurrent.LinkedBlockingQueue; 33 | 34 | /** 35 | * Takes a api/suggest query string and runs it against a random TSD node, 36 | * returns the result of the query 37 | * 38 | * TODO: needs to be refactored with HttpWorker, there is a lot of duplicate logic 39 | * TODO: error reporting differs from regular TSDB - no error message for invalid type 40 | */ 41 | public class SuggestHttpWorker implements Callable { 42 | private static final Logger LOG = LoggerFactory.getLogger(SuggestHttpWorker.class); 43 | 44 | private final String suggestQuery; 45 | 46 | private final String[] hosts; 47 | 48 | private static final Random RANDOM_GENERATOR = new Random(); 49 | 50 | public SuggestHttpWorker(String queryString) { 51 | this.suggestQuery = queryString; 52 | 53 | Set hosts = HttpWorker.TSDMap.keySet(); 54 | if (hosts.size() == 0) { 55 | throw new NullPointerException("No Query Hosts. TSDMap.size = 0"); 56 | } 57 | 58 | this.hosts = new String[hosts.size()]; 59 | int i=0; 60 | for (String h: hosts) { 61 | this.hosts[i] = h; 62 | i++; 63 | } 64 | } 65 | 66 | @Override 67 | public String call() throws Exception { 68 | LinkedBlockingQueue TSDs; 69 | 70 | //TODO: have it implement its own RegionChecker to get hbase locality looking for metric names 71 | //lets have it just pick a random host 72 | String hostname = getRandomHost(); 73 | TSDs = HttpWorker.TSDMap.get(hostname); 74 | 75 | if (TSDs == null) { 76 | LOG.error("We are not running TSDs on regionserver={}. Choosing a random host failed", hostname); 77 | return "{'error': 'Choice of hostname=" + hostname + " failed.'}"; 78 | } 79 | 80 | String server = TSDs.take(); 81 | String uri = "http://" + server + "/api/suggest?" + suggestQuery; 82 | 83 | CloseableHttpClient postman = HttpClientBuilder.create().build(); 84 | try { 85 | HttpGet getRequest = new HttpGet(uri); 86 | 87 | LOG.info("Sending query=" + uri + " to TSD running on host=" + hostname); 88 | 89 | HttpResponse response = postman.execute(getRequest); 90 | 91 | if (response.getStatusLine().getStatusCode() != 200) { 92 | throw new RuntimeException("Failed : HTTP error code : " 93 | + response.getStatusLine().getStatusCode()); 94 | } 95 | 96 | List dl = IOUtils.readLines(response.getEntity().getContent()); 97 | String result = StringUtils.join(dl, ""); 98 | LOG.info("Result={}", result); 99 | 100 | return result; 101 | } finally { 102 | IOUtils.closeQuietly(postman); 103 | 104 | TSDs.put(server); 105 | LOG.info("Returned {} into the available queue", server); 106 | } 107 | } 108 | 109 | /** 110 | * Chooses a random number between 0 - (host.length-1) inclusive 111 | * returns that index in the hosts list 112 | * @return 113 | */ 114 | private String getRandomHost() { 115 | return hosts[RANDOM_GENERATOR.nextInt(hosts.length)]; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/SuggestServlet.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer; 18 | 19 | import org.codehaus.jettison.json.JSONException; 20 | import org.codehaus.jettison.json.JSONObject; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import javax.servlet.http.HttpServlet; 25 | import javax.servlet.http.HttpServletRequest; 26 | import javax.servlet.http.HttpServletResponse; 27 | import java.io.BufferedReader; 28 | import java.io.IOException; 29 | 30 | /** 31 | * Implements the suggest api for requests with api/suggest by passing the 32 | * request on to a TSDB node and returning the result 33 | * for example: TSDB_HOST/api/suggest?type=metrics&q=tcollector" 34 | * should give you the first 25 metrics that start with "tcollector" 35 | * 36 | * Created by bpeltz on 10/19/15. 37 | */ 38 | public class SuggestServlet extends HttpServlet { 39 | 40 | private static final Logger LOG = LoggerFactory.getLogger(SuggestServlet.class); 41 | 42 | @Override 43 | protected void doGet(HttpServletRequest request, HttpServletResponse response) 44 | throws IOException { 45 | try { 46 | SuggestHttpWorker worker = new SuggestHttpWorker(request.getQueryString()); 47 | String json = worker.call(); 48 | 49 | response.setContentType("application/json"); 50 | response.getWriter().write(json); 51 | response.getWriter().flush(); 52 | } catch (IOException e) { 53 | LOG.error("IOException which processing GET request", e); 54 | } catch (Exception e) { 55 | LOG.error("Exception which processing GET request", e); 56 | } 57 | } 58 | 59 | @Override 60 | protected void doPost(HttpServletRequest request, HttpServletResponse response) 61 | throws IOException { 62 | //ok let's see if I can build the query string out of the request object 63 | BufferedReader reader = request.getReader(); 64 | String line; 65 | StringBuilder builder = new StringBuilder(); 66 | while ((line = reader.readLine()) != null) { 67 | builder.append(line); 68 | } 69 | 70 | String jsonPostRequest = builder.toString(); 71 | 72 | JSONObject obj = null; 73 | 74 | try { 75 | obj = new JSONObject(jsonPostRequest); 76 | } catch (JSONException e) { 77 | LOG.error("Exception reading POST data " + jsonPostRequest); 78 | response.getWriter().write("Exception reading POST data " + jsonPostRequest); 79 | return; 80 | } 81 | 82 | String type = null; 83 | 84 | try { 85 | type = obj.getString("type"); 86 | } catch (JSONException e) { 87 | LOG.error("No type provided for suggest request " + jsonPostRequest); 88 | response.getWriter().write("No type provided for suggest request " + jsonPostRequest); 89 | return; 90 | } 91 | 92 | StringBuilder getRequestString = new StringBuilder(); 93 | getRequestString.append("type="); 94 | getRequestString.append(type); 95 | 96 | String q = null; 97 | Integer max = null; 98 | //q is optional 99 | try { 100 | q = obj.getString("q"); 101 | } catch (JSONException e) {} 102 | 103 | //max is optional 104 | try { 105 | max = obj.getInt("max"); 106 | } catch (JSONException e) {} 107 | 108 | if (q != null) { 109 | getRequestString.append("&q="); 110 | getRequestString.append(q); 111 | } 112 | 113 | if (max != null) { 114 | getRequestString.append("&max="); 115 | getRequestString.append(max); 116 | } 117 | String requestString = getRequestString.toString(); 118 | 119 | SuggestHttpWorker worker = new SuggestHttpWorker(requestString); 120 | String json = null; 121 | try { 122 | json = worker.call(); 123 | } catch (Exception e) { 124 | LOG.error("Exception processing POST reqeust", e); 125 | response.getWriter().write("Exception processing POST request" + e); 126 | } 127 | 128 | response.setContentType("application/json"); 129 | response.getWriter().write(json); 130 | response.getWriter().flush(); 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/cache/JedisClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.cache; 18 | 19 | import com.turn.splicer.Config; 20 | 21 | import javax.annotation.Nullable; 22 | 23 | import redis.clients.jedis.Jedis; 24 | import redis.clients.jedis.JedisPool; 25 | import redis.clients.jedis.JedisPoolConfig; 26 | 27 | /** 28 | * @author sgangam 29 | */ 30 | public class JedisClient { 31 | 32 | private static final boolean CACHE_ENABLED = Config.get().getBoolean("caching.enabled"); 33 | 34 | protected final JedisPool jedisPool; 35 | 36 | private static final JedisClient CLIENT = new JedisClient(); 37 | 38 | private JedisClient() { 39 | if (CACHE_ENABLED) { 40 | String hostPortConfig = Config.get().getString("caching.hosts"); 41 | if (hostPortConfig == null) throw new NullPointerException("Could not find config"); 42 | 43 | String[] hp = hostPortConfig.split(":"); 44 | 45 | if (hp.length != 2) throw new IllegalArgumentException("Bad config for redis server"); 46 | jedisPool = new JedisPool(new JedisPoolConfig(), hp[0], Integer.parseInt(hp[1])); 47 | 48 | Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { 49 | @Override 50 | public void run() { 51 | jedisPool.close(); 52 | } 53 | })); 54 | 55 | } else { 56 | jedisPool = null; 57 | } 58 | } 59 | 60 | public static JedisClient get() { 61 | return CLIENT; 62 | } 63 | 64 | public void put(String key, String value) { 65 | if (CACHE_ENABLED && jedisPool != null) { 66 | try (Jedis jedis = jedisPool.getResource()) { 67 | jedis.set(key, value); 68 | } 69 | } 70 | } 71 | 72 | @Nullable 73 | public String get(String key) { 74 | if (CACHE_ENABLED && jedisPool != null) { 75 | try (Jedis jedis = jedisPool.getResource()) { 76 | return jedis.get(key); 77 | } 78 | } else { 79 | return null; 80 | } 81 | } 82 | 83 | public String config() { 84 | if (CACHE_ENABLED && jedisPool != null) { 85 | return "running at=" + Config.get().getString("caching.hosts") 86 | + ", numActive=" + jedisPool.getNumActive(); 87 | } else { 88 | return "not enabled"; 89 | } 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/hbase/MetricLookupException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.hbase; 18 | 19 | public class MetricLookupException extends RuntimeException { 20 | 21 | public MetricLookupException(String msg, Exception e) { 22 | super(msg, e); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/hbase/MetricsCache.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.hbase; 18 | 19 | import java.nio.charset.Charset; 20 | import java.util.Arrays; 21 | import java.util.concurrent.ExecutionException; 22 | 23 | import com.google.common.cache.CacheBuilder; 24 | import com.google.common.cache.CacheLoader; 25 | import com.google.common.cache.LoadingCache; 26 | import org.apache.hadoop.conf.Configuration; 27 | import org.apache.hadoop.hbase.client.Get; 28 | import org.apache.hadoop.hbase.client.HTable; 29 | import org.apache.hadoop.hbase.client.Result; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | 33 | public class MetricsCache { 34 | 35 | private static final Logger LOG = LoggerFactory.getLogger(MetricsCache.class); 36 | 37 | protected RegionUtil regionUtil = new RegionUtil(); 38 | 39 | protected LoadingCache cache = null; 40 | 41 | private static MetricsCache CACHE = new MetricsCache(); 42 | 43 | private MetricsCache() { 44 | cache = CacheBuilder.newBuilder() 45 | .maximumSize(1_000_000) 46 | .build(new KeyFetcher(regionUtil)); 47 | } 48 | 49 | public static MetricsCache get() { 50 | return CACHE; 51 | } 52 | 53 | static class KeyFetcher extends CacheLoader { 54 | 55 | private final RegionUtil regionUtil; 56 | 57 | public KeyFetcher(RegionUtil regionUtil) { 58 | this.regionUtil = regionUtil; 59 | } 60 | 61 | @Override 62 | public byte[] load(String metric) throws Exception { 63 | Configuration config = regionUtil.getConfig(); 64 | try (HTable table = new HTable(config, "tsdb-uid")) { 65 | Get get = new Get(toBytes(metric)); 66 | get.addColumn(toBytes("id"), toBytes("metrics")); 67 | Result result = table.get(get); 68 | LOG.info("Looking up key for metric={}. Found result={}", 69 | metric, Arrays.toString(result.value())); 70 | return result.value(); 71 | } 72 | } 73 | } 74 | 75 | static byte[] toBytes(String str) { 76 | return str.getBytes(Charset.forName("ISO-8859-1")); 77 | } 78 | 79 | public byte[] getMetricKey(String metric) { 80 | try { 81 | return cache.get(metric); 82 | } catch (ExecutionException e) { 83 | LOG.error("Error loading value to cache", e); 84 | throw new MetricLookupException("Error loading value to cache", e); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/hbase/RegionCheckException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.hbase; 18 | 19 | public class RegionCheckException extends RuntimeException { 20 | 21 | public RegionCheckException(String msg) { 22 | super(msg); 23 | } 24 | 25 | public RegionCheckException(String msg, Exception e) { 26 | super(msg, e); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/hbase/RegionChecker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.hbase; 18 | 19 | import java.io.Closeable; 20 | import java.io.IOException; 21 | import java.util.List; 22 | 23 | import org.apache.hadoop.conf.Configuration; 24 | import org.apache.hadoop.hbase.HRegionLocation; 25 | import org.apache.hadoop.hbase.client.HTable; 26 | import org.apache.hadoop.hbase.util.Bytes; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | public class RegionChecker implements Closeable { 31 | 32 | private static final Logger LOG = LoggerFactory.getLogger(RegionChecker.class); 33 | 34 | public static int METRIC_WIDTH = 4; 35 | public static int TS_HOUR_WIDTH = 4; 36 | 37 | private final HTable table; 38 | 39 | public RegionChecker(Configuration config) { 40 | try { 41 | this.table = new HTable(config, "tsdb"); 42 | } catch (IOException e) { 43 | LOG.error("Could not create connection", e); 44 | throw new RegionCheckException("Could not create connection", e); 45 | } 46 | } 47 | 48 | /** 49 | * Find the best region server for a given row key range 50 | * 51 | * @param metric name of the metric 52 | * @param startTime in seconds 53 | * @param endTime in seconds 54 | * @return the best region server to query for this metric, start, stop row key 55 | */ 56 | public String getBestRegionHost(String metric, long startTime, long endTime) { 57 | final byte[] startRowKey = new byte[METRIC_WIDTH + TS_HOUR_WIDTH]; 58 | final byte[] endRowKey = new byte[METRIC_WIDTH + TS_HOUR_WIDTH]; 59 | 60 | byte[] metricKey = MetricsCache.get().getMetricKey(metric); 61 | 62 | System.arraycopy(metricKey, 0, startRowKey, 0, METRIC_WIDTH); 63 | System.arraycopy(metricKey, 0, endRowKey, 0, METRIC_WIDTH); 64 | 65 | Bytes.putInt(startRowKey, METRIC_WIDTH, (int) startTime); 66 | Bytes.putInt(endRowKey, METRIC_WIDTH, (int) endTime); 67 | 68 | return getBestRegionHost(startRowKey, endRowKey); 69 | } 70 | 71 | public String getBestRegionHost(byte[] startRowKey, byte[] endRowKey) { 72 | try { 73 | List regions = table.getRegionsInRange(startRowKey, endRowKey); 74 | if (regions != null && regions.size() > 0) { 75 | HRegionLocation reg = regions.get(0); 76 | LOG.debug("Found region hostname: " + reg.getHostname()); 77 | return reg.getHostname(); 78 | } else { 79 | LOG.info("Regions is null"); 80 | throw new RegionCheckException("Could not find a host"); 81 | } 82 | } catch (IOException e) { 83 | throw new RegionCheckException("Could not handle region server lookup", e); 84 | } 85 | } 86 | 87 | public void close() { 88 | try { 89 | table.close(); 90 | } catch (IOException e) { 91 | LOG.error("Could not close connection", e); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/hbase/RegionUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.hbase; 18 | 19 | import com.turn.splicer.Config; 20 | 21 | import org.apache.hadoop.conf.Configuration; 22 | import org.apache.hadoop.hbase.HBaseConfiguration; 23 | 24 | public class RegionUtil { 25 | 26 | private static String HBASE_ZK = Config.get().getString("hbase.zookeeper.quorum"); 27 | private static String HBASE_ZNODE_PARENT = Config.get().getString("hbase.znode.parent"); 28 | 29 | protected Configuration config = HBaseConfiguration.create(); 30 | 31 | public RegionUtil() { 32 | init(); 33 | } 34 | 35 | protected void init() { 36 | config.set("hbase.zookeeper.quorum", HBASE_ZK); 37 | config.set("zookeeper.znode.parent", HBASE_ZNODE_PARENT); 38 | } 39 | 40 | public RegionChecker getRegionChecker() { 41 | return new RegionChecker(config); 42 | } 43 | 44 | public Configuration getConfig() { 45 | return config; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/merge/MergeException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.merge; 18 | 19 | public class MergeException extends RuntimeException { 20 | 21 | public MergeException(String msg) { 22 | super(msg); 23 | } 24 | 25 | public MergeException(String msg, Exception e) { 26 | super(msg, e); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/merge/QueryAwareResultsMerger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.merge; 18 | 19 | import com.turn.splicer.tsdbutils.TSSubQuery; 20 | import com.turn.splicer.tsdbutils.TsQuery; 21 | 22 | import java.util.ArrayList; 23 | import java.util.Arrays; 24 | import java.util.HashMap; 25 | import java.util.HashSet; 26 | import java.util.List; 27 | import java.util.Map; 28 | import java.util.Set; 29 | import java.util.TreeMap; 30 | 31 | import com.google.common.annotations.VisibleForTesting; 32 | import com.google.common.base.Preconditions; 33 | import org.slf4j.Logger; 34 | import org.slf4j.LoggerFactory; 35 | 36 | public class QueryAwareResultsMerger { 37 | 38 | private static final Logger LOG = LoggerFactory.getLogger(QueryAwareResultsMerger.class); 39 | 40 | static final String NO_TAGS = ""; 41 | 42 | private final TsQuery query; 43 | private final TSSubQuery subQuery; 44 | 45 | public QueryAwareResultsMerger(TsQuery query) 46 | { 47 | Preconditions.checkNotNull(query, "query is null"); 48 | Preconditions.checkNotNull(query.getQueries(), "null subqueries " + query); 49 | if (query.getQueries().size() > 1) { 50 | throw new IllegalArgumentException("too many subqueries. query=" + query); 51 | } 52 | 53 | this.query = query; 54 | this.subQuery = query.getQueries().get(0); 55 | } 56 | 57 | public TsdbResult[] merge(List slices) 58 | { 59 | if (slices == null || slices.size() == 0) { 60 | return new TsdbResult[]{}; 61 | } 62 | 63 | // if only slice, return it 64 | if (slices.size() == 1) return slices.get(0); 65 | 66 | // if two slices, merge them 67 | TsdbResult[] partialMerge = merge(slices.get(0), slices.get(1)); 68 | 69 | // for any remaining slices merge with partial result 70 | for (int i=2; i < slices.size(); i++) { 71 | partialMerge = merge(partialMerge, slices.get(i)); 72 | } 73 | 74 | // return result 75 | return partialMerge; 76 | } 77 | 78 | /** 79 | * Merge two results. The result is a new TsdbResult[] object. 80 | * 81 | * This method searches first creates pairs of TsdbResult objects 82 | * 83 | * @param left input result for a single slice 84 | * @param right another result for a different slice 85 | * @return a new object which is the result of merging two partial results. both 86 | * left and right are not mutated. 87 | */ 88 | protected TsdbResult[] merge(TsdbResult[] left, TsdbResult[] right) 89 | { 90 | if (left == null) left = new TsdbResult[]{}; 91 | if (right == null) right = new TsdbResult[]{}; 92 | 93 | Map leftIndex = index(left); 94 | List mergeResults = new ArrayList<>(); 95 | 96 | for (TsdbResult leftItem: right) { 97 | String ts = createTagString(leftItem); 98 | TsdbResult rightItem = leftIndex.remove(ts); 99 | if (rightItem == null) { 100 | LOG.info("Did not find counterpart for ts={}", ts); 101 | //continue; 102 | } 103 | TsdbResult merge = merge(leftItem, rightItem); 104 | mergeResults.add(merge); 105 | } 106 | 107 | // add any left over tags 108 | if (leftIndex.size() > 0) { 109 | mergeResults.addAll(leftIndex.values()); 110 | } 111 | 112 | TsdbResult[] r = new TsdbResult[mergeResults.size()]; 113 | mergeResults.toArray(r); 114 | return r; 115 | } 116 | 117 | /** 118 | * 119 | * Merge two results into one. It assumes that the inputs are have common tags. 120 | * 121 | * @param left a partial result. cannot be null. dps can be null 122 | * @param right a partial result. cannot be null. dps can be null 123 | * @return if both left and right is null, returns null, if either one is null, returns a 124 | * copy of the non-null result. else, returns a new TsdbResult whose points are a 125 | * union of the points in each input. the other fields are arbitrarily selected 126 | * from one of left or right. 127 | */ 128 | protected TsdbResult merge(TsdbResult left, TsdbResult right) 129 | { 130 | 131 | if (left == null && right == null) return new TsdbResult(); 132 | if (left == null) return right; 133 | if (right == null) return left; 134 | 135 | // initialize with same metadata, and empty point map 136 | TsdbResult m = TsdbResult.copyMeta(left); 137 | m.setDps(new TsdbResult.Points(new TreeMap())); 138 | 139 | // load left points 140 | if (left.getDps() != null && left.getDps().getMap() != null) { 141 | for (Map.Entry e : left.getDps().getMap().entrySet()) { 142 | m.getDps().getMap().put(e.getKey(), e.getValue()); 143 | } 144 | } 145 | 146 | // load right points 147 | if (right.getDps() != null && right.getDps().getMap() != null) { 148 | for (Map.Entry e : right.getDps().getMap().entrySet()) { 149 | m.getDps().getMap().put(e.getKey(), e.getValue()); 150 | } 151 | } 152 | 153 | // return 154 | return m; 155 | } 156 | 157 | /** 158 | * Make an index (string -> tsdbresult) given a TsdbResults[] array, such that each 159 | * string key is unique string (created by {@link #createTagString(TsdbResult)} for each 160 | * item in the input array. 161 | * 162 | * @param results input array 163 | * @return map of tag string with its respective tsdb result object 164 | */ 165 | protected Map index(TsdbResult[] results) 166 | { 167 | Map index = new HashMap<>(); 168 | for (TsdbResult result: results) { 169 | index.put(createTagString(result), result); 170 | } 171 | return index; 172 | } 173 | 174 | @VisibleForTesting 175 | protected String createTagString(TsdbResult result) 176 | { 177 | if (result.getTags() == null 178 | || result.getTags().getTags() == null 179 | || result.getTags().getTags().size() == 0) { 180 | return NO_TAGS; 181 | } 182 | 183 | final Set queryTags; 184 | if (subQuery.getTags() == null || subQuery.getTags().size() == 0) { 185 | queryTags = new HashSet<>(); 186 | } else { 187 | queryTags = subQuery.getTags().keySet(); 188 | } 189 | 190 | /** 191 | * we don't expect any tags in the query result. return {@link #NO_TAGS} 192 | */ 193 | if (queryTags.size() == 0) { 194 | return NO_TAGS; 195 | } 196 | 197 | /** 198 | * if we are expecting tags in our result, then creates string based on them only 199 | */ 200 | Map tagMap = result.getTags().getTags(); 201 | String[] tagKeys = tagMap.keySet().toArray(new String[tagMap.size()]); 202 | Arrays.sort(tagKeys); 203 | 204 | String tagString = ""; 205 | for (String tk: tagKeys) { 206 | if (queryTags.contains(tk)) { 207 | if (tagString.length() > 0) tagString += ","; 208 | tagString += tk + "=" + tagMap.get(tk); 209 | } 210 | } 211 | 212 | return tagString; 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/merge/ResultsMerger.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.merge; 18 | 19 | import java.nio.charset.Charset; 20 | import java.util.ArrayList; 21 | import java.util.Collections; 22 | import java.util.HashMap; 23 | import java.util.List; 24 | import java.util.Map; 25 | import java.util.TreeMap; 26 | 27 | import com.google.common.base.Preconditions; 28 | import com.google.common.collect.Lists; 29 | import com.google.common.hash.HashFunction; 30 | import com.google.common.hash.Hasher; 31 | import com.google.common.hash.Hashing; 32 | import org.slf4j.Logger; 33 | import org.slf4j.LoggerFactory; 34 | 35 | public class ResultsMerger { 36 | 37 | private static final Logger LOG = LoggerFactory.getLogger(ResultsMerger.class); 38 | 39 | /** 40 | * Merge two chunks of results into one piece 41 | * @param first Results for one slice 42 | * @param second Results for another slice from same query 43 | * @return a third chunk which should have points from first and second chunks 44 | */ 45 | public TsdbResult[] merge (TsdbResult[] first, TsdbResult[] second) { 46 | Preconditions.checkNotNull(first); 47 | Preconditions.checkNotNull(second); 48 | Preconditions.checkArgument(first.length > 0); 49 | Preconditions.checkArgument(second.length == first.length); 50 | 51 | String metricName = first[0].getMetric(); 52 | for (int i = 1; i < first.length; i++) { 53 | if (!first[i].getMetric().equals(metricName)) { 54 | throw new MergeException("Metric Names differ within first result. Metric=" 55 | + metricName + ", first has=" + first[i].getMetric()); 56 | } 57 | } 58 | 59 | for (TsdbResult s: second) { 60 | if (!s.getMetric().equals(metricName)) { 61 | throw new MergeException("Metric Names differ in second result. Metric=" 62 | + metricName + ", second has=" + s.getMetric()); 63 | } 64 | } 65 | 66 | Map> fTable = new HashMap<>(); 67 | for (TsdbResult f: first) { 68 | long signature = signatureOf(f); 69 | if (fTable.containsKey(signature)) { 70 | List list = fTable.get(signature); 71 | list.add(f); 72 | } else { 73 | fTable.put(signature, Lists.newArrayList(f)); 74 | } 75 | } 76 | 77 | List finalResults = new ArrayList<>(); 78 | for (TsdbResult s: second) { 79 | long signature = signatureOf(s); 80 | List fromFirst = fTable.get(signature); 81 | if (fromFirst == null || fromFirst.isEmpty()) { 82 | LOG.error("Could not find counterpart for " + s); 83 | continue; 84 | } 85 | 86 | boolean foundMatch = false; 87 | if (fromFirst.size() == 1) { 88 | finalResults.add(merge(fromFirst.get(0), s)); 89 | foundMatch = true; 90 | } else { 91 | for (TsdbResult r: fromFirst) { 92 | if (identical(r, s)) { 93 | finalResults.add(merge(r, s)); 94 | foundMatch = true; 95 | } 96 | } 97 | } 98 | 99 | if (!foundMatch) { 100 | throw new MergeException("Could not find match for s=" + s); 101 | } 102 | } 103 | 104 | return finalResults.toArray(new TsdbResult[finalResults.size()]); 105 | } 106 | 107 | /** 108 | * blind merge 109 | * @param first first result 110 | * @param second second result 111 | */ 112 | private TsdbResult merge(TsdbResult first, TsdbResult second) { 113 | TsdbResult result = new TsdbResult(); 114 | result.setMetric(first.getMetric()); 115 | result.setTags(first.getTags()); 116 | result.setAggregateTags(first.getAggregateTags()); 117 | result.setTsuids(first.getTsuids()); 118 | 119 | Map points = new TreeMap<>(); 120 | points.putAll(first.getDps().getMap()); 121 | points.putAll(second.getDps().getMap()); 122 | 123 | result.setDps(new TsdbResult.Points(points)); 124 | return result; 125 | } 126 | 127 | public boolean identical(TsdbResult one, TsdbResult two) 128 | { 129 | List aggTagsOne = one.getAggregateTags(); 130 | List aggTagsTwo = two.getAggregateTags(); 131 | if (aggTagsOne != null && aggTagsTwo != null && !aggTagsOne.containsAll(aggTagsTwo)) { 132 | return false; 133 | } else if (aggTagsOne == null && aggTagsTwo != null) { 134 | return false; 135 | } else if (aggTagsOne != null && aggTagsTwo == null) { 136 | return false; 137 | } 138 | 139 | Map tagsOne = one.getTags().getTags(); 140 | Map tagsTwo = two.getTags().getTags(); 141 | if (tagsOne != null && tagsTwo == null) { 142 | return false; 143 | } else if (tagsOne == null && tagsTwo != null) { 144 | return false; 145 | } 146 | 147 | if (tagsOne == null) tagsOne = new HashMap<>(); 148 | if (tagsTwo == null) tagsTwo = new HashMap<>(); 149 | 150 | if ( !tagsOne.keySet().containsAll(tagsTwo.keySet()) ) return false; 151 | if ( !tagsOne.values().containsAll(tagsTwo.values()) ) return false; 152 | 153 | List tsuidsOne = one.getTsuids(); 154 | List tsuidsTwo = two.getTsuids(); 155 | if (tsuidsOne != null && tsuidsTwo != null && !tsuidsOne.containsAll(tsuidsTwo)) { 156 | return false; 157 | } else if (tsuidsOne == null && tsuidsTwo != null) { 158 | return false; 159 | } else if (tsuidsOne != null && tsuidsTwo == null) { 160 | return false; 161 | } 162 | 163 | return true; 164 | } 165 | 166 | public long signatureOf(TsdbResult result) { 167 | HashFunction hf = Hashing.goodFastHash(64); 168 | Hasher hasher = hf.newHasher(); 169 | 170 | List aggTags = result.getAggregateTags(); 171 | if (aggTags != null) { 172 | List sortedAggTags = Lists.newArrayList(aggTags); 173 | Collections.sort(sortedAggTags); 174 | for (String aggTag: sortedAggTags) { 175 | hasher.putString(aggTag, Charset.forName("ISO-8859-1")); 176 | } 177 | } 178 | 179 | Map tags; 180 | if (result.getTags() != null && (tags = result.getTags().getTags()) != null) { 181 | List tagTokens = Lists.newArrayList(tags.keySet()); 182 | Collections.sort(tagTokens); 183 | for (String s: tagTokens) { 184 | hasher.putString(s, Charset.forName("ISO-8859-1")); 185 | hasher.putString(tags.get(s), Charset.forName("ISO-8859-1")); 186 | } 187 | } 188 | 189 | List tsuids = result.getTsuids(); 190 | if (tsuids != null) { 191 | List sortedTsUIDs = Lists.newArrayList(tsuids); 192 | Collections.sort(sortedTsUIDs); 193 | for (String tsuid: sortedTsUIDs) { 194 | hasher.putString(tsuid, Charset.forName("ISO-8859-1")); 195 | } 196 | } 197 | 198 | return hasher.hash().asLong(); 199 | } 200 | 201 | } 202 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/JSON.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils; 18 | 19 | import java.io.IOException; 20 | 21 | import com.fasterxml.jackson.core.JsonParser; 22 | import com.fasterxml.jackson.core.JsonProcessingException; 23 | import com.fasterxml.jackson.databind.ObjectMapper; 24 | import com.google.common.base.Preconditions; 25 | import net.opentsdb.utils.JSONException; 26 | 27 | public class JSON { 28 | 29 | /** 30 | * Jackson de/serializer initialized, configured and shared 31 | */ 32 | private static final ObjectMapper jsonMapper = new ObjectMapper(); 33 | 34 | static { 35 | // allows parsing NAN and such without throwing an exception. This is 36 | // important 37 | // for incoming data points with multiple points per put so that we can 38 | // toss only the bad ones but keep the good 39 | jsonMapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true); 40 | } 41 | 42 | public static T parseToObject(final String json, 43 | final Class pojo) { 44 | Preconditions.checkNotNull(json); 45 | Preconditions.checkArgument(!json.isEmpty(), "Incoming data was null or empty"); 46 | Preconditions.checkNotNull(pojo); 47 | 48 | try { 49 | return jsonMapper.readValue(json, pojo); 50 | } catch (IOException e) { 51 | throw new JSONException(e); 52 | } 53 | } 54 | 55 | /** 56 | * Serializes the given object to a JSON string 57 | * 58 | * @param object The object to serialize 59 | * @return A JSON formatted string 60 | * @throws IllegalArgumentException if the object was null 61 | * @throws JSONException if the object could not be serialized 62 | * @throws IOException Thrown when there was an issue reading the object 63 | */ 64 | public static String serializeToString(Object object) { 65 | if (object == null) 66 | throw new IllegalArgumentException("Object was null"); 67 | try { 68 | return jsonMapper.writeValueAsString(object); 69 | } catch (JsonProcessingException e) { 70 | throw new JSONException(e); 71 | } 72 | } 73 | 74 | /** 75 | * Serializes the given object to a JSON byte array 76 | * 77 | * @param object The object to serialize 78 | * @return A JSON formatted byte array 79 | * @throws IllegalArgumentException if the object was null 80 | * @throws JSONException if the object could not be serialized 81 | * @throws IOException Thrown when there was an issue reading the object 82 | */ 83 | public static byte[] serializeToBytes(Object object) { 84 | if (object == null) 85 | throw new IllegalArgumentException("Object was null"); 86 | try { 87 | return jsonMapper.writeValueAsBytes(object); 88 | } catch (JsonProcessingException e) { 89 | throw new JSONException(e); 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/RateOptions.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils; 18 | 19 | /** 20 | * Provides additional options that will be used when calculating rates. These 21 | * options are useful when working with metrics that are raw counter values, 22 | * where a counter is defined by a value that always increases until it hits 23 | * a maximum value and then it "rolls over" to start back at 0. 24 | *

25 | * These options will only be utilized if the query is for a rate calculation 26 | * and if the "counter" options is set to true. 27 | * 28 | * @since 2.0 29 | */ 30 | public class RateOptions { 31 | public static final long DEFAULT_RESET_VALUE = 0; 32 | 33 | /** 34 | * If true, then when calculating a rate of change assume that the metric 35 | * values are counters and thus non-zero, always increasing and wrap around at 36 | * some maximum 37 | */ 38 | private boolean counter; 39 | 40 | /** 41 | * If calculating a rate of change over a metric that is a counter, then this 42 | * value specifies the maximum value the counter will obtain before it rolls 43 | * over. This value will default to Long.MAX_VALUE. 44 | */ 45 | private long counter_max; 46 | 47 | /** 48 | * Specifies the the rate change value which, if exceeded, will be considered 49 | * a data anomaly, such as a system reset of the counter, and the rate will be 50 | * returned as a zero value for a given data point. 51 | */ 52 | private long reset_value; 53 | 54 | /** 55 | * Ctor 56 | */ 57 | public RateOptions() { 58 | this.counter = false; 59 | this.counter_max = Long.MAX_VALUE; 60 | this.reset_value = DEFAULT_RESET_VALUE; 61 | } 62 | 63 | /** 64 | * Ctor 65 | * 66 | * @param counter If true, indicates that the rate calculation should assume 67 | * that the underlying data is from a counter 68 | * @param counter_max Specifies the maximum value for the counter before it 69 | * will roll over and restart at 0 70 | * @param reset_value Specifies the largest rate change that is considered 71 | * acceptable, if a rate change is seen larger than this value then the 72 | * counter is assumed to have been reset 73 | */ 74 | public RateOptions(final boolean counter, final long counter_max, 75 | final long reset_value) { 76 | this.counter = counter; 77 | this.counter_max = counter_max; 78 | this.reset_value = reset_value; 79 | } 80 | 81 | /** 82 | * @return Whether or not the counter flag is set 83 | */ 84 | public boolean isCounter() { 85 | return counter; 86 | } 87 | 88 | /** 89 | * @return The counter max value 90 | */ 91 | public long getCounterMax() { 92 | return counter_max; 93 | } 94 | 95 | /** 96 | * @return The optional reset value for anomaly suppression 97 | */ 98 | public long getResetValue() { 99 | return reset_value; 100 | } 101 | 102 | /** 103 | * @param counter Whether or not the time series should be considered counters 104 | */ 105 | public void setIsCounter(boolean counter) { 106 | this.counter = counter; 107 | } 108 | 109 | /** 110 | * @param counter_max The value at which counters roll over 111 | */ 112 | public void setCounterMax(long counter_max) { 113 | this.counter_max = counter_max; 114 | } 115 | 116 | /** 117 | * @param reset_value A difference that may be an anomaly so suppress it 118 | */ 119 | public void setResetValue(long reset_value) { 120 | this.reset_value = reset_value; 121 | } 122 | 123 | /** 124 | * Generates a String version of the rate option instance in a format that 125 | * can be utilized in a query. 126 | * 127 | * @return string version of the rate option instance. 128 | */ 129 | public String toString() { 130 | StringBuilder buf = new StringBuilder(); 131 | buf.append('{'); 132 | buf.append(counter); 133 | buf.append(',').append(counter_max); 134 | buf.append(',').append(reset_value); 135 | buf.append('}'); 136 | return buf.toString(); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/SplicerQueryRunner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils; 18 | 19 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 20 | 21 | import com.turn.splicer.Const; 22 | import com.turn.splicer.HttpWorker; 23 | import com.turn.splicer.Splicer; 24 | import com.turn.splicer.hbase.RegionChecker; 25 | import com.turn.splicer.merge.QueryAwareResultsMerger; 26 | import com.turn.splicer.merge.ResultsMerger; 27 | import com.turn.splicer.merge.TsdbResult; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.io.IOException; 32 | import java.util.ArrayList; 33 | import java.util.List; 34 | import java.util.concurrent.*; 35 | import java.util.concurrent.atomic.AtomicInteger; 36 | 37 | /** 38 | * "Slices" a single TsQuery into multiple TsQuery objects that span the TsQuery 39 | * time range, runs those mulitple TsQuerys in parallel and splices 40 | * the results together into a TsdbResult[] 41 | */ 42 | public class SplicerQueryRunner { 43 | 44 | private static final Logger LOG = LoggerFactory.getLogger(SplicerQueryRunner.class); 45 | 46 | private static AtomicInteger POOL_NUMBER = new AtomicInteger(0); 47 | 48 | private static final int NUM_THREADS_PER_POOL = 10; 49 | 50 | private static ThreadFactoryBuilder THREAD_FACTORY_BUILDER = new ThreadFactoryBuilder() 51 | .setDaemon(false) 52 | .setPriority(Thread.NORM_PRIORITY); 53 | 54 | public TsdbResult[] sliceAndRunQuery(TsQuery tsQuery, RegionChecker checker) 55 | throws IOException 56 | { 57 | long duration = tsQuery.endTime() - tsQuery.startTime(); 58 | if (duration > TimeUnit.MILLISECONDS.convert(2, TimeUnit.HOURS)) { 59 | Splicer splicer = new Splicer(tsQuery); 60 | List slices = splicer.sliceQuery(); 61 | return runQuerySlices(tsQuery, slices, checker); 62 | } else { 63 | // only one query. run it in the servlet thread 64 | HttpWorker worker = new HttpWorker(tsQuery, checker); 65 | try { 66 | String json = worker.call(); 67 | return TsdbResult.fromArray(json); 68 | } catch (Exception e) { 69 | throw new RuntimeException(e); 70 | } 71 | } 72 | } 73 | 74 | private TsdbResult[] runQuerySlices(TsQuery query, List slices, RegionChecker checker) 75 | { 76 | String poolName = String.format("splice-pool-%d", POOL_NUMBER.incrementAndGet()); 77 | 78 | ThreadFactory factory = THREAD_FACTORY_BUILDER 79 | .setNameFormat(poolName + "-thread-%d") 80 | .build(); 81 | 82 | ExecutorService svc = Executors.newFixedThreadPool(NUM_THREADS_PER_POOL, factory); 83 | ResultsMerger merger = new ResultsMerger(); 84 | QueryAwareResultsMerger qamerger = new QueryAwareResultsMerger(query); 85 | try { 86 | List> results = new ArrayList<>(); 87 | for (TsQuery q : slices) { 88 | results.add(svc.submit(new HttpWorker(q, checker))); 89 | } 90 | 91 | List tmpResults = new ArrayList<>(); 92 | for (Future s: results) { 93 | String json = s.get(); 94 | LOG.debug("Got result={}", json); 95 | 96 | TsdbResult[] r = TsdbResult.fromArray(json); 97 | tmpResults.add(r); 98 | } 99 | 100 | TsdbResult[] qaResult = qamerger.merge(tmpResults); 101 | // respond with qaResult 102 | if (qaResult != null) { 103 | return qaResult; 104 | } else { 105 | return new TsdbResult[]{}; 106 | } 107 | 108 | } catch (Exception e) { 109 | LOG.error("Could not execute HTTP Queries", e); 110 | throw new RuntimeException(e); 111 | } finally { 112 | svc.shutdown(); 113 | LOG.info("Shutdown thread pool for query=" + stringify(query)); 114 | } 115 | } 116 | 117 | private String stringify(TsQuery query) 118 | { 119 | String subs = ""; 120 | for (TSSubQuery sub: query.getQueries()) { 121 | if (subs.length() > 0) subs += ","; 122 | subs += "m=[" + sub.getMetric() + sub.getTags() 123 | + ", downsample=" + sub.getDownsample() 124 | + ", rate=" + sub.getRate() 125 | + ", tags=" + sub.getTags() 126 | + "]"; 127 | } 128 | return "{" + query.startTime() + " to " + query.endTime() + ", " + subs + "}"; 129 | } 130 | 131 | } 132 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/TSSubQuery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils; 18 | 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | public class TSSubQuery { 25 | /** 26 | * User given name of an aggregation function to use 27 | */ 28 | private String aggregator; 29 | 30 | /** 31 | * User given name for a metric, e.g. "sys.cpu.0" 32 | */ 33 | private String metric; 34 | 35 | /** 36 | * User provided list of timeseries UIDs 37 | */ 38 | private List tsuids; 39 | 40 | /** 41 | * User supplied list of tags for specificity or grouping. May be null or 42 | * empty 43 | */ 44 | private HashMap tags; 45 | 46 | /** 47 | * User given downsampler 48 | */ 49 | private String downsample; 50 | 51 | /** 52 | * Whether or not the user wants to perform a rate conversion 53 | */ 54 | private boolean rate; 55 | 56 | /** 57 | * Rate options for counter rollover/reset 58 | */ 59 | private RateOptions rate_options; 60 | 61 | /** 62 | * Parsed downsample interval 63 | */ 64 | private long downsample_interval; 65 | 66 | /** 67 | * Default constructor necessary for POJO de/serialization 68 | */ 69 | public TSSubQuery() { 70 | 71 | } 72 | 73 | public String toString() { 74 | final StringBuilder buf = new StringBuilder(); 75 | buf.append("TSSubQuery(metric=") 76 | .append(metric == null || metric.isEmpty() ? "" : metric); 77 | buf.append(", tags=["); 78 | if (tags != null && !tags.isEmpty()) { 79 | int counter = 0; 80 | for (Map.Entry entry : tags.entrySet()) { 81 | if (counter > 0) { 82 | buf.append(", "); 83 | } 84 | buf.append(entry.getKey()) 85 | .append("=") 86 | .append(entry.getValue()); 87 | counter++; 88 | } 89 | } 90 | buf.append("], tsuids=["); 91 | if (tsuids != null && !tsuids.isEmpty()) { 92 | int counter = 0; 93 | for (String tsuid : tsuids) { 94 | if (counter > 0) { 95 | buf.append(", "); 96 | } 97 | buf.append(tsuid); 98 | counter++; 99 | } 100 | } 101 | buf.append("], agg=") 102 | .append(aggregator) 103 | .append(", downsample=") 104 | .append(downsample) 105 | .append(", ds_interval=") 106 | .append(downsample_interval) 107 | .append(", rate=") 108 | .append(rate) 109 | .append(", rate_options=") 110 | .append(rate_options); 111 | buf.append(")"); 112 | return buf.toString(); 113 | } 114 | 115 | /** 116 | * Runs through query parameters to make sure it's a valid request. 117 | * This includes parsing the aggregator, downsampling info, metrics, tags or 118 | * timeseries and setting the local parsed fields needed by the TSD for proper 119 | * execution. If no exceptions are thrown, the query is considered valid. 120 | * Note: You do not need to call this directly as it will be executed 121 | * by the {@link TsQuery} object the sub query is assigned to. 122 | * 123 | * @throws IllegalArgumentException if something is wrong with the query 124 | */ 125 | public void validateAndSetQuery() { 126 | if (aggregator == null || aggregator.isEmpty()) { 127 | throw new IllegalArgumentException("Missing the aggregation function"); 128 | } 129 | 130 | // we must have at least one TSUID OR a metric 131 | if ((tsuids == null || tsuids.isEmpty()) && 132 | (metric == null || metric.isEmpty())) { 133 | throw new IllegalArgumentException( 134 | "Missing the metric or tsuids, provide at least one"); 135 | } 136 | } 137 | 138 | /** 139 | * @return whether or not the user requested a rate conversion 140 | */ 141 | public boolean getRate() { 142 | return rate; 143 | } 144 | 145 | /** 146 | * @return options to use for rate calculations 147 | */ 148 | public RateOptions getRateOptions() { 149 | return rate_options; 150 | } 151 | 152 | /** 153 | * @return the parsed downsample interval in seconds 154 | */ 155 | public long downsampleInterval() { 156 | return this.downsample_interval; 157 | } 158 | 159 | /** 160 | * @return the user supplied aggregator 161 | */ 162 | public String getAggregator() { 163 | return aggregator; 164 | } 165 | 166 | /** 167 | * @return the user supplied metric 168 | */ 169 | public String getMetric() { 170 | return metric; 171 | } 172 | 173 | /** 174 | * @return the user supplied list of TSUIDs 175 | */ 176 | public List getTsuids() { 177 | return tsuids; 178 | } 179 | 180 | /** 181 | * @return the raw downsampling function request from the user, 182 | * e.g. "1h-avg" 183 | */ 184 | public String getDownsample() { 185 | return downsample; 186 | } 187 | 188 | /** 189 | * @return the user supplied list of query tags, may be empty 190 | */ 191 | public Map getTags() { 192 | if (tags == null) { 193 | return Collections.emptyMap(); 194 | } 195 | return tags; 196 | } 197 | 198 | /** 199 | * @param aggregator the name of an aggregation function 200 | */ 201 | public void setAggregator(String aggregator) { 202 | this.aggregator = aggregator; 203 | } 204 | 205 | /** 206 | * @param metric the name of a metric to fetch 207 | */ 208 | public void setMetric(String metric) { 209 | this.metric = metric; 210 | } 211 | 212 | /** 213 | * @param tsuids a list of timeseries UIDs as hex encoded strings to fetch 214 | */ 215 | public void setTsuids(List tsuids) { 216 | this.tsuids = tsuids; 217 | } 218 | 219 | /** 220 | * @param tags an optional list of tags for specificity or grouping 221 | */ 222 | public void setTags(HashMap tags) { 223 | this.tags = tags; 224 | } 225 | 226 | /** 227 | * @param downsample the downsampling function to use, e.g. "2h-avg" 228 | */ 229 | public void setDownsample(String downsample) { 230 | this.downsample = downsample; 231 | } 232 | 233 | /** 234 | * @param rate whether or not the result should be rate converted 235 | */ 236 | public void setRate(boolean rate) { 237 | this.rate = rate; 238 | } 239 | 240 | /** 241 | * @param options Options to set when calculating rates 242 | */ 243 | public void setRateOptions(RateOptions options) { 244 | this.rate_options = options; 245 | } 246 | 247 | } -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/TsQuerySerializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils; 18 | 19 | import com.google.common.base.Preconditions; 20 | 21 | public class TsQuerySerializer { 22 | 23 | public static TsQuery deserializeFromJson(String jsonContent) 24 | { 25 | Preconditions.checkNotNull(jsonContent); 26 | Preconditions.checkArgument(!jsonContent.isEmpty(), "Incoming data was null or empty"); 27 | 28 | return JSON.parseToObject(jsonContent, TsQuery.class); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/BadNumberException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | public class BadNumberException extends RuntimeException { 20 | 21 | public BadNumberException(String msg) { 22 | super(msg); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/EndpointAligningAggregationIterator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import net.opentsdb.core.AggregationIterator; 20 | import net.opentsdb.core.Aggregator; 21 | import net.opentsdb.core.Aggregators; 22 | import net.opentsdb.core.SeekableView; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | /** 27 | * Identical to AggregationIterator except for how it handles edges of a Span. 28 | * 29 | * In the constructor it will skip ahead to a point where each span has a nonzero 30 | * value for the next call to next(). 31 | * 32 | * In hasNext() it will return false if any span is out of points. 33 | * 34 | * This solves the problem with the difference function, where diff(A, B) 35 | * was producing spiking values if B did not have values for timestamps that 36 | * A had values for around the start and end time of a query. This caused us to 37 | * get a value diff(A, 0) which would set off alerts unnecessarily. 38 | * The solution was to shave off points at the boundary where one span didn't have values. 39 | * 40 | * @author Brian Peltz 41 | */ 42 | public class EndpointAligningAggregationIterator extends AggregationIterator { 43 | private static final Logger LOG = LoggerFactory.getLogger(AggregationIterator.class); 44 | 45 | public EndpointAligningAggregationIterator(SeekableView[] iterators, long start_time, long end_time, Aggregator aggregator, Aggregators.Interpolation method, boolean rate) { 46 | super(iterators, start_time, end_time, aggregator, method, rate); 47 | 48 | alignFirstTimestamps(); 49 | } 50 | 51 | /** 52 | * Goes through timestamp array looking for 0 values. If it finds a 0 it 53 | * checks to see if the next timestamp for that Span is the lowest of all next 54 | * timestamps (meaning it will not be zero after the first call to next()). If it 55 | * is not the lowest it calls next() until it is the lowest. Does this until every zero 56 | * value has the minimum next timestamp. 57 | */ 58 | private void alignFirstTimestamps() { 59 | int numSeries = iterators.length; 60 | //check for zeroes 61 | for(int i = 0; i < numSeries; i++) { 62 | if(timestamps[i] == 0) { 63 | long minTime = nextMinimumTimestamp(); 64 | //if next timestamp for this span is not the minimum 65 | while(timestamps[i + numSeries] > minTime) { 66 | if(hasNext()) { 67 | next(); 68 | } else { 69 | //no more data points so we're sunk 70 | break; 71 | } 72 | //recalculate min time 73 | minTime = nextMinimumTimestamp(); 74 | } 75 | } 76 | } 77 | } 78 | 79 | private long nextMinimumTimestamp() { 80 | //set min as first timestamp 81 | long minTime = timestamps[iterators.length]; 82 | for(int j = iterators.length; j < timestamps.length; j++) { 83 | if(timestamps[j] < minTime) { 84 | minTime = timestamps[j]; 85 | } 86 | } 87 | return minTime; 88 | } 89 | 90 | /** 91 | * Modified from AggregationIterator 92 | * If any next timestamp is greater than end_time we return false, 93 | * so if ANY series is out of DataPoints we don't produce values 94 | * @return 95 | */ 96 | @Override 97 | public boolean hasNext() { 98 | final int size = iterators.length; 99 | 100 | for (int i = 0; i < size; i++) { 101 | // As long as any of the iterators has a data point with a timestamp 102 | // that falls within our interval, we know we have at least one next. 103 | if ((timestamps[size + i] & TIME_MASK) > end_time) { 104 | //LOG.debug("Total aggregationTime=" + (aggregationTimeInNanos / (1000 * 1000)) + "ms."); 105 | //LOG.debug("Total interpolationTime=" + (interpolationTimeInNanos / (1000 * 1000)) + "ms."); 106 | //LOG.debug("Total downSampleTime=" + (downsampleTimeInNanos / (1000 * 1000)) + "ms."); 107 | //LOG.debug("Total moveToNextTime=" + (moveToNextTimeInNanos / (1000 * 1000)) + "ms."); 108 | //LOG.debug("No hasNext (return false)"); 109 | return false; 110 | } 111 | } 112 | return true; 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/ExprReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.google.common.base.Preconditions; 20 | 21 | public class ExprReader { 22 | 23 | protected final char[] chars; 24 | 25 | private int mark = 0; 26 | 27 | public ExprReader(char[] chars) { 28 | Preconditions.checkNotNull(chars); 29 | this.chars = chars; 30 | } 31 | 32 | public int getMark() { 33 | return mark; 34 | } 35 | 36 | public char peek() { 37 | return chars[mark]; 38 | } 39 | 40 | public char next() { 41 | return chars[mark++]; 42 | } 43 | 44 | public void skip(int num) { 45 | mark += num; 46 | } 47 | 48 | public boolean isNextChar(char c) { 49 | return peek() == c; 50 | } 51 | 52 | public boolean isNextSeq(CharSequence seq) { 53 | Preconditions.checkNotNull(seq); 54 | for (int i = 0; i < seq.length(); i++) { 55 | if (mark + i == chars.length) return false; 56 | if (chars[mark + i] != seq.charAt(i)) { 57 | return false; 58 | } 59 | } 60 | 61 | return true; 62 | } 63 | 64 | public String readFuncName() { 65 | StringBuilder builder = new StringBuilder(); 66 | while (peek() != '(' && !Character.isWhitespace(peek())) { 67 | builder.append(next()); 68 | } 69 | return builder.toString(); 70 | } 71 | 72 | public boolean isEOF() { 73 | return mark == chars.length; 74 | } 75 | 76 | public void skipWhitespaces() { 77 | for (int i = mark; i < chars.length; i++) { 78 | if (Character.isWhitespace(chars[i])) { 79 | mark++; 80 | } else { 81 | break; 82 | } 83 | } 84 | } 85 | 86 | public String readNextParameter() { 87 | StringBuilder builder = new StringBuilder(); 88 | int numNested = 0; 89 | while (!Character.isWhitespace(peek())) { 90 | char ch = peek(); 91 | if (ch == '(') numNested++; 92 | else if (ch == ')') numNested--; 93 | if (numNested < 0) { 94 | break; 95 | } 96 | if (numNested <= 0 && isNextSeq(",,")) { 97 | break; 98 | } 99 | builder.append(next()); 100 | } 101 | return builder.toString(); 102 | } 103 | 104 | @Override 105 | public String toString() { 106 | // make a copy 107 | return new String(chars); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/Expression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.turn.splicer.merge.TsdbResult; 20 | import com.turn.splicer.tsdbutils.TsQuery; 21 | 22 | import java.util.List; 23 | 24 | public interface Expression { 25 | 26 | TsdbResult[] evaluate(TsQuery dataQuery, List queryResults, List queryParams); 27 | 28 | String writeStringField(List queryParams, String innerExpression); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/ExpressionFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.google.common.annotations.VisibleForTesting; 20 | import com.google.common.base.Joiner; 21 | import com.google.common.collect.Maps; 22 | import com.turn.splicer.merge.TsdbResult; 23 | import com.turn.splicer.tsdbutils.Functions; 24 | import com.turn.splicer.tsdbutils.TsQuery; 25 | import org.apache.log4j.Logger; 26 | 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | public class ExpressionFactory { 31 | 32 | private static final Logger logger = Logger.getLogger(ExpressionFactory.class); 33 | 34 | private static Map availableFunctions = 35 | Maps.newHashMap(); 36 | 37 | static { 38 | availableFunctions.put("id", new IdentityExpression()); 39 | availableFunctions.put("alias", new AliasFunction()); 40 | availableFunctions.put("scale", new Functions.ScaleFunction()); 41 | availableFunctions.put("sumSeries", new Functions.SumSeriesFunction()); 42 | availableFunctions.put("sum", new Functions.SumSeriesFunction()); 43 | availableFunctions.put("difference", new Functions.DifferenceSeriesFunction()); 44 | availableFunctions.put("multiply", new Functions.MultiplySeriesFunction()); 45 | availableFunctions.put("divide", new Functions.DivideSeriesFunction()); 46 | availableFunctions.put("movingAverage", new Functions.MovingAverageFunction()); 47 | availableFunctions.put("abs", new Functions.AbsoluteValueFunction()); 48 | availableFunctions.put("timeShift", new Functions.TimeShiftFunction()); 49 | } 50 | 51 | @VisibleForTesting 52 | static void addFunction(String name, Expression expr) { 53 | availableFunctions.put(name, expr); 54 | } 55 | 56 | public static Expression getByName(String funcName) { 57 | return availableFunctions.get(funcName); 58 | } 59 | 60 | static class IdentityExpression implements Expression { 61 | @Override 62 | public TsdbResult[] evaluate(TsQuery dataQuery, 63 | List queryResults, List queryParams) { 64 | return queryResults.get(0); 65 | } 66 | 67 | @Override 68 | public String toString() { 69 | return "id"; 70 | } 71 | 72 | @Override 73 | public String writeStringField(List queryParams, String innerExpression) { 74 | return "id(" + innerExpression + ")"; 75 | } 76 | } 77 | 78 | static class AliasFunction implements Expression { 79 | 80 | static Joiner COMMA_JOINER = Joiner.on(',').skipNulls(); 81 | 82 | @Override 83 | public TsdbResult[] evaluate(TsQuery dataQuery, List queryResults, 84 | List queryParams) { 85 | if (queryResults == null || queryResults.size() == 0) { 86 | throw new NullPointerException("No query results"); 87 | } 88 | 89 | String aliasTemplate = "__default"; 90 | 91 | if (queryParams != null && queryParams.size() >= 0) { 92 | aliasTemplate = COMMA_JOINER.join(queryParams); 93 | } 94 | 95 | TsdbResult[] resultToAlias = queryResults.get(0); 96 | 97 | for(TsdbResult tsResult: resultToAlias) { 98 | String alias = aliasTemplate; 99 | for(Map.Entry tagKVPair: tsResult.getTags().getTags().entrySet()) { 100 | alias = alias.replace("@" + tagKVPair.getKey(), tagKVPair.getValue()); 101 | } 102 | tsResult.setAlias(alias); 103 | } 104 | 105 | return resultToAlias; 106 | } 107 | 108 | @Override 109 | public String writeStringField(List queryParams, String innerExpression) { 110 | if (queryParams == null || queryParams.size() == 0) { 111 | return "NULL"; 112 | } 113 | 114 | return queryParams.get(0); 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/ExpressionTree.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.turn.splicer.SplicerServlet; 20 | import com.turn.splicer.hbase.RegionChecker; 21 | import com.turn.splicer.merge.TsdbResult; 22 | 23 | import java.util.ArrayList; 24 | import java.util.Collection; 25 | import java.util.Collections; 26 | import java.util.List; 27 | import java.util.Map; 28 | import java.util.concurrent.ExecutionException; 29 | import java.util.concurrent.ExecutorService; 30 | import java.util.concurrent.Executors; 31 | import java.util.concurrent.Future; 32 | 33 | import com.google.common.base.Joiner; 34 | import com.google.common.collect.Lists; 35 | import com.google.common.collect.Maps; 36 | import com.turn.splicer.tsdbutils.Functions; 37 | import com.turn.splicer.tsdbutils.SplicerQueryRunner; 38 | import com.turn.splicer.tsdbutils.SplicerUtils; 39 | import com.turn.splicer.tsdbutils.TsQuery; 40 | 41 | public class ExpressionTree { 42 | 43 | private final Expression expr; 44 | private final TsQuery dataQuery; 45 | 46 | private List subExpressions; 47 | private List funcParams; 48 | private Map subMetricQueries; 49 | private Map parameterSourceIndex = Maps.newHashMap(); 50 | 51 | private static final Joiner DOUBLE_COMMA_JOINER = Joiner.on(",").skipNulls(); 52 | 53 | private SplicerQueryRunner queryRunner; 54 | 55 | private static ExecutorService pool = Executors.newCachedThreadPool(); 56 | 57 | 58 | enum Parameter { 59 | SUB_EXPRESSION, 60 | METRIC_QUERY 61 | } 62 | 63 | public ExpressionTree(String exprName, TsQuery dataQuery) { 64 | this(ExpressionFactory.getByName(exprName), dataQuery); 65 | } 66 | 67 | public ExpressionTree(Expression expr, TsQuery dataQuery) { 68 | this.expr = expr; 69 | this.dataQuery = dataQuery; 70 | this.queryRunner = new SplicerQueryRunner(); 71 | } 72 | 73 | 74 | public void addSubExpression(ExpressionTree child, int paramIndex) { 75 | if (subExpressions == null) { 76 | subExpressions = Lists.newArrayList(); 77 | } 78 | subExpressions.add(child); 79 | parameterSourceIndex.put(paramIndex, Parameter.SUB_EXPRESSION); 80 | } 81 | 82 | public void addSubMetricQuery(String metricQuery, int magic, 83 | int paramIndex) { 84 | if (subMetricQueries == null) { 85 | subMetricQueries = Maps.newHashMap(); 86 | } 87 | subMetricQueries.put(magic, metricQuery); 88 | parameterSourceIndex.put(paramIndex, Parameter.METRIC_QUERY); 89 | } 90 | 91 | public void addFunctionParameter(String param) { 92 | if (funcParams == null) { 93 | funcParams = Lists.newArrayList(); 94 | } 95 | funcParams.add(param); 96 | } 97 | 98 | public TsdbResult[] evaluateAll() throws ExecutionException, InterruptedException { 99 | 100 | List metricQueryKeys = null; 101 | 102 | if (subMetricQueries != null && subMetricQueries.size() > 0) { 103 | metricQueryKeys = Lists.newArrayList(subMetricQueries.keySet()); 104 | Collections.sort(metricQueryKeys); 105 | } 106 | 107 | int metricPointer = 0; 108 | int subExprPointer = 0; 109 | 110 | if(expr instanceof Functions.TimeShiftFunction) { 111 | String param = funcParams.get(0); 112 | if (param == null || param.length() == 0) { 113 | throw new NullPointerException("Invalid timeshift='" + param + "'"); 114 | } 115 | 116 | param = param.trim(); 117 | 118 | long timeshift = -1; 119 | if (param.startsWith("'") && param.endsWith("'")) { 120 | timeshift = Functions.parseParam(param); 121 | } else { 122 | throw new RuntimeException("Invalid timeshift parameter: eg '10min'"); 123 | } 124 | 125 | long newStart = dataQuery.startTime() - timeshift; 126 | long oldStart = dataQuery.startTime(); 127 | dataQuery.setStart(Long.toString(newStart)); 128 | long newEnd = dataQuery.endTime() - timeshift; 129 | long oldEnd = dataQuery.endTime(); 130 | dataQuery.setEnd(Long.toString(newEnd)); 131 | dataQuery.validateTimes(); 132 | } 133 | 134 | List> tsdbResultFutures = new ArrayList(parameterSourceIndex.size()); 135 | 136 | for (int i = 0; i < parameterSourceIndex.size(); i++) { 137 | Parameter p = parameterSourceIndex.get(i); 138 | 139 | if (p == Parameter.METRIC_QUERY) { 140 | if (metricQueryKeys == null) { 141 | throw new RuntimeException("Attempt to read metric " + 142 | "results when none exists"); 143 | } 144 | 145 | int ix = metricQueryKeys.get(metricPointer++); 146 | String query = subMetricQueries.get(ix); 147 | 148 | TsQuery realQuery = TsQuery.validCopyOf(dataQuery); 149 | 150 | SplicerUtils.parseMTypeSubQuery(query, realQuery); 151 | 152 | realQuery.validateAndSetQuery(); 153 | RegionChecker checker = SplicerServlet.REGION_UTIL.getRegionChecker(); 154 | 155 | tsdbResultFutures.add(pool.submit(new QueryRunnerWorker(queryRunner, realQuery, checker))); 156 | 157 | } else if (p == Parameter.SUB_EXPRESSION) { 158 | ExpressionTree nextExpression = subExpressions.get(subExprPointer++); 159 | tsdbResultFutures.add(pool.submit(new ExpressionTreeWorker(nextExpression))); 160 | } else { 161 | throw new RuntimeException("Unknown value: " + p); 162 | } 163 | } 164 | 165 | List orderedSubResults = Lists.newArrayList(); 166 | 167 | for (Future tsdbResultFuture : tsdbResultFutures) { 168 | orderedSubResults.add(tsdbResultFuture.get()); 169 | } 170 | 171 | return expr.evaluate(dataQuery, orderedSubResults, funcParams); 172 | } 173 | 174 | public String toString() { 175 | return writeStringField(); 176 | } 177 | 178 | public String writeStringField() { 179 | List strs = Lists.newArrayList(); 180 | if (subExpressions != null) { 181 | for (ExpressionTree sub : subExpressions) { 182 | strs.add(sub.toString()); 183 | } 184 | } 185 | 186 | if (subMetricQueries != null) { 187 | String subMetrics = clean(subMetricQueries.values()); 188 | if (subMetrics != null && subMetrics.length() > 0) { 189 | strs.add(subMetrics); 190 | } 191 | } 192 | 193 | String innerExpression = DOUBLE_COMMA_JOINER.join(strs); 194 | return expr.writeStringField(funcParams, innerExpression); 195 | } 196 | 197 | private String clean(Collection values) { 198 | if (values == null || values.size() == 0) { 199 | return ""; 200 | } 201 | 202 | List strs = Lists.newArrayList(); 203 | for (String v : values) { 204 | String tmp = v.replaceAll("\\{.*\\}", ""); 205 | int ix = tmp.lastIndexOf(':'); 206 | if (ix < 0) { 207 | strs.add(tmp); 208 | } else { 209 | strs.add(tmp.substring(ix + 1)); 210 | } 211 | } 212 | 213 | return DOUBLE_COMMA_JOINER.join(strs); 214 | } 215 | } -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/ExpressionTreeWorker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.turn.splicer.merge.TsdbResult; 20 | 21 | import java.util.concurrent.Callable; 22 | 23 | /** 24 | * Callable class that evaluates an expressionTree 25 | * Created by bpeltz on 10/21/15. 26 | */ 27 | public class ExpressionTreeWorker implements Callable { 28 | 29 | private final ExpressionTree expressionTree; 30 | 31 | public ExpressionTreeWorker(ExpressionTree et) { 32 | this.expressionTree = et; 33 | } 34 | 35 | @Override 36 | public TsdbResult[] call() throws Exception { 37 | return expressionTree.evaluateAll(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/Expressions.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.turn.splicer.tsdbutils.TsQuery; 20 | 21 | import java.util.List; 22 | 23 | import com.google.common.base.Preconditions; 24 | 25 | public class Expressions { 26 | 27 | public static ExpressionTree parse(String expr, 28 | List metricQueries, 29 | TsQuery dataQuery) { 30 | Preconditions.checkNotNull(expr); 31 | if (expr.indexOf('(') == -1 || expr.indexOf(')') == -1) { 32 | throw new RuntimeException("Invalid Expression: " + expr); 33 | } 34 | 35 | ExprReader reader = new ExprReader(expr.toCharArray()); 36 | reader.skipWhitespaces(); 37 | 38 | String funcName = reader.readFuncName(); 39 | Expression rootExpr = ExpressionFactory.getByName(funcName); 40 | if (rootExpr == null) { 41 | throw new RuntimeException("Could not find evaluator " + 42 | "for function '" + funcName + "'"); 43 | } 44 | 45 | ExpressionTree root = new ExpressionTree(rootExpr, dataQuery); 46 | 47 | reader.skipWhitespaces(); 48 | if (reader.peek() == '(') { 49 | reader.next(); 50 | parse(reader, metricQueries, root, dataQuery); 51 | } 52 | 53 | return root; 54 | } 55 | 56 | private static void parse(ExprReader reader, List metricQueries, 57 | ExpressionTree root, TsQuery dataQuery) { 58 | 59 | int parameterIndex = 0; 60 | reader.skipWhitespaces(); 61 | if (reader.peek() != ')') { 62 | String param = reader.readNextParameter(); 63 | parseParam(param, metricQueries, root, dataQuery, parameterIndex++); 64 | } 65 | 66 | while (true) { 67 | reader.skipWhitespaces(); 68 | if (reader.peek() == ')') { 69 | return; 70 | } else if (reader.isNextSeq(",,")) { 71 | reader.skip(2); //swallow the ",," delimiter 72 | reader.skipWhitespaces(); 73 | String param = reader.readNextParameter(); 74 | parseParam(param, metricQueries, root, dataQuery, parameterIndex++); 75 | } else { 76 | throw new RuntimeException("Invalid delimiter in parameter " + 77 | "list at pos=" + reader.getMark() + ", expr=" 78 | + reader.toString()); 79 | } 80 | } 81 | } 82 | 83 | private static void parseParam(String param, List metricQueries, 84 | ExpressionTree root, TsQuery dataQuery, int index) { 85 | if (param == null || param.length() == 0) { 86 | throw new RuntimeException("Invalid Parameter in " + 87 | "Expression"); 88 | } 89 | 90 | if (param.indexOf('(') > 0 && param.indexOf(')') > 0) { 91 | // sub expression 92 | ExpressionTree subTree = parse(param, metricQueries, dataQuery); 93 | root.addSubExpression(subTree, index); 94 | } else if (param.indexOf(':') >= 0) { 95 | // metric query 96 | metricQueries.add(param); 97 | root.addSubMetricQuery(param, metricQueries.size() - 1, index); 98 | } else { 99 | // expression parameter 100 | root.addFunctionParameter(param); 101 | } 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/QueryRunnerWorker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.turn.splicer.hbase.RegionChecker; 20 | import com.turn.splicer.merge.TsdbResult; 21 | import com.turn.splicer.tsdbutils.SplicerQueryRunner; 22 | import com.turn.splicer.tsdbutils.TsQuery; 23 | 24 | import java.util.concurrent.Callable; 25 | 26 | /** 27 | * Created by bpeltz on 10/22/15. 28 | */ 29 | public class QueryRunnerWorker implements Callable { 30 | 31 | private SplicerQueryRunner queryRunner; 32 | 33 | private TsQuery query; 34 | 35 | private RegionChecker checker; 36 | 37 | public QueryRunnerWorker(SplicerQueryRunner queryRunner, TsQuery query, RegionChecker checker) { 38 | this.queryRunner = queryRunner; 39 | this.query = query; 40 | this.checker = checker; 41 | } 42 | @Override 43 | public TsdbResult[] call() throws Exception { 44 | return queryRunner.sliceAndRunQuery(query, checker); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/SeekableViewDataPointImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import net.opentsdb.core.DataPoint; 20 | import net.opentsdb.core.SeekableView; 21 | 22 | /** 23 | * Implementation of SeekableView Interface to wrap around 24 | * a DataPoint[] 25 | */ 26 | public class SeekableViewDataPointImpl implements SeekableView { 27 | 28 | private DataPoint[] dps; 29 | private int currentIndex; 30 | 31 | public SeekableViewDataPointImpl(DataPoint[] dps) { 32 | 33 | if(dps == null || dps.length == 0) { 34 | throw new RuntimeException("Empty or null dps don't know what to do"); 35 | } 36 | 37 | this.dps = dps; 38 | currentIndex = 0; 39 | } 40 | 41 | @Override 42 | public boolean hasNext() { 43 | return currentIndex < dps.length; 44 | } 45 | 46 | @Override 47 | public DataPoint next() { 48 | return dps[currentIndex++]; 49 | } 50 | 51 | @Override 52 | public void remove() { 53 | //not needed yet? 54 | throw new RuntimeException("Not implemented yet"); 55 | } 56 | 57 | @Override 58 | public void seek(long timestamp) { 59 | for(int i = currentIndex; i < dps.length; i++) { 60 | if(dps[i].timestamp() >= timestamp) { 61 | break; 62 | } else { 63 | currentIndex++; 64 | 65 | } 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/parser/ParseException.java: -------------------------------------------------------------------------------- 1 | /* Generated By:JavaCC: Do not edit this line. ParseException.java Version 5.0 */ 2 | /* JavaCCOptions:KEEP_LINE_COL=null */ 3 | package com.turn.splicer.tsdbutils.expression.parser; 4 | 5 | /** 6 | * This exception is thrown when parse errors are encountered. 7 | * You can explicitly create objects of this exception type by 8 | * calling the method generateParseException in the generated 9 | * parser. 10 | *

11 | * You can modify this class to customize your error reporting 12 | * mechanisms so long as you retain the public fields. 13 | */ 14 | public class ParseException extends Exception { 15 | 16 | /** 17 | * The version identifier for this Serializable class. 18 | * Increment only if the serialized form of the 19 | * class changes. 20 | */ 21 | private static final long serialVersionUID = 1L; 22 | 23 | /** 24 | * This constructor is used by the method "generateParseException" 25 | * in the generated parser. Calling this constructor generates 26 | * a new object of this type with the fields "currentToken", 27 | * "expectedTokenSequences", and "tokenImage" set. 28 | */ 29 | public ParseException(Token currentTokenVal, 30 | int[][] expectedTokenSequencesVal, 31 | String[] tokenImageVal 32 | ) { 33 | super(initialise(currentTokenVal, expectedTokenSequencesVal, tokenImageVal)); 34 | currentToken = currentTokenVal; 35 | expectedTokenSequences = expectedTokenSequencesVal; 36 | tokenImage = tokenImageVal; 37 | } 38 | 39 | /** 40 | * The following constructors are for use by you for whatever 41 | * purpose you can think of. Constructing the exception in this 42 | * manner makes the exception behave in the normal way - i.e., as 43 | * documented in the class "Throwable". The fields "errorToken", 44 | * "expectedTokenSequences", and "tokenImage" do not contain 45 | * relevant information. The JavaCC generated code does not use 46 | * these constructors. 47 | */ 48 | 49 | public ParseException() { 50 | super(); 51 | } 52 | 53 | /** 54 | * Constructor with message. 55 | */ 56 | public ParseException(String message) { 57 | super(message); 58 | } 59 | 60 | 61 | /** 62 | * This is the last token that has been consumed successfully. If 63 | * this object has been created due to a parse error, the token 64 | * followng this token will (therefore) be the first error token. 65 | */ 66 | public Token currentToken; 67 | 68 | /** 69 | * Each entry in this array is an array of integers. Each array 70 | * of integers represents a sequence of tokens (by their ordinal 71 | * values) that is expected at this point of the parse. 72 | */ 73 | public int[][] expectedTokenSequences; 74 | 75 | /** 76 | * This is a reference to the "tokenImage" array of the generated 77 | * parser within which the parse error occurred. This array is 78 | * defined in the generated ...Constants interface. 79 | */ 80 | public String[] tokenImage; 81 | 82 | /** 83 | * It uses "currentToken" and "expectedTokenSequences" to generate a parse 84 | * error message and returns it. If this object has been created 85 | * due to a parse error, and you do not catch it (it gets thrown 86 | * from the parser) the correct error message 87 | * gets displayed. 88 | */ 89 | private static String initialise(Token currentToken, 90 | int[][] expectedTokenSequences, 91 | String[] tokenImage) { 92 | String eol = System.getProperty("line.separator", "\n"); 93 | StringBuffer expected = new StringBuffer(); 94 | int maxSize = 0; 95 | for (int i = 0; i < expectedTokenSequences.length; i++) { 96 | if (maxSize < expectedTokenSequences[i].length) { 97 | maxSize = expectedTokenSequences[i].length; 98 | } 99 | for (int j = 0; j < expectedTokenSequences[i].length; j++) { 100 | expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' '); 101 | } 102 | if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) { 103 | expected.append("..."); 104 | } 105 | expected.append(eol).append(" "); 106 | } 107 | String retval = "Encountered \""; 108 | Token tok = currentToken.next; 109 | for (int i = 0; i < maxSize; i++) { 110 | if (i != 0) retval += " "; 111 | if (tok.kind == 0) { 112 | retval += tokenImage[0]; 113 | break; 114 | } 115 | retval += " " + tokenImage[tok.kind]; 116 | retval += " \""; 117 | retval += add_escapes(tok.image); 118 | retval += " \""; 119 | tok = tok.next; 120 | } 121 | retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn; 122 | retval += "." + eol; 123 | if (expectedTokenSequences.length == 1) { 124 | retval += "Was expecting:" + eol + " "; 125 | } else { 126 | retval += "Was expecting one of:" + eol + " "; 127 | } 128 | retval += expected.toString(); 129 | return retval; 130 | } 131 | 132 | /** 133 | * The end of line string for this machine. 134 | */ 135 | protected String eol = System.getProperty("line.separator", "\n"); 136 | 137 | /** 138 | * Used to convert raw characters to their escaped version 139 | * when these raw version cannot be used as part of an ASCII 140 | * string literal. 141 | */ 142 | static String add_escapes(String str) { 143 | StringBuffer retval = new StringBuffer(); 144 | char ch; 145 | for (int i = 0; i < str.length(); i++) { 146 | switch (str.charAt(i)) { 147 | case 0: 148 | continue; 149 | case '\b': 150 | retval.append("\\b"); 151 | continue; 152 | case '\t': 153 | retval.append("\\t"); 154 | continue; 155 | case '\n': 156 | retval.append("\\n"); 157 | continue; 158 | case '\f': 159 | retval.append("\\f"); 160 | continue; 161 | case '\r': 162 | retval.append("\\r"); 163 | continue; 164 | case '\"': 165 | retval.append("\\\""); 166 | continue; 167 | case '\'': 168 | retval.append("\\\'"); 169 | continue; 170 | case '\\': 171 | retval.append("\\\\"); 172 | continue; 173 | default: 174 | if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { 175 | String s = "0000" + Integer.toString(ch, 16); 176 | retval.append("\\u" + s.substring(s.length() - 4, s.length())); 177 | } else { 178 | retval.append(ch); 179 | } 180 | continue; 181 | } 182 | } 183 | return retval.toString(); 184 | } 185 | 186 | } 187 | /* JavaCC - OriginalChecksum=2da0cd14b957a02229644f0387214240 (do not edit this line) */ 188 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/parser/SyntaxCheckerConstants.java: -------------------------------------------------------------------------------- 1 | /* Generated By:JavaCC: Do not edit this line. SyntaxCheckerConstants.java */ 2 | package com.turn.splicer.tsdbutils.expression.parser; 3 | 4 | 5 | /** 6 | * Token literal values and constants. 7 | * Generated by org.javacc.parser.OtherFilesGen#start() 8 | */ 9 | public interface SyntaxCheckerConstants { 10 | 11 | /** 12 | * End of File. 13 | */ 14 | int EOF = 0; 15 | /** 16 | * RegularExpression Id. 17 | */ 18 | int NAME = 5; 19 | /** 20 | * RegularExpression Id. 21 | */ 22 | int PARAM = 6; 23 | 24 | /** 25 | * Lexical state. 26 | */ 27 | int DEFAULT = 0; 28 | 29 | /** 30 | * Literal token values. 31 | */ 32 | String[] tokenImage = { 33 | "", 34 | "\" \"", 35 | "\"\\t\"", 36 | "\"\\n\"", 37 | "\"\\r\"", 38 | "", 39 | "\"&&\"", 40 | "\"(\"", 41 | "\",\"", 42 | "\")\"", 43 | "\":\"", 44 | "\"{\"", 45 | "\"=\"", 46 | "\"}\"", 47 | }; 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/parser/SyntaxCheckerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2015 Turn Inc. All Rights Reserved. 3 | * Proprietary and confidential. 4 | */ 5 | package com.turn.splicer.tsdbutils.expression.parser; 6 | 7 | 8 | import com.turn.splicer.tsdbutils.TsQuery; 9 | import com.turn.splicer.tsdbutils.expression.ExpressionTree; 10 | 11 | import java.io.StringReader; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | public class SyntaxCheckerTest { 16 | 17 | public static void main(String[] args) { 18 | try { 19 | StringReader r = new StringReader("sum(sum:proc.stat.cpu.percpu{cpu=1})"); 20 | SyntaxChecker checker = new SyntaxChecker(r); 21 | TsQuery query = new TsQuery(); 22 | List metrics = new ArrayList(); 23 | checker.setTsQuery(query); 24 | checker.setMetricQueries(metrics); 25 | ExpressionTree tree = checker.EXPRESSION(); 26 | System.out.println("Syntax is okay. ExprTree=" + tree); 27 | System.out.println("Metrics=" + metrics); 28 | } catch (Throwable e) { 29 | System.out.println("Syntax check failed: " + e); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/parser/SyntaxCheckerTokenManager.java: -------------------------------------------------------------------------------- 1 | /* Generated By:JavaCC: Do not edit this line. SyntaxCheckerTokenManager.java */ 2 | package com.turn.splicer.tsdbutils.expression.parser; 3 | 4 | /** 5 | * Token Manager. 6 | */ 7 | public class SyntaxCheckerTokenManager implements SyntaxCheckerConstants { 8 | 9 | /** 10 | * Debug output. 11 | */ 12 | public java.io.PrintStream debugStream = System.out; 13 | 14 | /** 15 | * Set debug output. 16 | */ 17 | public void setDebugStream(java.io.PrintStream ds) { 18 | debugStream = ds; 19 | } 20 | 21 | private final int jjStopStringLiteralDfa_0(int pos, long active0) { 22 | switch (pos) { 23 | default: 24 | return -1; 25 | } 26 | } 27 | 28 | private final int jjStartNfa_0(int pos, long active0) { 29 | return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0), pos + 1); 30 | } 31 | 32 | private int jjStopAtPos(int pos, int kind) { 33 | jjmatchedKind = kind; 34 | jjmatchedPos = pos; 35 | return pos + 1; 36 | } 37 | 38 | private int jjMoveStringLiteralDfa0_0() { 39 | switch (curChar) { 40 | case 38: 41 | return jjMoveStringLiteralDfa1_0(0x40L); 42 | case 40: 43 | return jjStopAtPos(0, 7); 44 | case 41: 45 | return jjStopAtPos(0, 9); 46 | case 44: 47 | return jjStopAtPos(0, 8); 48 | case 58: 49 | return jjStopAtPos(0, 10); 50 | case 61: 51 | return jjStopAtPos(0, 12); 52 | case 123: 53 | return jjStopAtPos(0, 11); 54 | case 125: 55 | return jjStopAtPos(0, 13); 56 | default: 57 | return jjMoveNfa_0(0, 0); 58 | } 59 | } 60 | 61 | private int jjMoveStringLiteralDfa1_0(long active0) { 62 | try { 63 | curChar = input_stream.readChar(); 64 | } catch (java.io.IOException e) { 65 | jjStopStringLiteralDfa_0(0, active0); 66 | return 1; 67 | } 68 | switch (curChar) { 69 | case 38: 70 | if ((active0 & 0x40L) != 0L) 71 | return jjStopAtPos(1, 6); 72 | break; 73 | default: 74 | break; 75 | } 76 | return jjStartNfa_0(0, active0); 77 | } 78 | 79 | private int jjMoveNfa_0(int startState, int curPos) { 80 | int startsAt = 0; 81 | jjnewStateCnt = 2; 82 | int i = 1; 83 | jjstateSet[0] = startState; 84 | int kind = 0x7fffffff; 85 | for (; ; ) { 86 | if (++jjround == 0x7fffffff) 87 | ReInitRounds(); 88 | if (curChar < 64) { 89 | long l = 1L << curChar; 90 | do { 91 | switch (jjstateSet[--i]) { 92 | case 0: 93 | if ((0x3ffe09800000000L & l) != 0L) { 94 | if (kind > 5) 95 | kind = 5; 96 | jjCheckNAdd(1); 97 | } else if (curChar == 42) { 98 | if (kind > 5) 99 | kind = 5; 100 | } 101 | break; 102 | case 1: 103 | if ((0x3ffe09800000000L & l) == 0L) 104 | break; 105 | kind = 5; 106 | jjCheckNAdd(1); 107 | break; 108 | default: 109 | break; 110 | } 111 | } while (i != startsAt); 112 | } else if (curChar < 128) { 113 | long l = 1L << (curChar & 077); 114 | do { 115 | switch (jjstateSet[--i]) { 116 | case 0: 117 | case 1: 118 | if ((0x17fffffeafffffffL & l) == 0L) 119 | break; 120 | kind = 5; 121 | jjCheckNAdd(1); 122 | break; 123 | default: 124 | break; 125 | } 126 | } while (i != startsAt); 127 | } else { 128 | int i2 = (curChar & 0xff) >> 6; 129 | long l2 = 1L << (curChar & 077); 130 | do { 131 | switch (jjstateSet[--i]) { 132 | default: 133 | break; 134 | } 135 | } while (i != startsAt); 136 | } 137 | if (kind != 0x7fffffff) { 138 | jjmatchedKind = kind; 139 | jjmatchedPos = curPos; 140 | kind = 0x7fffffff; 141 | } 142 | ++curPos; 143 | if ((i = jjnewStateCnt) == (startsAt = 2 - (jjnewStateCnt = startsAt))) 144 | return curPos; 145 | try { 146 | curChar = input_stream.readChar(); 147 | } catch (java.io.IOException e) { 148 | return curPos; 149 | } 150 | } 151 | } 152 | 153 | static final int[] jjnextStates = { 154 | }; 155 | 156 | /** 157 | * Token literal values. 158 | */ 159 | public static final String[] jjstrLiteralImages = { 160 | "", null, null, null, null, null, "\46\46", "\50", "\54", "\51", "\72", 161 | "\173", "\75", "\175",}; 162 | 163 | /** 164 | * Lexer state names. 165 | */ 166 | public static final String[] lexStateNames = { 167 | "DEFAULT", 168 | }; 169 | static final long[] jjtoToken = { 170 | 0x3fe1L, 171 | }; 172 | static final long[] jjtoSkip = { 173 | 0x1eL, 174 | }; 175 | protected SimpleCharStream input_stream; 176 | private final int[] jjrounds = new int[2]; 177 | private final int[] jjstateSet = new int[4]; 178 | protected char curChar; 179 | 180 | /** 181 | * Constructor. 182 | */ 183 | public SyntaxCheckerTokenManager(SimpleCharStream stream) { 184 | if (SimpleCharStream.staticFlag) 185 | throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer."); 186 | input_stream = stream; 187 | } 188 | 189 | /** 190 | * Constructor. 191 | */ 192 | public SyntaxCheckerTokenManager(SimpleCharStream stream, int lexState) { 193 | this(stream); 194 | SwitchTo(lexState); 195 | } 196 | 197 | /** 198 | * Reinitialise parser. 199 | */ 200 | public void ReInit(SimpleCharStream stream) { 201 | jjmatchedPos = jjnewStateCnt = 0; 202 | curLexState = defaultLexState; 203 | input_stream = stream; 204 | ReInitRounds(); 205 | } 206 | 207 | private void ReInitRounds() { 208 | int i; 209 | jjround = 0x80000001; 210 | for (i = 2; i-- > 0; ) 211 | jjrounds[i] = 0x80000000; 212 | } 213 | 214 | /** 215 | * Reinitialise parser. 216 | */ 217 | public void ReInit(SimpleCharStream stream, int lexState) { 218 | ReInit(stream); 219 | SwitchTo(lexState); 220 | } 221 | 222 | /** 223 | * Switch to specified lex state. 224 | */ 225 | public void SwitchTo(int lexState) { 226 | if (lexState >= 1 || lexState < 0) 227 | throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); 228 | else 229 | curLexState = lexState; 230 | } 231 | 232 | protected Token jjFillToken() { 233 | final Token t; 234 | final String curTokenImage; 235 | final int beginLine; 236 | final int endLine; 237 | final int beginColumn; 238 | final int endColumn; 239 | String im = jjstrLiteralImages[jjmatchedKind]; 240 | curTokenImage = (im == null) ? input_stream.GetImage() : im; 241 | beginLine = input_stream.getBeginLine(); 242 | beginColumn = input_stream.getBeginColumn(); 243 | endLine = input_stream.getEndLine(); 244 | endColumn = input_stream.getEndColumn(); 245 | t = Token.newToken(jjmatchedKind, curTokenImage); 246 | 247 | t.beginLine = beginLine; 248 | t.endLine = endLine; 249 | t.beginColumn = beginColumn; 250 | t.endColumn = endColumn; 251 | 252 | return t; 253 | } 254 | 255 | int curLexState = 0; 256 | int defaultLexState = 0; 257 | int jjnewStateCnt; 258 | int jjround; 259 | int jjmatchedPos; 260 | int jjmatchedKind; 261 | 262 | /** 263 | * Get the next Token. 264 | */ 265 | public Token getNextToken() { 266 | Token matchedToken; 267 | int curPos = 0; 268 | 269 | EOFLoop: 270 | for (; ; ) { 271 | try { 272 | curChar = input_stream.BeginToken(); 273 | } catch (java.io.IOException e) { 274 | jjmatchedKind = 0; 275 | matchedToken = jjFillToken(); 276 | return matchedToken; 277 | } 278 | 279 | try { 280 | input_stream.backup(0); 281 | while (curChar <= 32 && (0x100002600L & (1L << curChar)) != 0L) 282 | curChar = input_stream.BeginToken(); 283 | } catch (java.io.IOException e1) { 284 | continue EOFLoop; 285 | } 286 | jjmatchedKind = 0x7fffffff; 287 | jjmatchedPos = 0; 288 | curPos = jjMoveStringLiteralDfa0_0(); 289 | if (jjmatchedKind != 0x7fffffff) { 290 | if (jjmatchedPos + 1 < curPos) 291 | input_stream.backup(curPos - jjmatchedPos - 1); 292 | if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L) { 293 | matchedToken = jjFillToken(); 294 | return matchedToken; 295 | } else { 296 | continue EOFLoop; 297 | } 298 | } 299 | int error_line = input_stream.getEndLine(); 300 | int error_column = input_stream.getEndColumn(); 301 | String error_after = null; 302 | boolean EOFSeen = false; 303 | try { 304 | input_stream.readChar(); 305 | input_stream.backup(1); 306 | } catch (java.io.IOException e1) { 307 | EOFSeen = true; 308 | error_after = curPos <= 1 ? "" : input_stream.GetImage(); 309 | if (curChar == '\n' || curChar == '\r') { 310 | error_line++; 311 | error_column = 0; 312 | } else 313 | error_column++; 314 | } 315 | if (!EOFSeen) { 316 | input_stream.backup(1); 317 | error_after = curPos <= 1 ? "" : input_stream.GetImage(); 318 | } 319 | throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR); 320 | } 321 | } 322 | 323 | private void jjCheckNAdd(int state) { 324 | if (jjrounds[state] != jjround) { 325 | jjstateSet[jjnewStateCnt++] = state; 326 | jjrounds[state] = jjround; 327 | } 328 | } 329 | 330 | private void jjAddStates(int start, int end) { 331 | do { 332 | jjstateSet[jjnewStateCnt++] = jjnextStates[start]; 333 | } while (start++ != end); 334 | } 335 | 336 | private void jjCheckNAddTwoStates(int state1, int state2) { 337 | jjCheckNAdd(state1); 338 | jjCheckNAdd(state2); 339 | } 340 | 341 | } 342 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/parser/Tester.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (C) 2015 Turn Inc. All Rights Reserved. 3 | * Proprietary and confidential. 4 | */ 5 | package com.turn.splicer.tsdbutils.expression.parser; 6 | 7 | 8 | 9 | import com.turn.splicer.tsdbutils.TsQuery; 10 | import com.turn.splicer.tsdbutils.expression.ExpressionTree; 11 | 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | public class Tester { 16 | 17 | public static void main(String[] args) { 18 | try { 19 | String expr = "alias(sum:1m-avg:CapEnforcementControl.DailyBudgetSpend.value{allocType=manual,serverid=control1,capType=Total,currency=USD},1)"; 20 | SyntaxChecker checker = new SyntaxChecker(new java.io.StringReader(expr)); 21 | List metrics = new ArrayList(); 22 | checker.setMetricQueries(metrics); 23 | checker.setTsQuery(new TsQuery()); 24 | 25 | ExpressionTree tree = checker.EXPRESSION(); 26 | 27 | System.out.println("Syntax is okay, " + tree.toString()); 28 | System.out.println("Metrics=" + metrics); 29 | } catch (Throwable e) { 30 | // Catching Throwable is ugly but JavaCC throws Error objects! 31 | System.out.println("Syntax check failed: "); 32 | e.printStackTrace(); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/parser/Token.java: -------------------------------------------------------------------------------- 1 | /* Generated By:JavaCC: Do not edit this line. Token.java Version 5.0 */ 2 | /* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */ 3 | package com.turn.splicer.tsdbutils.expression.parser; 4 | 5 | /** 6 | * Describes the input token stream. 7 | */ 8 | 9 | public class Token implements java.io.Serializable { 10 | 11 | /** 12 | * The version identifier for this Serializable class. 13 | * Increment only if the serialized form of the 14 | * class changes. 15 | */ 16 | private static final long serialVersionUID = 1L; 17 | 18 | /** 19 | * An integer that describes the kind of this token. This numbering 20 | * system is determined by JavaCCParser, and a table of these numbers is 21 | * stored in the file ...Constants.java. 22 | */ 23 | public int kind; 24 | 25 | /** 26 | * The line number of the first character of this Token. 27 | */ 28 | public int beginLine; 29 | /** 30 | * The column number of the first character of this Token. 31 | */ 32 | public int beginColumn; 33 | /** 34 | * The line number of the last character of this Token. 35 | */ 36 | public int endLine; 37 | /** 38 | * The column number of the last character of this Token. 39 | */ 40 | public int endColumn; 41 | 42 | /** 43 | * The string image of the token. 44 | */ 45 | public String image; 46 | 47 | /** 48 | * A reference to the next regular (non-special) token from the input 49 | * stream. If this is the last token from the input stream, or if the 50 | * token manager has not read tokens beyond this one, this field is 51 | * set to null. This is true only if this token is also a regular 52 | * token. Otherwise, see below for a description of the contents of 53 | * this field. 54 | */ 55 | public Token next; 56 | 57 | /** 58 | * This field is used to access special tokens that occur prior to this 59 | * token, but after the immediately preceding regular (non-special) token. 60 | * If there are no such special tokens, this field is set to null. 61 | * When there are more than one such special token, this field refers 62 | * to the last of these special tokens, which in turn refers to the next 63 | * previous special token through its specialToken field, and so on 64 | * until the first special token (whose specialToken field is null). 65 | * The next fields of special tokens refer to other special tokens that 66 | * immediately follow it (without an intervening regular token). If there 67 | * is no such token, this field is null. 68 | */ 69 | public Token specialToken; 70 | 71 | /** 72 | * An optional attribute value of the Token. 73 | * Tokens which are not used as syntactic sugar will often contain 74 | * meaningful values that will be used later on by the compiler or 75 | * interpreter. This attribute value is often different from the image. 76 | * Any subclass of Token that actually wants to return a non-null value can 77 | * override this method as appropriate. 78 | */ 79 | public Object getValue() { 80 | return null; 81 | } 82 | 83 | /** 84 | * No-argument constructor 85 | */ 86 | public Token() { 87 | } 88 | 89 | /** 90 | * Constructs a new token for the specified Image. 91 | */ 92 | public Token(int kind) { 93 | this(kind, null); 94 | } 95 | 96 | /** 97 | * Constructs a new token for the specified Image and Kind. 98 | */ 99 | public Token(int kind, String image) { 100 | this.kind = kind; 101 | this.image = image; 102 | } 103 | 104 | /** 105 | * Returns the image. 106 | */ 107 | public String toString() { 108 | return image; 109 | } 110 | 111 | /** 112 | * Returns a new Token object, by default. However, if you want, you 113 | * can create and return subclass objects based on the value of ofKind. 114 | * Simply add the cases to the switch for all those special cases. 115 | * For example, if you have a subclass of Token called IDToken that 116 | * you want to create if ofKind is ID, simply add something like : 117 | *

118 | * case MyParserConstants.ID : return new IDToken(ofKind, image); 119 | *

120 | * to the following switch statement. Then you can cast matchedToken 121 | * variable to the appropriate type and use sit in your lexical actions. 122 | */ 123 | public static Token newToken(int ofKind, String image) { 124 | switch (ofKind) { 125 | default: 126 | return new Token(ofKind, image); 127 | } 128 | } 129 | 130 | public static Token newToken(int ofKind) { 131 | return newToken(ofKind, null); 132 | } 133 | 134 | } 135 | /* JavaCC - OriginalChecksum=85eeeff621e9c915d36cea81e480ac8d (do not edit this line) */ 136 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/parser/TokenMgrError.java: -------------------------------------------------------------------------------- 1 | /* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 5.0 */ 2 | /* JavaCCOptions: */ 3 | package com.turn.splicer.tsdbutils.expression.parser; 4 | 5 | /** 6 | * Token Manager Error. 7 | */ 8 | public class TokenMgrError extends Error { 9 | 10 | /** 11 | * The version identifier for this Serializable class. 12 | * Increment only if the serialized form of the 13 | * class changes. 14 | */ 15 | private static final long serialVersionUID = 1L; 16 | 17 | /* 18 | * Ordinals for various reasons why an Error of this type can be thrown. 19 | */ 20 | 21 | /** 22 | * Lexical error occurred. 23 | */ 24 | static final int LEXICAL_ERROR = 0; 25 | 26 | /** 27 | * An attempt was made to create a second instance of a static token manager. 28 | */ 29 | static final int STATIC_LEXER_ERROR = 1; 30 | 31 | /** 32 | * Tried to change to an invalid lexical state. 33 | */ 34 | static final int INVALID_LEXICAL_STATE = 2; 35 | 36 | /** 37 | * Detected (and bailed out of) an infinite loop in the token manager. 38 | */ 39 | static final int LOOP_DETECTED = 3; 40 | 41 | /** 42 | * Indicates the reason why the exception is thrown. It will have 43 | * one of the above 4 values. 44 | */ 45 | int errorCode; 46 | 47 | /** 48 | * Replaces unprintable characters by their escaped (or unicode escaped) 49 | * equivalents in the given string 50 | */ 51 | protected static final String addEscapes(String str) { 52 | StringBuffer retval = new StringBuffer(); 53 | char ch; 54 | for (int i = 0; i < str.length(); i++) { 55 | switch (str.charAt(i)) { 56 | case 0: 57 | continue; 58 | case '\b': 59 | retval.append("\\b"); 60 | continue; 61 | case '\t': 62 | retval.append("\\t"); 63 | continue; 64 | case '\n': 65 | retval.append("\\n"); 66 | continue; 67 | case '\f': 68 | retval.append("\\f"); 69 | continue; 70 | case '\r': 71 | retval.append("\\r"); 72 | continue; 73 | case '\"': 74 | retval.append("\\\""); 75 | continue; 76 | case '\'': 77 | retval.append("\\\'"); 78 | continue; 79 | case '\\': 80 | retval.append("\\\\"); 81 | continue; 82 | default: 83 | if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { 84 | String s = "0000" + Integer.toString(ch, 16); 85 | retval.append("\\u" + s.substring(s.length() - 4, s.length())); 86 | } else { 87 | retval.append(ch); 88 | } 89 | continue; 90 | } 91 | } 92 | return retval.toString(); 93 | } 94 | 95 | /** 96 | * Returns a detailed message for the Error when it is thrown by the 97 | * token manager to indicate a lexical error. 98 | * Parameters : 99 | * EOFSeen : indicates if EOF caused the lexical error 100 | * curLexState : lexical state in which this error occurred 101 | * errorLine : line number when the error occurred 102 | * errorColumn : column number when the error occurred 103 | * errorAfter : prefix that was seen before this error occurred 104 | * curchar : the offending character 105 | * Note: You can customize the lexical error message by modifying this method. 106 | */ 107 | protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) { 108 | return ("Lexical error at line " + 109 | errorLine + ", column " + 110 | errorColumn + ". Encountered: " + 111 | (EOFSeen ? " " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int) curChar + "), ") + 112 | "after : \"" + addEscapes(errorAfter) + "\""); 113 | } 114 | 115 | /** 116 | * You can also modify the body of this method to customize your error messages. 117 | * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not 118 | * of end-users concern, so you can return something like : 119 | *

120 | * "Internal Error : Please file a bug report .... " 121 | *

122 | * from this method for such cases in the release version of your parser. 123 | */ 124 | public String getMessage() { 125 | return super.getMessage(); 126 | } 127 | 128 | /* 129 | * Constructors of various flavors follow. 130 | */ 131 | 132 | /** 133 | * No arg constructor. 134 | */ 135 | public TokenMgrError() { 136 | } 137 | 138 | /** 139 | * Constructor with message and reason. 140 | */ 141 | public TokenMgrError(String message, int reason) { 142 | super(message); 143 | errorCode = reason; 144 | } 145 | 146 | /** 147 | * Full Constructor. 148 | */ 149 | public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) { 150 | this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); 151 | } 152 | } 153 | /* JavaCC - OriginalChecksum=f859635863fb4ecd57b5f86a4292d504 (do not edit this line) */ 154 | -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/parser/parser.jj: -------------------------------------------------------------------------------- 1 | PARSER_BEGIN(SyntaxChecker) 2 | package net.opentsdb.tsd.expression.parser; 3 | 4 | import net.opentsdb.tsd.expression.ExpressionTree; 5 | import net.opentsdb.core.TSQuery; 6 | 7 | import java.util.List; 8 | import com.google.common.base.Joiner; 9 | import com.google.common.collect.Lists; 10 | 11 | public class SyntaxChecker { 12 | 13 | private TSQuery dataQuery; 14 | private List metricQueries; 15 | 16 | public void setTSQuery(TSQuery dataQuery) { 17 | this.dataQuery = dataQuery; 18 | } 19 | 20 | public void setMetricQueries(List metricQueries) { 21 | this.metricQueries = metricQueries; 22 | } 23 | 24 | public static void main(String[] args) { 25 | try { 26 | new SyntaxChecker(new java.io.StringReader(args[0])).EXPRESSION(); 27 | System.out.println("Syntax is okay"); 28 | } catch (Throwable e) { 29 | // Catching Throwable is ugly but JavaCC throws Error objects! 30 | System.out.println("Syntax check failed: " + e.getMessage()); 31 | } 32 | } 33 | } 34 | 35 | PARSER_END(SyntaxChecker) 36 | 37 | SKIP: { " " | "\t" | "\n" | "\r" } 38 | TOKEN: { } 39 | TOKEN: { } 40 | 41 | ExpressionTree EXPRESSION(): {Token name; int paramIndex=0;} { 42 | name= { ExpressionTree tree=new ExpressionTree(name.image,dataQuery); } 43 | "(" PARAMETER(tree, paramIndex++) ("," PARAMETER(tree, paramIndex++))* ")" 44 | {return tree;} 45 | } 46 | 47 | void PARAMETER(ExpressionTree tree, int paramIndex): {String metric; Token param; ExpressionTree subTree;} { 48 | subTree=EXPRESSION() {tree.addSubExpression(subTree, paramIndex);} | 49 | metric=METRIC() {metricQueries.add(metric); tree.addSubMetricQuery(metric, metricQueries.size()-1, paramIndex);} | 50 | param= {tree.addFunctionParameter(param.image);} 51 | } 52 | 53 | // metric is agg:[interval-agg:][rate:]metric[{tag=value,...}] 54 | String METRIC() : {Token agg,itvl,rate,metric,tagk,tagv; StringBuilder builder = new StringBuilder(); 55 | Joiner JOINER = Joiner.on(",").skipNulls(); 56 | List tagPairs = Lists.newArrayList(); 57 | } { 58 | agg = ":" { builder.append(agg.image).append(":"); } 59 | (itvl= ":" { builder.append(itvl.image).append(":"); })? 60 | (rate= ":" { builder.append(rate.image).append(":"); })? 61 | metric= { builder.append(metric.image); } 62 | ("{" tagk= "=" tagv= {tagPairs.add(tagk+"="+tagv);} 63 | ("," tagk= "=" tagv= {tagPairs.add(tagk+"="+tagv);})* "}")? 64 | {if (tagPairs.size() > 0) builder.append("{").append(JOINER.join(tagPairs)).append("}"); return builder.toString();} } -------------------------------------------------------------------------------- /src/main/java/com/turn/splicer/tsdbutils/expression/parser/run_javacc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | javacc -STATIC:false -LOOKAHEAD:5 parser.jj 3 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} %-5level [%thread] %logger{0}: %msg%n 8 | 9 | 10 | 11 | 12 | 1024 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src/main/resources/splicer.conf: -------------------------------------------------------------------------------- 1 | ## the port on which the splicer listens for HTTP requests 2 | splicer.port = 4245 3 | 4 | ## disable connecting to TSDs 5 | tsd.connect.enable = true 6 | 7 | ## hosts on which the TSDs are running 8 | tsd.hosts = localhost 9 | 10 | ## start and end port for TSDs on data nodes. Start is inclusive, End is exclusive 11 | tsd.start.port = 4242 12 | tsd.end.port = 4243 13 | 14 | tsd.queries.per.port = 10 15 | 16 | ## enable overflow for slices 17 | slice.overflow.enable = false 18 | 19 | ## is redis caching enabled (disable this flag if Redis is not available) 20 | caching.enabled = true 21 | 22 | ## redis host:port setups for 23 | caching.hosts = localhost:6379 24 | 25 | ## hbase configuration 26 | hbase.zookeeper.quorum = localhost:2181 27 | hbase.znode.parent = /hbase-unsecure 28 | -------------------------------------------------------------------------------- /src/test/java/com/turn/splicer/merge/QueryAwareResultsMergerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.merge; 18 | 19 | import com.turn.splicer.tsdbutils.TSSubQuery; 20 | import com.turn.splicer.tsdbutils.TsQuery; 21 | 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | import org.testng.Assert; 28 | import org.testng.annotations.Test; 29 | 30 | public class QueryAwareResultsMergerTest { 31 | 32 | private static final Logger LOG = LoggerFactory.getLogger(QueryAwareResultsMergerTest.class); 33 | 34 | @Test 35 | public void testCreateTagString() 36 | { 37 | TsdbResult result = new TsdbResult(); 38 | TsQuery query = new TsQuery(); 39 | query.addSubQuery(new TSSubQuery()); 40 | QueryAwareResultsMerger merger = new QueryAwareResultsMerger(query); 41 | 42 | Assert.assertEquals(QueryAwareResultsMerger.NO_TAGS, merger.createTagString(result)); 43 | 44 | result = new TsdbResult(); 45 | Map m = new HashMap<>(); 46 | m.put("x", "b"); 47 | result.setTags(new TsdbResult.Tags(m)); 48 | 49 | // we should no tags inspite of there being tags in the 50 | Assert.assertEquals(QueryAwareResultsMerger.NO_TAGS, merger.createTagString(result)); 51 | 52 | // expect some tags in the result 53 | HashMap expectedTags = new HashMap<>(); 54 | expectedTags.put("x", "b"); 55 | expectedTags.put("y", "b"); 56 | expectedTags.put("z", "b"); 57 | query.getQueries().get(0).setTags(expectedTags); 58 | result = new TsdbResult(); 59 | m = new HashMap<>(); 60 | m.put("x", "b"); 61 | m.put("y", "b"); 62 | m.put("z", "b"); 63 | result.setTags(new TsdbResult.Tags(m)); 64 | 65 | // result should have all three tags 66 | Assert.assertEquals("x=b,y=b,z=b", merger.createTagString(result)); 67 | 68 | // expect fewer tags that what is sent back 69 | expectedTags = new HashMap<>(); 70 | expectedTags.put("x", "b"); 71 | expectedTags.put("z", "b"); 72 | query.getQueries().get(0).setTags(expectedTags); 73 | result = new TsdbResult(); 74 | m = new HashMap<>(); 75 | m.put("x", "b"); 76 | m.put("y", "b"); 77 | m.put("z", "b"); 78 | result.setTags(new TsdbResult.Tags(m)); 79 | 80 | // result should have all three tags 81 | Assert.assertEquals("x=b,z=b", merger.createTagString(result)); 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/test/java/com/turn/splicer/merge/TsdbResultTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.merge; 18 | 19 | import java.io.IOException; 20 | import java.util.Arrays; 21 | import java.util.HashMap; 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | import com.beust.jcommander.internal.Lists; 26 | import org.testng.Assert; 27 | import org.testng.annotations.Test; 28 | 29 | public class TsdbResultTest { 30 | 31 | @Test 32 | public void createTsdbResult() { 33 | TsdbResult obj = new TsdbResult(); 34 | obj.setMetric("metric.name"); 35 | obj.setAggregateTags(Arrays.asList("host", "domain")); 36 | } 37 | 38 | @Test 39 | public void deserializeTest() throws Exception 40 | { 41 | TsdbResult.JSON_MAPPER.readValue("{\"1438383600\":8499332.134660648, \"1438383601\":84}", 42 | TsdbResult.Points.class); 43 | 44 | TsdbResult.JSON_MAPPER.readValue("{\"metric\": \"name\"}}", TsdbResult.class); 45 | 46 | TsdbResult.JSON_MAPPER.readValue("{\"metric\": \"name\", \"dps\": {\"1438383600\":" 47 | + "8499332.134660648, \"1438383601\":84}}", TsdbResult.class); 48 | 49 | String js = "{\"metric\":\"turn.bid.rtb.bidrequest.bid\",\"tags\":{\"invsource\":\"adx\"}" 50 | + ",\"aggregateTags\":[\"serverid\",\"servertype\",\"host\",\"seat\"" 51 | + ",\"machine_class\",\"mediaChannelId\",\"domain\"]," 52 | + "\"dps\":{\"1438383600\":8499332.134660648,\"1438383660\":5189956.492815415," 53 | + "\"1438383720\":3625048.848030829,\"1438383780\":2767397.69178366," 54 | + "\"1438383840\":2259138.068511685,\"1438383900\":2259310.6599630998," 55 | + "\"1438383960\":2259482.384829848,\"1438384020\":2259648.478802788," 56 | + "\"1438384080\":2259814}}"; 57 | 58 | TsdbResult r = TsdbResult.from(js); 59 | 60 | Assert.assertEquals(r.getMetric(), "turn.bid.rtb.bidrequest.bid"); 61 | List aggTagsExp = Lists.newArrayList(); 62 | aggTagsExp.add("serverid"); 63 | aggTagsExp.add("servertype"); 64 | aggTagsExp.add("host"); 65 | aggTagsExp.add("seat"); 66 | aggTagsExp.add("machine_class"); 67 | aggTagsExp.add("mediaChannelId"); 68 | aggTagsExp.add("domain"); 69 | Assert.assertTrue(r.getAggregateTags().containsAll(aggTagsExp)); 70 | 71 | Map tags = r.getTags().getTags(); 72 | Assert.assertEquals(tags.size(), 1); 73 | Assert.assertEquals(tags.get("invsource"), "adx"); 74 | 75 | Assert.assertEquals(r.getDps().getMap().get("1438383660"), 5189956.492815415); 76 | Assert.assertEquals(r.getDps().getMap().get("1438383720"), 3625048.848030829); 77 | Assert.assertEquals(r.getDps().getMap().get("1438383960"), 2259482.384829848); 78 | Assert.assertEquals(r.getDps().getMap().get("1438384020"), 2259648.478802788); 79 | } 80 | 81 | @Test 82 | public void deserializeArray() throws IOException 83 | { 84 | String j = "[{\"metric\":\"turn.bid.rtb.bidrequest.bid\",\"tags\":{\"invsource\":\"adx\"}" 85 | + ",\"aggregateTags\":[\"serverid\",\"servertype\",\"host\",\"seat\"" 86 | + ",\"machine_class\",\"mediaChannelId\",\"domain\"]," 87 | + "\"dps\":{\"1438383600\":8499332.134660648,\"1438383660\":5189956.492815415," 88 | + "\"1438383720\":3625048.848030829,\"1438383780\":2767397.69178366," 89 | + "\"1438383840\":2259138.068511685,\"1438383900\":2259310.6599630998," 90 | + "\"1438383960\":2259482.384829848,\"1438384020\":2259648.478802788," 91 | + "\"1438384080\":2259814}}]"; 92 | 93 | TsdbResult[] results = TsdbResult.JSON_MAPPER.readValue(j, TsdbResult[].class); 94 | Assert.assertEquals(results.length, 1); 95 | 96 | TsdbResult r = results[0]; 97 | 98 | Assert.assertEquals(r.getMetric(), "turn.bid.rtb.bidrequest.bid"); 99 | List aggTagsExp = Lists.newArrayList(); 100 | aggTagsExp.add("serverid"); 101 | aggTagsExp.add("servertype"); 102 | aggTagsExp.add("host"); 103 | aggTagsExp.add("seat"); 104 | aggTagsExp.add("machine_class"); 105 | aggTagsExp.add("mediaChannelId"); 106 | aggTagsExp.add("domain"); 107 | Assert.assertTrue(r.getAggregateTags().containsAll(aggTagsExp)); 108 | 109 | Map tags = r.getTags().getTags(); 110 | Assert.assertEquals(tags.size(), 1); 111 | Assert.assertEquals(tags.get("invsource"), "adx"); 112 | 113 | Assert.assertEquals(r.getDps().getMap().get("1438383660"), 5189956.492815415); 114 | Assert.assertEquals(r.getDps().getMap().get("1438383720"), 3625048.848030829); 115 | Assert.assertEquals(r.getDps().getMap().get("1438383960"), 2259482.384829848); 116 | Assert.assertEquals(r.getDps().getMap().get("1438384020"), 2259648.478802788); 117 | 118 | } 119 | 120 | @Test 121 | public void testEquals() { 122 | TsdbResult result1 = new TsdbResult(); 123 | TsdbResult result2 = null; 124 | 125 | Assert.assertFalse(result1.equals(result2)); 126 | 127 | result2 = new TsdbResult(); 128 | Assert.assertTrue(result1.equals(result2)); 129 | 130 | result1.setMetric("dummy"); 131 | Assert.assertFalse(result1.equals(result2)); 132 | Assert.assertFalse(result2.equals(result1)); 133 | 134 | result2.setMetric("not_dummy"); 135 | Assert.assertFalse(result1.equals(result2)); 136 | Assert.assertFalse(result2.equals(result1)); 137 | 138 | result2.setMetric("dummy"); 139 | Assert.assertTrue(result1.equals(result2)); 140 | Assert.assertTrue(result2.equals(result1)); 141 | 142 | result1.setAlias("alias_dummy"); 143 | Assert.assertFalse(result1.equals(result2)); 144 | Assert.assertFalse(result2.equals(result1)); 145 | 146 | result2.setAlias("not_alias_dummy"); 147 | Assert.assertFalse(result1.equals(result2)); 148 | Assert.assertFalse(result2.equals(result1)); 149 | 150 | result2.setAlias("alias_dummy"); 151 | Assert.assertTrue(result1.equals(result2)); 152 | Assert.assertTrue(result2.equals(result1)); 153 | 154 | Map result1tags = new HashMap(); 155 | result1.setTags(new TsdbResult.Tags(result1tags)); 156 | Assert.assertFalse(result1.equals(result2)); 157 | Assert.assertFalse(result2.equals(result1)); 158 | 159 | result1tags.put("key1", "value1"); 160 | Assert.assertFalse(result1.equals(result2)); 161 | Assert.assertFalse(result2.equals(result1)); 162 | 163 | Map result2tags = new HashMap(); 164 | result2.setTags(new TsdbResult.Tags(result2tags)); 165 | Assert.assertFalse(result1.equals(result2)); 166 | Assert.assertFalse(result2.equals(result1)); 167 | 168 | result2tags.put("key1", "value1"); 169 | Assert.assertTrue(result1.equals(result2)); 170 | Assert.assertTrue(result2.equals(result1)); 171 | 172 | Map result1map = new HashMap(); 173 | result1.setDps(new TsdbResult.Points(result1map)); 174 | Assert.assertFalse(result1.equals(result2)); 175 | Assert.assertFalse(result2.equals(result1)); 176 | 177 | Map result2map = new HashMap(); 178 | result2.setDps(new TsdbResult.Points(result2map)); 179 | Assert.assertTrue(result1.equals(result2)); 180 | Assert.assertTrue(result2.equals(result1)); 181 | 182 | result1map.put("point1", "point1val"); 183 | Assert.assertFalse(result1.equals(result2)); 184 | Assert.assertFalse(result2.equals(result1)); 185 | 186 | result2map.put("point1", "point1val"); 187 | Assert.assertTrue(result1.equals(result2)); 188 | Assert.assertTrue(result2.equals(result1)); 189 | } 190 | 191 | } 192 | -------------------------------------------------------------------------------- /src/test/java/com/turn/splicer/tsdbutils/TsQueryDeserializeTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Collections; 21 | 22 | import org.testng.Assert; 23 | import org.testng.annotations.Test; 24 | 25 | @Test 26 | public class TsQueryDeserializeTest { 27 | 28 | @Test(expectedExceptions = RuntimeException.class) 29 | public void tryEmptyPostDeserialization() { 30 | TsQuerySerializer.deserializeFromJson(""); 31 | } 32 | 33 | @Test(expectedExceptions = NullPointerException.class) 34 | public void tryNullPostDeserialization() { 35 | TsQuerySerializer.deserializeFromJson(null); 36 | } 37 | 38 | @Test 39 | public void tryRealData() { 40 | String jsonContent = 41 | "{\"" + 42 | "start\":1438377954073,\"" + 43 | "queries\":" + 44 | "[" + 45 | "{\"metric\":\"tsd.jvm.ramused\"," + 46 | "\"aggregator\":\"avg\"," + 47 | "\"downsample\":\"1m-avg\"," + 48 | "\"tags\":" + 49 | "{\"host\":\"dwh-edge003\"}}" + 50 | "]}"; 51 | TsQuery query = TsQuerySerializer.deserializeFromJson(jsonContent); 52 | 53 | Assert.assertTrue(query.getStart().equals("1438377954073")); 54 | Assert.assertNotNull(query.getQueries()); 55 | Assert.assertEquals(query.getQueries().size(), 1); 56 | 57 | TSSubQuery subQuery = query.getQueries().get(0); 58 | Assert.assertEquals(subQuery.getMetric(), "tsd.jvm.ramused"); 59 | Assert.assertEquals(subQuery.getAggregator(), "avg"); 60 | Assert.assertEquals(subQuery.getDownsample(), "1m-avg"); 61 | Assert.assertEquals(subQuery.getTags().get("host"), "dwh-edge003"); 62 | Assert.assertEquals(subQuery.getTags().size(), 1); 63 | } 64 | 65 | @Test 66 | public void deserializeAndValidate() { 67 | String jsonContent = 68 | "{\"" + 69 | "start\":1438377954073,\"" + 70 | "queries\":" + 71 | "[" + 72 | "{\"metric\":\"tsd.jvm.ramused\"," + 73 | "\"aggregator\":\"avg\"," + 74 | "\"downsample\":\"1m-avg\"," + 75 | "\"tags\":" + 76 | "{\"host\":\"dwh-edge003\"}}" + 77 | "]}"; 78 | TsQuery query = TsQuerySerializer.deserializeFromJson(jsonContent); 79 | query.validateAndSetQuery(); 80 | 81 | Assert.assertEquals(query.startTime(), 1438377954073L); 82 | // end time time should be approximately equal to current time. 83 | Assert.assertTrue((System.currentTimeMillis() - query.endTime()) < 1000); 84 | } 85 | 86 | @Test 87 | public void serialize() { 88 | TsQuery query = new TsQuery(); 89 | query.setStart(String.valueOf(System.currentTimeMillis() - 7200000)); //2 hours 90 | query.setEnd(String.valueOf(System.currentTimeMillis())); 91 | TSSubQuery subQuery = new TSSubQuery(); 92 | subQuery.setDownsample("10m-avg"); 93 | subQuery.setAggregator("sum"); 94 | subQuery.setMetric("tsd.queries"); 95 | subQuery.setRate(true); 96 | query.setQueries(new ArrayList<>(Collections.singleton(subQuery))); 97 | } 98 | 99 | @Test 100 | public void serializeAndDeserialize() { 101 | String jsonContent = 102 | "{\"" + 103 | "start\":1438377954073,\"" + 104 | "queries\":" + 105 | "[" + 106 | "{\"metric\":\"tsd.jvm.ramused\"," + 107 | "\"aggregator\":\"avg\"," + 108 | "\"downsample\":\"1m-avg\"," + 109 | "\"tags\":" + 110 | "{\"host\":\"dwh-edge003\"}}" + 111 | "]}"; 112 | TsQuery query = TsQuerySerializer.deserializeFromJson(jsonContent); 113 | query.validateAndSetQuery(); 114 | 115 | long current = System.currentTimeMillis(); 116 | TsQuery slice = TsQuery.sliceOf(query, current - 3600, current); 117 | Assert.assertEquals(slice.startTime(), current - 3600); 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /src/test/java/com/turn/splicer/tsdbutils/expression/ExprReaderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | public class ExprReaderTest { 23 | 24 | @Test(expected = NullPointerException.class) 25 | public void badInit() { 26 | new ExprReader(null); 27 | } 28 | 29 | @Test 30 | public void initZero() { 31 | new ExprReader(new char[]{}); 32 | } 33 | 34 | @Test 35 | public void testPeekNext() { 36 | ExprReader reader = new ExprReader("hello".toCharArray()); 37 | 38 | Assert.assertEquals(reader.peek(), 'h'); 39 | Assert.assertEquals(reader.next(), 'h'); 40 | 41 | Assert.assertEquals(reader.peek(), 'e'); 42 | Assert.assertEquals(reader.next(), 'e'); 43 | 44 | Assert.assertEquals(reader.peek(), 'l'); 45 | Assert.assertEquals(reader.next(), 'l'); 46 | 47 | Assert.assertEquals(reader.peek(), 'l'); 48 | Assert.assertEquals(reader.next(), 'l'); 49 | 50 | Assert.assertEquals(reader.peek(), 'o'); 51 | Assert.assertEquals(reader.next(), 'o'); 52 | } 53 | 54 | @Test 55 | public void testEOF() { 56 | ExprReader reader = new ExprReader(new char[]{}); 57 | Assert.assertTrue(reader.isEOF()); 58 | } 59 | 60 | @Test 61 | public void testWhitespace() { 62 | ExprReader reader = new ExprReader("hello world".toCharArray()); 63 | for (int i=0; i<5; i++) { 64 | reader.next(); 65 | } 66 | 67 | Assert.assertFalse(reader.peek() == 'w'); 68 | reader.skipWhitespaces(); 69 | Assert.assertTrue(reader.peek() == 'w'); 70 | } 71 | 72 | @Test 73 | public void testAttemptWhitespace() { 74 | ExprReader reader = new ExprReader("hello world".toCharArray()); 75 | reader.skipWhitespaces(); 76 | Assert.assertTrue(reader.peek() == 'h'); 77 | } 78 | 79 | @Test 80 | public void testIsNextChar() { 81 | ExprReader reader = new ExprReader("hello".toCharArray()); 82 | Assert.assertTrue(reader.isNextChar('h')); 83 | Assert.assertTrue(reader.isNextChar('h')); 84 | Assert.assertTrue(reader.isNextChar('h')); 85 | } 86 | 87 | @Test 88 | public void testIsNextSequence() { 89 | ExprReader reader = new ExprReader("hello".toCharArray()); 90 | Assert.assertTrue(reader.isNextSeq("he")); 91 | Assert.assertTrue(reader.isNextSeq("he")); 92 | Assert.assertTrue(reader.isNextSeq("he")); 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/test/java/com/turn/splicer/tsdbutils/expression/ExpressionTreeTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.google.common.collect.Lists; 20 | import com.turn.splicer.merge.TsdbResult; 21 | import com.turn.splicer.tsdbutils.TsQuery; 22 | import org.junit.BeforeClass; 23 | import org.junit.Test; 24 | 25 | import java.util.List; 26 | 27 | public class ExpressionTreeTest { 28 | 29 | @BeforeClass 30 | public static void setup() { 31 | ExpressionFactory.addFunction("foo", new FooExpression()); 32 | } 33 | 34 | @Test 35 | public void parseSimple() { 36 | String expr = "foo(sum:proc.sys.cpu)"; 37 | List metricQueries = Lists.newArrayList(); 38 | ExpressionTree tree = Expressions.parse(expr, metricQueries, null); 39 | System.out.println(metricQueries); 40 | } 41 | 42 | @Test 43 | public void parseMultiParameter() { 44 | String expr = "foo(sum:proc.sys.cpu,, sum:proc.meminfo.memfree)"; 45 | List metricQueries = Lists.newArrayList(); 46 | ExpressionTree tree = Expressions.parse(expr, metricQueries, null); 47 | System.out.println(metricQueries); 48 | } 49 | 50 | @Test 51 | public void parseNestedExpr() { 52 | String expr = "foo(sum:proc.sys.cpu,, foo(sum:proc.a.b))"; 53 | List metricQueries = Lists.newArrayList(); 54 | ExpressionTree tree = Expressions.parse(expr, metricQueries, null); 55 | System.out.println(metricQueries); 56 | } 57 | 58 | @Test 59 | public void parseExprWithParam() { 60 | String expr = "foo(sum:proc.sys.cpu,, 100,, 3.1415)"; 61 | List metricQueries = Lists.newArrayList(); 62 | ExpressionTree tree = Expressions.parse(expr, metricQueries, null); 63 | System.out.println(metricQueries); 64 | } 65 | 66 | static class FooExpression implements Expression { 67 | 68 | @Override 69 | public TsdbResult[] evaluate(TsQuery data_query, List queryResults, List queryParams) { 70 | return new TsdbResult[0]; 71 | } 72 | 73 | @Override 74 | public String writeStringField(List queryParams, String innerExpression) { 75 | return "foo(" + innerExpression + ")"; 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/test/java/com/turn/splicer/tsdbutils/expression/FunctionsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.turn.splicer.hbase.RegionChecker; 20 | import com.turn.splicer.merge.TsdbResult; 21 | import com.turn.splicer.tsdbutils.SplicerQueryRunner; 22 | import com.turn.splicer.tsdbutils.SplicerUtils; 23 | import com.turn.splicer.tsdbutils.TsQuery; 24 | 25 | import java.io.IOException; 26 | import java.util.ArrayList; 27 | import java.util.Arrays; 28 | import java.util.HashMap; 29 | import java.util.List; 30 | import java.util.Map; 31 | import java.util.concurrent.ExecutionException; 32 | 33 | import org.mockito.InjectMocks; 34 | import org.mockito.Mock; 35 | import org.mockito.MockitoAnnotations; 36 | import org.testng.annotations.Test; 37 | 38 | import static org.mockito.Matchers.any; 39 | import static org.mockito.Mockito.when; 40 | import static org.testng.Assert.assertEquals; 41 | 42 | /** 43 | * Created by bpeltz on 9/15/15. 44 | */ 45 | public class FunctionsTest { 46 | 47 | final TsQuery dataQuery = new TsQuery(); 48 | 49 | @Mock 50 | private SplicerQueryRunner queryRunner; 51 | 52 | @InjectMocks 53 | private ExpressionTree expressionTree; 54 | 55 | /** 56 | * call before initMocks() is called so that I can inject the mocked SplicerQueryRunner 57 | * into expressionTree after its created (I think that is the right way to do this) 58 | */ 59 | private void setupExpressionTree(String query) { 60 | setupExpressionTree(query, "0", "0"); 61 | } 62 | 63 | private void setupExpressionTree(String query, String start, String end) { 64 | List expressionTrees = new ArrayList(); 65 | String[] expressions = {query}; 66 | List metricQueries = new ArrayList(); 67 | dataQuery.setStart(start); 68 | dataQuery.setEnd(end); 69 | 70 | SplicerUtils.syntaxCheck(expressions, dataQuery, metricQueries, expressionTrees); 71 | expressionTree = expressionTrees.get(0); 72 | } 73 | 74 | private TsdbResult getDefaultQueryResult() { 75 | TsdbResult defaultResult = new TsdbResult(); 76 | 77 | defaultResult.setMetric("tcollector.collector.lines_received"); 78 | Map dps = new HashMap(); 79 | dps.put("1", 10.0); 80 | dps.put("2", 20.0); 81 | dps.put("3", 30.0); 82 | 83 | defaultResult.setTags(new TsdbResult.Tags(new HashMap())); 84 | 85 | TsdbResult.Points points = new TsdbResult.Points(dps); 86 | defaultResult.setDps(points); 87 | 88 | return defaultResult; 89 | } 90 | 91 | private TsdbResult[] getDefaultQueryResultArray() { 92 | return new TsdbResult[]{ getDefaultQueryResult() }; 93 | } 94 | 95 | private TsdbResult[] getCorrectTimeShiftResult() { 96 | TsdbResult correctResult = new TsdbResult(); 97 | 98 | correctResult.setMetric("tcollector.collector.lines_received"); 99 | Map dps = new HashMap(); 100 | dps.put("2", 10.0); 101 | dps.put("3", 20.0); 102 | dps.put("4", 30.0); 103 | 104 | correctResult.setTags(new TsdbResult.Tags(new HashMap())); 105 | 106 | TsdbResult.Points points = new TsdbResult.Points(dps); 107 | correctResult.setDps(points); 108 | 109 | TsdbResult[] finalResult = {correctResult}; 110 | return finalResult; 111 | } 112 | 113 | private TsdbResult[] getCorrectResult() { 114 | TsdbResult correctResult = new TsdbResult(); 115 | 116 | correctResult.setMetric("tcollector.collector.lines_received"); 117 | Map dps = new HashMap(); 118 | dps.put("1", 100.0); 119 | dps.put("2", 200.0); 120 | dps.put("3", 300.0); 121 | 122 | correctResult.setTags(new TsdbResult.Tags(new HashMap())); 123 | 124 | TsdbResult.Points points = new TsdbResult.Points(dps); 125 | correctResult.setDps(points); 126 | 127 | TsdbResult[] finalResult = {correctResult}; 128 | return finalResult; 129 | } 130 | 131 | /** 132 | * builds an ExpressionTree off of query param, evaluates that expression tree 133 | * mocking results from Tsdb. Instead of calling tsdb in sliceAndRunQuery 134 | * it will just return mockQueryResult in all cases. toCompare is the result 135 | * to test equality against your expressions evaluation. 136 | * @param query 137 | * @param mockQueryResult 138 | * @param toCompare 139 | * @param outcome 140 | * @throws IOException 141 | */ 142 | private void evaluateQuery(String query, TsdbResult mockQueryResult, TsdbResult[] toCompare, boolean outcome) throws IOException, ExecutionException, InterruptedException { 143 | evaluateQuery(query, mockQueryResult, toCompare, outcome, null, null); 144 | } 145 | 146 | private void evaluateQuery(String query, TsdbResult mockQueryResult, TsdbResult[] toCompare, boolean outcome, String start, String end) throws IOException, ExecutionException, InterruptedException { 147 | setupExpressionTree(query, start, end); 148 | MockitoAnnotations.initMocks(this); 149 | 150 | TsdbResult[] mockQueryResultArray = new TsdbResult[1]; 151 | mockQueryResultArray[0] = mockQueryResult; 152 | 153 | when(queryRunner.sliceAndRunQuery(any(TsQuery.class), any(RegionChecker.class))).thenReturn(mockQueryResultArray); 154 | 155 | TsdbResult[] tsdbResults = null; 156 | 157 | tsdbResults = expressionTree.evaluateAll(); 158 | 159 | for(TsdbResult ts: tsdbResults) { 160 | System.out.println("res: " + ts.toString()); 161 | } 162 | 163 | for(TsdbResult tc: toCompare) { 164 | System.out.println("tc: " + tc.toString()); 165 | } 166 | assertEquals(Arrays.deepEquals(tsdbResults, toCompare), outcome); 167 | assertEquals(Arrays.deepEquals(toCompare, tsdbResults), outcome); 168 | } 169 | 170 | @Test 171 | public void evaluateScaleSimple() throws IOException, ExecutionException, InterruptedException { 172 | evaluateQuery("scale(sum:1m-avg:tcollector.collector.lines_received,10)", getDefaultQueryResult(), getCorrectResult(), true, "1000", "1001"); 173 | } 174 | 175 | @Test void evaluateAliasSimple() throws IOException, ExecutionException, InterruptedException { 176 | TsdbResult correctResult = getDefaultQueryResult(); 177 | 178 | TsdbResult[] correctResultArray = new TsdbResult[]{correctResult}; 179 | evaluateQuery("alias(sum:1m-avg:tcollector.collector.lines_received,dumpy)", getDefaultQueryResult(), correctResultArray, false, "1000", "1001"); 180 | 181 | correctResult.setAlias("dumpy"); 182 | evaluateQuery("alias(sum:1m-avg:tcollector.collector.lines_received,dumpy)", getDefaultQueryResult(), correctResultArray, true, "1000", "1001"); 183 | 184 | } 185 | 186 | @Test 187 | public void timeShiftFunctionSimple() throws IOException, ExecutionException, InterruptedException { 188 | //start end time given in millis 189 | evaluateQuery("timeShift(sum:1m-avg:tcollector.collector.lines_received,'1sec')", getDefaultQueryResult(), getCorrectTimeShiftResult(), true, "2000", "4000"); 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /src/test/java/com/turn/splicer/tsdbutils/expression/InterpolationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.google.common.collect.Lists; 20 | import com.google.common.collect.Maps; 21 | import com.stumbleupon.async.Deferred; 22 | import com.turn.splicer.merge.TsdbResult; 23 | import net.opentsdb.core.*; 24 | import net.opentsdb.meta.Annotation; 25 | import org.junit.Test; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.util.HashMap; 30 | import java.util.List; 31 | import java.util.Map; 32 | 33 | public class InterpolationTest { 34 | 35 | private static final Logger LOG = 36 | LoggerFactory.getLogger(AggregationIterator.class); 37 | 38 | @Test 39 | public void simpleTest() throws Exception { 40 | int count = 6; 41 | DataPoint[] points1 = new DataPoint[count]; 42 | points1[0] = MutableDataPoint.ofDoubleValue(0, 5); 43 | points1[1] = MutableDataPoint.ofDoubleValue(10, 5); 44 | points1[2] = MutableDataPoint.ofDoubleValue(20, 10); 45 | points1[3] = MutableDataPoint.ofDoubleValue(30, 15); 46 | points1[4] = MutableDataPoint.ofDoubleValue(40, 20); 47 | points1[5] = MutableDataPoint.ofDoubleValue(50, 5); 48 | 49 | DataPoint[] points2 = new DataPoint[count]; 50 | points2[0] = MutableDataPoint.ofDoubleValue(0, 10); 51 | points2[1] = MutableDataPoint.ofDoubleValue(10, 5); 52 | points2[2] = MutableDataPoint.ofDoubleValue(20, 20); 53 | points2[3] = MutableDataPoint.ofDoubleValue(30, 15); 54 | points2[4] = MutableDataPoint.ofDoubleValue(40, 10); 55 | points2[5] = MutableDataPoint.ofDoubleValue(50, 0); 56 | 57 | TsdbResult result1 = new TsdbResult(); 58 | result1.setDps(new TsdbResult.Points(new HashMap())); 59 | for(DataPoint p: points1) { 60 | result1.getDps().addPoint(p); 61 | } 62 | 63 | 64 | TsdbResult result2 = new TsdbResult(); 65 | result2.setDps(new TsdbResult.Points(new HashMap())); 66 | 67 | for(DataPoint p: points2) { 68 | result2.getDps().addPoint(p); 69 | } 70 | 71 | 72 | SeekableView[] views = new SeekableView[2]; 73 | views[0] = new SeekableViewDataPointImpl(result1.getDps().getDataPointsFromTreeMap()); 74 | views[1] = new SeekableViewDataPointImpl(result2.getDps().getDataPointsFromTreeMap()); 75 | 76 | 77 | 78 | AggregationIterator aggr = new AggregationIterator(views, 0, 100, 79 | Aggregators.SUM, Aggregators.Interpolation.LERP, false); 80 | 81 | while (aggr.hasNext()) { 82 | DataPoint dp = aggr.next(); 83 | LOG.info(dp.timestamp() + " " + (dp.isInteger()? dp.longValue() : dp.doubleValue())); 84 | } 85 | } 86 | 87 | @Test 88 | public void interpolateTest() throws Exception { 89 | DataPoint[] points1 = new DataPoint[] { 90 | MutableDataPoint.ofDoubleValue(10, 5), 91 | MutableDataPoint.ofDoubleValue(30, 15), 92 | MutableDataPoint.ofDoubleValue(50, 5) 93 | }; 94 | 95 | DataPoint[] points2 = new DataPoint[] { 96 | MutableDataPoint.ofDoubleValue(0, 10), 97 | MutableDataPoint.ofDoubleValue(20, 20), 98 | MutableDataPoint.ofDoubleValue(40, 10), 99 | MutableDataPoint.ofDoubleValue(60, 20) 100 | }; 101 | 102 | TsdbResult result1 = new TsdbResult(); 103 | result1.setDps(new TsdbResult.Points(new HashMap())); 104 | 105 | for(DataPoint p: points1) { 106 | result1.getDps().addPoint(p); 107 | } 108 | 109 | TsdbResult result2 = new TsdbResult(); 110 | result2.setDps(new TsdbResult.Points(new HashMap())); 111 | 112 | for(DataPoint p: points2) { 113 | result2.getDps().addPoint(p); 114 | } 115 | 116 | SeekableView[] views = new SeekableView[2]; 117 | views[0] = new SeekableViewDataPointImpl(result1.getDps().getDataPointsFromTreeMap()); 118 | views[1] = new SeekableViewDataPointImpl(result2.getDps().getDataPointsFromTreeMap()); 119 | 120 | AggregationIterator aggr = new AggregationIterator(views, 0, 100, 121 | Aggregators.SUM, Aggregators.Interpolation.LERP, false); 122 | 123 | while (aggr.hasNext()) { 124 | DataPoint dp = aggr.next(); 125 | LOG.info(dp.timestamp() + " " + (dp.isInteger()? dp.longValue() : dp.doubleValue())); 126 | } 127 | } 128 | 129 | static class MockDataPoints implements DataPoints { 130 | 131 | @Override 132 | public String metricName() { 133 | return "testMetric"; 134 | } 135 | 136 | @Override 137 | public Deferred metricNameAsync() { 138 | return Deferred.fromResult(metricName()); 139 | } 140 | 141 | @Override 142 | public byte[] metricUID() { 143 | return new byte[0]; 144 | } 145 | 146 | @Override 147 | public Map getTags() { 148 | Map p = Maps.newHashMap(); 149 | p.put("tagk", "tagv"); 150 | return p; 151 | } 152 | 153 | @Override 154 | public Deferred> getTagsAsync() { 155 | return Deferred.fromResult(getTags()); 156 | } 157 | 158 | @Override 159 | public org.hbase.async.Bytes.ByteMap getTagUids() { 160 | return null; 161 | } 162 | 163 | @Override 164 | public List getAggregatedTags() { 165 | return Lists.newArrayList("type"); 166 | } 167 | 168 | @Override 169 | public Deferred> getAggregatedTagsAsync() { 170 | return Deferred.fromResult(getAggregatedTags()); 171 | } 172 | 173 | @Override 174 | public List getAggregatedTagUids() { 175 | return null; 176 | } 177 | 178 | @Override 179 | public List getTSUIDs() { 180 | return null; 181 | } 182 | 183 | @Override 184 | public List getAnnotations() { 185 | return null; 186 | } 187 | 188 | @Override 189 | public int size() { 190 | throw new RuntimeException("not implemented"); 191 | } 192 | 193 | @Override 194 | public int aggregatedSize() { 195 | throw new RuntimeException("not implemented"); 196 | } 197 | 198 | @Override 199 | public SeekableView iterator() { 200 | throw new RuntimeException("not implemented"); 201 | } 202 | 203 | @Override 204 | public long timestamp(int i) { 205 | throw new RuntimeException("not implemented"); 206 | } 207 | 208 | @Override 209 | public boolean isInteger(int i) { 210 | throw new RuntimeException("not implemented"); 211 | } 212 | 213 | @Override 214 | public long longValue(int i) { 215 | throw new RuntimeException("not implemented"); 216 | } 217 | 218 | @Override 219 | public double doubleValue(int i) { 220 | throw new RuntimeException("not implemented"); 221 | } 222 | 223 | @Override 224 | public int getQueryIndex() { 225 | return 0; 226 | } 227 | } 228 | 229 | } 230 | -------------------------------------------------------------------------------- /src/test/java/com/turn/splicer/tsdbutils/expression/MovingAverageTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2015-2016 The Splicer Query Engine Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.turn.splicer.tsdbutils.expression; 18 | 19 | import com.turn.splicer.tsdbutils.Functions; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import java.util.concurrent.TimeUnit; 24 | 25 | public class MovingAverageTest { 26 | 27 | @Test 28 | public void testParseParam() { 29 | Functions.MovingAverageFunction func = new Functions.MovingAverageFunction(); 30 | long x; 31 | x = Functions.parseParam("'2min'"); 32 | Assert.assertEquals(TimeUnit.MILLISECONDS.convert(2, TimeUnit.MINUTES), x); 33 | 34 | x= Functions.parseParam("'1hr'"); 35 | Assert.assertEquals(TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS), x); 36 | 37 | x= Functions.parseParam("'1000hr'"); 38 | Assert.assertEquals(TimeUnit.MILLISECONDS.convert(1000,TimeUnit.HOURS), x); 39 | 40 | x= Functions.parseParam("'1sec'"); 41 | Assert.assertEquals(TimeUnit.MILLISECONDS.convert(1,TimeUnit.SECONDS), x); 42 | 43 | try { 44 | Functions.parseParam("'1sechr'"); 45 | Assert.assertTrue(false); 46 | } catch (RuntimeException e) { 47 | Assert.assertTrue(true); 48 | } 49 | 50 | try { 51 | Functions.parseParam("'1 sec'"); 52 | Assert.assertTrue(false); 53 | } catch (RuntimeException e) { 54 | Assert.assertTrue(true); 55 | } 56 | } 57 | } 58 | --------------------------------------------------------------------------------