├── IndexerAgent ├── pom.xml ├── sample_data │ ├── 1Nov.gz │ ├── 2Nov.gz │ ├── 31Oct.gz │ ├── 3Nov.gz │ └── 4Nov.gz ├── src │ └── main │ │ ├── java │ │ ├── com │ │ │ └── yahoo │ │ │ │ └── sql4d │ │ │ │ └── indexeragent │ │ │ │ ├── Agent.java │ │ │ │ ├── DruidMeta.java │ │ │ │ ├── actors │ │ │ │ ├── MainActor.java │ │ │ │ ├── MessageTypes.java │ │ │ │ ├── Throttler.java │ │ │ │ └── WorkerActor.java │ │ │ │ ├── meta │ │ │ │ ├── DBHandler.java │ │ │ │ ├── JobFreq.java │ │ │ │ ├── JobStatus.java │ │ │ │ ├── Utils.java │ │ │ │ └── beans │ │ │ │ │ ├── DataSource.java │ │ │ │ │ └── StatusTrail.java │ │ │ │ ├── sql │ │ │ │ ├── SqlFileSniffer.java │ │ │ │ └── SqlMeta.java │ │ │ │ └── util │ │ │ │ ├── FileSniffer.java │ │ │ │ └── UniqueOnlyQueue.java │ │ ├── crud.sql │ │ └── iagent.properties │ │ └── resources │ │ ├── META-INF │ │ └── persistence.xml │ │ └── log4j.properties └── startAgent.sh ├── LICENCE ├── README.md ├── Sql4DClient ├── .classpath ├── .project ├── dsql ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── yahoo │ │ └── sql4dclient │ │ ├── BeanGenUtil.java │ │ ├── CircularBuffer.java │ │ └── Main.java │ └── resources │ └── log4j.properties ├── Sql4DCompiler ├── .classpath ├── .project ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── yahoo │ └── sql4d │ ├── BaseStatementMeta.java │ ├── CrudProgram.java │ ├── CrudStatementMeta.java │ ├── DCompiler.java │ ├── DateFormatUtils.java │ ├── DeleteProgram.java │ ├── DropProgram.java │ ├── InsertProgram.java │ ├── Pair.java │ ├── Program.java │ ├── QueryProgram.java │ ├── beans │ └── EqualsToHolder.java │ ├── converter │ ├── druidG.tokens │ ├── druidGLexer.java │ └── druidGParser.java │ ├── delete │ ├── DeleteMeta.java │ └── SegmentSpec.java │ ├── drop │ └── DropMeta.java │ ├── druidG.g │ ├── insert │ ├── BasicInsertMeta.java │ ├── BatchInsertMeta.java │ ├── InsertMeta.java │ ├── RTInsertMeta.java │ └── nodes │ │ ├── GranularitySpec.java │ │ ├── InputSpec.java │ │ ├── PartitionsSpec.java │ │ └── RollupSpec.java │ ├── query │ ├── BaseAggQueryMeta.java │ ├── HintProcessor.java │ ├── PlainDimQueryMeta.java │ ├── QueryMeta.java │ ├── QueryUtils.java │ ├── RequestType.java │ ├── groupby │ │ ├── GroupByQueryMeta.java │ │ ├── Having.java │ │ └── LimitSpec.java │ ├── nodes │ │ ├── AggItem.java │ │ ├── Filter.java │ │ ├── Granularity.java │ │ ├── Interval.java │ │ └── PostAggItem.java │ ├── search │ │ └── SearchQueryMeta.java │ ├── select │ │ ├── PagingSpec.java │ │ └── SelectQueryMeta.java │ ├── timeboundary │ │ └── TimeBoundaryQueryMeta.java │ ├── timeseries │ │ └── TimeSeriesQueryMeta.java │ └── topn │ │ └── TopNQueryMeta.java │ └── utils │ ├── DruidUtils.java │ ├── TimeUtils.java │ └── Utils.java ├── Sql4Ddriver ├── .classpath ├── .project ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── yahoo │ │ └── sql4d │ │ └── sql4ddriver │ │ ├── BaseJoiner.java │ │ ├── BaseMapper.java │ │ ├── BrokerAccessor.java │ │ ├── CoordinatorAccessor.java │ │ ├── DDataSource.java │ │ ├── DruidNodeAccessor.java │ │ ├── Joiner4All.java │ │ ├── Joiner4Bean.java │ │ ├── Mapper4All.java │ │ ├── Mapper4Bean.java │ │ ├── NamedParameters.java │ │ ├── OverlordAccessor.java │ │ ├── PrettyPrint.java │ │ ├── TaskStatus.java │ │ ├── Util.java │ │ ├── rowmapper │ │ ├── DruidBaseBean.java │ │ └── TimeSeriesBean.java │ │ └── sql │ │ ├── DAODictionary.java │ │ └── MysqlAccessor.java │ └── test │ └── java │ └── com │ └── yahoo │ └── sql4d │ └── sql4ddriver │ ├── AnalyticsDruidTestBase.java │ ├── TimeSeriesTest.java │ └── rowmapper │ └── TestKryoDruidBaseBean.java └── pom.xml /IndexerAgent/sample_data/1Nov.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikalyc/Sql4D/2c052fe60ead5a16277c798a3440de7d4f6f24f6/IndexerAgent/sample_data/1Nov.gz -------------------------------------------------------------------------------- /IndexerAgent/sample_data/2Nov.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikalyc/Sql4D/2c052fe60ead5a16277c798a3440de7d4f6f24f6/IndexerAgent/sample_data/2Nov.gz -------------------------------------------------------------------------------- /IndexerAgent/sample_data/31Oct.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikalyc/Sql4D/2c052fe60ead5a16277c798a3440de7d4f6f24f6/IndexerAgent/sample_data/31Oct.gz -------------------------------------------------------------------------------- /IndexerAgent/sample_data/3Nov.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikalyc/Sql4D/2c052fe60ead5a16277c798a3440de7d4f6f24f6/IndexerAgent/sample_data/3Nov.gz -------------------------------------------------------------------------------- /IndexerAgent/sample_data/4Nov.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikalyc/Sql4D/2c052fe60ead5a16277c798a3440de7d4f6f24f6/IndexerAgent/sample_data/4Nov.gz -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/DruidMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent; 13 | import static com.yahoo.sql4d.indexeragent.Agent.*; 14 | 15 | /** 16 | * Assumes Agent is already initialized. 17 | * @author srikalyan 18 | */ 19 | public final class DruidMeta { 20 | private static final String LOCALHOST = "127.0.0.1"; 21 | private static final String COLON = ":"; 22 | 23 | private static String[] getBroker() { 24 | return getConfigAsStr("druid.broker", "127.0.0.1:8082").split(COLON); 25 | } 26 | private static String[] getCoordinator() { 27 | return getConfigAsStr("druid.coordinator", "127.0.0.1:8081").split(COLON); 28 | } 29 | private static String[] getOverlord() { 30 | return getConfigAsStr("druid.overlord", "127.0.0.1:8090").split(COLON); 31 | } 32 | 33 | public static String getBrokerHost() { 34 | String []sp = getBroker(); 35 | return (sp != null && sp.length > 0) ? sp[0] : LOCALHOST; 36 | } 37 | public static String getCoordinatorHost() { 38 | String []sp = getCoordinator(); 39 | return (sp != null && sp.length > 0) ? sp[0] : LOCALHOST; 40 | } 41 | public static String getOverlordHost() { 42 | String []sp = getOverlord(); 43 | return (sp != null && sp.length > 0) ? sp[0] : LOCALHOST; 44 | } 45 | 46 | public static int getBrokerPort() { 47 | String []sp = getBroker(); 48 | return (sp != null && sp.length > 1) ? Integer.valueOf(sp[1]) : 8082; 49 | } 50 | 51 | public static int getCoordinatorPort() { 52 | String []sp = getCoordinator(); 53 | return (sp != null && sp.length > 1) ? Integer.valueOf(sp[1]) : 8081; 54 | } 55 | 56 | public static int getOverlordPort() { 57 | String []sp = getOverlord(); 58 | return (sp != null && sp.length > 1) ? Integer.valueOf(sp[1]) : 8090; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/actors/MessageTypes.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.actors; 13 | 14 | /** 15 | * Akka Messages. 16 | * @author srikalyan 17 | */ 18 | public enum MessageTypes { 19 | BOOT_FROM_SQLS, START_TICKING, GENERATE_WORK, EXECUTE_WORK, TRACK_WORK, STOP_TICKING 20 | } 21 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/actors/Throttler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.actors; 13 | 14 | import akka.actor.Cancellable; 15 | import java.util.concurrent.TimeUnit; 16 | import java.util.concurrent.atomic.AtomicBoolean; 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | 20 | /** 21 | * 22 | * @author srikalyan 23 | */ 24 | public abstract class Throttler implements Cancellable, Runnable { 25 | 26 | private static final Logger log = LoggerFactory.getLogger(Throttler.class); 27 | private final int initialDelay; 28 | private final int msgsPerSecond; 29 | private final int maxMessagesAnyTime; 30 | 31 | private final AtomicBoolean cancel = new AtomicBoolean(false); 32 | private final Thread selfThread; 33 | 34 | public Throttler(int initialDelay, int msgsPerSecond, int maxMessagesAnyTime) { 35 | this.initialDelay = initialDelay; 36 | this.msgsPerSecond = msgsPerSecond; 37 | this.maxMessagesAnyTime = maxMessagesAnyTime; 38 | this.selfThread = new Thread(this); 39 | } 40 | 41 | public Cancellable startThrottling() { 42 | selfThread.start(); 43 | return this; 44 | } 45 | 46 | @Override 47 | public boolean cancel() { 48 | cancel.set(true); 49 | return cancel.get(); 50 | } 51 | 52 | @Override 53 | public boolean isCancelled() { 54 | return cancel.get(); 55 | } 56 | 57 | @Override 58 | public void run() { 59 | try { 60 | TimeUnit.SECONDS.sleep(initialDelay); 61 | } catch (InterruptedException ex) { 62 | log.warn("Throttler bootstrap interrupted ! {}", ex); 63 | } 64 | while (!isCancelled()) { 65 | try { 66 | int inProgressCount = getInProgressActionCount(); 67 | log.info("Current in progress {}, msgsPerSecond {}, maxMessagesAnyTime {}, ", inProgressCount, msgsPerSecond, maxMessagesAnyTime); 68 | if (inProgressCount >= maxMessagesAnyTime) { 69 | TimeUnit.SECONDS.sleep(1); 70 | } else { 71 | int p = (maxMessagesAnyTime - inProgressCount); 72 | if (p <= msgsPerSecond) { 73 | log.debug("Running action p={} times", p); 74 | runActionXTimes(p); 75 | } else { 76 | int q = p; // Lets say p = 11 and msgsPerSecond = 10 77 | while (q > 0) { // q = 11 78 | if (q > msgsPerSecond) { // All but last condition succeeds here. 79 | log.debug("Running action msgsPerSecond={} times", msgsPerSecond); 80 | runActionXTimes(msgsPerSecond); 81 | } else { // This occurs the last time i.e when q=1 82 | log.debug("Running action q={} times", q); 83 | runActionXTimes(q);//hit overlord q times 84 | } 85 | TimeUnit.SECONDS.sleep(1); 86 | q = q - msgsPerSecond; 87 | } 88 | } 89 | } 90 | } catch (InterruptedException ex) { 91 | log.warn("Throttler interrupted ! {}", ex); 92 | } 93 | } 94 | } 95 | 96 | private void runActionXTimes(int n) { 97 | for (int i = 0;i < n;i++) { 98 | runAction(); 99 | } 100 | } 101 | 102 | public abstract int getInProgressActionCount(); 103 | public abstract void runAction(); 104 | 105 | } 106 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/meta/JobFreq.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.meta; 13 | 14 | /** 15 | * 16 | * @author srikalyan 17 | */ 18 | public enum JobFreq { 19 | minute,fifteen_minute,thirty_minute,hour,day; 20 | 21 | public int inMinutes() { 22 | int mins = 0; 23 | switch(this) { 24 | case minute: 25 | mins = 1; 26 | break; 27 | case fifteen_minute: 28 | mins = 15; 29 | break; 30 | case thirty_minute: 31 | mins = 30; 32 | break; 33 | case hour: 34 | mins = 60; 35 | break; 36 | case day: 37 | mins = 1440; 38 | break; 39 | } 40 | return mins; 41 | } 42 | 43 | public int inMillis() { 44 | return inMinutes() * 60 * 1000; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/meta/JobStatus.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.meta; 13 | 14 | /** 15 | * 16 | * @author srikalyan 17 | */ 18 | public enum JobStatus { 19 | done, not_done, in_progress 20 | } 21 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/meta/Utils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.meta; 13 | 14 | import java.text.DecimalFormat; 15 | import java.text.ParseException; 16 | import org.apache.commons.lang.StringUtils; 17 | import org.joda.time.DateTime; 18 | import org.joda.time.DateTimeZone; 19 | 20 | /** 21 | * 22 | * @author srikalyan 23 | */ 24 | public class Utils { 25 | 26 | private static final DecimalFormat twoFormat = new DecimalFormat("00"); 27 | private static final DecimalFormat fourFormat = new DecimalFormat("0000"); 28 | 29 | /** 30 | * I/p -> O/p 31 | * 2015 -> 2015-01-01T00:00:00.000Z 32 | * 201504 -> 2015-04-01T00:00:00.000Z 33 | * 20150403 -> 2015-04-03T00:00:00.000Z 34 | * 201504030102 -> 2015-04-03T01:02:00.000Z 35 | * @param dateStr 36 | * @return 37 | */ 38 | public static DateTime dateOf(String dateStr) { 39 | return nthDateOf(dateStr, 0, 0); 40 | } 41 | 42 | public static String year(DateTime dt) { 43 | return fourFormat.format(dt.getYear()); 44 | } 45 | 46 | public static String month(DateTime dt) { 47 | return twoFormat.format(dt.getMonthOfYear()); 48 | } 49 | 50 | public static String day(DateTime dt) { 51 | return twoFormat.format(dt.getDayOfMonth()); 52 | } 53 | 54 | public static String hour(DateTime dt) { 55 | return twoFormat.format(dt.getHourOfDay()); 56 | } 57 | 58 | public static String mins(DateTime dt) { 59 | return twoFormat.format(dt.getMinuteOfHour()); 60 | } 61 | 62 | 63 | /** 64 | * 65 | * @param dateStr 66 | * @param n 67 | * @param freqInMinutes 68 | * @return 69 | */ 70 | public static DateTime nthDateOf(String dateStr, int n, int freqInMinutes) { 71 | if (StringUtils.isBlank(dateStr)) { 72 | throw new IllegalArgumentException("Invalid dateStr " + dateStr); 73 | } 74 | int year = 0; 75 | int month = 1; 76 | int day = 1; 77 | int hour = 0; 78 | int min = 0; 79 | 80 | try { 81 | if (dateStr.length() >= 4) { 82 | year = fourFormat.parse(dateStr.substring(0, 4)).intValue(); 83 | if (dateStr.length() >= 6) { 84 | month = twoFormat.parse(dateStr.substring(4, 6)).intValue(); 85 | if (dateStr.length() >= 8) { 86 | day = twoFormat.parse(dateStr.substring(6, 8)).intValue(); 87 | if (dateStr.length() >= 10) { 88 | hour = twoFormat.parse(dateStr.substring(8, 10)).intValue(); 89 | if (dateStr.length() >= 12) { 90 | min = twoFormat.parse(dateStr.substring(10, 12)).intValue(); 91 | } 92 | } 93 | } 94 | } 95 | } 96 | } catch (ParseException ex) { 97 | throw new IllegalArgumentException("Invalid dateStr " + dateStr + " " + ex); 98 | } 99 | return new DateTime(DateTimeZone.UTC).withYear(year).withMonthOfYear(month).withDayOfMonth(day).withHourOfDay(hour).withMinuteOfHour(min).withSecondOfMinute(0).withMillisOfSecond(0).plusMinutes(n * freqInMinutes); 100 | } 101 | 102 | public static void main(String[] args) { 103 | System.out.println(nthDateOf("201504030102", 5, 1440)); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/meta/beans/DataSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.meta.beans; 13 | 14 | import com.yahoo.sql4d.indexeragent.meta.JobFreq; 15 | import com.yahoo.sql4d.indexeragent.meta.JobStatus; 16 | import javax.persistence.Entity; 17 | import javax.persistence.GeneratedValue; 18 | import javax.persistence.Id; 19 | import javax.persistence.Table; 20 | 21 | /** 22 | * 23 | * @author srikalyan 24 | */ 25 | @Entity 26 | @Table(name = "DataSource") 27 | public class DataSource { 28 | @Id 29 | @GeneratedValue 30 | private int id; 31 | private String name; 32 | private long startTime; 33 | private long spinFromTime; 34 | private long endTime; 35 | private String frequency; 36 | private String status; 37 | private String templatePath; 38 | private String delimiter; 39 | private String listDelimiter; 40 | private String templateSql; 41 | 42 | public DataSource setId(int id) { 43 | this.id = id; 44 | return this; 45 | } 46 | 47 | public DataSource setName(String dataSource) { 48 | this.name = dataSource; 49 | return this; 50 | } 51 | 52 | public DataSource setStartTime(long startTime) { 53 | this.startTime = startTime; 54 | return this; 55 | } 56 | 57 | public DataSource setSpinFromTime(long spinFromTime) { 58 | this.spinFromTime = spinFromTime; 59 | return this; 60 | } 61 | 62 | public DataSource setEndTime(long endTime) { 63 | this.endTime = endTime; 64 | return this; 65 | } 66 | 67 | public DataSource setFrequency(JobFreq frequency) { 68 | this.frequency = frequency.name(); 69 | return this; 70 | } 71 | 72 | public DataSource setStatus(JobStatus status) { 73 | this.status = status.name(); 74 | return this; 75 | } 76 | 77 | public DataSource setTemplatePath(String templatePath) { 78 | this.templatePath = templatePath; 79 | return this; 80 | } 81 | 82 | public DataSource setDelimiter(String delimiter) { 83 | this.delimiter = delimiter; 84 | return this; 85 | } 86 | 87 | public DataSource setListDelimiter(String listDelimiter) { 88 | this.listDelimiter = listDelimiter; 89 | return this; 90 | } 91 | 92 | public DataSource setTemplateSql(String templateSql) { 93 | this.templateSql = templateSql; 94 | return this; 95 | } 96 | 97 | public int getId() { 98 | return id; 99 | } 100 | 101 | public String getName() { 102 | return name; 103 | } 104 | 105 | public long getStartTime() { 106 | return startTime; 107 | } 108 | 109 | public long getSpinFromTime() { 110 | return spinFromTime; 111 | } 112 | 113 | public long getEndTime() { 114 | return endTime; 115 | } 116 | 117 | public String getFrequency() { 118 | return frequency; 119 | } 120 | 121 | public String getStatus() { 122 | return status; 123 | } 124 | 125 | public String getTemplatePath() { 126 | return templatePath; 127 | } 128 | 129 | public String getDelimiter() { 130 | return delimiter; 131 | } 132 | 133 | public String getListDelimiter() { 134 | return listDelimiter; 135 | } 136 | 137 | public String getTemplateSql() { 138 | return templateSql; 139 | } 140 | 141 | public void updateFrom(DataSource ds) { 142 | setName(ds.getName()); 143 | setStartTime(ds.getStartTime()); 144 | setSpinFromTime(ds.getSpinFromTime()); 145 | setEndTime(ds.getEndTime()); 146 | setFrequency(JobFreq.valueOf(ds.getFrequency())); 147 | setStatus(JobStatus.valueOf(ds.getStatus())); 148 | setTemplatePath(ds.getTemplatePath()); 149 | setDelimiter(ds.getDelimiter()); 150 | setListDelimiter(ds.getListDelimiter()); 151 | setTemplateSql(ds.getTemplateSql()); 152 | } 153 | 154 | @Override 155 | public String toString() { 156 | return String.format("%d %s %d %d %d %s %s %s %s %s %s", id, name, startTime, spinFromTime, endTime, frequency, status, templatePath, delimiter, listDelimiter, templateSql); 157 | } 158 | 159 | } 160 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/meta/beans/StatusTrail.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.meta.beans; 13 | 14 | import com.yahoo.sql4d.indexeragent.meta.JobStatus; 15 | import javax.persistence.Entity; 16 | import javax.persistence.GeneratedValue; 17 | import javax.persistence.Id; 18 | import javax.persistence.Table; 19 | import static com.yahoo.sql4d.indexeragent.Agent.*; 20 | /** 21 | * 22 | * @author srikalyan 23 | */ 24 | @Entity 25 | @Table(name = "StatusTrail") 26 | public class StatusTrail implements Comparable { 27 | @Id 28 | @GeneratedValue 29 | private int id; 30 | private int dataSourceId; 31 | private long nominalTime; 32 | private String status; 33 | private int givenUp; 34 | private int attemptsDone; 35 | private String taskId;// This is the druid indexer taskId. 36 | 37 | public StatusTrail setId(int id) { 38 | this.id = id; 39 | return this; 40 | } 41 | 42 | public StatusTrail setDataSourceId(int dataSourceId) { 43 | this.dataSourceId = dataSourceId; 44 | return this; 45 | } 46 | 47 | public StatusTrail setNominalTime(long nominalTime) { 48 | this.nominalTime = nominalTime; 49 | return this; 50 | } 51 | 52 | public StatusTrail setStatus(JobStatus status) { 53 | this.status = status.name(); 54 | return this; 55 | } 56 | 57 | public StatusTrail setGivenUp(int givenUp) { 58 | this.givenUp = givenUp; 59 | return this; 60 | } 61 | 62 | public StatusTrail setAttemptsDone(int attemptsDone) { 63 | this.attemptsDone = attemptsDone; 64 | if (attemptsDone >= getMaxTaskAttempts()) { 65 | setGivenUp(1); 66 | } 67 | return this; 68 | } 69 | 70 | public StatusTrail setTaskId(String taskId) { 71 | this.taskId = taskId; 72 | return this; 73 | } 74 | 75 | public int getId() { 76 | return id; 77 | } 78 | 79 | public int getDataSourceId() { 80 | return dataSourceId; 81 | } 82 | 83 | public long getNominalTime() { 84 | return nominalTime; 85 | } 86 | 87 | public String getStatus() { 88 | return status; 89 | } 90 | 91 | public int getGivenUp() { 92 | return givenUp; 93 | } 94 | 95 | public int getAttemptsDone() { 96 | return attemptsDone; 97 | } 98 | 99 | public String getTaskId() { 100 | return taskId; 101 | } 102 | 103 | public void updateFrom(StatusTrail st) { 104 | setDataSourceId(st.getDataSourceId()); 105 | setNominalTime(st.getNominalTime()); 106 | setStatus(JobStatus.valueOf(st.getStatus())); 107 | setGivenUp(st.getGivenUp()); 108 | setAttemptsDone(st.getAttemptsDone()); 109 | setTaskId(st.getTaskId()); 110 | } 111 | 112 | @Override 113 | public String toString() { 114 | return String.format("%d %d %d %s %d %d %s", id, dataSourceId, nominalTime, status, givenUp, attemptsDone, taskId); 115 | } 116 | 117 | @Override 118 | public int compareTo(StatusTrail other) { 119 | return getId() - other.getId(); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/sql/SqlMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.sql; 13 | import static com.yahoo.sql4d.indexeragent.Agent.*; 14 | 15 | /** 16 | * Assumes Agent is already initialized. 17 | * @author srikalyan 18 | */ 19 | public final class SqlMeta { 20 | public static String getHost() { 21 | return getConfigAsStr("jdbc.host", "127.0.0.1"); 22 | } 23 | public static int getPort() { 24 | return getConfigAsInt("jdbc.port", 1527); 25 | } 26 | public static String getId() { 27 | return getConfigAsStr("jdbc.id", ""); 28 | } 29 | public static String getPassword() { 30 | return getConfigAsStr("jdbc.password", ""); 31 | } 32 | public static String getDbName() { 33 | return getConfigAsStr("jdbc.dbName", "indexerDB"); 34 | } 35 | public static String getDbType() { 36 | return getConfigAsStr("jdbc.dbType", "derby"); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/util/FileSniffer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.util; 13 | 14 | import java.io.IOException; 15 | import java.nio.file.Path; 16 | import java.nio.file.Paths; 17 | import static java.nio.file.StandardWatchEventKinds.*; 18 | import java.nio.file.WatchEvent; 19 | import java.nio.file.WatchKey; 20 | import java.nio.file.WatchService; 21 | import java.util.concurrent.TimeUnit; 22 | import java.util.concurrent.atomic.AtomicBoolean; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | /** 27 | * A generic file change observer. Looks at the file extension first and then the event. 28 | * @author srikalyan 29 | */ 30 | public abstract class FileSniffer { 31 | private static final Logger log = LoggerFactory.getLogger(FileSniffer.class); 32 | private final Path sniffPath; 33 | private final String fileExtension; 34 | private WatchService watcher; 35 | private final AtomicBoolean stopSniff = new AtomicBoolean(false); 36 | private Thread snifferThread; 37 | 38 | public FileSniffer(String pathToSniff, String fileExtension) { 39 | sniffPath = Paths.get(pathToSniff); 40 | this.fileExtension = fileExtension; 41 | try { 42 | watcher = sniffPath.getFileSystem().newWatchService(); 43 | sniffPath.register(watcher, ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY); 44 | } catch (IOException e) { 45 | log.error("Error: {}", e); 46 | } 47 | } 48 | 49 | public void startSniffing() { 50 | snifferThread = new Thread(new Runnable() { 51 | @Override 52 | public void run() { 53 | while (!stopSniff.get()) { 54 | try { 55 | // Could have done take() instead of poll() but that blocks indefinitely. 56 | WatchKey watckKey = watcher.poll(3, TimeUnit.SECONDS); 57 | if (stopSniff.get()) { 58 | break; 59 | } 60 | if (watckKey == null) { 61 | continue; 62 | } 63 | for (WatchEvent event : watckKey.pollEvents()) { 64 | log.info("Event Kind : {}", event.kind()); 65 | if (event.kind() == OVERFLOW) {// If event is lost/discarded. 66 | continue; 67 | } 68 | log.info("Event: {}", event.context().toString()); 69 | if (event.context().toString().endsWith(fileExtension) && (event.context() instanceof Path)) { 70 | Path fullPath = Paths.get(sniffPath.toString(), ((Path)event.context()).getFileName().toString()); 71 | if (event.kind() == ENTRY_CREATE) { 72 | log.info("New file found: {}", event.context().toString()); 73 | onCreate(fullPath); 74 | } 75 | if (event.kind() == ENTRY_DELETE) { 76 | log.info("File {} deleted.", event.context().toString()); 77 | onDelete(fullPath); 78 | } 79 | if (event.kind() == ENTRY_MODIFY) { 80 | log.info("File {} modified.", event.context().toString()); 81 | onModify(fullPath); 82 | } 83 | } 84 | } 85 | if (!watckKey.reset()) { 86 | log.error("Watch key has become invalid !!"); 87 | stopSniff.set(true); 88 | } 89 | } catch (InterruptedException ex) { 90 | log.error("Exception while sniffing {}", ex); 91 | stopSniff.set(true); 92 | } 93 | } 94 | } 95 | }); 96 | snifferThread.start(); 97 | } 98 | 99 | public void stopSniffing() { 100 | stopSniff.set(true); 101 | } 102 | 103 | public abstract void onCreate(Path path); 104 | public abstract void onDelete(Path path); 105 | public abstract void onModify(Path path); 106 | } 107 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/com/yahoo/sql4d/indexeragent/util/UniqueOnlyQueue.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.indexeragent.util; 13 | 14 | import java.util.ArrayList; 15 | import java.util.Collection; 16 | import java.util.Iterator; 17 | import java.util.LinkedHashSet; 18 | import java.util.List; 19 | import java.util.concurrent.locks.Lock; 20 | import java.util.concurrent.locks.ReentrantLock; 21 | 22 | /** 23 | * A Queue that disallows duplicates. Thread safe. 24 | * 25 | * @author srikalyan 26 | * @param 27 | */ 28 | public class UniqueOnlyQueue extends LinkedHashSet { 29 | 30 | private final Lock modifyLock = new ReentrantLock(); 31 | 32 | @Override 33 | public boolean add(T e) { 34 | try { 35 | modifyLock.lock(); 36 | return super.add(e); 37 | } finally { 38 | modifyLock.unlock(); 39 | } 40 | } 41 | 42 | @Override 43 | public boolean addAll(Collection c) { 44 | try { 45 | modifyLock.lock(); 46 | return super.addAll(c); 47 | } finally { 48 | modifyLock.unlock(); 49 | } 50 | } 51 | 52 | @Override 53 | public boolean remove(Object o) { 54 | try { 55 | modifyLock.lock(); 56 | return super.remove(o); 57 | } finally { 58 | modifyLock.unlock(); 59 | } 60 | } 61 | 62 | @Override 63 | public boolean removeAll(Collection c) { 64 | try { 65 | modifyLock.lock(); 66 | return super.removeAll(c); 67 | } finally { 68 | modifyLock.unlock(); 69 | } 70 | } 71 | 72 | /** 73 | * Special method that removes head element. 74 | * @return 75 | */ 76 | public T removeFirst() { 77 | try { 78 | modifyLock.lock(); 79 | Iterator it = iterator(); 80 | if (it.hasNext()) { 81 | T item = it.next(); 82 | it.remove(); 83 | return item; 84 | } 85 | } finally { 86 | modifyLock.unlock(); 87 | } 88 | return null; 89 | } 90 | 91 | /** 92 | * This method first removes all existing entries and zips together all the lists. For ex: 93 | * [[x1, x2, x3] [y1, y2] [z1] [d1, d2, d3]] -> [x1, y1, z1, d1, x2, y2, d2, x3, d3] 94 | * Assuming each list is per datasource and are sorted by id(latest on top) then 95 | * by zipping we are picking one latest from each data source. 96 | * @param kLists 97 | */ 98 | public void clearAndMergeKLists(List> kLists) { 99 | try { 100 | // Beauty of reentrant locks. The same thread is going to acquire the 101 | // same lock again multiple times during add. 102 | modifyLock.lock(); 103 | clear(); 104 | List> kListIters = new ArrayList<>(); 105 | for (List kThList:kLists) { 106 | kListIters.add(kThList.iterator()); 107 | } 108 | boolean hasNone = false; 109 | while (!hasNone) { 110 | hasNone = true; 111 | List> markedForRemoval = new ArrayList<>(); 112 | for (Iterator kThListIter:kListIters) { 113 | if (kThListIter.hasNext()) { 114 | hasNone = false; 115 | add(kThListIter.next()); 116 | } else { 117 | markedForRemoval.add(kThListIter); 118 | } 119 | } 120 | kListIters.removeAll(markedForRemoval); 121 | } 122 | } finally { 123 | modifyLock.unlock(); 124 | } 125 | } 126 | 127 | } 128 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/crud.sql: -------------------------------------------------------------------------------- 1 | ----------------------------- For Derby --------------------------------------- 2 | -- Start the Derby network server 3 | -- cd $DERBY_HOME/lib/; java -jar derbyrun.jar server start 4 | -- Start the Derby client 5 | -- ij 6 | -- connect 'jdbc:derby://localhost:1527/indexerDB;create=true'; 7 | -- Then run the following create statements. 8 | 9 | CREATE TABLE DataSource ( 10 | id INTEGER NOT NULL GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1), 11 | name VARCHAR(64) NOT NULL, 12 | startTime BIGINT, 13 | spinFromTime BIGINT, 14 | endTime BIGINT, 15 | frequency VARCHAR(64) NOT NULL, 16 | status VARCHAR(64) NOT NULL, 17 | templatePath VARCHAR(256) NOT NULL, 18 | delimiter VARCHAR(4) NOT NULL, 19 | listDelimiter VARCHAR(4) NOT NULL, 20 | templateSql VARCHAR(2000) NOT NULL, 21 | CONSTRAINT dataSource_pk PRIMARY KEY (id) 22 | ) ; 23 | 24 | CREATE TABLE StatusTrail ( 25 | id INTEGER NOT NULL GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1), 26 | dataSourceId INTEGER CONSTRAINT dataSource_fk REFERENCES DataSource (id) ON DELETE CASCADE ON UPDATE RESTRICT, 27 | nominalTime BIGINT, 28 | status VARCHAR(64) NOT NULL, 29 | givenUp INTEGER DEFAULT 0, -- non-zero value means we have 'givenUp' 30 | attemptsDone INTEGER DEFAULT 0, -- 'givenUp' is set to non-zero when attemptDone reaches max configured. 31 | taskId VARCHAR(128), 32 | CONSTRAINT statusTrail_pk PRIMARY KEY (id) 33 | ) ; 34 | 35 | --------------------------------- For MySQL -------------------------------------- 36 | -- Start the Mysql 37 | -- Add /usr/local/mysql/bin/ to PATH 38 | -- sudo mysqld_safe( do cntrl-Z and type bg command) 39 | -- Open Mysql client as 40 | -- mysql 41 | -- mysql> CREATE database indexerDB; 42 | -- mysql> GRANT ALL PRIVILEGES ON * . * TO ''@'localhost'; 43 | -- mysql> GRANT ALL PRIVILEGES ON * . * TO ''@'127.0.0.1'; 44 | -- Then run the following create statements. 45 | 46 | 47 | CREATE TABLE DataSource ( 48 | id INT NOT NULL AUTO_INCREMENT, 49 | name VARCHAR(64) NOT NULL, 50 | startTime BIGINT, 51 | spinFromTime BIGINT, 52 | endTime BIGINT, 53 | frequency VARCHAR(64) NOT NULL, 54 | status VARCHAR(64) NOT NULL, 55 | templatePath VARCHAR(256) NOT NULL, 56 | delimiter VARCHAR(4) NOT NULL, 57 | listDelimiter VARCHAR(4) NOT NULL, 58 | templateSql VARCHAR(2000) NOT NULL, 59 | PRIMARY KEY (id) 60 | ) ; 61 | 62 | CREATE TABLE StatusTrail ( 63 | id INT NOT NULL AUTO_INCREMENT, 64 | dataSourceId INT NOT NULL, 65 | nominalTime BIGINT NOT NULL, 66 | status VARCHAR(64) NOT NULL, 67 | givenUp INT DEFAULT 0, -- non-zero value means we have 'givenUp' 68 | attemptsDone INT DEFAULT 0, -- 'givenUp' is set to non-zero when attemptDone reaches max configured. 69 | taskId VARCHAR(128), 70 | PRIMARY KEY (id), 71 | FOREIGN KEY (dataSourceId) REFERENCES DataSource(id) ON DELETE CASCADE ON UPDATE RESTRICT 72 | ) ; 73 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/java/iagent.properties: -------------------------------------------------------------------------------- 1 | # All the time properties configured here are in minutes un less otherwise mentioned. 2 | # If delay is 60 minutes then first attempt to run 00 hour is 60 minutes later. 3 | taskAttemptDelay=60 4 | # Retry on a failed task is attempted after retryDelay. 5 | retryDelay=60 6 | # Maximum number of times the task run is attempted before setting givenUp=1 on that instance. 7 | maxRetries=3 8 | # If (Task nominal time + sla time) < current time then send an email. 9 | slaTime=300 10 | # Number of worker(s) - aka Akka actors 11 | numWorkers=3 12 | # Path(folder) where to find the druid sql files. (Default value is ${HOME}/dsqls) 13 | sqlsPath=/Users/srikalyan/dsqls/ 14 | 15 | # Work scheduler properties. 16 | work { 17 | # All the time(s) are in seconds. 18 | generate.interval=15 19 | execute.msgsPerSecond=1 20 | # Maximum number of parallel tasks (It depends on how many tasks the druid's middle managers can handle). 21 | execute.maxAtGivenTime=10 22 | track.interval=15 23 | } 24 | #JDBC properties for derby. For Mysql change port to 3306, dbType=mysql 25 | jdbc { 26 | host=127.0.0.1 27 | port=1527 28 | id="" 29 | password="" 30 | dbName=indexerDB 31 | # Supported types (mysql, derby) 32 | dbType=derby 33 | } 34 | 35 | #Druid cluster properties. 36 | druid { 37 | # default broker is localhost:8082 38 | broker="localhost:8082" 39 | # default coordinator is localhost:8081 40 | coordinator="localhost:8081" 41 | # default overlord is localhost:8090 42 | overlord="localhost:8090" 43 | } 44 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/resources/META-INF/persistence.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.hibernate.ejb.HibernatePersistence 5 | com.yahoo.sql4d.indexeragent.meta.beans.DataSource 6 | com.yahoo.sql4d.indexeragent.meta.beans.StatusTrail 7 | 8 | 9 | 10 | 11 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /IndexerAgent/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, STDOUT, LOGFILE 2 | log4j.logger.com.yahoo.sql4d=INFO 3 | log4j.logger.org.hibernate=ERROR 4 | log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender 5 | log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.STDOUT.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n 7 | 8 | log4j.appender.LOGFILE=org.apache.log4j.rolling.RollingFileAppender 9 | log4j.appender.LOGFILE.RollingPolicy=org.apache.log4j.rolling.TimeBasedRollingPolicy 10 | log4j.appender.LOGFILE.RollingPolicy.FileNamePattern=/tmp/dsql_indexer_agent_%d{yyyy-MM-dd}.log 11 | log4j.appender.LOGFILE.append=true 12 | log4j.appender.LOGFILE.threshold=INFO 13 | log4j.appender.LOGFILE.layout=org.apache.log4j.PatternLayout 14 | log4j.appender.LOGFILE.layout.ConversionPattern=%-5p %c: %m%n -------------------------------------------------------------------------------- /IndexerAgent/startAgent.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# -ne 1 ]; then 3 | echo "Need path to indexer configuration..." 4 | exit 1 5 | fi 6 | 7 | java -cp target/IndexerAgent-*.jar com.yahoo.sql4d.indexeragent.Agent $1 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Sql4D -> SQL interface to druid(see http://druid.io/ ), has 3 sub components 2 | 3 | 1) SQL -> Json compiler, 4 | 2) Terminal based SQL client interface, 5 | 3) Java JDBC like driver for druid. 6 | 7 | Tutors 8 | =========== 9 | | Description | Where to get from | 10 | | --------------------------------- |:-------------| 11 | | Getting started guide | https://github.com/srikalyc/Sql4D/wiki | 12 | | A first look at druid with Sql | http://druidwithsql.tumblr.com/post/98578718282/a-first-look-at-druid-with-sql | 13 | | Insert and deleting data in druid with Sql | http://druidwithsql.tumblr.com/post/108054375927/simple-insert-deleting-data-drop-table-in-druid | 14 | | Java JDBC access to druid | http://druidwithsql.tumblr.com/post/111014674922/accessing-druid-through-java-driver-interface | 15 | | Indexer Agent to ingest periodic data sets | https://github.com/srikalyc/Sql4D/wiki/Using-Indexer-Agent-for-batch-ingestion | 16 | 17 | 18 | Compatibiltiy with druid 19 | ========================= 20 | | Druid Version | SQL4D Version(Maven Central) | Comments | 21 | | ------------- |:-------------:| :-------------:| 22 | | 0.6.X | 3.X.X | You could also instead clone the Sql4D, and build against the tag v3.5.6 | 23 | | 0.7.X | 4.X.X | You could also instead clone the Sql4D, and build against the master | 24 | 25 | 26 | 27 | NOTE: Sql4D/sql4ddriver/.../PrettyPrint.java contains a code snippet modified from a publication here: http://stackoverflow.com/questions/11383070/pretty-print-2d-array-in-java . The 28 | text/code is licensed under Creative Commons-BY-SA 3.0. 29 | 30 | -------------------------------------------------------------------------------- /Sql4DClient/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /Sql4DClient/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | Sql4DClient 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /Sql4DClient/dsql: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NOW=`date +%Y-%m-%d` 3 | java -jar target/Sql4DClient.jar -bh $(BROKER_HOST) -bp $(BROKER_PORT) -ch $(COORDINATOR_HOST) -cp $(COORDINATOR_PORT) -ph $(PROXY_HOST) -pp $(PROXY_PORT) -i $(history_size) 4 | -------------------------------------------------------------------------------- /Sql4DClient/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.yahoo.sql4d 5 | Sql4DClient 6 | 4.1.0 7 | jar 8 | 9 | com.yahoo.sql4d 10 | Sql4D 11 | 4.1.0 12 | ../pom.xml 13 | 14 | 15 | 16 | 17 | org.apache.maven.plugins 18 | maven-shade-plugin 19 | 20 | 21 | package 22 | 23 | shade 24 | 25 | 26 | 27 | 29 | com.yahoo.sql4dclient.Main 30 | 31 | 33 | META-INF/spring.handlers 34 | 35 | 37 | META-INF/spring.schemas 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | commons-cli 49 | commons-cli 50 | 51 | 52 | com.yahoo.sql4d 53 | Sql4Ddriver 54 | 4.1.0 55 | 56 | 57 | org.apache.commons 58 | commons-lang3 59 | 60 | 61 | commons-io 62 | commons-io 63 | 64 | 65 | org.slf4j 66 | slf4j-api 67 | 68 | 69 | org.slf4j 70 | slf4j-log4j12 71 | 72 | 73 | log4j 74 | apache-log4j-extras 75 | 76 | 77 | commons-lang 78 | commons-lang 79 | 2.6 80 | 81 | 82 | 83 | UTF-8 84 | 1.7 85 | 1.7 86 | 87 | Sql4DClient 88 | -------------------------------------------------------------------------------- /Sql4DClient/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=DEBUG, LOGFILE 2 | log4j.logger.com.yahoo.sql4d=INFO 3 | log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender 4 | log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.STDOUT.layout.ConversionPattern=%5p [%t] (%F:%L) - %m%n 6 | 7 | log4j.appender.LOGFILE=org.apache.log4j.rolling.RollingFileAppender 8 | log4j.appender.LOGFILE.RollingPolicy=org.apache.log4j.rolling.TimeBasedRollingPolicy 9 | log4j.appender.LOGFILE.RollingPolicy.FileNamePattern=/tmp/dsql_client_%d{yyyy-MM-dd}.log 10 | log4j.appender.LOGFILE.threshold=INFO 11 | log4j.appender.LOGFILE.append=true 12 | log4j.appender.LOGFILE.layout=org.apache.log4j.PatternLayout 13 | log4j.appender.LOGFILE.layout.ConversionPattern=%-5p %c: %m%n -------------------------------------------------------------------------------- /Sql4DCompiler/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /Sql4DCompiler/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | Sql4DCompiler 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /Sql4DCompiler/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.yahoo.sql4d 5 | Sql4DCompiler 6 | 4.1.0 7 | jar 8 | 9 | com.yahoo.sql4d 10 | Sql4D 11 | 4.1.0 12 | ../pom.xml 13 | 14 | 15 | 16 | 17 | joda-time 18 | joda-time 19 | 20 | 21 | org.antlr 22 | antlr-runtime 23 | 24 | 25 | org.json 26 | json 27 | 28 | 29 | org.scala-lang 30 | scala-library 31 | 32 | 33 | org.testng 34 | testng 35 | 6.8.1 36 | test 37 | 38 | 39 | commons-io 40 | commons-io 41 | 42 | 43 | com.google.guava 44 | guava 45 | 46 | 47 | 48 | 49 | UTF-8 50 | 1.7 51 | 1.7 52 | 53 | Sql4DCompiler 54 | 55 | 56 | 57 | org.antlr 58 | antlr3-maven-plugin 59 | 3.5.2 60 | 61 | ${basedir}/src/main/java/com/yahoo/sql4d 62 | ${basedir}/src/main/java/com/yahoo/sql4d/converter 63 | 64 | 65 | 66 | 67 | antlr 68 | 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/BaseStatementMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | import java.util.LinkedHashMap; 14 | import java.util.Map; 15 | import org.json.JSONObject; 16 | 17 | /** 18 | * Base class for all sql statement types. 19 | * @author srikalyan 20 | */ 21 | public abstract class BaseStatementMeta { 22 | public String dataSource; 23 | public BaseStatementMeta queryDataSource; // http://druid.io/docs/latest/DataSource.html Query Datasource 24 | 25 | public BaseStatementMeta() { 26 | } 27 | 28 | public BaseStatementMeta(BaseStatementMeta meta) { 29 | this.dataSource = meta.dataSource; 30 | this.queryDataSource = meta.queryDataSource; 31 | } 32 | 33 | public BaseStatementMeta(String dataSource) { 34 | this.dataSource = dataSource; 35 | } 36 | 37 | @Override 38 | public String toString() { 39 | return getJson().toString(2); 40 | } 41 | 42 | public JSONObject getJson() { 43 | return new JSONObject(getDataMap()); 44 | } 45 | public Map getDataMap() { 46 | Map map = new LinkedHashMap<>(); 47 | JSONObject dataSourceJson = new JSONObject(); 48 | if (dataSource != null) { 49 | dataSourceJson.put("type", "table"); 50 | dataSourceJson.put("name", dataSource); 51 | } else if (queryDataSource != null) { 52 | dataSourceJson.put("type", "query"); 53 | dataSourceJson.put("query", queryDataSource.getDataMap()); 54 | } 55 | map.put("dataSource", dataSourceJson); 56 | return map; 57 | } 58 | 59 | public void postProcess(T anyContext) { 60 | 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/CrudProgram.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | /** 14 | * One or more insert meta. 15 | * @author srikalyan 16 | * @param 17 | */ 18 | public abstract class CrudProgram extends Program{ 19 | public boolean waitForCompletion = true; 20 | } 21 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/CrudStatementMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | /** 14 | * Base class for all CRUD sql statement types. 15 | * @author srikalyan 16 | */ 17 | public abstract class CrudStatementMeta extends BaseStatementMeta { 18 | public CrudStatementMeta() { 19 | } 20 | 21 | public CrudStatementMeta(String dataSource) { 22 | this.dataSource = dataSource; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/DCompiler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | import com.yahoo.sql4d.converter.druidGLexer; 14 | import com.yahoo.sql4d.converter.druidGParser; 15 | import java.util.logging.Level; 16 | import java.util.logging.Logger; 17 | import org.antlr.runtime.ANTLRStringStream; 18 | import org.antlr.runtime.CommonTokenStream; 19 | import org.antlr.runtime.RecognitionException; 20 | 21 | /** 22 | * A rudimentary compiler for Druid Dql -> Druid Json. 23 | * @author srikalyan 24 | */ 25 | public class DCompiler { 26 | /** 27 | * To avoid instantiation. 28 | */ 29 | private DCompiler() { 30 | } 31 | /** 32 | * Sql->Json. 33 | * @param query 34 | * @return 35 | */ 36 | public static Program compileSql(String query) { 37 | try { 38 | ANTLRStringStream in = new ANTLRStringStream(query); 39 | druidGLexer lexer = new druidGLexer(in); 40 | CommonTokenStream tokens = new CommonTokenStream(lexer); 41 | druidGParser parser = new druidGParser(tokens); 42 | Program pgm = parser.program(); 43 | return pgm; 44 | } catch (RecognitionException ex) { 45 | System.out.println(ex); 46 | Logger.getLogger(DCompiler.class.getName()).log(Level.SEVERE, null, ex); 47 | } 48 | return null; 49 | } 50 | 51 | public static void main(String[] args) { 52 | //Program p = compileSql("select UNIQUE(referer) AS refe, referer FROM miab_hourly_v2 WHERE interval BETWEEN 2014-11-05T08:18:00.000Z AND 2014-11-05T08:20:00.000Z BREAK BY 'day' GROUP BY referer;"); 53 | Program p = compileSql("INSERT INTO abf (timestamp , provider , title, uuid, DOUBLE_SUM(click) AS click) VALUES ('2014-10-31 00:00:00','sri','SE','fsd-sdf-dfgdf','2') WHERE interval BETWEEN '2013-08-31' AND '2023-09-01' BREAK BY 'day';"); 54 | System.out.println(p); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/DateFormatUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | import java.text.DateFormat; 14 | import java.text.ParseException; 15 | import java.text.SimpleDateFormat; 16 | import java.util.Date; 17 | import java.util.logging.Level; 18 | import java.util.logging.Logger; 19 | 20 | /** 21 | * Date formatting utilities. 22 | * @author srikalyan 23 | */ 24 | public class DateFormatUtils { 25 | private static final String DATE_HOUR = "yyyy-MM-dd'T'HH"; 26 | private static final String DATE_HOUR_MIN = "yyyy-MM-dd'T'HH:mm"; 27 | private static final String DATE_HOUR_MIN_SEC = "yyyy-MM-dd'T'HH:mm:ss"; 28 | private static final String DATE_TIME_ZONE = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX"; 29 | 30 | public static final DateFormat dateHourFormat = new SimpleDateFormat(DATE_HOUR); 31 | public static final DateFormat dateHourMinFormat = new SimpleDateFormat(DATE_HOUR_MIN); 32 | public static final DateFormat dateHourMinSecFormat = new SimpleDateFormat(DATE_HOUR_MIN_SEC); 33 | public static final DateFormat dateHourTimeZoneFormat = new SimpleDateFormat(DATE_TIME_ZONE); 34 | 35 | public static Date dateHour(String date) { 36 | return getDate(dateHourFormat, date); 37 | } 38 | 39 | public static Date dateHourMin(String date) { 40 | return getDate(dateHourMinFormat, date); 41 | } 42 | 43 | public static Date dateHourMinSec(String date) { 44 | return getDate(dateHourMinSecFormat, date); 45 | } 46 | 47 | public static Date dateHourTimeZone(String date) { 48 | return getDate(dateHourTimeZoneFormat, date); 49 | } 50 | 51 | private static Date getDate(DateFormat formatter, String date) { 52 | try { 53 | return formatter.parse(date); 54 | } catch (ParseException ex) { 55 | Logger.getLogger(DateFormatUtils.class.getName()).log(Level.SEVERE, null, ex); 56 | } 57 | return null; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/DeleteProgram.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | import com.yahoo.sql4d.delete.DeleteMeta; 14 | 15 | /** 16 | * One Delete Meta. 17 | * @author srikalyan 18 | */ 19 | public class DeleteProgram extends CrudProgram{ 20 | public DeleteProgram() { 21 | this.type = Type.DELETE; 22 | } 23 | 24 | @Override 25 | public String toString() { 26 | StringBuilder buffer = new StringBuilder(); 27 | for (DeleteMeta dMeta: getAllStmnts()) { 28 | System.out.println(dMeta.toString()); 29 | } 30 | return buffer.toString(); 31 | } 32 | 33 | /** 34 | * TODO: Do all semantic checks here. 1st field should be timestamp. 35 | * @throws java.lang.Exception 36 | */ 37 | @Override 38 | public void isValid() throws Exception { 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/DropProgram.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | import com.yahoo.sql4d.drop.DropMeta; 14 | 15 | /** 16 | * One Drop Meta. 17 | * @author srikalyan 18 | */ 19 | public class DropProgram extends CrudProgram{ 20 | public DropProgram() { 21 | this.type = Type.DROP; 22 | } 23 | 24 | @Override 25 | public String toString() { 26 | StringBuilder buffer = new StringBuilder(); 27 | for (DropMeta dMeta: getAllStmnts()) { 28 | System.out.println(dMeta.toString()); 29 | } 30 | return buffer.toString(); 31 | } 32 | 33 | /** 34 | * TODO: Do all semantic checks here. 1st field should be timestamp. 35 | * @throws java.lang.Exception 36 | */ 37 | @Override 38 | public void isValid() throws Exception { 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/InsertProgram.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | import com.yahoo.sql4d.insert.InsertMeta; 14 | 15 | /** 16 | * One or more insert meta. 17 | * @author srikalyan 18 | */ 19 | public class InsertProgram extends CrudProgram{ 20 | public static InsertProgram getInsertInstance() { 21 | return new InsertProgram(Type.INSERT); 22 | } 23 | 24 | public static InsertProgram getInsertHadoopInstance() { 25 | return new InsertProgram(Type.INSERT_HADOOP); 26 | } 27 | 28 | public static InsertProgram getInsertRTInstance() { 29 | return new InsertProgram(Type.INSERT_REALTIME); 30 | } 31 | 32 | private InsertProgram(Type type) { 33 | this.type = type; 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | StringBuilder buffer = new StringBuilder(); 39 | for (InsertMeta iMeta: getAllStmnts()) { 40 | System.out.println(iMeta.toString()); 41 | } 42 | return buffer.toString(); 43 | } 44 | 45 | /** 46 | * TODO: Do all semantic checks here. 1st field should be timestamp. 47 | * @throws java.lang.Exception 48 | */ 49 | @Override 50 | public void isValid() throws Exception { 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/Pair.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | /** 14 | * 15 | * @author srikalyan 16 | */ 17 | public class Pair { 18 | public T a; 19 | public U b; 20 | 21 | public Pair(T a, U b) { 22 | this.a = a; 23 | this.b = b; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/Program.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | 16 | /** 17 | * @author srikalyan 18 | * @param 19 | */ 20 | public abstract class Program { 21 | private final List stmnts = new ArrayList<>(); 22 | 23 | public enum Type { 24 | QUERY, INSERT, INSERT_HADOOP, INSERT_REALTIME, DROP, DELETE 25 | } 26 | 27 | protected Type type; 28 | 29 | public Type getStmntType() { 30 | return type; 31 | } 32 | 33 | public void addStmnt(T stmnt) { 34 | stmnts.add(stmnt); 35 | } 36 | 37 | public T nthStmnt(int i) { 38 | return stmnts.get(i); 39 | } 40 | 41 | public List getAllStmnts() { 42 | return stmnts; 43 | } 44 | 45 | public int numStmnts() { 46 | return stmnts.size(); 47 | } 48 | 49 | public void print(boolean toConsole) { 50 | if (toConsole) { 51 | System.out.println(toString()); 52 | } 53 | } 54 | /** 55 | * Do all semantic checks here. 56 | * @throws java.lang.Exception 57 | */ 58 | public abstract void isValid() throws Exception; 59 | } 60 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/QueryProgram.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d; 12 | 13 | import com.yahoo.sql4d.query.BaseAggQueryMeta; 14 | import com.yahoo.sql4d.query.PlainDimQueryMeta; 15 | import com.yahoo.sql4d.query.QueryMeta; 16 | import com.yahoo.sql4d.query.nodes.AggItem; 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | /** 22 | * One or more query meta which are joinable on primary key. 23 | * @author srikalyan 24 | */ 25 | public class QueryProgram extends Program{ 26 | public final List primaryJoinableHooks = new ArrayList<>();// Has 'N-1' items 27 | public final List joinTypes = new ArrayList<>();// Has 'N-1' items 28 | 29 | public QueryProgram() { 30 | this.type = Type.QUERY; 31 | } 32 | 33 | 34 | public void addJoinHook(String hook) { 35 | primaryJoinableHooks.add(hook); 36 | } 37 | 38 | public void addJoinType(String type) { 39 | joinTypes.add(type); 40 | } 41 | @Override 42 | public String toString() { 43 | StringBuilder buffer = new StringBuilder(); 44 | int i = 0; 45 | for (QueryMeta qMeta: getAllStmnts()) { 46 | buffer.append(qMeta.toString()); 47 | buffer.append("\n"); 48 | if (i == numStmnts() - 1) { 49 | break; 50 | } 51 | if (numStmnts() > 1) { 52 | buffer.append(" joined with "); 53 | buffer.append(primaryJoinableHooks.get(i++)); 54 | buffer.append("\n"); 55 | } 56 | 57 | } 58 | return buffer.toString(); 59 | } 60 | 61 | /** 62 | * TODO: This method should be a full blown functional but right now address 63 | * only partial semantics. 64 | * Do all semantic checks here. 65 | * @throws java.lang.Exception 66 | */ 67 | @Override 68 | public void isValid() throws Exception { 69 | if (numStmnts() >= 2) { 70 | if (primaryJoinableHooks.size() > 0) { 71 | for (String joinField:primaryJoinableHooks) { 72 | if (joinField.equals("timestamp")) {// timestamp is implicitly present in the result. 73 | continue; 74 | } 75 | for (QueryMeta query: getAllStmnts()) { 76 | boolean isPresent = false; 77 | if (query instanceof BaseAggQueryMeta) {// Since BaseAggQueryMeta is also a PlainDimQueryMeta we go with former first. 78 | for (AggItem item:((BaseAggQueryMeta)query).aggregations) { 79 | if (joinField.equals(item.fieldName) || (item.asName != null && joinField.equals(item.asName))) { 80 | isPresent = true; 81 | } 82 | } 83 | } 84 | if (query instanceof PlainDimQueryMeta) { 85 | Map dims = ((PlainDimQueryMeta)query).fetchDimensions; 86 | for (Map.Entry entry :dims.entrySet()) { 87 | if (joinField.equals(entry.getKey()) || (entry.getValue() != null && joinField.equals(entry.getValue()))) { 88 | isPresent = true; 89 | } 90 | } 91 | } 92 | if (!isPresent) { 93 | String error = joinField + " mentioned in the composite key should appear in all the data sources being queried and joined."; 94 | throw new Exception(error); 95 | } 96 | } 97 | } 98 | } else { 99 | String error = "Need atleast one field to join on"; 100 | throw new Exception(error); 101 | } 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/beans/EqualsToHolder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.beans; 12 | 13 | /** 14 | * Holds name = value 15 | * @author srikalyan 16 | */ 17 | public class EqualsToHolder { 18 | public String name; 19 | public String value; 20 | 21 | public EqualsToHolder(String name, String value) { 22 | this.name = name; 23 | this.value = value; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/converter/druidG.tokens: -------------------------------------------------------------------------------- 1 | T__89=89 2 | T__90=90 3 | T__91=91 4 | T__92=92 5 | AND=4 6 | ARITH_OPER=5 7 | AS=6 8 | ASC=7 9 | AUTO_ISO=8 10 | BETWEEN=9 11 | BREAK=10 12 | BY=11 13 | COMPARE_OPER=12 14 | CONTAINS=13 15 | COUNT=14 16 | DATE=15 17 | DATE_HOUR=16 18 | DATE_HOUR_MIN=17 19 | DATE_HOUR_MIN_SEC=18 20 | DATE_HOUR_MIN_SEC_SUB=19 21 | DATE_HOUR_MIN_SEC_SUB_TZ=20 22 | DATE_HOUR_MIN_SEC_SUB_UTC_TZ=21 23 | DATE_YEAR_MONTH_ONLY=22 24 | DATE_YEAR_ONLY=23 25 | DELETE=24 26 | DELIMITER=25 27 | DESC=26 28 | DOUBLE_SUM=27 29 | DROP=28 30 | DURATION=29 31 | EQUALS=30 32 | ESC_SEQ=31 33 | FIELD_ACCESS=32 34 | FLOAT=33 35 | FROM=34 36 | GROUP=35 37 | HAVING=36 38 | HEX_DIGIT=37 39 | HINT=38 40 | HYPER_UNIQUE=39 41 | ID=40 42 | INCLUDE=41 43 | INSERT=42 44 | INSERT_HADOOP=43 45 | INSERT_REALTIME=44 46 | INTO=45 47 | ISO=46 48 | JAVASCRIPT=47 49 | JOIN=48 50 | KAFKA=49 51 | LCURLY=50 52 | LEFT_JOIN=51 53 | LIKE=52 54 | LIMIT=53 55 | LONG=54 56 | LONG_SUM=55 57 | LPARAN=56 58 | LSQUARE=57 59 | MAX=58 60 | MAX_WINDOW=59 61 | MIN=60 62 | NEWLINE=61 63 | NOT=62 64 | NUM=63 65 | OCTAL_ESC=64 66 | ON=65 67 | OPT_AMPERSAND=66 68 | OPT_SEMI_COLON=67 69 | OR=68 70 | ORDER=69 71 | PARTITION=70 72 | PERIOD=71 73 | RCURLY=72 74 | RIGHT_JOIN=73 75 | ROLLUP=74 76 | RPARAN=75 77 | RSQUARE=76 78 | SELECT=77 79 | SINGLE_QUOTE_STRING=78 80 | SORT=79 81 | STRING=80 82 | TABLE=81 83 | THEN=82 84 | UNICODE_ESC=83 85 | UNIQUE=84 86 | VALUES=85 87 | WHERE=86 88 | WHICH=87 89 | WS=88 90 | '(*)'=89 91 | '*'=90 92 | ','=91 93 | 'interval'=92 94 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/delete/DeleteMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.delete; 13 | 14 | import com.google.common.base.Predicate; 15 | import com.google.common.collect.Iterables; 16 | import com.google.common.collect.Lists; 17 | import com.yahoo.sql4d.CrudStatementMeta; 18 | import com.yahoo.sql4d.query.nodes.Interval; 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import java.util.Map; 22 | import org.joda.time.DateTime; 23 | import org.json.JSONObject; 24 | 25 | /** 26 | * Delete statement's meta data. 27 | * 28 | * @author srikalyan 29 | */ 30 | public class DeleteMeta extends CrudStatementMeta { 31 | 32 | public String id;// UUid 33 | public Interval interval = null; 34 | public List segmentsToDelete = null;// This will be determined at runtime. 35 | public List dimensions = new ArrayList<>();// This will be determined at runtime. 36 | public List metrics = new ArrayList<>();// This will be determined at runtime. 37 | 38 | public DeleteMeta() { 39 | } 40 | 41 | public DeleteMeta(String dataSource, Interval interval) { 42 | super(dataSource); 43 | this.interval = interval; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return getJson().toString(2); 49 | } 50 | 51 | @Override 52 | public JSONObject getJson() { 53 | return new JSONObject(getDataMap()); 54 | } 55 | /** 56 | * Retain segments only overlapping with the range. 57 | * @param allSegments 58 | */ 59 | public void filterSegments(List allSegments) { 60 | segmentsToDelete = Lists.newArrayList(Iterables.filter(allSegments, new Predicate() { 61 | @Override 62 | public boolean apply(Interval i) { 63 | return i.fallsIn(interval); 64 | } 65 | })); 66 | } 67 | 68 | @Override 69 | public Map getDataMap() { 70 | // For each instance of data generation generate a new id. 71 | id = String.format("kill_%s_%s_%s_%s", dataSource, interval.getStartTime(), interval.getEndTime(), new DateTime().toString()); 72 | Map map = super.getDataMap(); 73 | map.put("type", "kill"); 74 | map.put("dataSource", dataSource); 75 | map.put("interval", interval.toString()); 76 | map.put("id", id); 77 | return map; 78 | } 79 | 80 | 81 | public void postProcess(T anyContext) { 82 | 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/delete/SegmentSpec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.delete; 12 | 13 | import com.yahoo.sql4d.query.nodes.*; 14 | import static com.yahoo.sql4d.utils.Utils.*; 15 | import java.util.Arrays; 16 | import java.util.LinkedHashMap; 17 | import java.util.List; 18 | import java.util.Map; 19 | import org.json.JSONArray; 20 | import org.json.JSONObject; 21 | 22 | /** 23 | * Ex: Used in data ingestion. 24 | *{ 25 | "dataSource" : "abf", 26 | "interval" : "2014-10-30T00:00:00.000-07:00/2014-11-30T00:00:00.000-07:00", 27 | "binaryVersion" : 9, 28 | "dimensions" : "provider,title,uuid", 29 | "metrics" : "count,click" 30 | } 31 | * 32 | * @author srikalyan 33 | */ 34 | public class SegmentSpec { 35 | private static final List validGSimpleStrings = Arrays.asList("all", "none", "minute", "fifteen_minute", "thirty_minute", "hour" , "day"); 36 | public String gran = "day"; 37 | public String type = "uniform";//TODO: Check out what are other types. 38 | public Interval interval = new Interval("2000-01-01", "2050-12-31"); 39 | 40 | public SegmentSpec(String gString) { 41 | this.gran = unquote(gString); 42 | isValid(this.gran); 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return String.format(getJson().toString(2)); 48 | } 49 | 50 | public JSONObject getJson() { 51 | return new JSONObject(getJsonMap()); 52 | } 53 | 54 | private void isValid(String gString) { 55 | if (!validGSimpleStrings.contains(gString)) { 56 | throw new IllegalArgumentException("Ivalid granularity " + gString); 57 | } 58 | this.gran = gString; 59 | } 60 | 61 | 62 | public Map getJsonMap() { 63 | Map map = new LinkedHashMap<>(); 64 | JSONArray intervalArr = new JSONArray(); 65 | intervalArr.put(interval.toString()); 66 | map.put("gran", gran); 67 | map.put("type", type); 68 | map.put("intervals", intervalArr); 69 | return map; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/drop/DropMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.drop; 13 | 14 | import com.yahoo.sql4d.CrudStatementMeta; 15 | import com.yahoo.sql4d.query.nodes.Interval; 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.util.Map; 19 | import org.joda.time.DateTime; 20 | import org.json.JSONObject; 21 | 22 | /** 23 | * Drop statement's meta data. 24 | * @author srikalyan 25 | */ 26 | public class DropMeta extends CrudStatementMeta { 27 | 28 | public String id;// UUid 29 | public Interval interval = null;// This should be filled with time boundary at runtime. 30 | public List dimensions = new ArrayList<>();// This will be determined at runtime. 31 | public List metrics = new ArrayList<>();// This will be determined at runtime. 32 | 33 | public DropMeta() { 34 | } 35 | 36 | public DropMeta(String dataSource, Interval interval) { 37 | super(dataSource); 38 | this.interval = interval; 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return getJson().toString(2); 44 | } 45 | 46 | @Override 47 | public JSONObject getJson() { 48 | return new JSONObject(getDataMap()); 49 | } 50 | 51 | @Override 52 | public Map getDataMap() { 53 | // For each instance of data generation generate a new id. 54 | id = String.format("kill_%s_%s_%s_%s", dataSource, interval.getStartTime(), interval.getEndTime(), new DateTime().toString()); 55 | Map map = super.getDataMap(); 56 | map.put("type", "kill"); 57 | map.put("dataSource", dataSource); 58 | map.put("interval", interval.toString()); 59 | map.put("id", id); 60 | return map; 61 | } 62 | 63 | 64 | public void postProcess(T anyContext) { 65 | 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/insert/BasicInsertMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.insert; 13 | 14 | import com.google.common.base.Joiner; 15 | import com.google.common.collect.ImmutableMap; 16 | import com.yahoo.sql4d.utils.TimeUtils; 17 | import java.io.File; 18 | import java.io.IOException; 19 | import java.util.ArrayList; 20 | import java.util.LinkedHashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.UUID; 24 | import java.util.logging.Level; 25 | import java.util.logging.Logger; 26 | import org.apache.commons.io.FileUtils; 27 | import org.json.JSONObject; 28 | 29 | /** 30 | * Insert Meta for simple insertions(non batch and non realtime) 31 | * 32 | * @author srikalyan 33 | */ 34 | public class BasicInsertMeta extends InsertMeta { 35 | 36 | public String dataPath = null;// Either dataPath is not null(or)values has some data. 37 | public List values = new ArrayList<>(); 38 | private final String tmpFolder = System.getProperty("java.io.tmpdir"); 39 | String timestampFormat = null; 40 | 41 | public BasicInsertMeta() { 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return getJson().toString(2); 47 | } 48 | 49 | @Override 50 | public JSONObject getJson() { 51 | return new JSONObject(getDataMap()); 52 | } 53 | 54 | @Override 55 | public Map getDataMap() { 56 | return ImmutableMap.of( 57 | "type", "index", 58 | "spec", getSpec()); 59 | } 60 | 61 | @Override 62 | public Map getTimestampSpec() { 63 | return ImmutableMap.of( 64 | "column", "timestamp", 65 | "format", (timestampFormat != null) ? timestampFormat : "iso");//TODO: Check if auto works instead of iso as default. 66 | } 67 | 68 | @Override 69 | public Map getIoConfig() { 70 | return ImmutableMap.of( 71 | "type", "index", 72 | "firehose", getFirehose()); 73 | } 74 | 75 | public Map getFirehose() { 76 | Map finalMap = new LinkedHashMap<>(); 77 | finalMap.put("type", "local"); 78 | if (dataPath != null) { 79 | int folderEndIndex = dataPath.lastIndexOf("/"); 80 | finalMap.put("baseDir", dataPath.substring(0, folderEndIndex + 1)); 81 | finalMap.put("filter", (folderEndIndex == dataPath.length() - 1)?"*":dataPath.substring(folderEndIndex + 1)); 82 | if (dataPath.endsWith("json")) { 83 | dataFormat = "json"; 84 | } else if (dataPath.endsWith("csv")) { 85 | dataFormat = "csv"; 86 | } 87 | } else { 88 | finalMap.put("baseDir", tmpFolder); 89 | String fileName = UUID.randomUUID().toString() + ".csv"; 90 | finalMap.put("filter", fileName); 91 | dataFormat = "csv"; 92 | if (values.isEmpty()) { 93 | throw new IllegalStateException("No values to insert !!"); 94 | } 95 | try { 96 | File file = new File(tmpFolder + File.separator + fileName); 97 | FileUtils.write(file, Joiner.on(",").join(values)); 98 | System.out.println("Written to " + file); 99 | Object timestamp = values.get(0); 100 | timestampFormat = TimeUtils.detectFormat(timestamp.toString()); 101 | } catch (IOException ex) { 102 | Logger.getLogger(BasicInsertMeta.class.getName()).log(Level.SEVERE, null, ex); 103 | } 104 | 105 | } 106 | return finalMap; 107 | } 108 | 109 | /** 110 | * Optional 111 | * @return 112 | */ 113 | @Override 114 | public Map getTuningConfig() { 115 | return ImmutableMap.of( 116 | "type", "index", 117 | "targetPartitionSize", -1, 118 | "rowFlushBoundary", 0, 119 | "numShards", 1); 120 | } 121 | public void postProcess(T anyContext) { 122 | 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/insert/BatchInsertMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.insert; 13 | 14 | import com.google.common.collect.ImmutableMap; 15 | import com.yahoo.sql4d.insert.nodes.PartitionsSpec; 16 | import com.yahoo.sql4d.insert.nodes.InputSpec; 17 | import com.yahoo.sql4d.insert.nodes.RollupSpec; 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | import java.util.Map; 21 | import org.json.JSONObject; 22 | import static java.lang.String.*; 23 | 24 | /** 25 | * Insert Meta for batch ingestions. 26 | * 27 | * @author srikalyan 28 | */ 29 | public class BatchInsertMeta extends InsertMeta { 30 | 31 | public InputSpec inputSpec = new InputSpec(); 32 | public PartitionsSpec partitionsSpec = new PartitionsSpec(); 33 | public RollupSpec rollupSpec = new RollupSpec();//TODO: Figure out its validity and where to put this ? 34 | public List dimensions = new ArrayList<>(); 35 | public List columns = new ArrayList<>(); 36 | 37 | public BatchInsertMeta() { 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return getJson().toString(2); 43 | } 44 | 45 | @Override 46 | public JSONObject getJson() { 47 | return new JSONObject(getDataMap()); 48 | } 49 | 50 | 51 | @Override 52 | public Map getDataMap() { 53 | return ImmutableMap.of( 54 | "id", format("index_hadoop_%s_%d_%d", dataSource, System.currentTimeMillis(), System.nanoTime()), 55 | "type", "index_hadoop", 56 | "spec", getSpec()); 57 | } 58 | 59 | public void inferFormat(String fileExtension) { 60 | if (fileExtension.endsWith("csv")) { 61 | dataFormat = "csv"; 62 | } else if (fileExtension.endsWith("json")) { 63 | dataFormat = "json"; 64 | } else { 65 | dataFormat = "tsv"; 66 | } 67 | } 68 | 69 | @Override 70 | public Map getTimestampSpec() { 71 | return ImmutableMap.of( 72 | "column", "timestamp", 73 | "format", "iso");//TODO: Check if auto works. We are setting to default iso. 74 | } 75 | 76 | @Override 77 | public Map getIoConfig() { 78 | return ImmutableMap.of( 79 | "type", "hadoop", 80 | "inputSpec", inputSpec.getDataMap()); 81 | } 82 | 83 | /** 84 | * Optional 85 | * @return 86 | */ 87 | @Override 88 | public Map getTuningConfig() { 89 | return ImmutableMap.of( 90 | "type", "hadoop", 91 | "partitionsSpec", partitionsSpec.getDataMap()); 92 | } 93 | 94 | 95 | public void postProcess(T anyContext) { 96 | 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/insert/RTInsertMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.insert; 13 | 14 | import com.yahoo.sql4d.insert.nodes.GranularitySpec; 15 | import com.yahoo.sql4d.query.nodes.AggItem; 16 | import java.util.ArrayList; 17 | import java.util.HashMap; 18 | import java.util.List; 19 | import java.util.Map; 20 | import org.json.JSONArray; 21 | import org.json.JSONObject; 22 | 23 | /** 24 | * Insert Meta for realtime ingestions. 25 | * TODO: Complete this implementation. 26 | * @author srikalyan 27 | */ 28 | public class RTInsertMeta extends InsertMeta { 29 | 30 | public List values = new ArrayList<>(); 31 | 32 | public RTInsertMeta() { 33 | } 34 | 35 | public RTInsertMeta(String dataSource, GranularitySpec granularity) { 36 | super(dataSource); 37 | this.granularitySpec = granularity; 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return getJson().toString(2); 43 | } 44 | 45 | @Override 46 | public JSONObject getJson() { 47 | return new JSONObject(getDataMap()); 48 | } 49 | 50 | @Override 51 | public Map getDataMap() { 52 | Map map = super.getDataMap(); 53 | map.put("type", "index_hadoop"); 54 | if (granularitySpec != null) { 55 | map.put("granularitySpec", granularitySpec.getJson()); 56 | } 57 | JSONArray aggregationsArray = new JSONArray(); 58 | for (AggItem item : aggregations) { 59 | aggregationsArray.put(item.getJson()); 60 | } 61 | map.put("aggregators", aggregationsArray); 62 | map.put("firehose", getFireHose()); 63 | return map; 64 | } 65 | 66 | private JSONObject getFireHose() { 67 | return new JSONObject(new HashMap()); 68 | } 69 | 70 | public void postProcess(T anyContext) { 71 | 72 | } 73 | 74 | @Override 75 | public Map getTimestampSpec() { 76 | throw new UnsupportedOperationException("Not supported yet."); 77 | } 78 | 79 | @Override 80 | public Map getIoConfig() { 81 | throw new UnsupportedOperationException("Not supported yet."); 82 | } 83 | 84 | @Override 85 | public Map getTuningConfig() { 86 | throw new UnsupportedOperationException("Not supported yet."); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/insert/nodes/GranularitySpec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.insert.nodes; 12 | 13 | import com.yahoo.sql4d.query.nodes.*; 14 | import static com.yahoo.sql4d.utils.Utils.*; 15 | import java.util.Arrays; 16 | import java.util.LinkedHashMap; 17 | import java.util.List; 18 | import java.util.Map; 19 | import org.json.JSONArray; 20 | import org.json.JSONObject; 21 | 22 | /** 23 | * Ex: Used in data ingestion. 24 | *"granularitySpec" : { 25 | "type" : "uniform", 26 | "gran" : "DAY", 27 | "interval" : [ "2010/2020" ] 28 | } 29 | * 30 | * @author srikalyan 31 | */ 32 | public class GranularitySpec { 33 | private static final List validGSimpleStrings = Arrays.asList("none", "minute", "fifteen_minute", "thirty_minute", "hour" , "day"); 34 | public String gran = "day"; 35 | public String type = "uniform";//TODO: Check out what are other types. 36 | public Interval interval = new Interval("2014-01-01", "2015-12-31"); 37 | 38 | public GranularitySpec(String gString) { 39 | this.gran = unquote(gString); 40 | isValid(this.gran); 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return String.format(getJson().toString(2)); 46 | } 47 | 48 | public JSONObject getJson() { 49 | return new JSONObject(getDataMap()); 50 | } 51 | 52 | private void isValid(String gString) { 53 | if (!validGSimpleStrings.contains(gString)) { 54 | throw new IllegalArgumentException("Ivalid granularity " + gString); 55 | } 56 | this.gran = gString; 57 | } 58 | 59 | 60 | public Map getDataMap() { 61 | Map map = new LinkedHashMap<>(); 62 | JSONArray intervalArr = new JSONArray(); 63 | intervalArr.put(interval.toString()); 64 | map.put("gran", gran); 65 | map.put("type", type); 66 | map.put("intervals", intervalArr); 67 | return map; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/insert/nodes/InputSpec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.insert.nodes; 12 | 13 | import java.util.LinkedHashMap; 14 | import java.util.Map; 15 | import org.json.JSONObject; 16 | 17 | /** 18 | * Ex: Used in hadoop data ingestion. 19 | *"inputSpec" : { 20 | "type" : "static", 21 | "paths" : "data.json",// This can be just a folder(comma separated list of paths) 22 | "filePattern": "*.gz"// This is optional if the paths has full path of file 23 | }, 24 | * 25 | * @author srikalyan 26 | */ 27 | public class InputSpec { 28 | private String rawPath = null; 29 | private String path = null;//TODO: This should be a list(right now we support only one path) 30 | public String type = "static";//TODO: Check out what are other types. 31 | private String filePattern = null; 32 | 33 | public void setPath(String p) { 34 | this.rawPath = p; 35 | path = p; 36 | int starIndex = p.indexOf("*"); 37 | if (starIndex >= 0) { 38 | filePattern = p.substring(starIndex); 39 | path = p.substring(0, starIndex); 40 | } 41 | } 42 | 43 | public String getPath() { 44 | return path; 45 | } 46 | 47 | public String getRawPath() { 48 | return rawPath; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return String.format(getJson().toString(2)); 54 | } 55 | 56 | public JSONObject getJson() { 57 | return new JSONObject(getDataMap()); 58 | } 59 | 60 | public Map getDataMap() { 61 | Map map = new LinkedHashMap<>(); 62 | map.put("paths", path); 63 | map.put("type", type); 64 | if (filePattern != null) { 65 | map.put("filePattern", filePattern); 66 | } 67 | return map; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/insert/nodes/PartitionsSpec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.insert.nodes; 12 | 13 | import java.util.LinkedHashMap; 14 | import java.util.Map; 15 | import org.json.JSONObject; 16 | 17 | /** 18 | * Ex: Used in batch data ingestion. 19 | * "partitionsSpec" : { 20 | "type" : "dimension", 21 | "partitionDimension" : null, 22 | "targetPartitionSize" : 5000000, 23 | "maxPartitionSize" : 7500000, 24 | "assumeGrouped" : false, 25 | "numShards" : -1 26 | } 27 | * 28 | * @author srikalyan 29 | */ 30 | public class PartitionsSpec { 31 | public long targetPartitionSize = 5000000; 32 | public String type = "hashed";// dimension/hashed 33 | 34 | @Override 35 | public String toString() { 36 | return String.format(getJson().toString(2)); 37 | } 38 | 39 | public JSONObject getJson() { 40 | return new JSONObject(getDataMap()); 41 | } 42 | 43 | public Map getDataMap() { 44 | Map map = new LinkedHashMap<>(); 45 | map.put("type", type); 46 | map.put("targetPartitionSize", targetPartitionSize); 47 | return map; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/insert/nodes/RollupSpec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.insert.nodes; 12 | 13 | import com.yahoo.sql4d.query.nodes.AggItem; 14 | import java.util.ArrayList; 15 | import java.util.LinkedHashMap; 16 | import java.util.List; 17 | import java.util.Map; 18 | import org.json.JSONArray; 19 | import org.json.JSONObject; 20 | 21 | /** 22 | * Ex: Used in data ingestion. 23 | *"rollupSpec" : { 24 | "aggs": [{ 25 | "type" : "count", 26 | "name" : "count" 27 | }, { 28 | "type" : "doubleSum", 29 | "name" : "added", 30 | "fieldName" : "added" 31 | }], 32 | "rowFlushBoundary": 100000,// THIS IS OPTIONAL 33 | "rollupGranularity" : "none"// can also be hour etc. NOTE this is different from segment granularity. 34 | } 35 | * 36 | * @author srikalyan 37 | */ 38 | public class RollupSpec { 39 | public String rollupGranularity = "none";//TODO: Can be hour etc. Validate for other types. 40 | public long rowFlushBoundary = 100000;//TODO: 41 | public List aggs = new ArrayList<>(); 42 | 43 | @Override 44 | public String toString() { 45 | return String.format(getJson().toString(2)); 46 | } 47 | 48 | public JSONObject getJson() { 49 | return new JSONObject(getDataMap()); 50 | } 51 | 52 | public Map getDataMap() { 53 | Map map = new LinkedHashMap<>(); 54 | map.put("rowFlushBoundary", rowFlushBoundary); 55 | map.put("rollupGranularity", rollupGranularity); 56 | JSONArray aggsArray = new JSONArray(); 57 | for (AggItem aggItem:aggs) { 58 | aggsArray.put(aggItem.getJson()); 59 | } 60 | map.put("aggs", aggsArray); 61 | return map; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/BaseAggQueryMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query; 12 | 13 | import com.yahoo.sql4d.query.nodes.AggItem; 14 | import com.yahoo.sql4d.query.nodes.PostAggItem; 15 | import java.util.List; 16 | import java.util.Map; 17 | import org.json.JSONArray; 18 | import org.json.JSONObject; 19 | 20 | /** 21 | * GroupBy, TimeSeries etc come under this category. 22 | * @author srikalyan 23 | */ 24 | public class BaseAggQueryMeta extends PlainDimQueryMeta { 25 | public List aggregations; 26 | public PostAggItem postAggregations; 27 | 28 | public BaseAggQueryMeta() { 29 | } 30 | 31 | 32 | public BaseAggQueryMeta(QueryMeta q) { 33 | super(q); 34 | if (q instanceof BaseAggQueryMeta) { 35 | aggregations = ((BaseAggQueryMeta)q).aggregations; 36 | postAggregations = ((BaseAggQueryMeta)q).postAggregations; 37 | } 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return getJson().toString(2); 43 | } 44 | 45 | @Override 46 | public JSONObject getJson() { 47 | return new JSONObject(getDataMap()); 48 | } 49 | 50 | @Override 51 | public Map getDataMap() { 52 | Map map = super.getDataMap(); 53 | 54 | JSONArray aggregationsArray = new JSONArray(); 55 | for (AggItem item:aggregations) { 56 | aggregationsArray.put(item.getJson()); 57 | } 58 | map.put("aggregations", aggregationsArray); 59 | 60 | if (postAggregations != null) { 61 | JSONArray jsonArr = new JSONArray(); 62 | jsonArr.put(postAggregations.getJson()); 63 | map.put("postAggregations", jsonArr); 64 | } else { 65 | map.put("postAggregations", new JSONArray()); 66 | } 67 | return map; 68 | } 69 | 70 | public static BaseAggQueryMeta promote(QueryMeta qMeta) { 71 | return new BaseAggQueryMeta(qMeta); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/HintProcessor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query; 12 | 13 | import com.yahoo.sql4d.query.timeseries.TimeSeriesQueryMeta; 14 | 15 | /** 16 | * Group by and timeseries are same(when grouping by timestamp we can hint as hint('timeseries') 17 | * so that the query is interpreted as timeseries by the compiler as well as druid 18 | * however technically this is not needed as druid still returns data considering 19 | * the query as groupby which may be inefficient. 20 | * @author srikalyan 21 | */ 22 | public class HintProcessor { 23 | 24 | private static final String TS = "timeseries"; 25 | 26 | public static QueryMeta process(QueryMeta qMeta, String hint) { 27 | hint = hint.replaceAll("'", ""); 28 | switch (hint.toLowerCase()) { 29 | case TS: 30 | return TimeSeriesQueryMeta.promote(qMeta); 31 | default: 32 | break; 33 | 34 | } 35 | return qMeta; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/PlainDimQueryMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query; 12 | 13 | import java.util.LinkedHashMap; 14 | import java.util.Map; 15 | import org.json.JSONArray; 16 | import org.json.JSONObject; 17 | 18 | /** 19 | * GroupBy, TimeSeries, Search, Select etc come under this category. 20 | * @author srikalyan 21 | */ 22 | public class PlainDimQueryMeta extends QueryMeta { 23 | public Map fetchDimensions = new LinkedHashMap<>();// Normal alias go into this (aggregation based alias go into AggItem) 24 | 25 | public PlainDimQueryMeta() { 26 | } 27 | 28 | 29 | public PlainDimQueryMeta(QueryMeta q) { 30 | super(q); 31 | if (q instanceof PlainDimQueryMeta) { 32 | fetchDimensions = ((PlainDimQueryMeta)q).fetchDimensions; 33 | } 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return getJson().toString(2); 39 | } 40 | 41 | @Override 42 | public JSONObject getJson() { 43 | return new JSONObject(getDataMap()); 44 | } 45 | 46 | @Override 47 | public Map getDataMap() { 48 | Map map = super.getDataMap(); 49 | if (fetchDimensions != null) { 50 | JSONArray dimensionsArray = new JSONArray(); 51 | for (String dim:fetchDimensions.keySet()) { 52 | String dimaAlias = fetchDimensions.get(dim); 53 | if (dimaAlias == null) { 54 | dimensionsArray.put(dim); 55 | } else { 56 | JSONObject json = new JSONObject(); 57 | json.put("type", "default"); 58 | json.put("dimension", dim); 59 | json.put("outputName", dimaAlias); 60 | dimensionsArray.put(json); 61 | } 62 | } 63 | map.put("dimensions", dimensionsArray); 64 | } 65 | return map; 66 | } 67 | 68 | public static PlainDimQueryMeta promote(QueryMeta qMeta) { 69 | return new PlainDimQueryMeta(qMeta); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/QueryMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query; 12 | 13 | import com.yahoo.sql4d.Pair; 14 | import com.yahoo.sql4d.BaseStatementMeta; 15 | import com.yahoo.sql4d.query.nodes.Interval; 16 | import com.yahoo.sql4d.query.nodes.Filter; 17 | import com.yahoo.sql4d.query.nodes.Granularity; 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | import java.util.Map; 21 | import org.json.JSONArray; 22 | import org.json.JSONObject; 23 | 24 | /** 25 | * Base class for all query types. 26 | * @author srikalyan 27 | */ 28 | public class QueryMeta extends BaseStatementMeta { 29 | public Granularity granularity = new Granularity("all"); 30 | public Filter filter; 31 | public List intervals = new ArrayList<>(); 32 | public List> microIntervals = new ArrayList<>(); // If empty or if intervals list size > 1 then this is not used. 33 | public RequestType queryType; 34 | 35 | public QueryMeta() { 36 | } 37 | 38 | public QueryMeta(QueryMeta q) { 39 | super(q); 40 | this.granularity = q.granularity; 41 | this.filter = q.filter; 42 | this.intervals = q.intervals; 43 | this.microIntervals = q.microIntervals; 44 | } 45 | 46 | public QueryMeta(String dataSource, Granularity granularity, Filter filter, List intervals, List> microIntervals) { 47 | super(dataSource); 48 | this.granularity = granularity; 49 | this.filter = filter; 50 | this.intervals = intervals; 51 | this.microIntervals = microIntervals; 52 | } 53 | 54 | 55 | @Override 56 | public String toString() { 57 | return getJson().toString(2); 58 | } 59 | 60 | @Override 61 | public JSONObject getJson() { 62 | return new JSONObject(getDataMap()); 63 | } 64 | @Override 65 | public Map getDataMap() { 66 | Map map = super.getDataMap(); 67 | map.put("queryType", queryType.getName()); 68 | if (queryType == RequestType.TIMEBOUNDARY) { 69 | return map; 70 | } 71 | if (granularity != null) { 72 | if (granularity.gComplex != null) { 73 | map.put("granularity", granularity.getJson()); 74 | } else { 75 | map.put("granularity", granularity.gSimple); 76 | } 77 | } 78 | if (filter != null) { 79 | map.put("filter", filter.getJson()); 80 | } 81 | 82 | JSONArray intervalsArray = new JSONArray(); 83 | if (!intervals.isEmpty()) { 84 | if (intervals.size() == 1 && !microIntervals.isEmpty()) { 85 | Interval grandInterval = intervals.get(0); 86 | // Break the single interval into micro intervals. 87 | for (int i = 0; i < grandInterval.getDays();i++) { 88 | for (Pair pair:microIntervals) { 89 | Interval microInterval = grandInterval.getInterval(i, pair.a, pair.a + pair.b);// a = start hour of day, b = number of hours from there of. 90 | intervalsArray.put(microInterval.toString()); 91 | } 92 | } 93 | } else { 94 | for (Interval interval:intervals) { 95 | intervalsArray.put(interval.toString()); 96 | } 97 | } 98 | JSONObject finalIntervalJson = new JSONObject(); 99 | finalIntervalJson.put("type", "intervals"); 100 | finalIntervalJson.put("intervals", intervalsArray); 101 | map.put("intervals", finalIntervalJson); 102 | } 103 | return map; 104 | } 105 | 106 | } 107 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/QueryUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | 12 | package com.yahoo.sql4d.query; 13 | 14 | import com.yahoo.sql4d.query.groupby.GroupByQueryMeta; 15 | import com.yahoo.sql4d.query.nodes.PostAggItem; 16 | import com.yahoo.sql4d.query.timeseries.TimeSeriesQueryMeta; 17 | 18 | /** 19 | * 20 | * @author srikalyan 21 | */ 22 | public class QueryUtils { 23 | public static void setPostAggregation(QueryMeta qMeta, PostAggItem pItem) { 24 | if (qMeta instanceof BaseAggQueryMeta) { 25 | ((BaseAggQueryMeta)qMeta).postAggregations = pItem; 26 | } 27 | } 28 | /** 29 | * Every query starts with GroupBy. Call this method after GROUP BY clause. 30 | * @param qMeta 31 | * @return 32 | */ 33 | public static QueryMeta checkAndPromoteToTimeSeries(QueryMeta qMeta) { 34 | if (qMeta instanceof GroupByQueryMeta) { 35 | if (((GroupByQueryMeta)qMeta).fetchDimensions == null) { 36 | return TimeSeriesQueryMeta.promote(qMeta); 37 | } 38 | } 39 | return qMeta; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/RequestType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query; 12 | 13 | /** 14 | * All possible Druid query types. 15 | * @author srikalyan 16 | */ 17 | public enum RequestType { 18 | GROUPBY, TIMESERIES, TOPN, SEARCH, SELECT, TIMEBOUNDARY, UNKNOWN; 19 | 20 | public String getName() { 21 | if (this == TIMEBOUNDARY) { 22 | return "timeBoundary"; 23 | } 24 | return this.name().toLowerCase(); 25 | } 26 | } 27 | 28 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/groupby/GroupByQueryMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.groupby; 12 | 13 | import com.yahoo.sql4d.query.BaseAggQueryMeta; 14 | import com.yahoo.sql4d.query.QueryMeta; 15 | import com.yahoo.sql4d.query.RequestType; 16 | import java.util.Map; 17 | import org.json.JSONObject; 18 | 19 | /** 20 | * { 21 | "queryType": "groupBy", 22 | "dataSource": "sample_datasource", 23 | "granularity": "day", 24 | "dimensions": ["dim1", "dim2", {"queryType":"default","dimension":"_content_thumbnail_url","outputName":"thumb"}], 25 | "limitSpec": { "queryType": "default", "limit": 5000, "columns": ["dim1", "metric1"] }, 26 | "filter": { 27 | "queryType": "and", 28 | "fields": [ 29 | { "queryType": "selector", "dimension": "sample_dimension1", "value": "sample_value1" }, 30 | { "queryType": "or", 31 | "fields": [ 32 | { "queryType": "selector", "dimension": "sample_dimension2", "value": "sample_value2" }, 33 | { "queryType": "selector", "dimension": "sample_dimension3", "value": "sample_value3" } 34 | ] 35 | } 36 | ] 37 | }, 38 | "aggregations": [ 39 | { "queryType": "longSum", "name": "sample_name1", "fieldName": "sample_fieldName1" }, 40 | { "queryType": "doubleSum", "name": "sample_name2", "fieldName": "sample_fieldName2" } 41 | ], 42 | "postAggregations": [ 43 | { "queryType": "arithmetic", 44 | "name": "sample_divide", 45 | "fn": "/", 46 | "fields": [ 47 | { "queryType": "fieldAccess", "name": "sample_name1", "fieldName": "sample_fieldName1" }, 48 | { "queryType": "fieldAccess", "name": "sample_name2", "fieldName": "sample_fieldName2" } 49 | ] 50 | } 51 | ], 52 | "intervals": [ "2012-01-01T00:00:00.000/2012-01-03T00:00:00.000" ], 53 | "having": { "queryType": "greaterThan", "aggregation": "sample_name1", "value": 0 } 54 | } 55 | 56 | * @author srikalyan 57 | */ 58 | public class GroupByQueryMeta extends BaseAggQueryMeta { 59 | public Having having; 60 | public LimitSpec limitSpec; 61 | 62 | public GroupByQueryMeta() { 63 | } 64 | 65 | public GroupByQueryMeta(QueryMeta q) { 66 | super(q); 67 | queryType = RequestType.GROUPBY; 68 | } 69 | 70 | @Override 71 | public String toString() { 72 | return getJson().toString(2); 73 | } 74 | 75 | @Override 76 | public JSONObject getJson() { 77 | return new JSONObject(getDataMap()); 78 | } 79 | 80 | @Override 81 | public Map getDataMap() { 82 | Map map = super.getDataMap(); 83 | 84 | map.put("queryType", "groupBy"); 85 | if (having != null) { 86 | map.put("having", having.getJson()); 87 | } 88 | if (limitSpec != null) { 89 | map.put("limitSpec", limitSpec.getJson()); 90 | } 91 | return map; 92 | } 93 | 94 | public boolean checkDimOrAlias(String dimOrAlias) { 95 | //TODO: Not sure if the alias for metric aggregate operations can be considered a dimension or not. Revisit later. 96 | // // First check if the dim(or) alias is in Aggregations . 97 | // for (AggItem item:aggregations) { 98 | // if (dimOrAlias.equals(item.fieldName) || dimOrAlias.equals(item.asName)) { 99 | // return true; 100 | // } 101 | // } 102 | // If not then check the fetchDimension which are just accessors(no Agg function) 103 | return fetchDimensions.containsKey(dimOrAlias) || fetchDimensions.containsValue(dimOrAlias); 104 | } 105 | 106 | public static GroupByQueryMeta promote(QueryMeta qMeta) { 107 | if (qMeta instanceof GroupByQueryMeta) 108 | return (GroupByQueryMeta)qMeta; 109 | return new GroupByQueryMeta(qMeta); 110 | } 111 | 112 | } 113 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/groupby/Having.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.groupby; 12 | 13 | import java.util.LinkedHashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | import org.json.JSONArray; 17 | import org.json.JSONObject; 18 | 19 | /** 20 | * HAVING clause for groupBy queries. 21 | * @author srikalyan 22 | */ 23 | public class Having { 24 | public String type; 25 | public String aggregation; 26 | public String value; 27 | public List havingSpecs; 28 | 29 | 30 | public Having() { 31 | } 32 | 33 | public Having(String type) { 34 | this.type = type; 35 | } 36 | 37 | public Having(String type, String aggregation, String value) { 38 | this.type = type; 39 | this.aggregation = aggregation; 40 | this.value = value; 41 | } 42 | @Override 43 | public String toString() { 44 | return getJson().toString(2); 45 | } 46 | 47 | public JSONObject getJson() { 48 | return new JSONObject(getJsonMap()); 49 | } 50 | 51 | public Map getJsonMap() { 52 | Map map = new LinkedHashMap<>(); 53 | map.put("type", getTypeForSymbol(type)); 54 | map.put("aggregation", aggregation); 55 | map.put("value", value); 56 | if (havingSpecs != null) { 57 | if (type.equals("not")) { 58 | map.put("havingSpecs", havingSpecs.get(0).getJson()); 59 | } else { 60 | JSONArray havingSpecsArray = new JSONArray(); 61 | for (Having item:havingSpecs) { 62 | havingSpecsArray.put(item.getJson()); 63 | } 64 | map.put("havingSpecs", havingSpecsArray); 65 | } 66 | } 67 | return map; 68 | } 69 | 70 | public static String getTypeForSymbol(String sym) { 71 | switch(sym) { 72 | case "=": 73 | return "equalTo"; 74 | case "<": 75 | return "lessThan"; 76 | case ">": 77 | return "greaterThan"; 78 | case "and": 79 | return "and"; 80 | case "or": 81 | return "or"; 82 | case "not": 83 | return "not"; 84 | } 85 | return null; 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/groupby/LimitSpec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.query.groupby; 13 | 14 | import java.util.HashMap; 15 | import java.util.LinkedHashMap; 16 | import java.util.Map; 17 | import org.json.JSONArray; 18 | import org.json.JSONObject; 19 | 20 | /** 21 | * Row limit spec. 22 | * 23 | * @author srikalyan 24 | */ 25 | public class LimitSpec { 26 | 27 | public String type = "default";//TODO: add support for more types(if there are any later) 28 | public long limit; 29 | public Map columns = new HashMap<>(); 30 | 31 | enum Direction { 32 | 33 | ASCENDING, 34 | DESCENDING 35 | } 36 | 37 | public LimitSpec() { 38 | } 39 | 40 | public LimitSpec(String type, long limit) { 41 | this.type = type; 42 | this.limit = limit; 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return String.format(getJson().toString(2)); 48 | } 49 | 50 | public void addColumn(String col, String asc) { 51 | if (asc.equalsIgnoreCase("DESC")) { 52 | columns.put(col, Direction.DESCENDING); 53 | } else { 54 | columns.put(col, Direction.ASCENDING); 55 | } 56 | } 57 | 58 | public JSONObject getJson() { 59 | return new JSONObject(getJsonMap()); 60 | } 61 | 62 | public Map getJsonMap() { 63 | Map map = new LinkedHashMap<>(); 64 | map.put("type", type); 65 | map.put("limit", limit); 66 | if (columns != null) { 67 | JSONArray columnsArray = new JSONArray(); 68 | for (Map.Entry item : columns.entrySet()) { 69 | JSONObject itemJson = new JSONObject(); 70 | itemJson.put("dimension", item.getKey()); 71 | itemJson.put("direction", item.getValue().name()); 72 | columnsArray.put(itemJson); 73 | } 74 | map.put("columns", columnsArray); 75 | } 76 | return map; 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/nodes/AggItem.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.nodes; 12 | 13 | import java.util.ArrayList; 14 | import java.util.LinkedHashMap; 15 | import java.util.List; 16 | import java.util.Map; 17 | import org.json.JSONArray; 18 | import org.json.JSONObject; 19 | 20 | /** 21 | * Ex: 22 | * "aggregations": [ 23 | { "type": "longSum", "name": "sample_name1", "fieldName": "sample_fieldName1" }, 24 | { "type": "doubleSum", "name": "sample_name2", "fieldName": "sample_fieldName2" }, 25 | {"type" : "count", "name" : "rows"}, 26 | {"type" : "hyperUnique", "name" : "unique_users", "fieldName" : "uniques"} 27 | {"type" : "cardinality", "name" : "unique_active_publishers", "fieldNames": ["active_publishers"]} 28 | { "type" : "min", "name" : "minValue", "fieldName" : "minV"} 29 | { "type" : "max", "name" : "maxValue", "fieldName" : "maxV"} 30 | { "type": "javascript", "name": "sum(log(x)/y) + 10", 31 | "fieldNames": ["x", "y"], 32 | "fnAggregate" : "function(current, a, b) { return current + (Math.log(a) * b); }", 33 | "fnCombine" : "function(partialA, partialB) { return partialA + partialB; }", 34 | "fnReset" : "function() { return 10; }" 35 | } 36 | 37 | 38 | .... 39 | ... 40 | ] 41 | * @author srikalyan 42 | */ 43 | public class AggItem { 44 | public String type;// longSum, doubleSum, hyperUnique(if pre computed), unique(cardinality), min, max, javascript 45 | public String fieldName; 46 | public String asName; 47 | 48 | public List fieldNames; 49 | public String fnAggregate; 50 | public String fnCombine; 51 | public String fnReset; 52 | 53 | public AggItem() { 54 | } 55 | 56 | 57 | public AggItem(String aggType, String fieldName, String asName) { 58 | this.type = aggType; 59 | this.fieldName = fieldName; 60 | this.asName = asName; 61 | } 62 | //Count, hyper unique etc are not direct metrics. 63 | public boolean isDirectMetric() { 64 | return !("hyperUnique".equals(type) || "unique".equals(type) || "count".equals(type) || "javascript".equals(type)); 65 | } 66 | public void setAggType(String aggType) { 67 | this.type = aggType; 68 | } 69 | 70 | public void setFieldName(String fieldName) { 71 | this.fieldName = fieldName; 72 | } 73 | 74 | public void setAsName(String asName) { 75 | this.asName = asName; 76 | } 77 | 78 | /** 79 | * 80 | * @return AS name (alias) if not null (else the fieldName) 81 | */ 82 | public String getCanonicalName() { 83 | return (asName != null)?asName:fieldName; 84 | } 85 | 86 | @Override 87 | public String toString() { 88 | return String.format(getJson().toString(2)); 89 | } 90 | 91 | public JSONObject getJson() { 92 | return new JSONObject(getJsonMap()); 93 | } 94 | 95 | public Map getJsonMap() { 96 | Map map = new LinkedHashMap<>(); 97 | if ("unique".equals(type)) { 98 | type = "cardinality";// Cardinality requires fieldNames instead of fieldName 99 | if (fieldName != null) { 100 | fieldNames = new ArrayList<>(); 101 | fieldNames.add(fieldName); 102 | fieldName = null; 103 | } 104 | } else if ("hyperUnique".equals(type)) { 105 | type = "hyperUnique"; 106 | } 107 | 108 | map.put("type", type); 109 | map.put("name", asName); 110 | if (fieldName != null) { 111 | map.put("fieldName", fieldName); 112 | } 113 | 114 | if (fieldNames != null) { 115 | JSONArray fieldNamesArray = new JSONArray(); 116 | for (String item:fieldNames) { 117 | fieldNamesArray.put(item); 118 | } 119 | map.put("fieldNames", fieldNamesArray); 120 | } 121 | if (fnAggregate != null) { 122 | map.put("fnAggregate", fnAggregate); 123 | } 124 | if (fnCombine != null) { 125 | map.put("fnCombine", fnCombine); 126 | } 127 | if (fnReset != null) { 128 | map.put("fnReset", fnReset); 129 | } 130 | return map; 131 | } 132 | 133 | } 134 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/nodes/Filter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.nodes; 12 | 13 | import java.util.LinkedHashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | import org.json.JSONArray; 17 | import org.json.JSONObject; 18 | 19 | /** 20 | * Ex: 21 | * "filter": { "type": "selector", "dimension": , "value": } 22 | "filter": { "type": "regex", "dimension": , "pattern": } 23 | "filter": { "type": "and", "fields": [, , ...] } 24 | "filter": { "type": "or", "fields": [, , ...] } 25 | "filter": { "type": "not", "field": } 26 | "filter": { "type" : "javascript", "dimension" : "name", 27 | "function" : "function(x) { return(x >= 'bar' && x <= 'foo') }" 28 | } 29 | 30 | * @author srikalyan 31 | */ 32 | public class Filter { 33 | public String type; 34 | public String dimension; 35 | public String value; 36 | public String pattern; 37 | 38 | public List fields; 39 | public String function; 40 | 41 | public Filter(String type) { 42 | this.type = type; 43 | } 44 | 45 | 46 | @Override 47 | public String toString() { 48 | return getJson().toString(2); 49 | } 50 | 51 | public JSONObject getJson() { 52 | return new JSONObject(getJsonMap()); 53 | } 54 | 55 | public Map getJsonMap() { 56 | Map map = new LinkedHashMap<>(); 57 | map.put("type", type); 58 | if (dimension != null) { 59 | map.put("dimension", dimension); 60 | } 61 | if (value != null) { 62 | map.put("value", value); 63 | } 64 | if (pattern != null) { 65 | map.put("pattern", pattern); 66 | } 67 | if (fields != null) { 68 | if (type.equals("not")) {// For not we call it field (not fields because we have only one filter) 69 | map.put("field", fields.get(0).getJson()); 70 | } else { 71 | JSONArray fieldArray = new JSONArray(); 72 | for (Filter item:fields) { 73 | fieldArray.put(item.getJson()); 74 | } 75 | map.put("fields", fieldArray); 76 | } 77 | } 78 | if (function != null) { 79 | map.put("fnAggregate", function); 80 | } 81 | return map; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/nodes/Granularity.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.nodes; 12 | 13 | import static com.yahoo.sql4d.utils.Utils.*; 14 | import java.util.Arrays; 15 | import java.util.LinkedHashMap; 16 | import java.util.List; 17 | import java.util.Map; 18 | import org.json.JSONObject; 19 | import scala.util.Either; 20 | import scala.util.Left; 21 | import scala.util.Right; 22 | 23 | /** 24 | * Ex: 25 | * "granularity": "all" (OR) "none" (OR) "minute" (OR) "fifteen_minute" (OR) 26 | * "thirty_minute" (OR) "hour" (OR) "day" 27 | * "granularity": {"type": "duration", "duration": "7200000"} 28 | * "granularity": {"type": "duration", "duration": "3600000", "origin": "2012-01-01T00:30:00Z"} 29 | * "granularity": {"type": "period", "period": "P2D", "timeZone": "America/Los_Angeles"} 30 | * "granularity": {"type": "period", "period": "P3M", "timeZone": "America/Los_Angeles", "origin": "2012-02-01T00:00:00-08:00"} 31 | * @author srikalyan 32 | */ 33 | public class Granularity { 34 | private static final List validGSimpleStrings = Arrays.asList("all", "none", "minute", "fifteen_minute", "thirty_minute", "hour" , "day"); 35 | public String gSimple = "all"; 36 | 37 | public Either gComplex;// Duration/Period types. 38 | 39 | public String origin; 40 | public String timeZone; 41 | 42 | public Granularity() { 43 | gSimple = null;// Because this constructor is intented to be used for typed granularity(duration/period) 44 | } 45 | 46 | 47 | public Granularity(String gString) { 48 | this.gSimple = unquote(gString); 49 | } 50 | 51 | public void setDuration(String duration) { 52 | this.gComplex = new Left<>(unquote(duration)); 53 | } 54 | 55 | public void setOrigin(String origin) { 56 | this.origin = unquote(origin); 57 | } 58 | 59 | public void setPeriod(String period) { 60 | this.gComplex = new Right<>(unquote(period)); 61 | } 62 | 63 | public void setTimeZone(String timeZone) { 64 | this.timeZone = unquote(timeZone); 65 | } 66 | 67 | 68 | @Override 69 | public String toString() { 70 | return String.format(getJson().toString(2)); 71 | } 72 | 73 | public JSONObject getJson() { 74 | return new JSONObject(getJsonMap()); 75 | } 76 | 77 | public void setgString(String gString) { 78 | if (!validGSimpleStrings.contains(gString)) { 79 | System.err.println("Ivalid granularity " + gString); 80 | } 81 | this.gSimple = gString; 82 | } 83 | 84 | 85 | /** 86 | * Basic type when granularity is just string is taken care in QueryMeta class. 87 | * For duration and period types we get Json from here. 88 | * @return 89 | */ 90 | public Map getJsonMap() { 91 | Map map = new LinkedHashMap<>(); 92 | 93 | map.put("type", gComplex!=null && gComplex.isLeft()? "duration":"period"); 94 | if (gComplex != null) { 95 | if (gComplex.isLeft()) { 96 | map.put("duration", gComplex.left().get()); 97 | } else { 98 | map.put("period", gComplex.right().get()); 99 | } 100 | if (origin != null) { 101 | map.put("origin", origin); 102 | } 103 | if (timeZone != null) { 104 | map.put("timeZone", timeZone); 105 | } 106 | } 107 | return map; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/nodes/Interval.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.nodes; 12 | 13 | import com.google.common.base.Preconditions; 14 | import com.yahoo.sql4d.utils.TimeUtils; 15 | import org.joda.time.DateTime; 16 | import org.joda.time.Days; 17 | 18 | /** 19 | * Interval (start,end) times as text. Can construct new intervals out of existing. 20 | * Supports parsing time text in most of ISO formats. 21 | * @author srikalyan 22 | */ 23 | public class Interval { 24 | public String startTime; 25 | public String endTime; 26 | public int days;// Includes partial days as well. 27 | 28 | public Interval(String startTime, String endTime) { 29 | this.startTime = startTime.replaceAll("'", ""); 30 | this.endTime = endTime.replaceAll("'", ""); 31 | days = Days.daysBetween(getStartTime().withTimeAtStartOfDay() , getEndTime().withTimeAtStartOfDay() ).getDays() + 1; 32 | } 33 | 34 | /** 35 | * Interval should be of form 2014-10-31T00:00:00.000-07:00/2014-11-01T00:00:00.000-07:00 36 | * @param interval 37 | */ 38 | public Interval(String interval) { 39 | String[] dates = interval.split("/"); 40 | Preconditions.checkArgument(dates.length == 2); 41 | this.startTime = dates[0].replaceAll("'", ""); 42 | this.endTime = dates[1].replaceAll("'", ""); 43 | days = Days.daysBetween(getStartTime().withTimeAtStartOfDay() , getEndTime().withTimeAtStartOfDay() ).getDays() + 1; 44 | } 45 | 46 | public DateTime getStartTime() { 47 | return TimeUtils.getDateTime(startTime); 48 | } 49 | 50 | public DateTime getEndTime() { 51 | return TimeUtils.getDateTime(endTime); 52 | } 53 | 54 | public int getDays() { 55 | return days; 56 | } 57 | 58 | public Interval getInterval(int daysOffset, int startHourOffset, int endHourOffset) { 59 | DateTime baseDateTime = getStartTime().withTime(0, 0, 0, 0).plusDays(daysOffset); 60 | return new Interval(baseDateTime.plusHours(startHourOffset).toString(), 61 | baseDateTime.plusHours(endHourOffset).minusSeconds(1).toString()); 62 | } 63 | 64 | public Interval expandIntervalByDay(int days) { 65 | return new Interval(getStartTime().minusDays(days).toString(), 66 | getEndTime().plusDays(days).toString()); 67 | } 68 | 69 | public Interval expandEndTimeByDay(int days) { 70 | return new Interval(getStartTime().toString(), 71 | getEndTime().plusDays(days).toString()); 72 | } 73 | 74 | public Interval expandIntervalBySec(int secs) { 75 | return new Interval(getStartTime().minusSeconds(secs).toString(), 76 | getEndTime().plusSeconds(secs).toString()); 77 | } 78 | 79 | @Override 80 | public String toString() { 81 | return String.format("%s/%s", startTime, endTime); 82 | } 83 | 84 | public boolean fallsIn(Interval range) { 85 | return range.getStartTime().isBefore(getStartTime()) && range.getEndTime().isAfter(getEndTime()); 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/search/SearchQueryMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.search; 12 | 13 | import com.yahoo.sql4d.query.PlainDimQueryMeta; 14 | import com.yahoo.sql4d.query.QueryMeta; 15 | import com.yahoo.sql4d.query.RequestType; 16 | import java.util.ArrayList; 17 | import java.util.HashMap; 18 | import java.util.List; 19 | import java.util.Map; 20 | import org.json.JSONArray; 21 | import org.json.JSONObject; 22 | 23 | /** 24 | * { "queryType": "search", "dataSource": "sample_datasource", 25 | "granularity": "day", 26 | "searchDimensions": [ 27 | "dim1", 28 | "dim2" 29 | ], 30 | "query": { 31 | "type": "insensitive_contains", 32 | "value": "Ke" 33 | }, 34 | "sort" : { 35 | "type": "lexicographic" 36 | }, 37 | "intervals": [ 38 | "2013-01-01T00:00:00.000/2013-01-03T00:00:00.000" 39 | ] 40 | } 41 | 42 | * @author srikalyan 43 | */ 44 | public class SearchQueryMeta extends PlainDimQueryMeta { 45 | 46 | public String type;// insensitive_contains, fragment 47 | public List values; 48 | 49 | public String sort;// lexicographic, strlen 50 | 51 | public SearchQueryMeta() { 52 | queryType = RequestType.SEARCH; 53 | } 54 | 55 | public void addValue(String v) { 56 | if (values != null) { 57 | values.add(v.replace("'", "")); 58 | } 59 | } 60 | 61 | public void setSort(String sort) { 62 | sort = sort.replace("'", ""); 63 | if (sort.equals("lexicographic") || sort.equals("strlen")) { 64 | this.sort = sort; 65 | } else { 66 | System.out.println("Invalid sort type for the search query."); 67 | } 68 | } 69 | 70 | 71 | 72 | public SearchQueryMeta(QueryMeta q) { 73 | super(q); 74 | if (q instanceof SearchQueryMeta) { 75 | fetchDimensions = ((SearchQueryMeta)q).fetchDimensions; 76 | values = ((SearchQueryMeta)q).values; 77 | } else { 78 | fetchDimensions = new HashMap<>(); 79 | values = new ArrayList<>(); 80 | } 81 | } 82 | 83 | @Override 84 | public String toString() { 85 | return getJson().toString(2); 86 | } 87 | 88 | @Override 89 | public JSONObject getJson() { 90 | return new JSONObject(getDataMap()); 91 | } 92 | @Override 93 | public Map getDataMap() { 94 | Map map = super.getDataMap(); 95 | map.put("queryType", "search"); 96 | map.remove("dimensions");// Because it is called searchDimensions here. 97 | if (fetchDimensions != null) { 98 | JSONArray dimensionsArray = new JSONArray(); 99 | for (String dim:fetchDimensions.keySet()) { 100 | dimensionsArray.put(dim); 101 | } 102 | map.put("searchDimensions", dimensionsArray); 103 | } 104 | JSONObject queryJson = new JSONObject(); 105 | queryJson.put("type", type); 106 | // TODO: more validation needed here. 107 | if (type.equals("insensitive_contains")) { 108 | queryJson.put("value", values.get(0)); 109 | } else if (type != null && type.equals("fragment")) { 110 | JSONArray valuesArray = new JSONArray(); 111 | for (String val:values) { 112 | valuesArray.put(val); 113 | } 114 | queryJson.put("values", valuesArray); 115 | } 116 | map.put("query", queryJson); 117 | if (sort != null) { 118 | JSONObject sortJson = new JSONObject(); 119 | sortJson.put("type", sort); 120 | map.put("sort", sortJson); 121 | } 122 | return map; 123 | } 124 | 125 | public static SearchQueryMeta promote(QueryMeta qMeta) { 126 | return new SearchQueryMeta(qMeta); 127 | } 128 | 129 | } 130 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/select/PagingSpec.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.select; 12 | 13 | import java.util.LinkedHashMap; 14 | import java.util.Map; 15 | 16 | /** 17 | * pagingIdentifiers is empty for first request. From the response we extract 18 | * pagingIdentifiers and use that in the subsequent request. 19 | * @author srikalyan 20 | */ 21 | public class PagingSpec { 22 | public Map pagingIdentifiers = new LinkedHashMap<>();// Right now we use only 1st set of values for subsequent requests in pagination. 23 | public int threshold;// Number of rows expected in the response. 24 | 25 | public PagingSpec(Map pagingIdentifiers, int threshold) { 26 | this.pagingIdentifiers = pagingIdentifiers; 27 | this.threshold = threshold; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/select/SelectQueryMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.select; 12 | 13 | import com.yahoo.sql4d.query.PlainDimQueryMeta; 14 | import com.yahoo.sql4d.query.QueryMeta; 15 | import com.yahoo.sql4d.query.RequestType; 16 | import java.util.ArrayList; 17 | import java.util.Iterator; 18 | import java.util.LinkedHashMap; 19 | import java.util.List; 20 | import java.util.Map; 21 | import org.json.JSONArray; 22 | import org.json.JSONObject; 23 | import scala.Tuple2; 24 | 25 | /** 26 | * { 27 | "queryType": "select", 28 | "dataSource": { 29 | "type": "table", 30 | "name": "SomeTable" 31 | }, 32 | "dimensions": [], 33 | "metrics": [], 34 | "granularity": "all", 35 | "intervals": [ 36 | "2014-07-01/2014-07-02" 37 | ], 38 | "pagingSpec": { 39 | "pagingIdentifiers": {}, 40 | "threshold": 5 41 | } 42 | } 43 | 44 | * @author srikalyan 45 | */ 46 | public class SelectQueryMeta extends PlainDimQueryMeta { 47 | 48 | public final PagingSpec pagingSpec = new PagingSpec(null, 5); 49 | public List fetchMetrics = new ArrayList<>(); 50 | 51 | 52 | private static final JSONObject EMPTY_JSON = new JSONObject(); 53 | 54 | public SelectQueryMeta() { 55 | queryType = RequestType.SELECT; 56 | } 57 | 58 | public SelectQueryMeta(QueryMeta q) { 59 | super(q); 60 | queryType = RequestType.SELECT; 61 | } 62 | 63 | @Override 64 | public String toString() { 65 | return getJson().toString(2); 66 | } 67 | 68 | @Override 69 | public JSONObject getJson() { 70 | return new JSONObject(getDataMap()); 71 | } 72 | 73 | @Override 74 | public Map getDataMap() { 75 | Map map = super.getDataMap(); 76 | Map pagingSpecMap = new LinkedHashMap<>(); 77 | if (pagingSpec != null) { 78 | if (pagingSpec.pagingIdentifiers == null) { 79 | pagingSpecMap.put("pagingIdentifiers", EMPTY_JSON); 80 | } else { 81 | pagingSpecMap.put("pagingIdentifiers", pagingSpec.pagingIdentifiers); 82 | } 83 | pagingSpecMap.put("threshold", pagingSpec.threshold); 84 | } 85 | map.put("pagingSpec", pagingSpecMap); 86 | if (fetchMetrics != null) { 87 | map.put("metrics", new JSONArray(fetchMetrics)); 88 | } 89 | return map; 90 | } 91 | 92 | /** 93 | * This method must be called prior to calling {@link SelectQueryMeta#getDataMap() }. 94 | * Essentially splits columns into Dimensions and Metrics. 95 | * @param 96 | * @param anyContext 97 | */ 98 | @Override 99 | public void postProcess(T anyContext) { 100 | Tuple2, List> dimsAndMetrics = (Tuple2, List>)anyContext; 101 | List dims = dimsAndMetrics._1(); 102 | List metrics = dimsAndMetrics._2(); 103 | Iterator> colIter = fetchDimensions.entrySet().iterator(); 104 | while (colIter.hasNext()) { 105 | Map.Entry entry = colIter.next(); 106 | if (dims.contains(entry.getKey())) { 107 | //No action. 108 | } else if (metrics.contains(entry.getKey())) { 109 | fetchMetrics.add(entry.getKey()); 110 | colIter.remove(); 111 | } else {//TODO: Handle error 112 | 113 | } 114 | } 115 | } 116 | 117 | public static SelectQueryMeta promote(QueryMeta qMeta) { 118 | return new SelectQueryMeta(qMeta); 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/timeboundary/TimeBoundaryQueryMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.timeboundary; 12 | 13 | import com.yahoo.sql4d.query.QueryMeta; 14 | import com.yahoo.sql4d.query.RequestType; 15 | import java.util.Map; 16 | import org.json.JSONObject; 17 | 18 | /** 19 | * { 20 | "queryType": "timeBoundary", 21 | "dataSource": "SomeTable" 22 | } 23 | } 24 | [ { 25 | "timestamp" : "2013-09-04T21:44:00.000Z", 26 | "result" : { 27 | "minTime" : "2013-09-04T21:44:00.000Z", 28 | "maxTime" : "2013-09-04T21:47:00.000Z" 29 | } 30 | } ] 31 | 32 | * @author srikalyan 33 | */ 34 | public class TimeBoundaryQueryMeta extends QueryMeta { 35 | 36 | public TimeBoundaryQueryMeta() { 37 | queryType = RequestType.TIMEBOUNDARY; 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return getJson().toString(2); 43 | } 44 | 45 | @Override 46 | public JSONObject getJson() { 47 | return new JSONObject(getDataMap()); 48 | } 49 | 50 | @Override 51 | public Map getDataMap() { 52 | Map map = super.getDataMap(); 53 | return map; 54 | } 55 | 56 | 57 | public static TimeBoundaryQueryMeta promote(QueryMeta qMeta) { 58 | TimeBoundaryQueryMeta newQueryMeta = new TimeBoundaryQueryMeta(); 59 | newQueryMeta.dataSource = qMeta.dataSource; 60 | return newQueryMeta; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/timeseries/TimeSeriesQueryMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.timeseries; 12 | 13 | import com.yahoo.sql4d.query.QueryMeta; 14 | import com.yahoo.sql4d.query.RequestType; 15 | import com.yahoo.sql4d.query.groupby.GroupByQueryMeta; 16 | import java.util.Map; 17 | import org.json.JSONObject; 18 | 19 | /** 20 | * { 21 | "queryType": "timeseries", 22 | "dataSource": "sample_datasource", 23 | "granularity": "day", 24 | "filter": { 25 | "type": "and", 26 | "fields": [ 27 | { "type": "selector", "dimension": "sample_dimension1", "value": "sample_value1" }, 28 | { "type": "or", 29 | "fields": [ 30 | { "type": "selector", "dimension": "sample_dimension2", "value": "sample_value2" }, 31 | { "type": "selector", "dimension": "sample_dimension3", "value": "sample_value3" } 32 | ] 33 | } 34 | ] 35 | }, 36 | "aggregations": [ 37 | { "type": "longSum", "name": "sample_name1", "fieldName": "sample_fieldName1" }, 38 | { "type": "doubleSum", "name": "sample_name2", "fieldName": "sample_fieldName2" } 39 | ], 40 | "postAggregations": [ 41 | { "type": "arithmetic", 42 | "name": "sample_divide", 43 | "fn": "/", 44 | "fields": [ 45 | { "type": "fieldAccess", "name": "sample_name1", "fieldName": "sample_fieldName1" }, 46 | { "type": "fieldAccess", "name": "sample_name2", "fieldName": "sample_fieldName2" } 47 | ] 48 | } 49 | ], 50 | "intervals": [ "2012-01-01T00:00:00.000/2012-01-03T00:00:00.000" ] 51 | } 52 | 53 | The fundamental difference between Group by and Timeseries is the fetchDimensions is 54 | mandatory in former and not in latter. 55 | * @author srikalyan 56 | */ 57 | public class TimeSeriesQueryMeta extends GroupByQueryMeta { 58 | 59 | public String context;// TODO: Address this later. Right now we just fill null in query. 60 | public TimeSeriesQueryMeta() { 61 | queryType = RequestType.TIMESERIES; 62 | } 63 | 64 | 65 | public TimeSeriesQueryMeta(QueryMeta q) { 66 | super(q); 67 | if (q instanceof GroupByQueryMeta) { 68 | fetchDimensions = ((GroupByQueryMeta)q).fetchDimensions; 69 | having = ((GroupByQueryMeta)q).having; 70 | } 71 | queryType = RequestType.TIMESERIES; 72 | } 73 | 74 | @Override 75 | public String toString() { 76 | return getJson().toString(2); 77 | } 78 | 79 | @Override 80 | public JSONObject getJson() { 81 | JSONObject json = new JSONObject(getDataMap()); 82 | json.put("context", JSONObject.NULL); 83 | return json; 84 | } 85 | 86 | @Override 87 | public Map getDataMap() { 88 | Map map = super.getDataMap(); 89 | map.put("queryType", "timeseries"); 90 | map.remove("dimensions"); 91 | return map; 92 | } 93 | 94 | public static TimeSeriesQueryMeta promote(QueryMeta qMeta) { 95 | return new TimeSeriesQueryMeta(qMeta); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/query/topn/TopNQueryMeta.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.query.topn; 12 | 13 | import com.yahoo.sql4d.query.BaseAggQueryMeta; 14 | import com.yahoo.sql4d.query.PlainDimQueryMeta; 15 | import com.yahoo.sql4d.query.QueryMeta; 16 | import com.yahoo.sql4d.query.RequestType; 17 | import java.util.Map; 18 | import org.json.JSONObject; 19 | 20 | /** 21 | * { 22 | "queryType": "topN", 23 | "dataSource": "sample_data", 24 | "dimension": "sample_dim", 25 | "threshold": 5, 26 | "metric": "count", 27 | "granularity": "all", 28 | "filter": { 29 | "type": "and", 30 | "fields": [ 31 | { 32 | "type": "selector", 33 | "dimension": "dim1", 34 | "value": "some_value" 35 | }, 36 | { 37 | "type": "selector", 38 | "dimension": "dim2", 39 | "value": "some_other_val" 40 | } 41 | ] 42 | }, 43 | "aggregations": [ 44 | { 45 | "type": "longSum", 46 | "name": "count", 47 | "fieldName": "count" 48 | }, 49 | { 50 | "type": "doubleSum", 51 | "name": "some_metric", 52 | "fieldName": "some_metric" 53 | } 54 | ], 55 | "postAggregations": [ 56 | { 57 | "type": "arithmetic", 58 | "name": "sample_divide", 59 | "fn": "/", 60 | "fields": [ 61 | { 62 | "type": "fieldAccess", 63 | "name": "some_metric", 64 | "fieldName": "some_metric" 65 | }, 66 | { 67 | "type": "fieldAccess", 68 | "name": "count", 69 | "fieldName": "count" 70 | } 71 | ] 72 | } 73 | ], 74 | "intervals": [ 75 | "2013-08-31T00:00:00.000/2013-09-03T00:00:00.000" 76 | ] 77 | } 78 | 79 | 80 | * @author srikalyan 81 | */ 82 | public class TopNQueryMeta extends BaseAggQueryMeta { 83 | public String dimension;// group by 84 | public int threshold = 10;// limit 85 | public String metric;// order by 86 | 87 | public TopNQueryMeta() { 88 | queryType = RequestType.TOPN; 89 | } 90 | 91 | public TopNQueryMeta(QueryMeta q) { 92 | super(q); 93 | queryType = RequestType.TOPN; 94 | } 95 | 96 | @Override 97 | public String toString() { 98 | return getJson().toString(2); 99 | } 100 | 101 | @Override 102 | public JSONObject getJson() { 103 | return new JSONObject(getDataMap()); 104 | } 105 | 106 | @Override 107 | public Map getDataMap() { 108 | Map map = super.getDataMap(); 109 | map.put("queryType", "topN"); 110 | map.remove("dimensions");// During promotion from GroupBy dimensions may have sneaked in so remove it. 111 | if (dimension != null) { 112 | map.put("dimension", dimension); 113 | } 114 | map.put("threshold", threshold); 115 | if (metric != null) { 116 | map.put("metric", metric); 117 | } 118 | return map; 119 | } 120 | 121 | public static TopNQueryMeta promote(QueryMeta qMeta) { 122 | TopNQueryMeta topN = new TopNQueryMeta(qMeta); 123 | if (qMeta instanceof PlainDimQueryMeta) { 124 | PlainDimQueryMeta gBQMeta = ((PlainDimQueryMeta)qMeta); 125 | if (gBQMeta.fetchDimensions != null && gBQMeta.fetchDimensions.size() == 1) { 126 | for (String dim:gBQMeta.fetchDimensions.keySet()) {// We care only about 1st dimension 127 | String dimaAlias = gBQMeta.fetchDimensions.get(dim); 128 | if (dimaAlias == null) {//Directly use the dim 129 | topN.dimension = dim; 130 | } else { 131 | topN.dimension = dimaAlias; 132 | } 133 | 134 | break;// Break after the first dimension. 135 | } 136 | } else { 137 | System.out.println("Wrong(OR) no dimensions specified in GROUP BY(Need 1 dimension)"); 138 | } 139 | } 140 | return topN; 141 | } 142 | 143 | } 144 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/utils/DruidUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.utils; 12 | 13 | import com.google.common.base.Function; 14 | import static com.google.common.collect.Iterables.concat; 15 | import com.google.common.collect.Lists; 16 | import com.yahoo.sql4d.query.nodes.AggItem; 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | /** 22 | * 23 | * @author srikalyan 24 | */ 25 | public class DruidUtils { 26 | // This method actually returns everything(including timestamp which is first field. 27 | public static List getDimensions(Map fetchDimensions) { 28 | return new ArrayList<>(fetchDimensions.keySet()); 29 | } 30 | public static List getMetrics(List aggItems) { 31 | return Lists.transform(aggItems, new Function() { 32 | @Override 33 | public String apply(AggItem f) { 34 | return f.fieldName; 35 | } 36 | }); 37 | } 38 | public static List getColumns(Map fetchDimensions, List aggItems) { 39 | return Lists.newArrayList(concat(getDimensions(fetchDimensions), getMetrics(aggItems))); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Sql4DCompiler/src/main/java/com/yahoo/sql4d/utils/Utils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.utils; 12 | 13 | /** 14 | * TODO: Improve the implementation. 15 | * @author srikalyan 16 | */ 17 | public class Utils { 18 | public static String unquote(String ipStr) { 19 | if (ipStr == null) return ipStr; 20 | if (ipStr.startsWith("'") && ipStr.endsWith("'")) { 21 | ipStr = ipStr.substring(1, ipStr.length() - 1); 22 | } 23 | return ipStr; 24 | } 25 | public static String unescape(String ipStr) { 26 | if (ipStr == null) return ipStr; 27 | return ipStr.replace("\\\\", "\\"); 28 | } 29 | 30 | //TODO: Handles only the most popular cases now. 31 | public static String unicode(String ipStr) { 32 | if (ipStr == null) return ipStr; 33 | switch(ipStr) { 34 | case "\\u0001": 35 | return "\u0001"; 36 | case "\\u0002": 37 | return "\u0002"; 38 | case "\\u0003": 39 | return "\u0003"; 40 | case "\\u0004": 41 | return "\u0004"; 42 | case "\\u0005": 43 | return "\u0005"; 44 | case "\\u0006": 45 | return "\u0006"; 46 | case "\\n"://TODO: This is not a unicode , find a proper home for this. 47 | return "\n"; 48 | } 49 | return ipStr; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Sql4Ddriver/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /Sql4Ddriver/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | Sql4Ddriver 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /Sql4Ddriver/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.yahoo.sql4d 5 | Sql4Ddriver 6 | 4.1.0 7 | jar 8 | 9 | com.yahoo.sql4d 10 | Sql4D 11 | 4.1.0 12 | ../pom.xml 13 | 14 | 15 | UTF-8 16 | 1.7 17 | 1.7 18 | 19 | 20 | 21 | com.yahoo.sql4d 22 | Sql4DCompiler 23 | 4.1.0 24 | 25 | 26 | org.testng 27 | testng 28 | 6.8.1 29 | test 30 | 31 | 32 | org.apache.httpcomponents 33 | httpclient 34 | 35 | 36 | org.apache.httpcomponents 37 | fluent-hc 38 | 39 | 40 | mysql 41 | mysql-connector-java 42 | 43 | 44 | org.springframework 45 | spring-jdbc 46 | 4.1.2.RELEASE 47 | 48 | 49 | com.fasterxml.jackson.core 50 | jackson-core 51 | 2.5.0 52 | 53 | 54 | com.fasterxml.jackson.core 55 | jackson-databind 56 | 2.5.0 57 | 58 | 59 | com.fasterxml.jackson.core 60 | jackson-annotations 61 | 2.5.0 62 | 63 | 64 | 65 | commons-pool 66 | commons-pool 67 | 1.6 68 | 69 | 70 | 71 | com.esotericsoftware.kryo 72 | kryo 73 | 2.24.0 74 | 75 | 76 | 77 | org.slf4j 78 | slf4j-api 79 | 80 | 81 | org.slf4j 82 | slf4j-log4j12 83 | 84 | 85 | log4j 86 | apache-log4j-extras 87 | 88 | 89 | commons-lang 90 | commons-lang 91 | 2.6 92 | 93 | 94 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/BaseJoiner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver; 12 | 13 | import com.yahoo.sql4d.query.RequestType; 14 | import java.util.Arrays; 15 | import java.util.LinkedHashSet; 16 | import java.util.List; 17 | import java.util.Set; 18 | import org.json.JSONArray; 19 | import org.json.JSONObject; 20 | 21 | /** 22 | * Base Joiner with basic join functionality. The actual join functionality goes into 23 | * {@link #extractAndTakeAction(java.lang.String, org.json.JSONArray, java.lang.String, com.yahoo.sql4d.sql4ddriver.BaseMapper.RequestType, com.yahoo.sql4d.sql4ddriver.BaseJoiner.ActionType) } 24 | * 25 | * @author srikalyan 26 | */ 27 | public abstract class BaseJoiner extends BaseMapper { 28 | 29 | public static enum ActionType { 30 | FIRST_CUT, JOIN, LEFT_JOIN, RIGHT_JOIN 31 | } 32 | 33 | /** 34 | * If action if FIRST_CUT then the values are added directly to the result else 35 | * rows will be filtered on key. 36 | * @param jsonAllRows 37 | * @param joinFields 38 | * @param action 39 | */ 40 | public void join(JSONArray jsonAllRows, List joinFields, ActionType action) { 41 | if (jsonAllRows.length() == 0) { 42 | return; 43 | } 44 | JSONObject sample = jsonAllRows.getJSONObject(0); 45 | if (sample.has("event")) {// GroupBy 46 | extractAndTakeAction(null, jsonAllRows, joinFields, RequestType.GROUPBY, action); 47 | } else if (sample.has("result")) {// Could be timeseries/topN 48 | if (sample.optJSONObject("result") != null) {// Timeseries 49 | extractAndTakeAction(null, jsonAllRows, joinFields, RequestType.TIMESERIES, action); 50 | } else if (sample.optJSONArray("result") != null) {// TopN 51 | // There should be only 1 item in here. 52 | JSONObject jsonItem = jsonAllRows.getJSONObject(0); 53 | JSONArray result = jsonItem.getJSONArray("result"); 54 | extractAndTakeAction(jsonItem.optString("timestamp"), result, joinFields, RequestType.TOPN, action); 55 | } 56 | } 57 | } 58 | 59 | public static boolean isJoinType(ActionType action) { 60 | return action == ActionType.JOIN || action == ActionType.LEFT_JOIN ||action == ActionType.RIGHT_JOIN; 61 | } 62 | 63 | /** 64 | * Extract the data item from json(based on the request type) and also fill in the 65 | * headers as a side affect. 66 | * @param jsonItem 67 | * @param requestType 68 | * @param action 69 | * @return 70 | */ 71 | protected JSONObject dataItemAt(JSONObject jsonItem, RequestType requestType, ActionType action) { 72 | JSONObject eachRow = null; 73 | if (requestType == RequestType.GROUPBY) { 74 | eachRow = jsonItem.getJSONObject("event"); 75 | } else if (requestType == RequestType.TIMESERIES) { 76 | eachRow = jsonItem.getJSONObject("result"); 77 | } else if (requestType == RequestType.TOPN) { 78 | eachRow = jsonItem; 79 | } 80 | return eachRow; 81 | } 82 | //TODO: Make this like tryRefillHeaders() method in BaseMapper because some items in the json resultset may not have all the fields. 83 | protected void fillHeaders(JSONObject eachRow, List joinFields, ActionType action) { 84 | if (action == ActionType.FIRST_CUT) { 85 | baseFieldNames.add("timestamp"); 86 | } 87 | if (eachRow != null) { 88 | if (isJoinType(action)) { 89 | Set keysCopy = new LinkedHashSet<>(eachRow.keySet()); 90 | keysCopy.removeAll(joinFields); 91 | baseFieldNames.addAll(keysCopy); 92 | } else { 93 | baseFieldNames.addAll(eachRow.keySet()); 94 | } 95 | } 96 | } 97 | protected List splitCompositeKeys(String keys) { 98 | String keysArr[] = keys.split("\u0001"); 99 | if (keysArr != null && keysArr.length > 0) 100 | return Arrays.asList(keysArr); 101 | return Arrays.asList(keys); 102 | } 103 | 104 | protected abstract void extractAndTakeAction(String timestamp, JSONArray jsonAllRows, List joinFields, RequestType requestType, ActionType action); 105 | } 106 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/BaseMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver; 12 | 13 | import com.yahoo.sql4d.query.RequestType; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | import org.json.JSONArray; 17 | import org.json.JSONObject; 18 | 19 | /** 20 | * Base class for bean mapping. 21 | * @author srikalyan 22 | */ 23 | public class BaseMapper { 24 | public List baseFieldNames = new ArrayList<>(); 25 | 26 | /** 27 | * @param jsonAllRows 28 | */ 29 | public void map(JSONArray jsonAllRows) { 30 | if (jsonAllRows.length() == 0) { 31 | return; 32 | } 33 | JSONObject sample = jsonAllRows.getJSONObject(0); 34 | if (sample.has("event")) {// GroupBy 35 | extractAndMap(null, jsonAllRows, RequestType.GROUPBY); 36 | } else if (sample.has("result")) {// Could be timeseries/topN/select 37 | if (sample.optJSONObject("result") != null) {// Timeseries/select 38 | if (sample.optJSONObject("result").optJSONArray("events") != null) {//select 39 | extractAndMap(null, sample.optJSONObject("result").optJSONArray("events"), RequestType.SELECT); 40 | } else {// Timeseries. 41 | extractAndMap(null, jsonAllRows, RequestType.TIMESERIES); 42 | } 43 | } else if (sample.optJSONArray("result") != null) {// TopN 44 | // There should be only 1 item in here. 45 | JSONObject jsonItem = jsonAllRows.getJSONObject(0); 46 | JSONArray result = jsonItem.getJSONArray("result"); 47 | extractAndMap(jsonItem.optString("timestamp"), result, RequestType.TOPN); 48 | } 49 | } 50 | } 51 | 52 | protected void extractAndMap(String timestamp, JSONArray jsonAllRows, RequestType requestType) { 53 | // Empty 54 | } 55 | 56 | protected JSONObject dataItemAt(JSONObject jsonItem, RequestType requestType) { 57 | JSONObject eachRow = null; 58 | if (requestType == RequestType.GROUPBY || requestType == RequestType.SELECT) { 59 | eachRow = jsonItem.getJSONObject("event"); 60 | } else if (requestType == RequestType.TIMESERIES) { 61 | eachRow = jsonItem.getJSONObject("result"); 62 | } else if (requestType == RequestType.TOPN) { 63 | eachRow = jsonItem; 64 | } 65 | return eachRow; 66 | } 67 | /** 68 | * Will attempt to refill headers if we see more fields. 69 | * @param eachRow 70 | */ 71 | protected void tryRefillHeaders(JSONObject eachRow) { 72 | if (eachRow == null) { 73 | return; 74 | } 75 | if (eachRow.keySet().size() > baseFieldNames.size() - 1) {//i.e excluding timestamp, if the sizes are not same then we have not covered all field names 76 | baseFieldNames.clear(); 77 | baseFieldNames.add("timestamp"); 78 | baseFieldNames.addAll(eachRow.keySet()); 79 | } 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/BrokerAccessor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.sql4ddriver; 13 | 14 | import com.yahoo.sql4d.BaseStatementMeta; 15 | import com.yahoo.sql4d.DCompiler; 16 | import com.yahoo.sql4d.Program; 17 | import com.yahoo.sql4d.query.nodes.Interval; 18 | import java.io.IOException; 19 | import static java.lang.String.format; 20 | import java.util.List; 21 | import java.util.Map; 22 | import org.apache.commons.io.IOUtils; 23 | import org.apache.http.client.methods.CloseableHttpResponse; 24 | import org.json.JSONArray; 25 | import org.json.JSONException; 26 | import scala.util.Either; 27 | import scala.util.Left; 28 | import scala.util.Right; 29 | 30 | /** 31 | * Query interface to druid broker. 32 | * @author srikalyan 33 | */ 34 | public class BrokerAccessor extends DruidNodeAccessor { 35 | private final String brokerHost; 36 | private int brokerPort = 4080; 37 | private final String brokerUrl = "http://%s:%d/druid/v2/?pretty"; 38 | public BrokerAccessor(String host, int port, int maxConns) { 39 | super(host, port, maxConns); 40 | this.brokerHost = host; 41 | this.brokerPort = port; 42 | } 43 | 44 | /** 45 | * For firing simple queries(i.e non join queries). 46 | * 47 | * @param jsonQuery 48 | * @param reqHeaders 49 | * @param requiresMapping 50 | * @return 51 | */ 52 | public Either> fireQuery(String jsonQuery, Map reqHeaders, boolean requiresMapping) { 53 | CloseableHttpResponse resp = null; 54 | String respStr; 55 | String url = format(brokerUrl, brokerHost, brokerPort); 56 | try { 57 | resp = postJson(url, jsonQuery, reqHeaders); 58 | respStr = IOUtils.toString(resp.getEntity().getContent()); 59 | } catch (IOException ex) { 60 | return new Left<>(format("Http %s, faced exception %s\n", resp, ex)); 61 | } finally { 62 | returnClient(resp); 63 | } 64 | JSONArray possibleResArray = null; 65 | try { 66 | possibleResArray = new JSONArray(respStr); 67 | } catch (JSONException je) { 68 | return new Left<>(format("Recieved data %s not in json format. \n", respStr)); 69 | } 70 | if (requiresMapping) { 71 | return new Right>(new Left(new Mapper4All(possibleResArray))); 72 | } 73 | return new Right>(new Right(possibleResArray)); 74 | } 75 | 76 | /** 77 | * Get timeboundary. 78 | * 79 | * @param dataSource 80 | * @param reqHeaders 81 | * @return 82 | * @throws java.lang.IllegalAccessException 83 | */ 84 | public Interval getTimeBoundary(String dataSource, Map reqHeaders) throws IllegalAccessException { 85 | Program pgm = DCompiler.compileSql(format("SELECT FROM %s", dataSource)); 86 | Either> res = fireQuery(pgm.nthStmnt(0).toString(), reqHeaders, true); 87 | if (res.isLeft()) { 88 | throw new IllegalAccessException(format("DataSource %s does not exist(or) check if druid is accessible, faced exception %s", dataSource, res.left().get())); 89 | } 90 | Mapper4All finalRes = res.right().get().left().get();// Thats because we know Time boundary cannot be a Join result!! 91 | int min = finalRes.baseFieldNames.indexOf("minTime"); 92 | int max = finalRes.baseFieldNames.indexOf("maxTime"); 93 | if (finalRes.baseAllRows.isEmpty()) {// Possible when table does not exist. 94 | throw new IllegalAccessException("Either table does not exist(or) druid is not accessible"); 95 | } 96 | List row = finalRes.baseAllRows.get(0);// Only 1 element is returned in Timeboundary. 97 | return new Interval(row.get(min).toString(), row.get(max).toString()); 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/Mapper4All.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver; 12 | 13 | import com.yahoo.sql4d.query.RequestType; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | import org.json.JSONArray; 17 | import org.json.JSONObject; 18 | 19 | /** 20 | * Simply maps JSON to bean . 21 | * 22 | * @author srikalyan 23 | */ 24 | public class Mapper4All extends BaseMapper { 25 | 26 | public List> baseAllRows = new ArrayList<>(); 27 | 28 | public Mapper4All() { 29 | } 30 | 31 | public Mapper4All(JSONArray jsonAllRows) { 32 | map(jsonAllRows); 33 | } 34 | 35 | /** 36 | * @param timestamp 37 | * @param jsonAllRows 38 | * @param requestType 39 | */ 40 | @Override 41 | protected void extractAndMap(String timestamp, JSONArray jsonAllRows, RequestType requestType) { 42 | JSONObject eachRow = null; 43 | for (int i = 0; i < jsonAllRows.length(); i++) { 44 | JSONObject jsonItem = jsonAllRows.getJSONObject(i); 45 | eachRow = dataItemAt(jsonItem, requestType); 46 | String actualTimestamp = timestamp; 47 | if (timestamp == null) { 48 | if (requestType == RequestType.SELECT) { 49 | actualTimestamp = eachRow.optString("timestamp"); 50 | // Because the timestamp is within each row remove them once you extract it. 51 | eachRow.remove("timestamp"); 52 | } else { 53 | actualTimestamp = jsonItem.optString("timestamp"); 54 | } 55 | } 56 | // Try refill headers for every row simply because we may not have result for that field in the response(which may skip that field) worst 57 | // case is some may have and some rows may not have values for that field. 58 | tryRefillHeaders(eachRow); 59 | List row = extractKeyAndRow(actualTimestamp, eachRow); 60 | baseAllRows.add(row); 61 | } 62 | } 63 | 64 | /** 65 | * Extract v = all fields from json. The first field is always timestamp and 66 | * is passed to the method not extracted. 67 | * @param timestamp 68 | * @param jsonRow 69 | * @return 70 | */ 71 | private List extractKeyAndRow(String timestamp, JSONObject jsonRow) { 72 | List rowValues = new ArrayList<>(); 73 | rowValues.add(timestamp); 74 | for (Object key : jsonRow.keySet()) { 75 | rowValues.add(jsonRow.get(key.toString())); 76 | } 77 | return rowValues; 78 | } 79 | 80 | @Override 81 | public String toString() { 82 | return baseFieldNames.toString() + "\n" + baseAllRows.toString(); 83 | } 84 | 85 | 86 | } 87 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/Mapper4Bean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver; 12 | 13 | import com.yahoo.sql4d.query.RequestType; 14 | import com.yahoo.sql4d.sql4ddriver.rowmapper.DruidBaseBean; 15 | import java.lang.reflect.InvocationTargetException; 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.util.logging.Level; 19 | import java.util.logging.Logger; 20 | import org.json.JSONArray; 21 | import org.json.JSONObject; 22 | 23 | /** 24 | * Simply maps JSON to bean . 25 | * 26 | * @author srikalyan 27 | * @param 28 | */ 29 | public class Mapper4Bean extends BaseMapper { 30 | 31 | public List baseAllRows = new ArrayList<>();// Each entry = Bean of type T 32 | private Class rowMapper; 33 | 34 | public Mapper4Bean(JSONArray jsonAllRows, Class rowMapper) { 35 | this.rowMapper = rowMapper; 36 | map(jsonAllRows); 37 | } 38 | 39 | 40 | /** 41 | * Generate a Tuple 42 | * A = list of field names B = map of from jsonArray. 43 | * 44 | * @param timestamp 45 | * @param jsonAllRows 46 | * @param requestType 47 | */ 48 | @Override 49 | protected void extractAndMap(String timestamp, JSONArray jsonAllRows, RequestType requestType) { 50 | JSONObject eachRow = null; 51 | for (int i = 0; i < jsonAllRows.length(); i++) { 52 | JSONObject jsonItem = jsonAllRows.getJSONObject(i); 53 | eachRow = dataItemAt(jsonItem, requestType); 54 | String actualTimestamp = timestamp; 55 | if (timestamp == null) { 56 | if (requestType == RequestType.SELECT) { 57 | actualTimestamp = eachRow.optString("timestamp"); 58 | // Because the timestamp is within each row remove them once you extract it. 59 | eachRow.remove("timestamp"); 60 | } else { 61 | actualTimestamp = jsonItem.optString("timestamp"); 62 | } 63 | } 64 | if (i == 0) {// Fill headers (only once) 65 | tryRefillHeaders(eachRow); 66 | } 67 | T row = extractKeyAndRow(actualTimestamp, eachRow); 68 | baseAllRows.add(row); 69 | } 70 | } 71 | 72 | /** 73 | * Extract v = all fields from json. The first field is always timestamp and 74 | * is not extracted here. 75 | * 76 | * @param timestamp 77 | * @param jsonRow 78 | * @return 79 | */ 80 | private T extractKeyAndRow(String timestamp, JSONObject jsonRow) { 81 | T rowValues = null; 82 | try { 83 | rowValues = rowMapper.newInstance(); 84 | rowValues.getClass().getMethod("setTimestamp", String.class).invoke(rowValues, timestamp); 85 | for (Object key : jsonRow.keySet()) { 86 | Util.applyKVToBean(rowValues, key.toString(), jsonRow.get(key.toString())); 87 | } 88 | 89 | } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | SecurityException | IllegalArgumentException | InvocationTargetException ex) { 90 | Logger.getLogger(Mapper4Bean.class.getName()).log(Level.SEVERE, null, ex); 91 | } 92 | return rowValues; 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/NamedParameters.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver; 12 | 13 | import java.math.BigDecimal; 14 | import java.text.DateFormat; 15 | import java.text.SimpleDateFormat; 16 | import java.util.Date; 17 | import java.util.HashMap; 18 | import java.util.Map; 19 | import java.util.TimeZone; 20 | import org.joda.time.DateTime; 21 | import org.joda.time.DateTimeZone; 22 | import org.joda.time.format.DateTimeFormatter; 23 | import org.joda.time.format.ISODateTimeFormat; 24 | 25 | /** 26 | * Simple named parameters. 27 | * @author srikalyan 28 | */ 29 | public class NamedParameters { 30 | 31 | private final Map namedParams = new HashMap<>(); 32 | private static final TimeZone utcTz = TimeZone.getTimeZone("UTC"); 33 | private static final TimeZone estTz = TimeZone.getTimeZone("EST"); 34 | private static final TimeZone est5edtTz = TimeZone.getTimeZone("EST5EDT"); 35 | private static final DateFormat javaDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); 36 | 37 | private static final DateTimeZone jodaUtcTz = DateTimeZone.forTimeZone(utcTz); 38 | private static final DateTimeZone jodaEstTz = DateTimeZone.forTimeZone(estTz); 39 | private static final DateTimeZone jodaEst5edtTz = DateTimeZone.forTimeZone(est5edtTz); 40 | private static final DateTimeFormatter jodaDateFormat = ISODateTimeFormat.dateTime().withZone(jodaUtcTz); 41 | private static final DateTimeFormatter jodaEstDateFormat = ISODateTimeFormat.dateTime().withZone(jodaEstTz); 42 | private static final DateTimeFormatter jodaEst5EDTDateFormat = ISODateTimeFormat.dateTime().withZone(jodaEst5edtTz); 43 | 44 | private final DateFormat currentJavaDateFormat; 45 | private final DateTimeFormatter currentJodaDateFormat; 46 | 47 | 48 | public NamedParameters() { 49 | javaDateFormat.setTimeZone(utcTz); 50 | currentJavaDateFormat = javaDateFormat; 51 | currentJodaDateFormat = jodaDateFormat; 52 | } 53 | //TODO: Must be able to handle all timezones types. 54 | public NamedParameters(String timeZone) { 55 | if (timeZone.equalsIgnoreCase("est")) { 56 | javaDateFormat.setTimeZone(estTz); 57 | currentJodaDateFormat = jodaEstDateFormat; 58 | } else if (timeZone.equalsIgnoreCase("est5edt")) { 59 | javaDateFormat.setTimeZone(est5edtTz); 60 | currentJodaDateFormat = jodaEst5EDTDateFormat; 61 | } else { 62 | currentJodaDateFormat = jodaDateFormat; 63 | } 64 | currentJavaDateFormat = javaDateFormat; 65 | } 66 | 67 | public void add(String key, Object value) { 68 | namedParams.put(key, value); 69 | } 70 | 71 | /** 72 | * TODO: Make it more efficient. 73 | * @param sqlQuery 74 | * @return 75 | */ 76 | public String deParameterize(String sqlQuery) { 77 | String result = sqlQuery; 78 | for (String key:namedParams.keySet()) { 79 | Object value = namedParams.get(key); 80 | if (value instanceof String) {// This could be combined with last else(but mostly we encounter String) 81 | result = result.replaceAll(String.format(":%s", key), String.format("%s", value)); 82 | } else if (value instanceof BigDecimal) { 83 | result = result.replaceAll(String.format(":%s", key), String.format("%f", ((BigDecimal)value).doubleValue())); 84 | } else if (value instanceof Date) { 85 | result = result.replaceAll(String.format(":%s", key), currentJavaDateFormat.format(value)).replace("Z", ""); 86 | } else if (value instanceof DateTime) { 87 | result = result.replaceAll(String.format(":%s", key), currentJodaDateFormat.print(((DateTime)value).getMillis())).replace("Z", ""); 88 | } else { 89 | result = result.replaceAll(String.format(":%s", key), value.toString()); 90 | } 91 | } 92 | return result; 93 | } 94 | public static void main(String[] args) { 95 | 96 | 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/TaskStatus.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.sql4ddriver; 13 | 14 | /** 15 | * 16 | * @author srikalyan 17 | */ 18 | public enum TaskStatus { 19 | SUCCESS, FAILED, RUNNING, UNKNOWN 20 | } 21 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/rowmapper/DruidBaseBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver.rowmapper; 12 | 13 | import com.esotericsoftware.kryo.Kryo; 14 | import com.esotericsoftware.kryo.Registration; 15 | import com.esotericsoftware.kryo.Serializer; 16 | import com.esotericsoftware.kryo.io.Input; 17 | import com.esotericsoftware.kryo.io.Output; 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | import com.yahoo.sql4d.sql4ddriver.Util; 20 | import java.lang.reflect.InvocationTargetException; 21 | import java.util.LinkedHashMap; 22 | import java.util.Map; 23 | import java.util.logging.Level; 24 | import java.util.logging.Logger; 25 | import org.joda.time.format.DateTimeFormat; 26 | import org.joda.time.format.DateTimeFormatter; 27 | 28 | /** 29 | * Has timestamp which is always available in druid response. All the subclasses 30 | * must ensure they provide getter and setter for every instance variable. Do not 31 | * use Boxed types like Integer, Long etc. Must define a constructor without 32 | * arguments in case you define one with arguments. Any bean of this type is Kryo 33 | * serializable. 34 | * @author srikalyan 35 | * @param 36 | */ 37 | public class DruidBaseBean extends Serializer { 38 | private static final DateTimeFormatter dateOnlyFormat = DateTimeFormat.forPattern("yyyy-MM-dd"); 39 | private static final DateTimeFormatter dateTimeFormat = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss"); 40 | private static final DateTimeFormatter dateTimeWithSubSecFormat = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS"); 41 | private static final DateTimeFormatter dateTimeAndTZFormat = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ssZZ"); 42 | private static final DateTimeFormatter dateTimeWithSubSecAndTZFormat = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZZ"); 43 | 44 | public static void main(String[] args) { 45 | System.out.println(dateTimeFormat.parseDateTime("2014-05-25T16:45:49")); 46 | System.out.println(dateTimeAndTZFormat.parseDateTime("2014-05-25T16:45:49+00:00")); 47 | System.out.println(dateTimeAndTZFormat.parseDateTime("2014-05-25T16:45:49Z")); 48 | System.out.println(dateTimeWithSubSecFormat.parseDateTime("2014-05-25T16:45:49.100")); 49 | System.out.println(dateTimeWithSubSecAndTZFormat.parseDateTime("2014-05-25T16:45:49.000Z")); 50 | } 51 | 52 | public String timestamp;// TODO: Crazy,though getDeclaredMethods can be used to access private fields, it is not possible on subclasses. 53 | 54 | public void setTimestamp(String timestamp) { 55 | this.timestamp = timestamp; 56 | } 57 | 58 | public String getTimestamp() { 59 | return timestamp; 60 | } 61 | 62 | public Map toMap() { 63 | Map map = new ObjectMapper().convertValue(this, Map.class); 64 | // The following 2 comes from Serializer. 65 | map.remove("acceptsNull"); 66 | map.remove("immutable"); 67 | return map; 68 | } 69 | 70 | @Override 71 | public String toString() { 72 | return toMap().toString(); 73 | } 74 | 75 | @Override 76 | public void write(Kryo kryo, Output output, T t) { 77 | Map map = toMap(); 78 | for (String key:map.keySet()) { 79 | Object value = map.get(key); 80 | kryo.writeClass(output, value.getClass()); 81 | kryo.writeObjectOrNull(output, key, String.class); 82 | kryo.writeObjectOrNull(output, value, value.getClass()); 83 | } 84 | } 85 | 86 | @Override 87 | public T read(Kryo kryo, Input input, Class type) { 88 | T retValue = null; 89 | try { 90 | retValue = type.newInstance(); 91 | Registration reg = kryo.readClass(input); 92 | String key = kryo.readObjectOrNull(input, String.class); 93 | Object value = kryo.readObjectOrNull(input, reg.getType()); 94 | Util.applyKVToBean(retValue, key, value); 95 | } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | IllegalArgumentException | InvocationTargetException ex) { 96 | Logger.getLogger(DruidBaseBean.class.getName()).log(Level.SEVERE, null, ex); 97 | } 98 | return retValue; 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/rowmapper/TimeSeriesBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver.rowmapper; 12 | 13 | /** AUTO-GENERATED **/ 14 | import com.yahoo.sql4d.sql4ddriver.rowmapper.DruidBaseBean; 15 | 16 | public class TimeSeriesBean extends DruidBaseBean { 17 | private long shares ; 18 | private long content_seen ; 19 | private long content_views ; 20 | public long getShares() { 21 | return this.shares; 22 | } 23 | public void setShares(long shares) { 24 | this.shares = shares; 25 | } 26 | public long getContentSeen() { 27 | return this.content_seen; 28 | } 29 | public void setContentSeen(long content_seen) { 30 | this.content_seen = content_seen; 31 | } 32 | public long getContentViews() { 33 | return this.content_views; 34 | } 35 | public void setContentViews(long content_views) { 36 | this.content_views = content_views; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return shares + "," + content_views + "," + content_seen; 42 | } 43 | 44 | } 45 | /** AUTO-GENERATED **/ 46 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/main/java/com/yahoo/sql4d/sql4ddriver/sql/DAODictionary.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 3 | * (the "License"); you may not use this file except in compliance with the 4 | * License. You may obtain a copy of the License at 5 | * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 | * or agreed to in writing, software distributed under the License is 7 | * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 | * KIND, either express or implied. See the License for the specific language 9 | * governing permissions and limitations under the License. See accompanying 10 | * LICENSE file. 11 | */ 12 | package com.yahoo.sql4d.sql4ddriver.sql; 13 | 14 | /** 15 | * 16 | * @author srikalyan 17 | */ 18 | public interface DAODictionary { 19 | public static final String SEGMENTS_IN_RANGE = "SELECT * FROM druid_segments WHERE dataSource = :dataSource AND ((start >= :iStart && start < :iEnd) || (end > :iStart && end <= :iEnd))"; 20 | public static final String USED_SEGMENTS_IN_RANGE = SEGMENTS_IN_RANGE + " AND used = :used"; 21 | public static final String DISABLE_SEGMENTS_IN_RANGE = "UPDATE druid_segments SET used = 0 WHERE dataSource = :dataSource AND ((start >= :iStart && start < :iEnd) || (end > :iStart && end <= :iEnd))"; 22 | public static final String DISABLE_ALL_SEGMENTS = "UPDATE druid_segments SET used = 0 WHERE dataSource = :dataSource"; 23 | } 24 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/test/java/com/yahoo/sql4d/sql4ddriver/AnalyticsDruidTestBase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver; 12 | 13 | import org.testng.annotations.AfterClass; 14 | import org.testng.annotations.BeforeClass; 15 | import scala.util.Either; 16 | 17 | /** 18 | * TestBase for druid based API. 19 | * @author srikalyan 20 | */ 21 | public abstract class AnalyticsDruidTestBase { 22 | 23 | DDataSource source = null; 24 | 25 | public void testPositive(String query) throws Exception { 26 | Either> result = source.query(query, null); 27 | if (result.isLeft()) { 28 | throw new Exception(result.left().get()); 29 | } 30 | Either goodResult = result.right().get(); 31 | if (goodResult.isLeft()) { 32 | PrettyPrint.print(goodResult.left().get()); 33 | } else { 34 | PrettyPrint.print(goodResult.right().get()); 35 | } 36 | } 37 | 38 | @BeforeClass 39 | public void setUpClass() throws Exception { 40 | source = new DDataSource("yourbroker.com", 4080, "yourcoordinator.com", 8082, "proxyIfAny", 3128); 41 | } 42 | 43 | @AfterClass 44 | public void tearDownClass() throws Exception { 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/test/java/com/yahoo/sql4d/sql4ddriver/TimeSeriesTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver; 12 | 13 | import com.yahoo.sql4d.sql4ddriver.rowmapper.TimeSeriesBean; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | import java.util.Map; 17 | import org.testng.annotations.Test; 18 | import scala.util.Either; 19 | 20 | /** 21 | * Timeseries test. 22 | * @author srikalyan 23 | */ 24 | public class TimeSeriesTest extends AnalyticsDruidTestBase { 25 | 26 | String timeseriesContentAggHourly = "SELECT timestamp , LONG_SUM(content_views) AS content_views, LONG_SUM(shares) AS shares FROM AggTable WHERE interval BETWEEN 2014-05-20T00:00:00.000-04:00 AND 2014-05-31T23:00:00.000-04:00 AND provider_id='superprovider' AND content_type='cavideo' BREAK BY PERIOD('P1D', 'EST5EDT') GROUP BY timestamp, content_views, shares HINT('timeseries')"; 27 | String timeseriesUniqCount = "SELECT timestamp , LONG_SUM(all_content_seen) AS content_seen FROM UniqueCountTable WHERE interval BETWEEN 2014-05-20T00:00:00.000-04:00 AND 2014-05-31T23:00:00.000-04:00 AND provider_id='superprovider' AND content_type='cavideo' BREAK BY PERIOD('P1D', 'EST5EDT') GROUP BY timestamp,content_seen HINT('timeseries')"; 28 | String tsJoin = String.format("%s JOIN (%s) ON (timestamp)", timeseriesContentAggHourly, timeseriesUniqCount); 29 | 30 | @Test 31 | public void testTimeSeriesBean() throws Exception { 32 | Either, Map>> mapperRes = source.query(tsJoin, null, TimeSeriesBean.class, null, false); 33 | if (mapperRes.isLeft()) { 34 | throw new Exception(mapperRes.left().get()); 35 | } 36 | Either, Map> goodResult = mapperRes.right().get(); 37 | if (goodResult.isLeft()) { 38 | PrettyPrint.print(goodResult.left().get()); 39 | } else { 40 | PrettyPrint.print(new ArrayList<>(goodResult.right().get().values())); 41 | } 42 | } 43 | 44 | @Test 45 | public void testTimeSeries() throws Exception { 46 | testPositive(tsJoin); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /Sql4Ddriver/src/test/java/com/yahoo/sql4d/sql4ddriver/rowmapper/TestKryoDruidBaseBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0 (the 3 | * "License"); you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 5 | * Unless required by applicable law or agreed to in writing, software distributed 6 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 7 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 8 | * specific language governing permissions and limitations under the License. 9 | * See accompanying LICENSE file. 10 | */ 11 | package com.yahoo.sql4d.sql4ddriver.rowmapper; 12 | 13 | import com.esotericsoftware.kryo.Kryo; 14 | import com.esotericsoftware.kryo.io.Input; 15 | import com.esotericsoftware.kryo.io.Output; 16 | import org.testng.Assert; 17 | import org.testng.annotations.Test; 18 | 19 | /** 20 | * 21 | * @author srikalyan 22 | */ 23 | public class TestKryoDruidBaseBean { 24 | @Test 25 | public void testKryoSerDe() { 26 | String expected = "{timestamp=null, b=yoyoy, a=1, c=5.0}"; 27 | byte[] bytes = new byte[32]; 28 | Kryo kryo = new Kryo(); 29 | kryo.register(A.class); 30 | Output output = new Output(bytes); 31 | kryo.writeObject(output, new A()); 32 | Assert.assertEquals(kryo.readObject(new Input(bytes), A.class).toString(), expected); 33 | } 34 | } 35 | class A extends DruidBaseBean { 36 | private String b = "yoyoy"; 37 | private int a = 1; 38 | private double c = 5L; 39 | 40 | public int getA() { 41 | return a; 42 | } 43 | 44 | public void setA(int a) { 45 | this.a = a; 46 | } 47 | 48 | public String getB() { 49 | return b; 50 | } 51 | 52 | public void setB(String b) { 53 | this.b = b; 54 | } 55 | 56 | public double getC() { 57 | return c; 58 | } 59 | 60 | public void setC(double c) { 61 | this.c = c; 62 | } 63 | } 64 | 65 | --------------------------------------------------------------------------------