├── README.md ├── code ├── TestDuff.java └── test ├── jstack ├── brokerProcessThread ├── groupby ├── historicalProcessThread └── historicalResponse ├── note ├── Vectorized_Query_Execution ├── accept_query ├── cache ├── datasketches ├── groupby-multi-valued-dimensions ├── issue ├── liaotian ├── patch ├── query_analysis ├── query_execute ├── segment ├── sql ├── storage └── storage_format ├── pdf └── abadi-column-stores.pdf └── query ├── count.json ├── count_count.json ├── getDataSource ├── run_query_broker.sh ├── run_query_historical.sh └── search_segment.json /README.md: -------------------------------------------------------------------------------- 1 | # druid-learn 2 | druid学习笔记 3 | 4 | -------------------------------------------------------------------------------- /code/TestDuff.java: -------------------------------------------------------------------------------- 1 | 2 | // https://en.wikipedia.org/wiki/Duff%27s_device 3 | // http://stackoverflow.com/questions/514118/how-does-duffs-device-work 4 | // PooledTopNAlgorithm 5 | public class TestDuff { 6 | 7 | public void test1() { 8 | int[] myArray = new int[100]; 9 | for (int i = 0; i < 100; i++) { 10 | myArray[i] += 1; 11 | } 12 | } 13 | 14 | public void test2() { 15 | int[] myArray = new int[100]; 16 | for (int i = 0; i < 100; i += 10) { 17 | myArray[i] += 1; 18 | myArray[i+1] += 1; 19 | myArray[i+2] += 1; 20 | myArray[i+3] += 1; 21 | myArray[i+4] += 1; 22 | myArray[i+5] += 1; 23 | myArray[i+6] += 1; 24 | myArray[i+7] += 1; 25 | myArray[i+8] += 1; 26 | myArray[i+9] += 1; 27 | } 28 | } 29 | 30 | public static void main(String[] args) { 31 | TestDuff duff = new TestDuff(); 32 | long start = System.nanoTime(); 33 | for (int i = 0; i < 100; i++) { 34 | duff.test1(); 35 | } 36 | long end = System.nanoTime(); 37 | System.out.println("Test1 cost " + (end - start)); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /code/test: -------------------------------------------------------------------------------- 1 | 2 | io.druid.segment.incremental.IncrementalIndexStorageAdapterTest debug GroupByQueryEngine 3 | 4 | io.druid.query.topn.TopNQueryRunnerTest 5 | 6 | 7 | -------------------------------------------------------------------------------- /jstack/brokerProcessThread: -------------------------------------------------------------------------------- 1 | 2 | "qtp1401285607-171" daemon prio=10 tid=0x00007f97a1ba2800 nid=0xf3ab waiting on condition [0x00007f977a89d000] 3 | java.lang.Thread.State: WAITING (parking) 4 | at sun.misc.Unsafe.park(Native Method) 5 | - parking to wait for <0x0000000285e8a0c8> (a com.google.common.util.concurrent.AbstractFuture$Sync) 6 | at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) 7 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) 8 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) 9 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) 10 | at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:285) 11 | at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) 12 | at io.druid.client.DirectDruidClient$JsonParserIterator.init(DirectDruidClient.java:475) 13 | at io.druid.client.DirectDruidClient$JsonParserIterator.hasNext(DirectDruidClient.java:438) 14 | at com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:103) 15 | at com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81) 16 | at com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46) 17 | at com.metamx.common.guava.MergeSequence$2.accumulate(MergeSequence.java:66) 18 | at com.metamx.common.guava.MergeSequence$2.accumulate(MergeSequence.java:62) 19 | at com.metamx.common.guava.YieldingAccumulators$1.accumulate(YieldingAccumulators.java:32) 20 | at com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:104) 21 | at com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81) 22 | at com.metamx.common.guava.BaseSequence.accumulate(BaseSequence.java:67) 23 | at com.metamx.common.guava.MergeSequence.toYielder(MergeSequence.java:59) 24 | at com.metamx.common.guava.LazySequence.toYielder(LazySequence.java:43) 25 | at io.druid.query.RetryQueryRunner$1.toYielder(RetryQueryRunner.java:97) 26 | at io.druid.common.guava.CombiningSequence.toYielder(CombiningSequence.java:76) 27 | at com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46) 28 | at io.druid.query.CPUTimeMetricQueryRunner$1.toYielder(CPUTimeMetricQueryRunner.java:95) 29 | at com.metamx.common.guava.Sequences$1.toYielder(Sequences.java:98) 30 | at io.druid.server.QueryResource.doPost(QueryResource.java:162) 31 | at sun.reflect.GeneratedMethodAccessor64.invoke(Unknown Source) 32 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 33 | at java.lang.reflect.Method.invoke(Method.java:606) 34 | at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) 35 | at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) 36 | at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) 37 | at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) 38 | at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) 39 | at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) 40 | at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) 41 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) 42 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) 43 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) 44 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) 45 | at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) 46 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558) 47 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733) 48 | at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) 49 | at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:278) 50 | at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:268) 51 | at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:180) 52 | at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:93) 53 | at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85) 54 | at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) 55 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:132) 56 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:129) 57 | at com.google.inject.servlet.GuiceFilter$Context.call(GuiceFilter.java:206) 58 | at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:129) 59 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 60 | at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83) 61 | at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:364) 62 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 63 | at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) 64 | at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) 65 | at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1125) 66 | at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) 67 | at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) 68 | at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1059) 69 | at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 70 | at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) 71 | at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) 72 | at org.eclipse.jetty.server.Server.handle(Server.java:497) 73 | at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) 74 | at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:248) 75 | at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) 76 | at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:620) 77 | at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:540) 78 | at java.lang.Thread.run(Thread.java:744) 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /jstack/groupby: -------------------------------------------------------------------------------- 1 | 2 | "groupBy_dws_tb_log_wl_part0_[2015-11-29T00:00:00.000+08:00/2015-11-30T00:00:00.000+08:00]" daemon prio=5 tid=0x00007f9b5b35c000 nid=0x7207 runnable [0x000000011bf99000] 3 | java.lang.Thread.State: RUNNABLE 4 | at io.druid.segment.incremental.IncrementalIndex.add(IncrementalIndex.java:446) 5 | - locked <0x00000007e3680098> (a java.util.LinkedHashMap) 6 | at io.druid.query.groupby.GroupByQueryHelper$3.accumulate(GroupByQueryHelper.java:115) 7 | at io.druid.query.groupby.GroupByQueryHelper$3.accumulate(GroupByQueryHelper.java:107) 8 | at com.metamx.common.guava.YieldingAccumulators$1.accumulate(YieldingAccumulators.java:32) 9 | at com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:104) 10 | at com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81) 11 | at com.metamx.common.guava.ConcatSequence.makeYielder(ConcatSequence.java:93) 12 | at com.metamx.common.guava.ConcatSequence.toYielder(ConcatSequence.java:72) 13 | at com.metamx.common.guava.ResourceClosingSequence.toYielder(ResourceClosingSequence.java:41) 14 | at com.metamx.common.guava.YieldingSequenceBase.accumulate(YieldingSequenceBase.java:34) 15 | at io.druid.query.MetricsEmittingQueryRunner$1.accumulate(MetricsEmittingQueryRunner.java:118) 16 | at io.druid.query.MetricsEmittingQueryRunner$1.accumulate(MetricsEmittingQueryRunner.java:118) 17 | at io.druid.query.spec.SpecificSegmentQueryRunner$2$1.call(SpecificSegmentQueryRunner.java:85) 18 | at io.druid.query.spec.SpecificSegmentQueryRunner.doNamed(SpecificSegmentQueryRunner.java:169) 19 | at io.druid.query.spec.SpecificSegmentQueryRunner.access$400(SpecificSegmentQueryRunner.java:39) 20 | at io.druid.query.spec.SpecificSegmentQueryRunner$2.doItNamed(SpecificSegmentQueryRunner.java:160) 21 | at io.druid.query.spec.SpecificSegmentQueryRunner$2.accumulate(SpecificSegmentQueryRunner.java:78) 22 | at io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:116) 23 | at io.druid.query.GroupByParallelQueryRunner$1$1.call(GroupByParallelQueryRunner.java:107) 24 | at java.util.concurrent.FutureTask.run(FutureTask.java:262) 25 | at io.druid.query.PrioritizedExecutorService$PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:222) 26 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) 27 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) 28 | at java.lang.Thread.run(Thread.java:745) 29 | 30 | "qtp1438241568-16" daemon prio=5 tid=0x00007f9b5a272800 nid=0x5d03 waiting on condition [0x00000001183fd000] 31 | java.lang.Thread.State: TIMED_WAITING (parking) 32 | at sun.misc.Unsafe.park(Native Method) 33 | - parking to wait for <0x00000007e3a64140> (a com.google.common.util.concurrent.AbstractFuture$Sync) 34 | at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) 35 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1033) 36 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1326) 37 | at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:268) 38 | at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:96) 39 | at io.druid.query.GroupByParallelQueryRunner.run(GroupByParallelQueryRunner.java:145) 40 | at io.druid.query.groupby.GroupByQueryQueryToolChest.mergeGroupByResults(GroupByQueryQueryToolChest.java:190) 41 | at io.druid.query.groupby.GroupByQueryQueryToolChest.access$000(GroupByQueryQueryToolChest.java:81) 42 | at io.druid.query.groupby.GroupByQueryQueryToolChest$3.run(GroupByQueryQueryToolChest.java:130) 43 | at io.druid.query.FinalizeResultsQueryRunner.run(FinalizeResultsQueryRunner.java:100) 44 | at io.druid.query.BaseQuery.run(BaseQuery.java:78) 45 | at io.druid.query.BaseQuery.run(BaseQuery.java:73) 46 | at io.druid.server.QueryResource.doPost(QueryResource.java:154) 47 | at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source) 48 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 49 | at java.lang.reflect.Method.invoke(Method.java:606) 50 | at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) 51 | at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) 52 | at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) 53 | at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) 54 | at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) 55 | at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) 56 | at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) 57 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) 58 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) 59 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) 60 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) 61 | at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) 62 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558) 63 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733) 64 | at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) 65 | at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:278) 66 | at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:268) 67 | at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:180) 68 | at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:93) 69 | at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85) 70 | at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) 71 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:132) 72 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:129) 73 | at com.google.inject.servlet.GuiceFilter$Context.call(GuiceFilter.java:206) 74 | at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:129) 75 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 76 | at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83) 77 | at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:364) 78 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 79 | at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) 80 | at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) 81 | at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1125) 82 | at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) 83 | at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) 84 | at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1059) 85 | at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 86 | at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) 87 | at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) 88 | at org.eclipse.jetty.server.Server.handle(Server.java:497) 89 | at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) 90 | at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:248) 91 | at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) 92 | at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:620) 93 | at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:540) 94 | at java.lang.Thread.run(Thread.java:745) 95 | 96 | -------------------------------------------------------------------------------- /jstack/historicalProcessThread: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | "qtp793840173-116" daemon prio=10 tid=0x00007fac98aca800 nid=0xc2ca waiting on condition [0x00007fac75cd5000] 6 | java.lang.Thread.State: TIMED_WAITING (parking) 7 | at sun.misc.Unsafe.park(Native Method) 8 | - parking to wait for <0x0000000681fc0a58> (a com.google.common.util.concurrent.AbstractFuture$Sync) 9 | at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) 10 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1033) 11 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1326) 12 | at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:268) 13 | at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:96) 14 | at io.druid.query.ChainedExecutionQueryRunner$1.make(ChainedExecutionQueryRunner.java:157) 15 | at com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:78) 16 | at io.druid.common.guava.CombiningSequence.toYielder(CombiningSequence.java:76) 17 | at com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46) 18 | at io.druid.query.CPUTimeMetricQueryRunner$1.toYielder(CPUTimeMetricQueryRunner.java:95) 19 | at com.metamx.common.guava.Sequences$1.toYielder(Sequences.java:98) 20 | at io.druid.server.QueryResource.doPost(QueryResource.java:162) 21 | at sun.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) 22 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 23 | at java.lang.reflect.Method.invoke(Method.java:606) 24 | at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) 25 | at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) 26 | at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) 27 | at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) 28 | at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) 29 | at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) 30 | at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) 31 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) 32 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) 33 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) 34 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) 35 | at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) 36 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558) 37 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733) 38 | at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) 39 | at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:278) 40 | at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:268) 41 | at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:180) 42 | at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:93) 43 | at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85) 44 | at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) 45 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:132) 46 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:129) 47 | at com.google.inject.servlet.GuiceFilter$Context.call(GuiceFilter.java:206) 48 | at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:129) 49 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 50 | at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83) 51 | at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:364) 52 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 53 | at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) 54 | at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) 55 | at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1125) 56 | at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) 57 | at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) 58 | at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1059) 59 | at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 60 | at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) 61 | at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) 62 | at org.eclipse.jetty.server.Server.handle(Server.java:497) 63 | at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) 64 | at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:248) 65 | at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) 66 | at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:620) 67 | at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:540) 68 | at java.lang.Thread.run(Thread.java:744) 69 | 70 | 71 | 72 | 73 | 74 | "timeseries_dws_wl_sketch_online1_[2016-02-01T00:00:00.000+08:00/2016-02-01T23:59:59.999+08:00]" daemon prio=10 tid=0x00007facb14da800 nid=0xc414 runnable [0x00007fac756d1000] 75 | java.lang.Thread.State: RUNNABLE 76 | at com.yahoo.sketches.theta.HashOperations.hashInsert(HashOperations.java:133) 77 | at com.yahoo.sketches.theta.HeapQuickSelectSketch.hashUpdate(HeapQuickSelectSketch.java:224) 78 | at com.yahoo.sketches.theta.UnionImpl.processVer3(UnionImpl.java:225) 79 | at com.yahoo.sketches.theta.UnionImpl.update(UnionImpl.java:128) 80 | at io.druid.query.aggregation.datasketches.theta.SynchronizedUnion.update(SynchronizedUnion.java:47) 81 | - locked <0x00000006547d0510> (a io.druid.query.aggregation.datasketches.theta.SynchronizedUnion) 82 | at io.druid.query.aggregation.datasketches.theta.SketchAggregator.updateUnion(SketchAggregator.java:107) 83 | at io.druid.query.aggregation.datasketches.theta.SketchAggregator.aggregate(SketchAggregator.java:60) 84 | at io.druid.query.timeseries.TimeseriesQueryEngine$1.apply(TimeseriesQueryEngine.java:67) 85 | at io.druid.query.timeseries.TimeseriesQueryEngine$1.apply(TimeseriesQueryEngine.java:51) 86 | at io.druid.query.QueryRunnerHelper$1.apply(QueryRunnerHelper.java:74) 87 | at io.druid.query.QueryRunnerHelper$1.apply(QueryRunnerHelper.java:69) 88 | at com.metamx.common.guava.MappingYieldingAccumulator.accumulate(MappingYieldingAccumulator.java:57) 89 | at com.metamx.common.guava.FilteringYieldingAccumulator.accumulate(FilteringYieldingAccumulator.java:69) 90 | at com.metamx.common.guava.MappingYieldingAccumulator.accumulate(MappingYieldingAccumulator.java:57) 91 | at com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:104) 92 | at com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81) 93 | at com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46) 94 | at com.metamx.common.guava.ResourceClosingSequence.toYielder(ResourceClosingSequence.java:41) 95 | at com.metamx.common.guava.FilteredSequence.toYielder(FilteredSequence.java:52) 96 | at com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46) 97 | at com.metamx.common.guava.FilteredSequence.toYielder(FilteredSequence.java:52) 98 | at com.metamx.common.guava.ResourceClosingSequence.toYielder(ResourceClosingSequence.java:41) 99 | at com.metamx.common.guava.YieldingSequenceBase.accumulate(YieldingSequenceBase.java:34) 100 | at io.druid.query.MetricsEmittingQueryRunner$1.accumulate(MetricsEmittingQueryRunner.java:118) 101 | at com.metamx.common.guava.MappedSequence.accumulate(MappedSequence.java:40) 102 | at com.metamx.common.guava.Sequences$1.accumulate(Sequences.java:90) 103 | at io.druid.query.MetricsEmittingQueryRunner$1.accumulate(MetricsEmittingQueryRunner.java:118) 104 | at io.druid.query.spec.SpecificSegmentQueryRunner$2$1.call(SpecificSegmentQueryRunner.java:85) 105 | at io.druid.query.spec.SpecificSegmentQueryRunner.doNamed(SpecificSegmentQueryRunner.java:169) 106 | at io.druid.query.spec.SpecificSegmentQueryRunner.access$400(SpecificSegmentQueryRunner.java:39) 107 | at io.druid.query.spec.SpecificSegmentQueryRunner$2.doItNamed(SpecificSegmentQueryRunner.java:160) 108 | at io.druid.query.spec.SpecificSegmentQueryRunner$2.accumulate(SpecificSegmentQueryRunner.java:78) 109 | at io.druid.query.CPUTimeMetricQueryRunner$1.accumulate(CPUTimeMetricQueryRunner.java:83) 110 | at com.metamx.common.guava.Sequences$1.accumulate(Sequences.java:90) 111 | at com.metamx.common.guava.Sequences.toList(Sequences.java:113) 112 | at io.druid.query.ChainedExecutionQueryRunner$1$1$1.call(ChainedExecutionQueryRunner.java:130) 113 | at io.druid.query.ChainedExecutionQueryRunner$1$1$1.call(ChainedExecutionQueryRunner.java:120) 114 | at java.util.concurrent.FutureTask.run(FutureTask.java:262) 115 | at io.druid.query.PrioritizedExecutorService$PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:222) 116 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) 117 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) 118 | at java.lang.Thread.run(Thread.java:744) 119 | 120 | 121 | 122 | 123 | -------------------------------------------------------------------------------- /jstack/historicalResponse: -------------------------------------------------------------------------------- 1 | 2 | "qtp1930776538-86" daemon prio=10 tid=0x00007fb5e8a87800 nid=0x2efb1 waiting on condition [0x00007fb5c64f1000] 3 | java.lang.Thread.State: TIMED_WAITING (parking) 4 | at sun.misc.Unsafe.park(Native Method) 5 | - parking to wait for <0x000000067ea410a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) 6 | at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) 7 | at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2176) 8 | at org.eclipse.jetty.util.SharedBlockingCallback$Blocker.block(SharedBlockingCallback.java:213) 9 | at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:133) 10 | at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:347) 11 | at org.eclipse.jetty.servlets.gzip.DeflatedOutputStream.deflate(DeflatedOutputStream.java:74) 12 | at org.eclipse.jetty.servlets.gzip.DeflatedOutputStream.write(DeflatedOutputStream.java:64) 13 | at org.eclipse.jetty.servlets.gzip.GzipOutputStream.write(GzipOutputStream.java:46) 14 | - locked <0x0000000654a94310> (a org.eclipse.jetty.servlets.gzip.GzipOutputStream) 15 | at org.eclipse.jetty.servlets.gzip.AbstractCompressedStream.write(AbstractCompressedStream.java:226) 16 | at com.sun.jersey.spi.container.servlet.WebComponent$Writer.write(WebComponent.java:300) 17 | at com.sun.jersey.spi.container.ContainerResponse$CommittingOutputStream.write(ContainerResponse.java:135) 18 | at com.fasterxml.jackson.dataformat.smile.SmileGenerator._flushBuffer(SmileGenerator.java:2227) 19 | at com.fasterxml.jackson.dataformat.smile.SmileGenerator._writeBytesLong(SmileGenerator.java:1930) 20 | at com.fasterxml.jackson.dataformat.smile.SmileGenerator._writeBytes(SmileGenerator.java:1891) 21 | at com.fasterxml.jackson.dataformat.smile.SmileGenerator.writeBinary(SmileGenerator.java:1173) 22 | at com.fasterxml.jackson.core.JsonGenerator.writeBinary(JsonGenerator.java:845) 23 | at io.druid.query.aggregation.datasketches.theta.SketchJsonSerializer.serialize(SketchJsonSerializer.java:36) 24 | at io.druid.query.aggregation.datasketches.theta.SketchJsonSerializer.serialize(SketchJsonSerializer.java:30) 25 | at com.fasterxml.jackson.databind.ser.std.MapSerializer.serializeFields(MapSerializer.java:491) 26 | at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:412) 27 | at com.fasterxml.jackson.databind.ser.std.MapSerializer.serialize(MapSerializer.java:27) 28 | at com.fasterxml.jackson.databind.ser.std.JsonValueSerializer.serialize(JsonValueSerializer.java:170) 29 | at com.fasterxml.jackson.databind.ser.BeanPropertyWriter.serializeAsField(BeanPropertyWriter.java:505) 30 | at com.fasterxml.jackson.databind.ser.std.BeanSerializerBase.serializeFields(BeanSerializerBase.java:639) 31 | at com.fasterxml.jackson.databind.ser.BeanSerializer.serialize(BeanSerializer.java:152) 32 | at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:128) 33 | at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:1902) 34 | at com.fasterxml.jackson.core.base.GeneratorBase.writeObject(GeneratorBase.java:280) 35 | at io.druid.jackson.DruidDefaultSerializersModule$5.serialize(DruidDefaultSerializersModule.java:137) 36 | at io.druid.jackson.DruidDefaultSerializersModule$5.serialize(DruidDefaultSerializersModule.java:128) 37 | at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:128) 38 | at com.fasterxml.jackson.databind.ObjectWriter._configAndWriteValue(ObjectWriter.java:800) 39 | at com.fasterxml.jackson.databind.ObjectWriter.writeValue(ObjectWriter.java:642) 40 | at io.druid.server.QueryResource$2.write(QueryResource.java:185) 41 | at com.sun.jersey.core.impl.provider.entity.StreamingOutputProvider.writeTo(StreamingOutputProvider.java:71) 42 | at com.sun.jersey.core.impl.provider.entity.StreamingOutputProvider.writeTo(StreamingOutputProvider.java:57) 43 | at com.sun.jersey.spi.container.ContainerResponse.write(ContainerResponse.java:302) 44 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1510) 45 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) 46 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) 47 | at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) 48 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558) 49 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733) 50 | at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) 51 | at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:278) 52 | at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:268) 53 | at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:180) 54 | at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:93) 55 | at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85) 56 | at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) 57 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:132) 58 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:129) 59 | at com.google.inject.servlet.GuiceFilter$Context.call(GuiceFilter.java:206) 60 | at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:129) 61 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 62 | at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83) 63 | at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:364) 64 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 65 | at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) 66 | at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) 67 | at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1125) 68 | at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) 69 | at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) 70 | at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1059) 71 | at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 72 | at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) 73 | at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) 74 | at org.eclipse.jetty.server.Server.handle(Server.java:497) 75 | at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) 76 | at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:248) 77 | at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) 78 | at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:620) 79 | at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:540) 80 | at java.lang.Thread.run(Thread.java:744) 81 | 82 | 83 | -------------------------------------------------------------------------------- /note/Vectorized_Query_Execution: -------------------------------------------------------------------------------- 1 | 2 | https://cwiki.apache.org/confluence/display/Hive/Vectorized+Query+Execution 3 | 4 | https://issues.apache.org/jira/browse/HIVE-4160 5 | 6 | -------------------------------------------------------------------------------- /note/accept_query: -------------------------------------------------------------------------------- 1 | 2 | "qtp615751745-32" daemon prio=5 tid=0x00007ffd942dd800 nid=0x7c03 waiting on condition [0x0000000114163000] 3 | java.lang.Thread.State: WAITING (parking) 4 | at sun.misc.Unsafe.park(Native Method) 5 | - parking to wait for <0x00000007e2458a90> (a com.google.common.util.concurrent.AbstractFuture$Sync) 6 | at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) 7 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) 8 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) 9 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) 10 | at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:285) 11 | at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) 12 | at io.druid.client.DirectDruidClient$JsonParserIterator.init(DirectDruidClient.java:474) 13 | at io.druid.client.DirectDruidClient$JsonParserIterator.hasNext(DirectDruidClient.java:437) 14 | at com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:103) 15 | at com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81) 16 | at com.metamx.common.guava.MappedSequence.toYielder(MappedSequence.java:46) 17 | at com.metamx.common.guava.MergeSequence$2.accumulate(MergeSequence.java:66) 18 | at com.metamx.common.guava.MergeSequence$2.accumulate(MergeSequence.java:62) 19 | at com.metamx.common.guava.YieldingAccumulators$1.accumulate(YieldingAccumulators.java:32) 20 | at com.metamx.common.guava.BaseSequence.makeYielder(BaseSequence.java:104) 21 | at com.metamx.common.guava.BaseSequence.toYielder(BaseSequence.java:81) 22 | at com.metamx.common.guava.BaseSequence.accumulate(BaseSequence.java:67) 23 | at com.metamx.common.guava.MergeSequence.toYielder(MergeSequence.java:59) 24 | at com.metamx.common.guava.LazySequence.toYielder(LazySequence.java:43) 25 | at io.druid.query.RetryQueryRunner$1.toYielder(RetryQueryRunner.java:97) 26 | at com.metamx.common.guava.YieldingSequenceBase.accumulate(YieldingSequenceBase.java:34) 27 | at io.druid.query.groupby.GroupByQueryQueryToolChest.makeIncrementalIndex(GroupByQueryQueryToolChest.java:246) 28 | at io.druid.query.groupby.GroupByQueryQueryToolChest.mergeGroupByResults(GroupByQueryQueryToolChest.java:190) 29 | at io.druid.query.groupby.GroupByQueryQueryToolChest.access$000(GroupByQueryQueryToolChest.java:81) 30 | at io.druid.query.groupby.GroupByQueryQueryToolChest$3.run(GroupByQueryQueryToolChest.java:130) 31 | at io.druid.query.FinalizeResultsQueryRunner.run(FinalizeResultsQueryRunner.java:100) 32 | at io.druid.query.BaseQuery.run(BaseQuery.java:78) 33 | at io.druid.query.BaseQuery.run(BaseQuery.java:73) 34 | at io.druid.server.QueryResource.doPost(QueryResource.java:154) 35 | at sun.reflect.GeneratedMethodAccessor33.invoke(Unknown Source) 36 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 37 | at java.lang.reflect.Method.invoke(Method.java:606) 38 | at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) 39 | at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) 40 | at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) 41 | at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) 42 | at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) 43 | at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) 44 | at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) 45 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) 46 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) 47 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) 48 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) 49 | at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) 50 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558) 51 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733) 52 | at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) 53 | at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:278) 54 | at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:268) 55 | at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:180) 56 | at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:93) 57 | at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85) 58 | at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) 59 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:132) 60 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:129) 61 | at com.google.inject.servlet.GuiceFilter$Context.call(GuiceFilter.java:206) 62 | at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:129) 63 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 64 | at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83) 65 | at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:364) 66 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 67 | at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) 68 | at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) 69 | at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1125) 70 | at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) 71 | at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) 72 | at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1059) 73 | at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 74 | at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) 75 | at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) 76 | at org.eclipse.jetty.server.Server.handle(Server.java:497) 77 | at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) 78 | at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:248) 79 | at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) 80 | at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:620) 81 | at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:540) 82 | at java.lang.Thread.run(Thread.java:745) 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | "qtp1438241568-16" daemon prio=5 tid=0x00007f9b5a272800 nid=0x5d03 runnable [0x00000001183fe000] 92 | java.lang.Thread.State: RUNNABLE 93 | at java.util.zip.Deflater.deflateBytes(Native Method) 94 | at java.util.zip.Deflater.deflate(Deflater.java:430) 95 | - locked <0x00000007e67e5ac0> (a java.util.zip.ZStreamRef) 96 | at java.util.zip.Deflater.deflate(Deflater.java:352) 97 | at org.eclipse.jetty.servlets.gzip.DeflatedOutputStream.deflate(DeflatedOutputStream.java:71) 98 | at org.eclipse.jetty.servlets.gzip.DeflatedOutputStream.write(DeflatedOutputStream.java:64) 99 | at org.eclipse.jetty.servlets.gzip.GzipOutputStream.write(GzipOutputStream.java:46) 100 | - locked <0x00000007e0407978> (a org.eclipse.jetty.servlets.gzip.GzipOutputStream) 101 | at org.eclipse.jetty.servlets.gzip.AbstractCompressedStream.write(AbstractCompressedStream.java:226) 102 | at com.sun.jersey.spi.container.servlet.WebComponent$Writer.write(WebComponent.java:300) 103 | at com.sun.jersey.spi.container.ContainerResponse$CommittingOutputStream.write(ContainerResponse.java:135) 104 | at com.fasterxml.jackson.dataformat.smile.SmileGenerator._flushBuffer(SmileGenerator.java:2227) 105 | at com.fasterxml.jackson.dataformat.smile.SmileGenerator.flush(SmileGenerator.java:1524) 106 | at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:1904) 107 | at com.fasterxml.jackson.core.base.GeneratorBase.writeObject(GeneratorBase.java:280) 108 | at io.druid.jackson.DruidDefaultSerializersModule$5.serialize(DruidDefaultSerializersModule.java:137) 109 | at io.druid.jackson.DruidDefaultSerializersModule$5.serialize(DruidDefaultSerializersModule.java:128) 110 | at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:128) 111 | at com.fasterxml.jackson.databind.ObjectWriter._configAndWriteValue(ObjectWriter.java:800) 112 | at com.fasterxml.jackson.databind.ObjectWriter.writeValue(ObjectWriter.java:642) 113 | at io.druid.server.QueryResource$2.write(QueryResource.java:185) 114 | at com.sun.jersey.core.impl.provider.entity.StreamingOutputProvider.writeTo(StreamingOutputProvider.java:71) 115 | at com.sun.jersey.core.impl.provider.entity.StreamingOutputProvider.writeTo(StreamingOutputProvider.java:57) 116 | at com.sun.jersey.spi.container.ContainerResponse.write(ContainerResponse.java:302) 117 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1510) 118 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) 119 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) 120 | at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) 121 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558) 122 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733) 123 | at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) 124 | at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:278) 125 | at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:268) 126 | at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:180) 127 | at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:93) 128 | at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85) 129 | at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) 130 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:132) 131 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:129) 132 | at com.google.inject.servlet.GuiceFilter$Context.call(GuiceFilter.java:206) 133 | at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:129) 134 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 135 | at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83) 136 | at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:364) 137 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 138 | at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) 139 | at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) 140 | at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1125) 141 | at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) 142 | at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) 143 | at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1059) 144 | at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 145 | at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) 146 | at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) 147 | at org.eclipse.jetty.server.Server.handle(Server.java:497) 148 | at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) 149 | at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:248) 150 | at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) 151 | at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:620) 152 | at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:540) 153 | at java.lang.Thread.run(Thread.java:745) 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | "qtp1438241568-16" daemon prio=5 tid=0x00007f9b5a272800 nid=0x5d03 waiting on condition [0x00000001183fd000] 163 | java.lang.Thread.State: TIMED_WAITING (parking) 164 | at sun.misc.Unsafe.park(Native Method) 165 | - parking to wait for <0x00000007e3a64140> (a com.google.common.util.concurrent.AbstractFuture$Sync) 166 | at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) 167 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1033) 168 | at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1326) 169 | at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:268) 170 | at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:96) 171 | at io.druid.query.GroupByParallelQueryRunner.run(GroupByParallelQueryRunner.java:145) 172 | at io.druid.query.groupby.GroupByQueryQueryToolChest.mergeGroupByResults(GroupByQueryQueryToolChest.java:190) 173 | at io.druid.query.groupby.GroupByQueryQueryToolChest.access$000(GroupByQueryQueryToolChest.java:81) 174 | at io.druid.query.groupby.GroupByQueryQueryToolChest$3.run(GroupByQueryQueryToolChest.java:130) 175 | at io.druid.query.FinalizeResultsQueryRunner.run(FinalizeResultsQueryRunner.java:100) 176 | at io.druid.query.BaseQuery.run(BaseQuery.java:78) 177 | at io.druid.query.BaseQuery.run(BaseQuery.java:73) 178 | at io.druid.server.QueryResource.doPost(QueryResource.java:154) 179 | at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source) 180 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 181 | at java.lang.reflect.Method.invoke(Method.java:606) 182 | at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) 183 | at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) 184 | at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) 185 | at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) 186 | at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) 187 | at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) 188 | at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) 189 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) 190 | at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) 191 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) 192 | at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) 193 | at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) 194 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558) 195 | at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733) 196 | at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) 197 | at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:278) 198 | at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:268) 199 | at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:180) 200 | at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:93) 201 | at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85) 202 | at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) 203 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:132) 204 | at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:129) 205 | at com.google.inject.servlet.GuiceFilter$Context.call(GuiceFilter.java:206) 206 | at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:129) 207 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 208 | at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83) 209 | at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:364) 210 | at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) 211 | at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) 212 | at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) 213 | at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1125) 214 | at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) 215 | at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) 216 | at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1059) 217 | at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) 218 | at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) 219 | at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) 220 | at org.eclipse.jetty.server.Server.handle(Server.java:497) 221 | at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) 222 | at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:248) 223 | at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) 224 | at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:620) 225 | at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:540) 226 | at java.lang.Thread.run(Thread.java:745) 227 | 228 | 229 | 230 | 231 | 232 | -------------------------------------------------------------------------------- /note/cache: -------------------------------------------------------------------------------- 1 | 2 | 官方推荐： 3 | if your cluster is getting large, disable the cache on the broker and turn on the cache on the historical, it will allow historicals to merge their own results 4 | otherwise, the broker has to do the merge for all the segments 5 | 6 | (1) GROUP_BY 和 SELECT 两种query不使用cache 7 | (2) Broker开启了cache，那么它会设置传给historical的populateCache为false，bySegment为true。 8 | 9 | 10 | -------------------------------------------------------------------------------- /note/datasketches: -------------------------------------------------------------------------------- 1 | 2 | https://github.com/druid-io/druid/blob/master/docs/content/development/datasketches-aggregators.md 3 | 4 | 5 | -------------------------------------------------------------------------------- /note/groupby-multi-valued-dimensions: -------------------------------------------------------------------------------- 1 | 2 | https://github.com/druid-io/druid/blob/master/docs/content/querying/multi-valued-dimensions.md 3 | 4 | https://github.com/druid-io/druid/pull/2130/files 5 | 6 | -------------------------------------------------------------------------------- /note/issue: -------------------------------------------------------------------------------- 1 | 2 | Why groupBy is slower than topN 3 | https://groups.google.com/forum/#!searchin/druid-development/topn/druid-development/zUKGEkSAAMo/uLIGuuYjTO4J 4 | 5 | 6 | -------------------------------------------------------------------------------- /note/liaotian: -------------------------------------------------------------------------------- 1 | 2 | 3 | [13:09] @fj http请求怎么在broker和historical里面排队的？ 4 | [13:10] 我们发现请求实际处理时间只有几百ms，但是整体时间很长 5 | [13:12] <@fj> binlijin, look at the metric: 6 | [13:13] <@fj> "segment/scan/pending" 7 | [13:13] <@fj> on historical 8 | [13:13] <@fj> it tells you how many segments are pending waiting to be scanned 9 | [13:14] <@fj> binlijin, segments are stored in a priority queue waiting to be scanned 10 | [13:14] ok 11 | [13:15] 我看看 12 | [13:15] <@fj> look at how PrioritizedExecutorService.java is used 13 | [13:16] thanks 14 | [13:16] <@fj> if segment/scan/pending is high, it means too many concurrent queries (needs more CPUs) or segment/scan/time taking too long 15 | [13:19] <@fj> binlijin, main reasons things are slow 16 | [13:19] <@fj> 1) segment scans taking too long 17 | [13:19] <@fj> 2) too many pending segments 18 | [13:19] <@fj> 3) merge time on broker is slow 19 | [13:19] <@fj> 4) network time 20 | [13:19] <@fj> 5) too much paging of segments in and out of memory 21 | [13:20] <@fj> binlijin: http://druid.io/docs/latest/operations/metrics.html 22 | [13:21] Good! 23 | [13:21] <@fj> binlijin: sys/disk/read/size is a goo dmetric to see how much is being read from disk 24 | [13:23] {“feed":"metrics","timestamp":"2016-01-07T17:46:58.005+08:00","service":"broker","host":"host060:8082","metric":"query/node/time","value":3286,"dataSource":"dws4_pc_sketch","duration":"PT86400S","hasFilters":"false","id":"ea7e64bb-403e-418c-a4b9-d93566853c3c","interval":["2016-01-04T00:00:00.000+08:00/2016-01-05T00:00:00.000+08:00"],"numComplexMetrics":"0","numMetrics":"1","server":"host074:8083","type":"timeseries","cluster":"dru 25 | [13:23] {"feed":"metrics","timestamp":"2016-01-07T17:46:58.004+08:00","service":"historical","host":"host074:8083","metric":"query/time","value":282,"context":"{\"bySegment\":true,\"finalize\":false,\"intermediate\":true,\"populateCache\":false,\"priority\":0,\"queryId\":\"ea7e64bb-403e-418c-a4b9-d93566853c3c\",\"timeout\":300000}","dataSource":"dws4_pc_sketch","duration":"PT86400S","hasFilters":"false","id":"ea7e64bb-403e-418c-a4b9-d93566853c3c","interval 26 | [13:23] 这是一个请求里面的两个metrics 27 | [13:23] broker这边看到的是"metric":"query/node/time","value":3286 28 | [13:24] 而historical上面的metric是"metric":"query/time","value":282 29 | [13:24] 所以应该是这个请求在排队。 30 | [13:26] 从historical的日志来看"segment/scan/pending"基本上未0，偶尔是1 31 | [13:27] 貌似还是在其他地方。 32 | [13:28] <@fj> binlijin, how many historical were hit? 33 | [13:28] <@fj> binlijin, if your cluster is getting large, disable the cache on the broker and turn on the cache on the historical, it will allow historicals to merge their own results 34 | [13:29] <@fj> otherwise, the broker has to do the merge for all the segments 35 | [13:31] current we turn on the cache on both broker and historical 36 | [13:31] <@fj> okay 37 | [13:31] <@fj> so i see this 38 | [13:31] <@fj> \"bySegment\":true 39 | [13:31] <@fj> this means historical is returning per segment results and not doing local merging 40 | [13:31] <@fj> and broker is doing all the merging 41 | [13:32] ok 42 | [13:32] <@fj> druid will cache per segment results 43 | [13:32] <@fj> so if you turn on caching on the broker 44 | [13:32] <@fj> no merging occurs at historical level 45 | [13:32] <@fj> because broker needs to cache every segment's results 46 | [13:32] <@fj> quick test: 47 | [13:32] <@fj> issue same query with this "context" 48 | [13:33] <@fj> "context" : {"useCache":"false", "populateCache":"false"} 49 | [13:33] <@fj> this will disable caching 50 | [13:33] <@fj> for the query 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /note/patch: -------------------------------------------------------------------------------- 1 | 重要的patch跟踪 2 | 3 | Towards consistent null handling 4 | https://github.com/druid-io/druid/pull/995 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /note/query_analysis: -------------------------------------------------------------------------------- 1 | 2 | 3 | （1）StorageAdapter（QueryableIndexStorageAdapter/IncrementalIndexStorageAdapter）返回 Sequence 4 | 输入参数是(Filter filter, Interval interval, QueryGranularity gran) 5 | 6 | （2）然后各种query根据Sequence进行操作。 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /note/query_execute: -------------------------------------------------------------------------------- 1 | 2 | (1) QueryResource.doPost 3 | $ grep -R "QuerySegmentWalker.class" . 4 | ./services/src/main/java/io/druid/cli/CliBridge.java: binder.bind(QuerySegmentWalker.class).to(BridgeQuerySegmentWalker.class).in(LazySingleton.class); 5 | ./services/src/main/java/io/druid/cli/CliBroker.java: binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); 6 | ./services/src/main/java/io/druid/cli/CliHistorical.java: binder.bind(QuerySegmentWalker.class).to(ServerManager.class).in(LazySingleton.class); 7 | ./services/src/main/java/io/druid/cli/CliPeon.java: binder.bind(QuerySegmentWalker.class).to(ThreadPoolTaskRunner.class); 8 | ./services/src/main/java/io/druid/guice/RealtimeModule.java: binder.bind(QuerySegmentWalker.class).to(RealtimeManager.class).in(ManageLifecycle.class); 9 | (2) Query.run(QuerySegmentWalker walker, Map context) 10 | (3) BaseQuery.run(QuerySegmentWalker walker, Map context) 11 | @Override 12 | public Sequence run(QuerySegmentWalker walker, Map context) 13 | { 14 | return run(querySegmentSpec.lookup(this, walker), context); 15 | } 16 | 17 | public Sequence run(QueryRunner runner, Map context) 18 | { 19 | return runner.run(this, context); 20 | } 21 | 22 | 23 | -------------------------------------------------------------------------------- /note/segment: -------------------------------------------------------------------------------- 1 | IncrementalIndexSegment 未持久化，在内存中的。 2 | QueryableIndexSegment 已经写到文件中，持久化后的数据。 3 | 4 | 5 | -------------------------------------------------------------------------------- /note/sql: -------------------------------------------------------------------------------- 1 | GroupBy 2 | Count(distinct) 3 | Filter 4 | Sum 5 | 6 | http://druidwithsql.tumblr.com/post/98578718282/a-first-look-at-druid-with-sql 7 | 8 | SELECT page, LONG_SUM(count) AS edit_count 9 | FROM wikipedia 10 | WHERE interval BETWEEN 2010-01-01T00:00:00.000Z AND 2020-01-01T00:00:00.000Z AND country='United States' 11 | BREAK BY 'minute' 12 | GROUP BY page; 13 | 14 | https://github.com/srikalyc/Sql4D/wiki/Sql4DCompiler 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /note/storage: -------------------------------------------------------------------------------- 1 | V9 2 | (1)index.drd 3 | 第一个string集合，所有的columns：包括dimensions和metrics 4 | 第二个string集合，所有的dimensions 5 | 最大最小时间 6 | bitmap类型 7 | (2)time column 8 | 所有的LONG 9 | (3)dimension columns 10 | ValueType目前都是String 11 | hasMultipleValues是否有多值 12 | Column value dictionary，value按照大小排序编号。 13 | 正排，存每个doc该字段的value对应的Integer id。 14 | 倒排，目前只支持bitmap，就是各个value在哪些doc里面出现过。 15 | Segment格式可以见http://druid.io/docs/latest/design/segments.html 16 | (4)metric columns 17 | 三种数据类型(LONG,FLOAT,COMPLEX) 18 | (5)version.bin 19 | 只存版本号 20 | -------------------------------------------------------------------------------- /note/storage_format: -------------------------------------------------------------------------------- 1 | 2 | 数据存储格式： 3 | （1）通用存储格式 GenericIndexed/GenericIndexedWriter，可以Streams方式生成。 4 | 分两部分：header/values, header存offset，values存实际的数据，所以根据index可以很快定位到读取出所在位置的数据。 5 | 写的时候是两个文件单独生成，最后面会把两个文件合到一起，加上一个头部meta文件。 6 | 说明: 7 | 往GenericIndexedWriter里面写3个String："", "100", "200" 8 | 通过GenericIndexed读上来的结果是：null, "100", "200" 9 | 注意这里写下去的是""空字符串，读上来的是null。 10 | 11 | （2）time column存储成CompressedLongs 65536/8=8192 12 | 详见CompressedLongsIndexedSupplier，默认LZ4压缩，底层基于GenericIndexed格式，但是做了压缩。 13 | 详细存储格式: 14 | 分成8192个long一段，这8192个long使用LZ4压缩(https://github.com/jpountz/lz4-java), 压缩完后,写到GenericIndexed里。 15 | 读取的时候先根据index/8192算出来在那个段里面，然后index%8192算出在这个段里面的偏移，最终获取到数据。 16 | 17 | （3）metric columns 18 | long类型，跟time column一样的存储格式 19 | float类型，CompressedFloats（CompressedFloatsIndexedSupplier）跟CompressedLongs类似存储，只不过float是4个byte，所以一个段里面有16384个元素。 20 | complex类型，直接用GenericIndexed存储，不分段。 21 | 22 | (4) dimension columns 23 | Column value dictionary: 存成GenericIndexed 24 | 倒排: 存成GenericIndexed 25 | 正排: 26 | 单值 CompressedVSizeIntsIndexedSupplier 27 | 多值 CompressedVSizeIndexedSupplier 28 | Spatial indexes: ImmutableRTree 29 | 30 | CompressedVSizeIntsIndexedSupplier分析: 31 | 1 chunkFactor 65536 32 | 1 chunkBytes 65536 33 | 2 chunkFactor 32768 34 | 2 chunkBytes 65536 35 | 3 chunkFactor 16384 36 | 3 chunkBytes 49153 37 | 4 chunkFactor 16384 38 | 4 chunkBytes 65536 39 | 分段存储压缩，跟CompressedLongs存储一样，底层是GenericIndexed。 40 | 这里针对内容大小根据最大的int需要几个byte有优化，如果只要一个byte一个段里面可以存储65536个，如果是2个byte那么可以存32768个，如果3byte那么存16384个，如果4个byte那么存16384个。 41 | 每个段都是65536个byte大小。 42 | 读取的时候根据index直接读取到该位置上面的值。 43 | 44 | CompressedVSizeIndexedSupplier分析: 45 | 底层存储有两块: 46 | offsetSupplier (CompressedVSizeIntsIndexedSupplier) 47 | //offsets - indexed integers of length num of rows + 1 representing offsets of starting index of first element of each row in values index 48 | // last element represents the length of values column 49 | valueSupplier (CompressedVSizeIntsIndexedSupplier) values - indexed integers representing actual values in each row 50 | List offsetList = new ArrayList<>(); 51 | List values = new ArrayList<>(); 52 | 读取的时候很简单： 53 | @Override 54 | public IndexedInts get(int index) 55 | { 56 | final int offset = offsets.get(index); 57 | final int size = offsets.get(index + 1) - offset; 58 | return new IndexedInts() 59 | { 60 | @Override 61 | public int size() 62 | { 63 | return size; 64 | } 65 | 66 | @Override 67 | public int get(int index) 68 | { 69 | if (index >= size) { 70 | throw new IllegalArgumentException(String.format("Index[%s] >= size[%s]", index, size)); 71 | } 72 | return values.get(index + offset); 73 | } 74 | } 75 | } 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /pdf/abadi-column-stores.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binlijin/druid-learn/3b6e2aecb1f5892b694ec3a79807d43587cb7525/pdf/abadi-column-stores.pdf -------------------------------------------------------------------------------- /query/count.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataSource":"tableName", 3 | "queryType":"timeseries", 4 | "granularity":"day", 5 | "intervals":["2016-03-01/2100-01-01"], 6 | "aggregations":[ 7 | { 8 | "fieldName":"count", 9 | "name":"count", 10 | "type":"longSum" 11 | } 12 | ] 13 | } 14 | 15 | -------------------------------------------------------------------------------- /query/count_count.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataSource":"tableName", 3 | "queryType":"timeseries", 4 | "granularity":"day", 5 | "intervals":["2015-01-01/2100-01-01"], 6 | "aggregations":[ 7 | { 8 | "fieldName":"count", 9 | "name":"count", 10 | "type":"count" 11 | } 12 | ] 13 | } 14 | 15 | -------------------------------------------------------------------------------- /query/getDataSource: -------------------------------------------------------------------------------- 1 | 2 | curl -X 'GET' http://localhost:8081/druid/coordinator/v1/metadata/datasources/wap_benchmark/segments?full 3 | 4 | 5 | -------------------------------------------------------------------------------- /query/run_query_broker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | time curl -X POST 'http://localhost:8082/druid/v2/?pretty' -H 'content-type: application/json' -d@$1 4 | 5 | -------------------------------------------------------------------------------- /query/run_query_historical.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | time curl -X POST 'http://localhost:8083/druid/v2/?pretty' -H 'content-type: application/json' -d@$1 4 | 5 | -------------------------------------------------------------------------------- /query/search_segment.json: -------------------------------------------------------------------------------- 1 | { 2 | "queryType": "search", 3 | "dataSource": { 4 | "type": "table", 5 | "name": "log_wl" 6 | }, 7 | "filter": null, 8 | "granularity": { 9 | "type": "all" 10 | }, 11 | "limit": 2147483647, 12 | "intervals": { 13 | "type": "segments", 14 | "segments": [ 15 | { 16 | "itvl": "2015-11-29T00:00:00.000+08:00/2015-11-30T00:00:00.000+08:00", 17 | "ver": "2016-03-09T13:16:34.206+08:00", 18 | "part": 0 19 | } 20 | ] 21 | }, 22 | "searchDimensions": [ 23 | { 24 | "type": "default", 25 | "dimension": "visitor_id", 26 | "outputName": "visitor_id" 27 | } 28 | ], 29 | "query": { 30 | "type": "insensitive_contains", 31 | "value": "" 32 | }, 33 | "sort": { 34 | "type": "lexicographic" 35 | }, 36 | "context": { 37 | "useCache":"false", 38 | "populateCache":"false", 39 | "queryId": "291825be-3cc9-1795-995c-ed74127833bf:3:1" 40 | } 41 | } 42 | 43 | --------------------------------------------------------------------------------