├── .gitignore ├── src ├── test │ ├── resources │ │ ├── empty_schema.json │ │ ├── custom_config.yaml │ │ ├── storage_config.yaml │ │ ├── log4j2.properties │ │ ├── suppressions.xml │ │ └── test_config.yaml │ └── java │ │ └── com │ │ └── yahoo │ │ └── bullet │ │ ├── common │ │ ├── BulletErrorTest.java │ │ └── BulletExceptionTest.java │ │ ├── windowing │ │ └── ClosableStrategy.java │ │ ├── query │ │ ├── aggregations │ │ │ ├── DistributionTypeTest.java │ │ │ ├── CountDistinctTest.java │ │ │ ├── LinearDistributionTest.java │ │ │ ├── GroupAllTest.java │ │ │ └── TopKTest.java │ │ ├── WindowUtils.java │ │ ├── postaggregations │ │ │ ├── HavingTest.java │ │ │ ├── CullingTest.java │ │ │ ├── ComputationTest.java │ │ │ └── OrderByTest.java │ │ ├── tablefunctions │ │ │ ├── LateralViewTest.java │ │ │ └── ExplodeTest.java │ │ ├── expressions │ │ │ ├── ExpressionUtils.java │ │ │ ├── ListExpressionTest.java │ │ │ ├── UnaryExpressionTest.java │ │ │ ├── ValueExpressionTest.java │ │ │ ├── NAryExpressionTest.java │ │ │ ├── CastExpressionTest.java │ │ │ └── BinaryExpressionTest.java │ │ ├── FieldTest.java │ │ └── ProjectionTest.java │ │ ├── querying │ │ ├── partitioning │ │ │ └── MockPartitioner.java │ │ ├── aggregations │ │ │ ├── NoSerDeBulletRecord.java │ │ │ ├── AggregationUtils.java │ │ │ ├── StrategyTest.java │ │ │ └── MockStrategy.java │ │ ├── evaluators │ │ │ ├── EvaluatorUtils.java │ │ │ ├── ValueEvaluatorTest.java │ │ │ ├── CastEvaluatorTest.java │ │ │ ├── ListEvaluatorTest.java │ │ │ ├── UnaryEvaluatorTest.java │ │ │ ├── BinaryEvaluatorTest.java │ │ │ └── NAryEvaluatorTest.java │ │ ├── FilterTest.java │ │ ├── RateLimitErrorTest.java │ │ ├── postaggregations │ │ │ ├── HavingStrategyTest.java │ │ │ └── CullingStrategyTest.java │ │ └── RunningQueryTest.java │ │ ├── pubsub │ │ ├── PubSubExceptionTest.java │ │ ├── PubSubResponderTest.java │ │ ├── PublisherTest.java │ │ ├── SubscriberTest.java │ │ ├── rest │ │ │ └── RESTMetadataTest.java │ │ ├── MockPubSub.java │ │ └── IdentityPubSubMessageSerDeTest.java │ │ ├── result │ │ └── JSONFormatterTest.java │ │ └── storage │ │ └── StorageConfigTest.java └── main │ ├── resources │ ├── copyright.txt │ ├── log4j2.properties │ └── rest_pubsub_defaults.yaml │ └── java │ └── com │ └── yahoo │ └── bullet │ ├── query │ ├── tablefunctions │ │ ├── TableFunctionType.java │ │ ├── OuterableTableFunction.java │ │ ├── TableFunction.java │ │ ├── LateralView.java │ │ └── Explode.java │ ├── aggregations │ │ ├── AggregationType.java │ │ ├── DistributionType.java │ │ ├── Raw.java │ │ ├── Distribution.java │ │ ├── LinearDistribution.java │ │ ├── GroupAll.java │ │ ├── CountDistinct.java │ │ ├── Aggregation.java │ │ ├── TopK.java │ │ └── ManualDistribution.java │ ├── postaggregations │ │ ├── PostAggregationType.java │ │ ├── PostAggregation.java │ │ ├── Having.java │ │ ├── Culling.java │ │ └── Computation.java │ ├── expressions │ │ ├── Expression.java │ │ ├── ListExpression.java │ │ ├── CastExpression.java │ │ ├── UnaryExpression.java │ │ ├── ValueExpression.java │ │ ├── NAryExpression.java │ │ └── BinaryExpression.java │ ├── Field.java │ └── Projection.java │ ├── common │ ├── Closable.java │ ├── Configurable.java │ ├── Initializable.java │ ├── BulletException.java │ ├── metrics │ │ ├── MetricEventPublisher.java │ │ └── MetricEvent.java │ ├── RandomPool.java │ ├── BulletError.java │ └── SerializerDeserializer.java │ ├── querying │ ├── postaggregations │ │ ├── PostStrategy.java │ │ ├── CullingStrategy.java │ │ ├── ComputationStrategy.java │ │ └── HavingStrategy.java │ ├── aggregations │ │ ├── grouping │ │ │ ├── GroupDataSummarySetOperations.java │ │ │ └── GroupDataSummaryFactory.java │ │ ├── Strategy.java │ │ ├── KMVStrategy.java │ │ └── GroupAllStrategy.java │ ├── partitioning │ │ └── Partitioner.java │ ├── evaluators │ │ ├── Evaluator.java │ │ ├── ValueEvaluator.java │ │ ├── CastEvaluator.java │ │ ├── UnaryEvaluator.java │ │ ├── BinaryEvaluator.java │ │ ├── ListEvaluator.java │ │ └── NAryEvaluator.java │ ├── tablefunctors │ │ ├── TableFunctor.java │ │ ├── OuterableTableFunctor.java │ │ └── LateralViewFunctor.java │ ├── Filter.java │ ├── RunningQuery.java │ └── RateLimitError.java │ ├── pubsub │ ├── IdentityPubSubMessageSerDe.java │ ├── rest │ │ ├── RESTMetadata.java │ │ ├── RESTResultPublisher.java │ │ └── RESTQueryPublisher.java │ ├── PubSubResponder.java │ ├── Publisher.java │ ├── Subscriber.java │ ├── PubSubException.java │ └── BulletPubSubResponder.java │ ├── windowing │ ├── AdditiveTumbling.java │ └── Tumbling.java │ ├── result │ └── JSONFormatter.java │ └── storage │ ├── NullStorageManager.java │ ├── MemoryStorageManager.java │ └── StorageConfig.java ├── screwdriver ├── pubring.gpg.enc ├── secring.gpg.enc ├── release.sh └── settings.xml ├── screwdriver.yaml ├── Makefile └── Contributing.md /.gitignore: -------------------------------------------------------------------------------- 1 | /target/** 2 | pom.xml.bak 3 | -------------------------------------------------------------------------------- /src/test/resources/empty_schema.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /screwdriver/pubring.gpg.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bullet-db/bullet-core/HEAD/screwdriver/pubring.gpg.enc -------------------------------------------------------------------------------- /screwdriver/secring.gpg.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bullet-db/bullet-core/HEAD/screwdriver/secring.gpg.enc -------------------------------------------------------------------------------- /src/test/resources/custom_config.yaml: -------------------------------------------------------------------------------- 1 | my.custom.list: 2 | - foo 3 | - bar 4 | my.custom.map: 5 | first: 10 6 | second: 42 7 | empty.string: "" 8 | -------------------------------------------------------------------------------- /src/test/resources/storage_config.yaml: -------------------------------------------------------------------------------- 1 | bullet.storage.partition.count: 15 2 | bullet.storage.fake.setting: foo 3 | bullet.storage.namespaces: 4 | - one 5 | - two 6 | - three 7 | - eight 8 | -------------------------------------------------------------------------------- /src/main/resources/copyright.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | status=warn 2 | 3 | appender.console.type=Console 4 | appender.console.name=STDOUT 5 | appender.console.layout.type=PatternLayout 6 | appender.console.layout.pattern=[%-5p] %d{ISO8601} %c - %m%n 7 | 8 | rootLogger.level=info 9 | rootLogger.appenderRef.stdout.ref=STDOUT 10 | -------------------------------------------------------------------------------- /src/test/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | status=warn 2 | 3 | appender.console.type=Console 4 | appender.console.name=STDOUT 5 | appender.console.layout.type=PatternLayout 6 | appender.console.layout.pattern=[%-5p] %d{ISO8601} %c - %m%n 7 | 8 | rootLogger.level=fatal 9 | rootLogger.appenderRef.stdout.ref=STDOUT 10 | -------------------------------------------------------------------------------- /src/test/resources/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /screwdriver/release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | export GPG_TTY=$(tty) 6 | 7 | openssl aes-256-cbc -pass pass:$GPG_ENCPHRASE -in screwdriver/pubring.gpg.enc -out screwdriver/pubring.gpg -d 8 | openssl aes-256-cbc -pass pass:$GPG_ENCPHRASE -in screwdriver/secring.gpg.enc -out screwdriver/secring.gpg -d 9 | 10 | mvn clean deploy --settings screwdriver/settings.xml -DskipTests=true -Possrh -Prelease 11 | 12 | rm -rf screwdriver/*.gpg 13 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/tablefunctions/TableFunctionType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.tablefunctions; 7 | 8 | /** 9 | * Represents the type of the {@link TableFunction}. 10 | */ 11 | public enum TableFunctionType { 12 | LATERAL_VIEW, 13 | EXPLODE 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/common/Closable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common; 7 | 8 | public interface Closable { 9 | /** 10 | * Returns true if this is currently closed. 11 | * 12 | * @return A boolean denoting whether this object is current closed. 13 | */ 14 | boolean isClosed(); 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/AggregationType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | /** 9 | * Represents the type of the {@link Aggregation}. 10 | */ 11 | public enum AggregationType { 12 | GROUP, 13 | COUNT_DISTINCT, 14 | TOP_K, 15 | DISTRIBUTION, 16 | RAW 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/postaggregations/PostAggregationType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.postaggregations; 7 | 8 | /** 9 | * Represents the type of the {@link PostAggregation}. 10 | */ 11 | public enum PostAggregationType { 12 | HAVING, 13 | COMPUTATION, 14 | ORDER_BY, 15 | CULLING; 16 | } 17 | -------------------------------------------------------------------------------- /src/test/resources/test_config.yaml: -------------------------------------------------------------------------------- 1 | bullet.query.max.duration.ms: 10000 2 | bullet.query.default.duration.ms: 1000 3 | bullet.query.aggregation.max.size: 100 4 | # Some random fake setting 5 | fake.setting: null 6 | bullet.pubsub.context.name: "QUERY_SUBMISSION" 7 | bullet.pubsub.class.name: "com.yahoo.bullet.pubsub.MockPubSub" 8 | bullet.pubsub.rest.subscriber.connect.timeout.ms: 88 9 | bullet.pubsub.rest.query.urls: 10 | - "http://localhost:9901/CUSTOM/query" 11 | - "http://localhost:9902/CUSTOM/query" 12 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/DistributionType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import lombok.AllArgsConstructor; 9 | import lombok.Getter; 10 | 11 | @Getter @AllArgsConstructor 12 | public enum DistributionType { 13 | QUANTILE("QUANTILE"), 14 | PMF("FREQ"), 15 | CDF("CUMFREQ"); 16 | 17 | private String name; 18 | } 19 | -------------------------------------------------------------------------------- /screwdriver.yaml: -------------------------------------------------------------------------------- 1 | cache: 2 | pipeline: ["~/.m2"] 3 | 4 | shared: 5 | image: maven:3.6.3-jdk-8 6 | 7 | jobs: 8 | main: 9 | requires: [~pr, ~commit] 10 | secrets: 11 | - COVERALLS_TOKEN 12 | steps: 13 | - build: mvn -B clean verify 14 | - coverage: mvn coveralls:report 15 | 16 | release: 17 | requires: [~tag:/^bullet-core-\d+\.\d+\.\d+/] 18 | secrets: 19 | - SONATYPE_USERNAME 20 | - SONATYPE_PASSWORD 21 | - GPG_PASSPHRASE 22 | - GPG_ENCPHRASE 23 | steps: 24 | - publish: screwdriver/release.sh 25 | 26 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/common/Configurable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common; 7 | 8 | public interface Configurable { 9 | /** 10 | * Takes a {@link BulletConfig} containing configuration and applies it to itself. 11 | * 12 | * @param configuration The configuration containing the settings. 13 | */ 14 | void configure(BulletConfig configuration); 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/postaggregations/PostStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.postaggregations; 7 | 8 | import com.yahoo.bullet.result.Clip; 9 | 10 | public interface PostStrategy { 11 | /** 12 | * Executes the post aggregation. 13 | * 14 | * @param clip The input {@link Clip}. 15 | * @return The output {@link Clip}. 16 | */ 17 | Clip execute(Clip clip); 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/common/BulletErrorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common; 7 | 8 | import org.testng.Assert; 9 | import org.testng.annotations.Test; 10 | 11 | public class BulletErrorTest { 12 | @Test 13 | public void testToString() { 14 | BulletError error = BulletError.makeError("foo", "bar"); 15 | Assert.assertEquals(error.toString(), "{error: foo, resolutions: [bar]}"); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/common/Initializable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common; 7 | 8 | import java.util.List; 9 | import java.util.Optional; 10 | 11 | public interface Initializable { 12 | /** 13 | * Validates and initializes this object. 14 | * 15 | * @return An {@link Optional} {@link List} of {@link BulletError} in this object or its constituents. 16 | */ 17 | Optional> initialize(); 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/windowing/ClosableStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.windowing; 7 | 8 | import com.yahoo.bullet.querying.aggregations.MockStrategy; 9 | import lombok.Setter; 10 | 11 | public class ClosableStrategy extends MockStrategy { 12 | @Setter 13 | private boolean closed = false; 14 | 15 | @Override 16 | public boolean isClosed() { 17 | super.isClosed(); 18 | return closed; 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/aggregations/DistributionTypeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import org.testng.Assert; 9 | import org.testng.annotations.Test; 10 | 11 | public class DistributionTypeTest { 12 | @Test 13 | public void testGetName() { 14 | Assert.assertEquals(DistributionType.QUANTILE.getName(), "QUANTILE"); 15 | Assert.assertEquals(DistributionType.PMF.getName(), "FREQ"); 16 | Assert.assertEquals(DistributionType.CDF.getName(), "CUMFREQ"); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/partitioning/MockPartitioner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.partitioning; 7 | 8 | import com.yahoo.bullet.query.Query; 9 | import com.yahoo.bullet.record.BulletRecord; 10 | 11 | import java.util.Set; 12 | 13 | public class MockPartitioner implements Partitioner { 14 | @Override 15 | public Set getKeys(Query query) { 16 | return null; 17 | } 18 | 19 | @Override 20 | public Set getKeys(BulletRecord record) { 21 | return null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: full 2 | 3 | full: 4 | mvn clean javadoc:jar package 5 | 6 | clean: 7 | mvn clean 8 | 9 | test: 10 | mvn clean verify 11 | 12 | jar: 13 | mvn clean package 14 | 15 | release: 16 | mvn -B release:prepare release:clean 17 | 18 | coverage: 19 | mvn clean clover2:setup test clover2:aggregate clover2:clover 20 | 21 | doc: 22 | mvn clean javadoc:javadoc 23 | 24 | cc: see-coverage 25 | 26 | see-coverage: coverage 27 | cd target/site/clover; python -m SimpleHTTPServer 28 | 29 | cd: see-doc 30 | 31 | see-doc: doc 32 | cd target/site/apidocs; python -m SimpleHTTPServer 33 | 34 | fix-javadocs: 35 | mvn javadoc:fix -DfixClassComment=false -DfixFieldComment=false 36 | 37 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/common/BulletExceptionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common; 7 | 8 | import org.testng.Assert; 9 | import org.testng.annotations.Test; 10 | 11 | public class BulletExceptionTest { 12 | @Test 13 | public void testWrappingExceptions() { 14 | BulletException exception1 = new BulletException(new BulletError("foo", "bar")); 15 | BulletException exception2 = new BulletException("foo", "bar"); 16 | 17 | Assert.assertNotNull(exception1.getError()); 18 | Assert.assertNotNull(exception2.getError()); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/tablefunctions/OuterableTableFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.tablefunctions; 7 | 8 | import lombok.Getter; 9 | 10 | /** 11 | * The OuterableTableFunction class adds the outer option. When outer is specified, a table function that generates no 12 | * records from a Bullet record will generate an empty record instead. 13 | */ 14 | @Getter 15 | public abstract class OuterableTableFunction extends TableFunction { 16 | private static final long serialVersionUID = 3175147159913255802L; 17 | 18 | protected final boolean outer; 19 | 20 | public OuterableTableFunction(boolean outer, TableFunctionType type) { 21 | super(type); 22 | this.outer = outer; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/WindowUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query; 7 | 8 | public class WindowUtils { 9 | public static Window makeTumblingWindow(Integer emitValue) { 10 | return new Window(emitValue, Window.Unit.TIME); 11 | } 12 | 13 | public static Window makeSlidingWindow(Integer emitValue) { 14 | return new Window(emitValue, Window.Unit.RECORD); 15 | } 16 | 17 | public static Window makeWindow(Window.Unit emitUnit, Integer emitValue, Window.Unit includeUnit, Integer includeValue) { 18 | if (includeUnit != null) { 19 | return new Window(emitValue, emitUnit, includeUnit, includeValue); 20 | } 21 | return new Window(emitValue, emitUnit); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/postaggregations/PostAggregation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.postaggregations; 7 | 8 | import com.yahoo.bullet.querying.postaggregations.PostStrategy; 9 | import lombok.AllArgsConstructor; 10 | import lombok.Getter; 11 | 12 | import java.io.Serializable; 13 | 14 | @Getter @AllArgsConstructor 15 | public abstract class PostAggregation implements Serializable { 16 | private static final long serialVersionUID = -3083946184345104820L; 17 | 18 | protected final PostAggregationType type; 19 | 20 | /** 21 | * Returns a new {@link PostStrategy} that handles this post-aggregation. 22 | 23 | * @return A new instance of a strategy that handles this post-aggregation. 24 | */ 25 | public abstract PostStrategy getPostStrategy(); 26 | } 27 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/aggregations/NoSerDeBulletRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.aggregations; 7 | 8 | import com.yahoo.bullet.record.avro.TypedAvroBulletRecord; 9 | 10 | import java.io.IOException; 11 | import java.io.Serializable; 12 | 13 | public class NoSerDeBulletRecord extends TypedAvroBulletRecord implements Serializable { 14 | private static final long serialVersionUID = 4138653240854288567L; 15 | 16 | private void writeObject(java.io.ObjectOutputStream out) throws IOException { 17 | throw new IOException("Forced test serialization failure"); 18 | } 19 | 20 | private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { 21 | throw new IOException("Forced test deserialization failure"); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/pubsub/IdentityPubSubMessageSerDe.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | 10 | public class IdentityPubSubMessageSerDe extends PubSubMessageSerDe { 11 | private static final long serialVersionUID = -1709000962888195381L; 12 | 13 | /** 14 | * Constructor. 15 | * 16 | * @param config The {@link BulletConfig} to configure this class. 17 | */ 18 | public IdentityPubSubMessageSerDe(BulletConfig config) { 19 | super(config); 20 | } 21 | 22 | @Override 23 | public PubSubMessage toMessage(PubSubMessage message) { 24 | return message; 25 | } 26 | 27 | @Override 28 | public PubSubMessage fromMessage(PubSubMessage message) { 29 | return message; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/common/BulletException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common; 7 | 8 | import lombok.AllArgsConstructor; 9 | import lombok.Getter; 10 | 11 | @Getter @AllArgsConstructor 12 | public class BulletException extends RuntimeException { 13 | private static final long serialVersionUID = 2868933191828758133L; 14 | 15 | private BulletError error; 16 | 17 | /** 18 | * Creates a BulletException from an error and resolution. 19 | * 20 | * @param error The error message. 21 | * @param resolution The resolution message. 22 | */ 23 | public BulletException(String error, String resolution) { 24 | this.error = new BulletError(error, resolution); 25 | } 26 | 27 | @Override 28 | public String getMessage() { 29 | return error.getError(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/Raw.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | import com.yahoo.bullet.querying.aggregations.RawStrategy; 10 | import com.yahoo.bullet.querying.aggregations.Strategy; 11 | 12 | public class Raw extends Aggregation { 13 | private static final long serialVersionUID = -589592577885076012L; 14 | 15 | /** 16 | * Constructor that creates a RAW aggregation with a specified max size. 17 | * 18 | * @param size The max size of the RAW aggregation. Can be null. 19 | */ 20 | public Raw(Integer size) { 21 | super(size, AggregationType.RAW); 22 | } 23 | 24 | @Override 25 | public Strategy getStrategy(BulletConfig config) { 26 | return new RawStrategy(this, config); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/aggregations/grouping/GroupDataSummarySetOperations.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.aggregations.grouping; 7 | 8 | import com.yahoo.sketches.tuple.SummarySetOperations; 9 | 10 | /** 11 | * The stateless implementation of the summary set operations for a {@link GroupDataSummary}. Intersection is not 12 | * supported. 13 | */ 14 | public class GroupDataSummarySetOperations implements SummarySetOperations { 15 | @Override 16 | public GroupDataSummary union(GroupDataSummary a, GroupDataSummary b) { 17 | return GroupDataSummary.mergeInPlace(a, b); 18 | } 19 | 20 | @Override 21 | public GroupDataSummary intersection(GroupDataSummary a, GroupDataSummary b) { 22 | throw new UnsupportedOperationException("Intersection is not supported at the moment."); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /screwdriver/settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ossrh 5 | ${env.SONATYPE_USERNAME} 6 | ${env.SONATYPE_PASSWORD} 7 | 8 | 9 | 10 | 11 | 12 | ossrh 13 | 14 | true 15 | 16 | 17 | gpg 18 | ${env.GPG_PASSPHRASE} 19 | false 20 | ${env.SD_SOURCE_DIR}/screwdriver 21 | pubring.gpg 22 | secring.gpg 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/partitioning/Partitioner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.partitioning; 7 | 8 | import com.yahoo.bullet.query.Query; 9 | import com.yahoo.bullet.record.BulletRecord; 10 | 11 | import java.util.Set; 12 | 13 | public interface Partitioner { 14 | /** 15 | * Returns the partitioning keys for this {@link Query} instance. 16 | * 17 | * @param query The query to partition for. 18 | * @return A non-null {@link Set} of Strings representing the keys for this query. 19 | */ 20 | Set getKeys(Query query); 21 | 22 | /** 23 | * Returns the partitioning keys for this {@link BulletRecord} instance. 24 | * 25 | * @param record The record to partition. 26 | * @return A non-null {@link Set} of Strings representing the keys for this record. 27 | */ 28 | Set getKeys(BulletRecord record); 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/aggregations/AggregationUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.aggregations; 7 | 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | import java.util.function.Function; 12 | import java.util.stream.Collectors; 13 | 14 | import static java.util.Arrays.asList; 15 | 16 | public class AggregationUtils { 17 | public static Map makeGroupFields(List fields) { 18 | if (fields != null) { 19 | return fields.stream().collect(Collectors.toMap(Function.identity(), Function.identity())); 20 | } 21 | return new HashMap<>(); 22 | } 23 | 24 | public static Map makeGroupFields(String... fields) { 25 | if (fields == null) { 26 | return new HashMap<>(); 27 | } 28 | return makeGroupFields(asList(fields)); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/postaggregations/CullingStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.postaggregations; 7 | 8 | import com.yahoo.bullet.query.postaggregations.Culling; 9 | import com.yahoo.bullet.result.Clip; 10 | 11 | import java.util.Set; 12 | 13 | public class CullingStrategy implements PostStrategy { 14 | private Set transientFields; 15 | 16 | /** 17 | * Constructor that creates a Culling post-strategy. 18 | * 19 | * @param culling The culling post-aggregation to create a strategy for. 20 | */ 21 | public CullingStrategy(Culling culling) { 22 | transientFields = culling.getTransientFields(); 23 | } 24 | 25 | @Override 26 | public Clip execute(Clip clip) { 27 | for (String field : transientFields) { 28 | clip.getRecords().forEach(record -> record.remove(field)); 29 | } 30 | return clip; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/pubsub/rest/RESTMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub.rest; 7 | 8 | import com.yahoo.bullet.pubsub.Metadata; 9 | import lombok.Getter; 10 | import lombok.RequiredArgsConstructor; 11 | 12 | @RequiredArgsConstructor 13 | class RESTMetadata extends Metadata { 14 | private static final long serialVersionUID = 5718947090573796171L; 15 | @Getter 16 | private final String url; 17 | 18 | /** 19 | * Only for use within the RESTPubSub. 20 | * 21 | * @param url The URL to use to identify where to send messages to. 22 | * @param metadata An instance of a {@link Metadata} to wrap. 23 | */ 24 | RESTMetadata(String url, Metadata metadata) { 25 | super(metadata.getSignal(), metadata.getContent()); 26 | this.url = url; 27 | } 28 | 29 | @Override 30 | public Metadata copy() { 31 | return new RESTMetadata(url, this); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/postaggregations/ComputationStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.postaggregations; 7 | 8 | import com.yahoo.bullet.query.postaggregations.Computation; 9 | import com.yahoo.bullet.querying.Projection; 10 | import com.yahoo.bullet.result.Clip; 11 | import lombok.extern.slf4j.Slf4j; 12 | 13 | @Slf4j 14 | public class ComputationStrategy implements PostStrategy { 15 | private Projection projection; 16 | 17 | /** 18 | * Constructor that creates a Computation post-strategy. 19 | * 20 | * @param computation The computation post-aggregation to create a strategy for. 21 | */ 22 | public ComputationStrategy(Computation computation) { 23 | projection = new Projection(computation.getFields()); 24 | } 25 | 26 | @Override 27 | public Clip execute(Clip clip) { 28 | clip.getRecords().forEach(projection::project); 29 | return clip; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/evaluators/Evaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.record.BulletRecord; 9 | import com.yahoo.bullet.typesystem.TypedObject; 10 | 11 | import java.io.Serializable; 12 | 13 | /** 14 | * Evaluators are built from expressions. They are evaluated given a {@link BulletRecord} and will throw exceptions on 15 | * any errors which are most likely to be the result of missing fields or incorrect types. 16 | */ 17 | public abstract class Evaluator implements Serializable { 18 | private static final long serialVersionUID = 8998958368200061680L; 19 | 20 | /** 21 | * Evaluates this evaluator on the given {@link BulletRecord}. 22 | * 23 | * @param record The Bullet record to evaluate this evaluator on. 24 | * @return The result of this evaluator on the given Bullet record. 25 | */ 26 | public abstract TypedObject evaluate(BulletRecord record); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/aggregations/Strategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.aggregations; 7 | 8 | import com.yahoo.bullet.common.Monoidal; 9 | import com.yahoo.bullet.result.Meta; 10 | 11 | public interface Strategy extends Monoidal { 12 | /** 13 | * Returns false if more data should not be consumed or combined. This method can be used to avoid passing more 14 | * data into this Strategy. By default, returns false unless overridden. 15 | * 16 | * @return A boolean denoting whether the next consumption or combination should not occur. 17 | */ 18 | @Override 19 | default boolean isClosed() { 20 | return false; 21 | } 22 | 23 | /** 24 | * Get the {@link Meta} so far. By default, returns an empty one. 25 | * 26 | * @return The resulting metadata of the data aggregated so far. 27 | */ 28 | @Override 29 | default Meta getMetadata() { 30 | return new Meta(); 31 | } 32 | } 33 | 34 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/pubsub/PubSubResponder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | import lombok.extern.slf4j.Slf4j; 10 | 11 | /** 12 | * This can be extended by any class that needs to respond to a {@link PubSubMessage}. 13 | */ 14 | @Slf4j 15 | public abstract class PubSubResponder implements AutoCloseable { 16 | protected BulletConfig config; 17 | 18 | /** 19 | * Constructor. 20 | * 21 | * @param config The {@link BulletConfig} to use. 22 | */ 23 | public PubSubResponder(BulletConfig config) { 24 | this.config = config; 25 | } 26 | 27 | /** 28 | * Respond to a {@link PubSubMessage}. 29 | * 30 | * @param id The id of the response. 31 | * @param message The actual {@link PubSubMessage} containing the response. 32 | */ 33 | public abstract void respond(String id, PubSubMessage message); 34 | 35 | @Override 36 | public void close() { 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/pubsub/rest/RESTResultPublisher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub.rest; 7 | 8 | import com.yahoo.bullet.pubsub.PubSubMessage; 9 | import lombok.extern.slf4j.Slf4j; 10 | import org.apache.http.impl.client.CloseableHttpClient; 11 | 12 | @Slf4j 13 | public class RESTResultPublisher extends RESTPublisher { 14 | /** 15 | * Create a RESTQueryPublisher from a {@link CloseableHttpClient}. 16 | * 17 | * @param client The client. 18 | * @param connectTimeout The minimum time (ms) to wait for a connection to be made. 19 | */ 20 | public RESTResultPublisher(CloseableHttpClient client, int connectTimeout) { 21 | super(client, connectTimeout); 22 | } 23 | 24 | @Override 25 | public PubSubMessage send(PubSubMessage message) { 26 | String url = ((RESTMetadata) message.getMetadata()).getUrl(); 27 | log.debug("Extracted url to send results to: {}", url); 28 | sendToURL(url, message); 29 | return message; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/evaluators/EvaluatorUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.FieldExpression; 9 | import com.yahoo.bullet.query.expressions.ListExpression; 10 | import com.yahoo.bullet.query.expressions.ValueExpression; 11 | 12 | import java.io.Serializable; 13 | import java.util.ArrayList; 14 | import java.util.stream.Collectors; 15 | import java.util.stream.Stream; 16 | 17 | public class EvaluatorUtils { 18 | static ValueEvaluator valueEvaluator(Serializable value) { 19 | return new ValueEvaluator(new ValueExpression(value)); 20 | } 21 | 22 | static ListEvaluator listEvaluator(Serializable... values) { 23 | return new ListEvaluator(new ListExpression(Stream.of(values).map(ValueExpression::new).collect(Collectors.toCollection(ArrayList::new)))); 24 | } 25 | 26 | static FieldEvaluator fieldEvaluator(String field) { 27 | return new FieldEvaluator(new FieldExpression(field)); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/evaluators/ValueEvaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.ValueExpression; 9 | import com.yahoo.bullet.record.BulletRecord; 10 | import com.yahoo.bullet.typesystem.TypedObject; 11 | 12 | /** 13 | * An evaluator that returns a constant value. 14 | */ 15 | public class ValueEvaluator extends Evaluator { 16 | private static final long serialVersionUID = -1689526286716310223L; 17 | 18 | final TypedObject value; 19 | 20 | /** 21 | * Constructor that creates a value evaluator from a {@link ValueExpression}. 22 | * 23 | * @param valueExpression The value expression to construct the evaluator from. 24 | */ 25 | public ValueEvaluator(ValueExpression valueExpression) { 26 | value = new TypedObject(valueExpression.getType(), valueExpression.getValue()); 27 | } 28 | 29 | @Override 30 | public TypedObject evaluate(BulletRecord record) { 31 | return value; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/evaluators/ValueEvaluatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.ValueExpression; 9 | import com.yahoo.bullet.result.RecordBox; 10 | import com.yahoo.bullet.typesystem.Type; 11 | import com.yahoo.bullet.typesystem.TypedObject; 12 | import org.testng.Assert; 13 | import org.testng.annotations.Test; 14 | 15 | public class ValueEvaluatorTest { 16 | @Test 17 | public void testConstructor() { 18 | ValueEvaluator evaluator = new ValueEvaluator(new ValueExpression(5)); 19 | Assert.assertEquals(evaluator.value, new TypedObject(Type.INTEGER, 5)); 20 | Assert.assertEquals(evaluator.evaluate(RecordBox.get().getRecord()), evaluator.value); 21 | 22 | evaluator = new ValueEvaluator(new ValueExpression("5")); 23 | Assert.assertEquals(evaluator.value, new TypedObject(Type.STRING, "5")); 24 | Assert.assertEquals(evaluator.evaluate(RecordBox.get().getRecord()), evaluator.value); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/evaluators/CastEvaluatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.CastExpression; 9 | import com.yahoo.bullet.query.expressions.ValueExpression; 10 | import com.yahoo.bullet.result.RecordBox; 11 | import com.yahoo.bullet.typesystem.Type; 12 | import com.yahoo.bullet.typesystem.TypedObject; 13 | import org.testng.Assert; 14 | import org.testng.annotations.Test; 15 | 16 | public class CastEvaluatorTest { 17 | @Test 18 | public void testConstructor() { 19 | CastExpression expression = new CastExpression(new ValueExpression(5), Type.STRING); 20 | expression.setType(Type.STRING); 21 | 22 | CastEvaluator evaluator = new CastEvaluator(expression); 23 | Assert.assertTrue(evaluator.value instanceof ValueEvaluator); 24 | Assert.assertEquals(evaluator.castType, Type.STRING); 25 | Assert.assertEquals(evaluator.evaluate(RecordBox.get().getRecord()), new TypedObject(Type.STRING, "5")); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/pubsub/Publisher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import java.io.Serializable; 9 | 10 | public interface Publisher extends AutoCloseable { 11 | /** 12 | * Send a message with an ID and content. 13 | * 14 | * @param id The ID associated with the message. 15 | * @param content The content of the message. 16 | * @return The sent {@link PubSubMessage}. 17 | * @throws PubSubException if the messaging system throws an error. 18 | */ 19 | default PubSubMessage send(String id, Serializable content) throws PubSubException { 20 | return send(new PubSubMessage(id, content)); 21 | } 22 | 23 | /** 24 | * Sends a {@link PubSubMessage}. The message might be modified so the sent message is returned. 25 | * 26 | * @param message The {@link PubSubMessage} to be sent. 27 | * @return The sent {@link PubSubMessage}. 28 | * @throws PubSubException if the messaging system throws an error. 29 | */ 30 | PubSubMessage send(PubSubMessage message) throws PubSubException; 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/pubsub/PubSubExceptionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import org.testng.Assert; 9 | import org.testng.annotations.Test; 10 | 11 | public class PubSubExceptionTest { 12 | @Test 13 | public void testGetMessage() { 14 | String randomMessage = "foo"; 15 | PubSubException ex = new PubSubException(randomMessage); 16 | Assert.assertTrue(ex.getMessage().equals(randomMessage)); 17 | } 18 | 19 | @Test 20 | public void testGetArgumentFailedWithoutCause() { 21 | PubSubException ex = PubSubException.forArgument("bar", null); 22 | Assert.assertEquals(ex.getMessage(), "Could not read required argument: bar"); 23 | Assert.assertNull(ex.getCause()); 24 | } 25 | 26 | @Test 27 | public void testGetArgumentFailedWithCause() { 28 | Throwable cause = new NullPointerException(); 29 | PubSubException ex = PubSubException.forArgument("bar", cause); 30 | Assert.assertEquals(ex.getMessage(), "Could not read required argument: bar"); 31 | Assert.assertEquals(ex.getCause(), cause); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/postaggregations/HavingStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.postaggregations; 7 | 8 | import com.yahoo.bullet.query.postaggregations.Having; 9 | import com.yahoo.bullet.querying.evaluators.Evaluator; 10 | import com.yahoo.bullet.result.Clip; 11 | import com.yahoo.bullet.typesystem.Type; 12 | 13 | public class HavingStrategy implements PostStrategy { 14 | private final Evaluator evaluator; 15 | 16 | /** 17 | * Constructor that creates a Having post-strategy. 18 | * 19 | * @param having The Having post-aggregation to create a strategy for. 20 | */ 21 | public HavingStrategy(Having having) { 22 | evaluator = having.getExpression().getEvaluator(); 23 | } 24 | 25 | @Override 26 | public Clip execute(Clip clip) { 27 | clip.getRecords().removeIf(record -> { 28 | try { 29 | return !((Boolean) evaluator.evaluate(record).forceCast(Type.BOOLEAN).getValue()); 30 | } catch (Exception ignored) { 31 | return true; 32 | } 33 | }); 34 | return clip; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/postaggregations/HavingTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.postaggregations; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.postaggregations.HavingStrategy; 10 | import com.yahoo.bullet.query.expressions.ValueExpression; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | public class HavingTest { 15 | @Test 16 | public void testHaving() { 17 | Having having = new Having(new ValueExpression(true)); 18 | 19 | Assert.assertEquals(having.getExpression(), new ValueExpression(true)); 20 | Assert.assertEquals(having.getType(), PostAggregationType.HAVING); 21 | Assert.assertEquals(having.toString(), "{type: HAVING, expression: {value: true, type: BOOLEAN}}"); 22 | Assert.assertTrue(having.getPostStrategy() instanceof HavingStrategy); 23 | } 24 | 25 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "The HAVING post-aggregation requires an expression\\.") 26 | public void testConstructorMissingExpression() { 27 | new Having(null); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/windowing/AdditiveTumbling.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.windowing; 7 | 8 | import com.yahoo.bullet.querying.aggregations.Strategy; 9 | import com.yahoo.bullet.common.BulletConfig; 10 | import com.yahoo.bullet.query.Window; 11 | 12 | public class AdditiveTumbling extends Tumbling { 13 | /** 14 | * Creates an instance of this windowing scheme with the provided {@link Strategy} and {@link BulletConfig}. 15 | * 16 | * @param aggregation The non-null initialized aggregation strategy that this window will operate. 17 | * @param window The initialized, configured window to use. 18 | * @param config The validated config to use. 19 | */ 20 | public AdditiveTumbling(Strategy aggregation, Window window, BulletConfig config) { 21 | super(aggregation, window, config); 22 | } 23 | 24 | @Override 25 | public void reset() { 26 | nextCloseTime = nextCloseTime + windowLength; 27 | windowCount++; 28 | } 29 | 30 | @Override 31 | public void resetForPartition() { 32 | // Do reset the strategy. 33 | aggregation.reset(); 34 | reset(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/evaluators/CastEvaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.CastExpression; 9 | import com.yahoo.bullet.record.BulletRecord; 10 | import com.yahoo.bullet.typesystem.Type; 11 | import com.yahoo.bullet.typesystem.TypedObject; 12 | 13 | /** 14 | * An evaluator that force casts the result of an evaluator to a given type. 15 | */ 16 | public class CastEvaluator extends Evaluator { 17 | private static final long serialVersionUID = -7527404049388459123L; 18 | 19 | final Evaluator value; 20 | final Type castType; 21 | 22 | /** 23 | * Constructor that creates a cast evaluator from a {@link CastExpression}. 24 | * 25 | * @param castExpression The cast expression to construct the evaluator from. 26 | */ 27 | public CastEvaluator(CastExpression castExpression) { 28 | value = castExpression.getValue().getEvaluator(); 29 | castType = castExpression.getCastType(); 30 | } 31 | 32 | @Override 33 | public TypedObject evaluate(BulletRecord record) { 34 | return value.evaluate(record).forceCast(castType); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/evaluators/UnaryEvaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.UnaryExpression; 9 | import com.yahoo.bullet.record.BulletRecord; 10 | import com.yahoo.bullet.typesystem.TypedObject; 11 | 12 | /** 13 | * An evaluator that applies a unary operator to the result of an evaluator. 14 | */ 15 | public class UnaryEvaluator extends Evaluator { 16 | private static final long serialVersionUID = -4008832651046022947L; 17 | 18 | final Evaluator operand; 19 | final UnaryOperations.UnaryOperator op; 20 | 21 | /** 22 | * Constructor that creates a unary evaluator from a {@link UnaryExpression}. 23 | * 24 | * @param unaryExpression The unary expression to construct the evaluator from. 25 | */ 26 | public UnaryEvaluator(UnaryExpression unaryExpression) { 27 | operand = unaryExpression.getOperand().getEvaluator(); 28 | op = UnaryOperations.UNARY_OPERATORS.get(unaryExpression.getOp()); 29 | } 30 | 31 | @Override 32 | public TypedObject evaluate(BulletRecord record) { 33 | return op.apply(operand, record); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/common/metrics/MetricEventPublisher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common.metrics; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | 10 | import java.util.Map; 11 | 12 | /** 13 | * This class concretizes {@link MetricPublisher} with {@link MetricEvent}. 14 | */ 15 | public abstract class MetricEventPublisher extends MetricPublisher { 16 | public static String DEFAULT_GROUP = "default"; 17 | 18 | /** 19 | * Constructor taking a {@link BulletConfig}. 20 | * 21 | * @param config The config to use containing the necessary information. 22 | */ 23 | public MetricEventPublisher(BulletConfig config) { 24 | super(config); 25 | } 26 | 27 | /** 28 | * Get the group to use for the {@link MetricEvent}. By default, uses {@link #DEFAULT_GROUP}. 29 | * 30 | * @return The group to use for the {@link MetricEvent}. 31 | */ 32 | public String getGroup() { 33 | return DEFAULT_GROUP; 34 | } 35 | 36 | @Override 37 | public MetricEvent convert(Map dimensions, Map metrics) { 38 | return new MetricEvent(getGroup(), dimensions, metrics); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/evaluators/ListEvaluatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.ListExpression; 9 | import com.yahoo.bullet.query.expressions.ValueExpression; 10 | import com.yahoo.bullet.result.RecordBox; 11 | import com.yahoo.bullet.typesystem.Type; 12 | import com.yahoo.bullet.typesystem.TypedObject; 13 | import org.testng.Assert; 14 | import org.testng.annotations.Test; 15 | 16 | import java.util.ArrayList; 17 | import java.util.Arrays; 18 | 19 | public class ListEvaluatorTest { 20 | @Test 21 | public void testConstructor() { 22 | ListExpression expression = new ListExpression(Arrays.asList(new ValueExpression(1), new ValueExpression(2))); 23 | expression.setType(Type.INTEGER_LIST); 24 | 25 | ListEvaluator evaluator = new ListEvaluator(expression); 26 | Assert.assertTrue(evaluator.evaluators.get(0) instanceof ValueEvaluator); 27 | Assert.assertTrue(evaluator.evaluators.get(1) instanceof ValueEvaluator); 28 | Assert.assertEquals(evaluator.evaluate(RecordBox.get().getRecord()), new TypedObject(Type.INTEGER_LIST, new ArrayList<>(Arrays.asList(1, 2)))); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/result/JSONFormatterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.result; 7 | 8 | import lombok.AllArgsConstructor; 9 | import org.testng.Assert; 10 | import org.testng.annotations.Test; 11 | 12 | import java.util.Collections; 13 | import java.util.Map; 14 | 15 | import static com.yahoo.bullet.TestHelpers.assertJSONEquals; 16 | 17 | public class JSONFormatterTest { 18 | @AllArgsConstructor 19 | private static class Foo implements JSONFormatter { 20 | private int bar; 21 | private Map baz; 22 | 23 | @Override 24 | public String asJSON() { 25 | return JSONFormatter.asJSON(this); 26 | } 27 | } 28 | 29 | @Test 30 | public void testToJSON() { 31 | String json = new Foo(42, Collections.singletonMap("foo", 42.0)).asJSON(); 32 | assertJSONEquals(json, "{'bar': 42, 'baz': {'foo': 42.0 }}"); 33 | } 34 | 35 | @Test 36 | public void testFromJSON() { 37 | Foo foo = JSONFormatter.fromJSON("{'bar': 42, 'baz': {'foo': 42.0 }}", Foo.class); 38 | Assert.assertEquals(foo.bar, 42); 39 | Assert.assertEquals(foo.baz, Collections.singletonMap("foo", 42.0)); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/pubsub/Subscriber.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | public interface Subscriber extends AutoCloseable { 9 | /** 10 | * Gets a new {@link PubSubMessage} from the assigned partition/partitions (Here a partition is a unit of 11 | * parallelism in the Pub/Sub queue, See {@link PubSub}). 12 | * 13 | * @return the received {@link PubSubMessage}. 14 | * @throws PubSubException when a receive fails. 15 | */ 16 | PubSubMessage receive() throws PubSubException; 17 | 18 | /** 19 | * Commits allow clients to implement at least once, at most once or exactly once semantics when processing messages. 20 | * 21 | * Common implementations might 22 | * 1. Ack all received messages. 23 | * 2. Commit current read offset to persistent/fault tolerant storage. 24 | * 25 | * @param id The ID of the message to be marked as committed. 26 | */ 27 | void commit(String id); 28 | 29 | /** 30 | * Marks the processing of the {@link PubSubMessage} with the given id as failed. 31 | * 32 | * @param id The ID of the PubSubMessage to mark as a processing failure. 33 | */ 34 | void fail(String id); 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/tablefunctions/TableFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.tablefunctions; 7 | 8 | import com.yahoo.bullet.querying.tablefunctors.TableFunctor; 9 | import lombok.AllArgsConstructor; 10 | import lombok.Getter; 11 | 12 | import java.io.Serializable; 13 | 14 | /** 15 | * Table functions are used in Bullet queries to generate virtual records from incoming Bullet records. The generated 16 | * records are then fed to the rest of the query (filter, projection, aggregation, etc.) 17 | * 18 | * Currently, the supported table function types are Lateral View and Explode. 19 | * 20 | * Look at {@link TableFunctor} to see how table functions are applied in the {@link com.yahoo.bullet.querying.Querier}. 21 | */ 22 | @Getter @AllArgsConstructor 23 | public abstract class TableFunction implements Serializable { 24 | private static final long serialVersionUID = 4126801547249854808L; 25 | 26 | protected final TableFunctionType type; 27 | 28 | /** 29 | * Gets a new instance of a table functor for this table function. 30 | * 31 | * @return A newly-constructed table functor for this table function. 32 | */ 33 | public abstract TableFunctor getTableFunctor(); 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/evaluators/UnaryEvaluatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.Operation; 9 | import com.yahoo.bullet.query.expressions.UnaryExpression; 10 | import com.yahoo.bullet.query.expressions.ValueExpression; 11 | import com.yahoo.bullet.result.RecordBox; 12 | import com.yahoo.bullet.typesystem.Type; 13 | import com.yahoo.bullet.typesystem.TypedObject; 14 | import org.testng.Assert; 15 | import org.testng.annotations.Test; 16 | 17 | import static com.yahoo.bullet.querying.evaluators.UnaryOperations.UNARY_OPERATORS; 18 | 19 | public class UnaryEvaluatorTest { 20 | @Test 21 | public void testConstructor() { 22 | UnaryExpression expression = new UnaryExpression(new ValueExpression(1), Operation.IS_NOT_NULL); 23 | expression.setType(Type.BOOLEAN); 24 | 25 | UnaryEvaluator evaluator = new UnaryEvaluator(expression); 26 | Assert.assertTrue(evaluator.operand instanceof ValueEvaluator); 27 | Assert.assertEquals(evaluator.op, UNARY_OPERATORS.get(Operation.IS_NOT_NULL)); 28 | Assert.assertEquals(evaluator.evaluate(RecordBox.get().getRecord()), new TypedObject(Type.BOOLEAN, true)); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/tablefunctions/LateralViewTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.tablefunctions; 7 | 8 | import com.yahoo.bullet.query.expressions.FieldExpression; 9 | import com.yahoo.bullet.querying.tablefunctors.LateralViewFunctor; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | public class LateralViewTest { 14 | @Test 15 | public void testLateralViewTableFunction() { 16 | Explode explode = new Explode(new FieldExpression("abc"), "foo", "bar", true); 17 | LateralView tableFunction = new LateralView(explode); 18 | 19 | Assert.assertEquals(tableFunction.getType(), TableFunctionType.LATERAL_VIEW); 20 | Assert.assertEquals(tableFunction.getTableFunctions().get(0), explode); 21 | Assert.assertTrue(tableFunction.getTableFunctor() instanceof LateralViewFunctor); 22 | } 23 | 24 | @Test 25 | public void testToString() { 26 | LateralView tableFunction = new LateralView(new Explode(new FieldExpression("abc"), "foo", null, true)); 27 | 28 | Assert.assertEquals(tableFunction.toString(), "{type: LATERAL_VIEW, tableFunctions: [{outer: true, type: EXPLODE, field: {field: abc, type: null}, keyAlias: foo, valueAlias: null}]}"); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/storage/StorageConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.storage; 7 | 8 | import com.yahoo.bullet.common.Config; 9 | import org.testng.Assert; 10 | import org.testng.annotations.Test; 11 | 12 | import java.util.Arrays; 13 | import java.util.Collections; 14 | import java.util.HashSet; 15 | 16 | public class StorageConfigTest { 17 | @Test 18 | public void testDefaults() { 19 | StorageConfig config = new StorageConfig((Config) null); 20 | Assert.assertEquals(config.get(StorageConfig.NAMESPACES), 21 | Collections.singleton(StorageConfig.DEFAULT_NAMESPACE)); 22 | Assert.assertNull(config.getAs(StorageConfig.PARTITION_COUNT, Integer.class)); 23 | } 24 | 25 | @Test 26 | public void testCreation() { 27 | StorageConfig config = new StorageConfig("src/test/resources/storage_config.yaml"); 28 | Assert.assertEquals(config.get(StorageConfig.NAMESPACES), 29 | new HashSet<>(Arrays.asList("one", "two", "three", "eight"))); 30 | Assert.assertEquals(config.getAs(StorageConfig.PARTITION_COUNT, Integer.class), (Integer) 15); 31 | Assert.assertEquals(config.getAs(StorageConfig.PREFIX + "fake.setting", String.class), "foo"); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/common/metrics/MetricEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common.metrics; 7 | 8 | import com.yahoo.bullet.result.JSONFormatter; 9 | import lombok.Getter; 10 | 11 | import java.util.Map; 12 | 13 | /** 14 | * Simple wrapper that represents a metric event being published. 15 | */ 16 | @Getter 17 | public class MetricEvent implements JSONFormatter { 18 | private final String group; 19 | private final long timestamp; 20 | private final Map dimensions; 21 | private final Map metrics; 22 | 23 | /** 24 | * Constructor. 25 | * 26 | * @param group The identity of the entity being measured by the metrics. 27 | * @param dimensions Static metadata or tags describing the entity being measured. 28 | * @param metrics Numeric data describing values of the measured entity. 29 | */ 30 | public MetricEvent(String group, Map dimensions, Map metrics) { 31 | this.group = group; 32 | this.dimensions = dimensions; 33 | this.metrics = metrics; 34 | this.timestamp = System.currentTimeMillis(); 35 | } 36 | 37 | @Override 38 | public String asJSON() { 39 | return JSONFormatter.asJSON(this); 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/tablefunctors/TableFunctor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.tablefunctors; 7 | 8 | import com.yahoo.bullet.record.BulletRecord; 9 | import com.yahoo.bullet.record.BulletRecordProvider; 10 | import lombok.AllArgsConstructor; 11 | 12 | import java.io.Serializable; 13 | import java.util.List; 14 | 15 | /** 16 | * Table functors are built from table functions. A table functor is applied to a {@link BulletRecord} and generates a 17 | * list of {@link BulletRecord}. 18 | * 19 | * Classes inheriting from this class must implement {@link TableFunctor#apply(BulletRecord, BulletRecordProvider)}. 20 | */ 21 | @AllArgsConstructor 22 | public abstract class TableFunctor implements Serializable { 23 | private static final long serialVersionUID = 5843896863161739195L; 24 | 25 | /** 26 | * Applies this table functor to the given {@link BulletRecord}. 27 | * 28 | * @param record The Bullet record to apply this table functor to. 29 | * @param provider The provider used in generating new Bullet records. 30 | * @return The generated list of records from applying this table functor to the given Bullet record. 31 | */ 32 | public abstract List apply(BulletRecord record, BulletRecordProvider provider); 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/tablefunctions/ExplodeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.tablefunctions; 7 | 8 | import com.yahoo.bullet.query.expressions.FieldExpression; 9 | import com.yahoo.bullet.querying.tablefunctors.ExplodeFunctor; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | public class ExplodeTest { 14 | @Test 15 | public void testExplodeTableFunction() { 16 | Explode tableFunction = new Explode(new FieldExpression("abc"), "foo", "bar", true); 17 | 18 | Assert.assertEquals(tableFunction.getType(), TableFunctionType.EXPLODE); 19 | Assert.assertEquals(tableFunction.getField(), new FieldExpression("abc")); 20 | Assert.assertEquals(tableFunction.getKeyAlias(), "foo"); 21 | Assert.assertEquals(tableFunction.getValueAlias(), "bar"); 22 | Assert.assertTrue(tableFunction.isOuter()); 23 | Assert.assertTrue(tableFunction.getTableFunctor() instanceof ExplodeFunctor); 24 | } 25 | 26 | @Test 27 | public void testToString() { 28 | Explode tableFunction = new Explode(new FieldExpression("abc"), "foo", null, true); 29 | 30 | Assert.assertEquals(tableFunction.toString(), "{outer: true, type: EXPLODE, field: {field: abc, type: null}, keyAlias: foo, valueAlias: null}"); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/resources/rest_pubsub_defaults.yaml: -------------------------------------------------------------------------------- 1 | # Http connection timout for subscribers 2 | bullet.pubsub.rest.subscriber.connect.timeout.ms: 5000 3 | # Http connection timout for publishers 4 | bullet.pubsub.rest.publisher.connect.timeout.ms: 5000 5 | # Maxiumum number of uncommitted messages allowed before read requests will wait for commits (used by both the web service and the backend) 6 | bullet.pubsub.rest.subscriber.max.uncommitted.messages: 100 7 | # Minimum time (ms) between http calls to the result subscriber REST endpoint. This can be used to limit the number of http requests to the REST endpoints 8 | bullet.pubsub.rest.result.subscriber.min.wait.ms: 10 9 | # Minimum time (ms) between http calls to the query subscriber REST endpoint. This can be used to limit the number of http requests to the REST endpoints 10 | bullet.pubsub.rest.query.subscriber.min.wait.ms: 10 11 | # The uri of the result http endpoint (this is only used in the web service - the backend loads the uri from the message metadata) 12 | bullet.pubsub.rest.result.url: "http://localhost:9901/api/bullet/pubsub/result" 13 | # A list of url(s) for the query endpoint. In the web service, this should contain a single URL for the query endpoint 14 | # of the in-memory pubsub instance running on that web service. For the backend it should contain the urls of all 15 | # the pubsub instances. 16 | bullet.pubsub.rest.query.urls: 17 | - "http://localhost:9901/api/bullet/pubsub/query" 18 | - "http://localhost:9902/api/bullet/pubsub/query" 19 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/postaggregations/CullingTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.postaggregations; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.postaggregations.CullingStrategy; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | import java.util.Collections; 14 | 15 | public class CullingTest { 16 | @Test 17 | public void testCulling() { 18 | Culling culling = new Culling(Collections.singleton("abc")); 19 | 20 | Assert.assertEquals(culling.getTransientFields(), Collections.singleton("abc")); 21 | Assert.assertEquals(culling.getType(), PostAggregationType.CULLING); 22 | Assert.assertEquals(culling.toString(), "{type: CULLING, transientFields: [abc]}"); 23 | Assert.assertTrue(culling.getPostStrategy() instanceof CullingStrategy); 24 | } 25 | 26 | @Test(expectedExceptions = NullPointerException.class) 27 | public void testConstructorNullFields() { 28 | new Culling(null); 29 | } 30 | 31 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "The CULLING post-aggregation requires at least one field\\.") 32 | public void testConstructorMissingFields() { 33 | new Culling(Collections.emptySet()); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/evaluators/BinaryEvaluatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.BinaryExpression; 9 | import com.yahoo.bullet.query.expressions.Operation; 10 | import com.yahoo.bullet.query.expressions.ValueExpression; 11 | import com.yahoo.bullet.result.RecordBox; 12 | import com.yahoo.bullet.typesystem.Type; 13 | import com.yahoo.bullet.typesystem.TypedObject; 14 | import org.testng.Assert; 15 | import org.testng.annotations.Test; 16 | 17 | import static com.yahoo.bullet.querying.evaluators.BinaryOperations.BINARY_OPERATORS; 18 | 19 | public class BinaryEvaluatorTest { 20 | @Test 21 | public void testConstructor() { 22 | BinaryExpression expression = new BinaryExpression(new ValueExpression(1), new ValueExpression(2), Operation.ADD); 23 | expression.setType(Type.INTEGER); 24 | 25 | BinaryEvaluator evaluator = new BinaryEvaluator(expression); 26 | Assert.assertTrue(evaluator.left instanceof ValueEvaluator); 27 | Assert.assertTrue(evaluator.right instanceof ValueEvaluator); 28 | Assert.assertEquals(evaluator.op, BINARY_OPERATORS.get(Operation.ADD)); 29 | Assert.assertEquals(evaluator.evaluate(RecordBox.get().getRecord()), new TypedObject(Type.INTEGER, 3)); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/evaluators/BinaryEvaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.BinaryExpression; 9 | import com.yahoo.bullet.record.BulletRecord; 10 | import com.yahoo.bullet.typesystem.TypedObject; 11 | 12 | /** 13 | * An evaluator that applies a binary operator to the result of a left evaluator and the result of a right evaluator. 14 | */ 15 | public class BinaryEvaluator extends Evaluator { 16 | private static final long serialVersionUID = -467853226398830498L; 17 | 18 | final Evaluator left; 19 | final Evaluator right; 20 | final BinaryOperations.BinaryOperator op; 21 | 22 | /** 23 | * Constructor that creates a binary evaluator from a {@link BinaryExpression}. 24 | * 25 | * @param binaryExpression The binary expression to construct the evaluator from. 26 | */ 27 | public BinaryEvaluator(BinaryExpression binaryExpression) { 28 | left = binaryExpression.getLeft().getEvaluator(); 29 | right = binaryExpression.getRight().getEvaluator(); 30 | op = BinaryOperations.BINARY_OPERATORS.get(binaryExpression.getOp()); 31 | } 32 | 33 | @Override 34 | public TypedObject evaluate(BulletRecord record) { 35 | return op.apply(left, right, record); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/expressions/ExpressionUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import org.testng.Assert; 9 | 10 | import java.util.function.Supplier; 11 | 12 | public class ExpressionUtils { 13 | /** 14 | * Helper for testing equals() and hashCode() in classes that extend {@link Expression}. 15 | * 16 | * @param supplier A supplier that constructs the expression to compare to. 17 | * @param expressions The other expressions to compare to that should be not equal. 18 | */ 19 | public static void testEqualsAndHashCode(Supplier supplier, Expression... expressions) { 20 | Expression expression = supplier.get(); 21 | Assert.assertEquals(expression, expression); 22 | Assert.assertEquals(expression.hashCode(), expression.hashCode()); 23 | 24 | for (Expression other : expressions) { 25 | Assert.assertNotEquals(expression, other); 26 | Assert.assertNotEquals(expression.hashCode(), other.hashCode()); 27 | } 28 | 29 | Expression other = supplier.get(); 30 | Assert.assertEquals(expression, other); 31 | Assert.assertEquals(expression.hashCode(), other.hashCode()); 32 | 33 | // coverage 34 | Assert.assertFalse(expression.equals(null)); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/FilterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying; 7 | 8 | import com.yahoo.bullet.query.expressions.BinaryExpression; 9 | import com.yahoo.bullet.query.expressions.FieldExpression; 10 | import com.yahoo.bullet.query.expressions.Operation; 11 | import com.yahoo.bullet.query.expressions.ValueExpression; 12 | import com.yahoo.bullet.record.BulletRecord; 13 | import com.yahoo.bullet.result.RecordBox; 14 | import org.testng.Assert; 15 | import org.testng.annotations.Test; 16 | 17 | public class FilterTest { 18 | @Test 19 | public void testFilterMatch() { 20 | Filter filter = new Filter(new BinaryExpression(new FieldExpression("abc"), new ValueExpression(0), Operation.GREATER_THAN)); 21 | 22 | BulletRecord recordA = RecordBox.get().add("abc", 1).getRecord(); 23 | BulletRecord recordB = RecordBox.get().add("abc", 0).getRecord(); 24 | BulletRecord recordC = RecordBox.get().getRecord(); 25 | 26 | Assert.assertTrue(filter.match(recordA)); 27 | Assert.assertFalse(filter.match(recordB)); 28 | Assert.assertFalse(filter.match(recordC)); 29 | } 30 | 31 | @Test 32 | public void testFilterMatchException() { 33 | Filter filter = new Filter(new FieldExpression("abc")); 34 | 35 | Assert.assertFalse(filter.match(null)); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/evaluators/ListEvaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.Expression; 9 | import com.yahoo.bullet.query.expressions.ListExpression; 10 | import com.yahoo.bullet.record.BulletRecord; 11 | import com.yahoo.bullet.typesystem.TypedObject; 12 | 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | import java.util.stream.Collectors; 16 | 17 | /** 18 | * An evaluator that returns a list of the results of a list of evaluators. 19 | */ 20 | public class ListEvaluator extends Evaluator { 21 | private static final long serialVersionUID = -2945310695360542354L; 22 | 23 | final List evaluators; 24 | 25 | /** 26 | * Constructor that creates a list evaluator from a {@link ListExpression}. 27 | * 28 | * @param listExpression The list expression to construct the evaluator from. 29 | */ 30 | public ListEvaluator(ListExpression listExpression) { 31 | evaluators = listExpression.getValues().stream().map(Expression::getEvaluator).collect(Collectors.toList()); 32 | } 33 | 34 | @Override 35 | public TypedObject evaluate(BulletRecord record) { 36 | return new TypedObject(evaluators.stream().map(e -> e.evaluate(record).getValue()).collect(Collectors.toCollection(ArrayList::new))); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/evaluators/NAryEvaluator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.Expression; 9 | import com.yahoo.bullet.query.expressions.NAryExpression; 10 | import com.yahoo.bullet.record.BulletRecord; 11 | import com.yahoo.bullet.typesystem.TypedObject; 12 | 13 | import java.util.List; 14 | import java.util.stream.Collectors; 15 | 16 | /** 17 | * An evaluator that applies an n-ary operator to the results of a list of evaluators. 18 | */ 19 | public class NAryEvaluator extends Evaluator { 20 | private static final long serialVersionUID = 54879052369401372L; 21 | 22 | final List operands; 23 | final NAryOperations.NAryOperator op; 24 | 25 | /** 26 | * Constructor that creates an n-ary evaluator from a {@link NAryExpression}. 27 | * 28 | * @param nAryExpression The n-ary expression to construct the evaluator from. 29 | */ 30 | public NAryEvaluator(NAryExpression nAryExpression) { 31 | operands = nAryExpression.getOperands().stream().map(Expression::getEvaluator).collect(Collectors.toList()); 32 | op = NAryOperations.N_ARY_OPERATORS.get(nAryExpression.getOp()); 33 | } 34 | 35 | @Override 36 | public TypedObject evaluate(BulletRecord record) { 37 | return op.apply(operands, record); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/pubsub/PubSubResponderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import org.testng.Assert; 9 | import org.testng.annotations.Test; 10 | 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | 14 | public class PubSubResponderTest { 15 | private static class TestResponder extends PubSubResponder { 16 | private Map store = new HashMap<>(); 17 | 18 | TestResponder() { 19 | super(null); 20 | } 21 | 22 | @Override 23 | public void respond(String id, PubSubMessage message) { 24 | store.put(id, message); 25 | } 26 | 27 | @Override 28 | public void close() { 29 | super.close(); 30 | store.clear(); 31 | } 32 | } 33 | 34 | @Test 35 | public void testResponding() { 36 | PubSubResponder responder = new TestResponder(); 37 | responder.respond("id1", null); 38 | responder.respond("id2", new PubSubMessage("id2", new byte[0])); 39 | 40 | TestResponder testResponder = (TestResponder) responder; 41 | Assert.assertNull(testResponder.store.get("id1")); 42 | Assert.assertEquals(testResponder.store.get("id2"), new PubSubMessage("id2", new byte[0])); 43 | responder.close(); 44 | Assert.assertTrue(testResponder.store.isEmpty()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/aggregations/StrategyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.aggregations; 7 | 8 | import com.yahoo.bullet.record.BulletRecord; 9 | import com.yahoo.bullet.record.avro.TypedAvroBulletRecord; 10 | import com.yahoo.bullet.result.Clip; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | import java.util.List; 15 | 16 | public class StrategyTest { 17 | private static class EmptyStrategy implements Strategy { 18 | @Override 19 | public void consume(BulletRecord data) { 20 | } 21 | 22 | @Override 23 | public void combine(byte[] data) { 24 | } 25 | 26 | @Override 27 | public byte[] getData() { 28 | return new byte[0]; 29 | } 30 | 31 | @Override 32 | public Clip getResult() { 33 | return null; 34 | } 35 | 36 | @Override 37 | public List getRecords() { 38 | return null; 39 | } 40 | 41 | @Override 42 | public void reset() { 43 | } 44 | } 45 | @Test 46 | public void testDefaultClosed() { 47 | EmptyStrategy strategy = new EmptyStrategy(); 48 | Assert.assertFalse(strategy.isClosed()); 49 | strategy.consume(new TypedAvroBulletRecord()); 50 | Assert.assertFalse(strategy.isClosed()); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/Filter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying; 7 | 8 | import com.yahoo.bullet.query.expressions.Expression; 9 | import com.yahoo.bullet.querying.evaluators.Evaluator; 10 | import com.yahoo.bullet.record.BulletRecord; 11 | import com.yahoo.bullet.typesystem.Type; 12 | import com.yahoo.bullet.typesystem.TypedObject; 13 | 14 | /** 15 | * Filter consists of an evaluator built from the filter expression in the bullet query. 16 | * 17 | * Note, the filter expression does not necessarily have to have boolean type as it will be force-casted anyways. 18 | * Also note that if the evaluator throws an exception, the filter will not match. 19 | */ 20 | public class Filter { 21 | private Evaluator evaluator; 22 | 23 | public Filter(Expression filter) { 24 | evaluator = filter.getEvaluator(); 25 | } 26 | 27 | /** 28 | * Checks whether the given record matches this filter. 29 | * 30 | * @param record The BulletRecord to check. 31 | * @return True if the record matches this filter and false otherwise. 32 | */ 33 | public boolean match(BulletRecord record) { 34 | try { 35 | TypedObject value = evaluator.evaluate(record); 36 | return !value.isNull() && (Boolean) value.forceCast(Type.BOOLEAN).getValue(); 37 | } catch (Exception e) { 38 | return false; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/pubsub/PublisherTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import com.yahoo.bullet.common.SerializerDeserializer; 9 | import org.testng.Assert; 10 | import org.testng.annotations.Test; 11 | 12 | import java.util.UUID; 13 | 14 | public class PublisherTest { 15 | private static class MockPublisher implements Publisher { 16 | PubSubMessage sentMessage; 17 | 18 | @Override 19 | public PubSubMessage send(PubSubMessage message) { 20 | sentMessage = message; 21 | return message; 22 | } 23 | 24 | @Override 25 | public void close() { 26 | throw new UnsupportedOperationException(); 27 | } 28 | } 29 | 30 | @Test 31 | public void testDefaultSend() throws PubSubException { 32 | String randomId = UUID.randomUUID().toString(); 33 | byte[] randomMessage = SerializerDeserializer.toBytes(UUID.randomUUID()); 34 | MockPublisher mockPublisher = new MockPublisher(); 35 | PubSubMessage message = mockPublisher.send(randomId, randomMessage); 36 | 37 | Assert.assertEquals(message.getContent(), randomMessage); 38 | Assert.assertEquals(message.getId(), randomId); 39 | Assert.assertEquals(mockPublisher.sentMessage.getContent(), randomMessage); 40 | Assert.assertEquals(mockPublisher.sentMessage.getId(), randomId); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/expressions/Expression.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.querying.evaluators.Evaluator; 9 | import com.yahoo.bullet.typesystem.Type; 10 | import lombok.Getter; 11 | import lombok.Setter; 12 | 13 | import java.io.Serializable; 14 | 15 | /** 16 | * Expressions are used in Bullet queries to filter on, project, and compute potentially complex values based on 17 | * Bullet record fields. 18 | * 19 | * The supported expressions are: 20 | * - ValueExpression 21 | * - FieldExpression 22 | * - UnaryExpression 23 | * - BinaryExpression 24 | * - NAryExpression 25 | * - ListExpression 26 | * - CastExpression 27 | * 28 | * Look at {@link Evaluator} to see how expressions are evaluated. 29 | */ 30 | @Getter @Setter 31 | public abstract class Expression implements Serializable { 32 | private static final long serialVersionUID = -769774785327135375L; 33 | 34 | protected Type type; 35 | 36 | /** 37 | * Gets a new instance of an evaluator for this expression. 38 | * 39 | * @return A newly-constructed evaluator for this expression. 40 | */ 41 | public abstract Evaluator getEvaluator(); 42 | 43 | @Override 44 | public abstract boolean equals(Object obj); 45 | 46 | @Override 47 | public abstract int hashCode(); 48 | 49 | @Override 50 | public String toString() { 51 | return "type: " + type; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/Field.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query; 7 | 8 | import com.yahoo.bullet.query.expressions.Expression; 9 | import lombok.Getter; 10 | 11 | import java.io.Serializable; 12 | import java.util.Objects; 13 | 14 | @Getter 15 | public class Field implements Serializable { 16 | private static final long serialVersionUID = -2084429671585261042L; 17 | 18 | private String name; 19 | private Expression value; 20 | 21 | /** 22 | * Constructor that creates a field used in projection and computation. 23 | * 24 | * @param name The non-null name of the field. 25 | * @param value The non-null value of the field. 26 | */ 27 | public Field(String name, Expression value) { 28 | this.name = Objects.requireNonNull(name); 29 | this.value = Objects.requireNonNull(value); 30 | } 31 | 32 | @Override 33 | public boolean equals(Object obj) { 34 | if (obj == this) { 35 | return true; 36 | } 37 | if (!(obj instanceof Field)) { 38 | return false; 39 | } 40 | Field other = (Field) obj; 41 | return Objects.equals(name, other.name) && Objects.equals(value, other.value); 42 | } 43 | 44 | @Override 45 | public int hashCode() { 46 | return Objects.hash(name, value); 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return "{name: " + name + ", value: " + value + "}"; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/postaggregations/Having.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.postaggregations; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.postaggregations.HavingStrategy; 10 | import com.yahoo.bullet.querying.postaggregations.PostStrategy; 11 | import com.yahoo.bullet.query.expressions.Expression; 12 | import lombok.Getter; 13 | 14 | @Getter 15 | public class Having extends PostAggregation { 16 | private static final long serialVersionUID = -123184459098221770L; 17 | 18 | public static final BulletException HAVING_REQUIRES_EXPRESSION = 19 | new BulletException("The HAVING post-aggregation requires an expression.", "Please add an expression."); 20 | 21 | private Expression expression; 22 | 23 | /** 24 | * Constructor that creates a Having post-aggregation. 25 | * 26 | * @param expression The non-null expression to filter by after aggregation. 27 | */ 28 | public Having(Expression expression) { 29 | super(PostAggregationType.HAVING); 30 | if (expression == null) { 31 | throw HAVING_REQUIRES_EXPRESSION; 32 | } 33 | this.expression = expression; 34 | } 35 | 36 | @Override 37 | public PostStrategy getPostStrategy() { 38 | return new HavingStrategy(this); 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return "{type: " + type + ", expression: " + expression + "}"; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/pubsub/SubscriberTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import lombok.NoArgsConstructor; 9 | import org.testng.Assert; 10 | import org.testng.annotations.Test; 11 | 12 | import java.util.UUID; 13 | 14 | public class SubscriberTest { 15 | @NoArgsConstructor 16 | private static class MockSubscriber implements Subscriber { 17 | private String commitID; 18 | private String failID; 19 | 20 | public PubSubMessage receive() { 21 | throw new UnsupportedOperationException(); 22 | } 23 | 24 | @Override 25 | public void commit(String id) { 26 | commitID = id; 27 | } 28 | 29 | @Override 30 | public void fail(String id) { 31 | failID = id; 32 | } 33 | 34 | public void close() { 35 | throw new UnsupportedOperationException(); 36 | } 37 | } 38 | 39 | @Test 40 | public void testCommitWithNoSequenceNumber() { 41 | String randomID = UUID.randomUUID().toString(); 42 | MockSubscriber subscriber = new MockSubscriber(); 43 | subscriber.commit(randomID); 44 | Assert.assertEquals(randomID, subscriber.commitID); 45 | } 46 | 47 | @Test 48 | public void testFailWithNoSequenceNumber() { 49 | String randomID = UUID.randomUUID().toString(); 50 | MockSubscriber subscriber = new MockSubscriber(); 51 | subscriber.fail(randomID); 52 | Assert.assertEquals(randomID, subscriber.failID); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/RateLimitErrorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying; 7 | 8 | import com.yahoo.bullet.common.BulletError; 9 | import com.yahoo.bullet.result.Meta; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | import static com.yahoo.bullet.TestHelpers.assertJSONEquals; 17 | import static com.yahoo.bullet.common.BulletConfig.DEFAULT_RATE_LIMIT_MAX_EMIT_COUNT; 18 | import static com.yahoo.bullet.common.BulletConfig.DEFAULT_RATE_LIMIT_TIME_INTERVAL; 19 | 20 | public class RateLimitErrorTest { 21 | @Test 22 | public void testMetaAndRateConversion() { 23 | double defaultRate = ((double) DEFAULT_RATE_LIMIT_MAX_EMIT_COUNT / DEFAULT_RATE_LIMIT_TIME_INTERVAL); 24 | RateLimitError error = new RateLimitError(19.34, defaultRate); 25 | Map actual = error.makeMeta().asMap(); 26 | Assert.assertTrue(actual.containsKey(Meta.ERROR_KEY)); 27 | Assert.assertTrue(((List) actual.get(Meta.ERROR_KEY)).get(0) == error); 28 | 29 | String asJSON = error.asJSON(); 30 | double actualRate = 19.34 * RateLimiter.SECOND; 31 | assertJSONEquals(asJSON, "{'error': '" + String.format(RateLimitError.ERROR_FORMAT, defaultRate * RateLimiter.SECOND, actualRate) + "', " + 32 | "'resolutions': ['" + RateLimitError.NARROW_FILTER + "', '" + RateLimitError.TIME_WINDOW + "']" + 33 | "}"); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/expressions/ListExpressionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.querying.evaluators.ListEvaluator; 9 | import com.yahoo.bullet.typesystem.Type; 10 | import org.testng.Assert; 11 | import org.testng.annotations.Test; 12 | 13 | import java.util.ArrayList; 14 | import java.util.Arrays; 15 | 16 | public class ListExpressionTest { 17 | @Test 18 | public void testConstructor() { 19 | ListExpression expression = new ListExpression(Arrays.asList(new ValueExpression(1), new ValueExpression(2))); 20 | Assert.assertEquals(expression.toString(), "{values: [{value: 1, type: INTEGER}, {value: 2, type: INTEGER}], type: null}"); 21 | } 22 | 23 | @Test(expectedExceptions = NullPointerException.class) 24 | public void testConstructorThrows() { 25 | new ListExpression(null); 26 | } 27 | 28 | @Test 29 | public void testGetEvaluator() { 30 | Assert.assertTrue(new ListExpression(new ArrayList<>()).getEvaluator() instanceof ListEvaluator); 31 | } 32 | 33 | @Test 34 | public void testEqualsAndHashCode() { 35 | ListExpression expression = new ListExpression(new ArrayList<>()); 36 | expression.setType(Type.INTEGER_LIST); 37 | 38 | ExpressionUtils.testEqualsAndHashCode(() -> new ListExpression(new ArrayList<>()), 39 | new ListExpression(Arrays.asList(new ValueExpression(1))), 40 | expression); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/pubsub/rest/RESTMetadataTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub.rest; 7 | 8 | import com.yahoo.bullet.pubsub.Metadata; 9 | import org.testng.Assert; 10 | import org.testng.annotations.Test; 11 | 12 | import java.util.Collections; 13 | import java.util.HashMap; 14 | 15 | public class RESTMetadataTest { 16 | @Test 17 | public void testCreationWithoutMetadata() { 18 | RESTMetadata metadata = new RESTMetadata("foo"); 19 | Assert.assertNull(metadata.getContent()); 20 | Assert.assertNull(metadata.getSignal()); 21 | Assert.assertEquals(metadata.getUrl(), "foo"); 22 | } 23 | 24 | @Test 25 | public void testCreationWithMetadata() { 26 | RESTMetadata metadata = new RESTMetadata("foo", new Metadata(Metadata.Signal.CUSTOM, new HashMap<>())); 27 | Assert.assertEquals(metadata.getSignal(), Metadata.Signal.CUSTOM); 28 | Assert.assertEquals(metadata.getContent(), Collections.emptyMap()); 29 | Assert.assertEquals(metadata.getUrl(), "foo"); 30 | } 31 | 32 | @Test 33 | public void testCopy() { 34 | RESTMetadata metadata = new RESTMetadata("foo", new Metadata(Metadata.Signal.CUSTOM, new HashMap<>())); 35 | RESTMetadata copy = (RESTMetadata) metadata.copy(); 36 | Assert.assertNotEquals(metadata, copy); 37 | Assert.assertEquals(metadata.getSignal(), copy.getSignal()); 38 | Assert.assertEquals(metadata.getContent(), copy.getContent()); 39 | Assert.assertEquals(metadata.getUrl(), copy.getUrl()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Contributing.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | First, thanks for taking the time to contribute to our project! The following information provides a guide for making contributions. 3 | 4 | ## Code of Conduct 5 | 6 | By participating in this project, you agree to abide by the [Yahoo Code of Conduct](Code-of-Conduct.md). Everyone is welcome to submit a pull request or open an issue to improve the documentation, add improvements, or report bugs. 7 | 8 | ## How to Ask a Question 9 | 10 | If you simply have a question that needs an answer, [create an issue](https://help.github.com/articles/creating-an-issue/), and label it as a question. 11 | 12 | ## How To Contribute 13 | 14 | ### Report a Bug or Request a Feature 15 | 16 | If you encounter any bugs while using this software, or want to request a new feature or enhancement, feel free to [create an issue](https://help.github.com/articles/creating-an-issue/) to report it, make sure you add a label to indicate what type of issue it is. 17 | 18 | ### Contribute Code 19 | Pull requests are welcome for bug fixes. If you want to implement something new, please [request a feature first](#report-a-bug-or-request-a-feature) so we can discuss it. 20 | 21 | #### Creating a Pull Request 22 | Before you submit any code, we need you to agree to our [Contributor License Agreement](https://yahoocla.herokuapp.com/); this ensures we can continue to protect your contributions under an open source license well into the future. 23 | 24 | Please follow [best practices](https://github.com/trein/dev-best-practices/wiki/Git-Commit-Best-Practices) for creating git commits. 25 | 26 | When your code is ready to be submitted, you can [submit a pull request](https://help.github.com/articles/creating-a-pull-request/) to begin the code review process. 27 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/postaggregations/ComputationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.postaggregations; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.postaggregations.ComputationStrategy; 10 | import com.yahoo.bullet.query.Field; 11 | import com.yahoo.bullet.query.expressions.ValueExpression; 12 | import org.testng.Assert; 13 | import org.testng.annotations.Test; 14 | 15 | import java.util.Collections; 16 | 17 | public class ComputationTest { 18 | @Test 19 | public void testComputation() { 20 | Computation computation = new Computation(Collections.singletonList(new Field("abc", new ValueExpression(1)))); 21 | 22 | Assert.assertEquals(computation.getFields(), Collections.singletonList(new Field("abc", new ValueExpression(1)))); 23 | Assert.assertEquals(computation.getType(), PostAggregationType.COMPUTATION); 24 | Assert.assertEquals(computation.toString(), "{type: COMPUTATION, fields: [{name: abc, value: {value: 1, type: INTEGER}}]}"); 25 | Assert.assertTrue(computation.getPostStrategy() instanceof ComputationStrategy); 26 | } 27 | 28 | @Test(expectedExceptions = NullPointerException.class) 29 | public void testConstructorNullFields() { 30 | new Computation(null); 31 | } 32 | 33 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "The COMPUTATION post-aggregation requires at least one field\\.") 34 | public void testConstructorMissingFields() { 35 | new Computation(Collections.emptyList()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/pubsub/PubSubException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | /** 9 | * Exception to be thrown if there is an error in {@link PubSub}, {@link Publisher} or {@link Subscriber}. 10 | */ 11 | public class PubSubException extends Exception { 12 | /** 13 | * Constructor to initialize PubSubException with a message. 14 | * 15 | * @param message The error message to be associated with the PubSubException. 16 | */ 17 | public PubSubException(String message) { 18 | super(message); 19 | } 20 | 21 | /** 22 | * Constructor to initialize PubSubException with a message and a {@link Throwable} cause. 23 | * 24 | * @param message The error message to be associated with the PubSubException. 25 | * @param cause The reason for the PubSubException. 26 | */ 27 | public PubSubException(String message, Throwable cause) { 28 | super(message, cause); 29 | } 30 | 31 | /** 32 | * Method to create a PubSubException when a required argument could not be read. 33 | * 34 | * @param name The name of the argument that could not be read. 35 | * @param cause The optional {@link Throwable} that caused the exception. 36 | * @return A PubSubException indicating failure to read a required argument. 37 | */ 38 | public static PubSubException forArgument(String name, Throwable cause) { 39 | String message = "Could not read required argument: " + name; 40 | return cause == null ? new PubSubException(message) : new PubSubException(message, cause); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/Distribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.querying.aggregations.QuantileSketchingStrategy; 9 | import com.yahoo.bullet.querying.aggregations.Strategy; 10 | import com.yahoo.bullet.common.BulletConfig; 11 | import lombok.Getter; 12 | 13 | import java.util.Collections; 14 | import java.util.List; 15 | import java.util.Objects; 16 | 17 | @Getter 18 | public abstract class Distribution extends Aggregation { 19 | private static final long serialVersionUID = -7862051610403543796L; 20 | 21 | protected final String field; 22 | protected final DistributionType distributionType; 23 | 24 | /** 25 | * Constructor that creates a DISTRIBUTION aggregation with a specified max size. 26 | * 27 | * @param field The non-null field 28 | * @param type The non-null distribution type 29 | * @param size The max size of the DISTRIBUTION aggregation. Can be null. 30 | */ 31 | protected Distribution(String field, DistributionType type, Integer size) { 32 | super(size, AggregationType.DISTRIBUTION); 33 | this.field = Objects.requireNonNull(field, "The field must be non-null."); 34 | this.distributionType = Objects.requireNonNull(type, "The distribution type must be non-null."); 35 | } 36 | 37 | @Override 38 | public Strategy getStrategy(BulletConfig config) { 39 | return new QuantileSketchingStrategy(this, config); 40 | } 41 | 42 | @Override 43 | public List getFields() { 44 | return Collections.singletonList(field); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/postaggregations/Culling.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.postaggregations; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.common.Utilities; 10 | import com.yahoo.bullet.querying.postaggregations.CullingStrategy; 11 | import com.yahoo.bullet.querying.postaggregations.PostStrategy; 12 | import lombok.Getter; 13 | 14 | import java.util.Set; 15 | 16 | @Getter 17 | public class Culling extends PostAggregation { 18 | private static final long serialVersionUID = -4606818164037391850L; 19 | 20 | public static final BulletException CULLING_REQUIRES_FIELDS = 21 | new BulletException("The CULLING post-aggregation requires at least one field.", "Please add at least one field."); 22 | 23 | private Set transientFields; 24 | 25 | /** 26 | * Constructor that creates a Culling post-aggregation. 27 | * 28 | * @param transientFields The non-null set of fields to remove after aggregation. 29 | */ 30 | public Culling(Set transientFields) { 31 | super(PostAggregationType.CULLING); 32 | Utilities.requireNonNull(transientFields); 33 | if (transientFields.isEmpty()) { 34 | throw CULLING_REQUIRES_FIELDS; 35 | } 36 | this.transientFields = transientFields; 37 | } 38 | 39 | @Override 40 | public PostStrategy getPostStrategy() { 41 | return new CullingStrategy(this); 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return "{type: " + type + ", transientFields: " + transientFields + "}"; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/RunningQuery.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying; 7 | 8 | import com.yahoo.bullet.pubsub.Metadata; 9 | import com.yahoo.bullet.query.Query; 10 | import lombok.Getter; 11 | 12 | /** 13 | * A wrapper for a running query. 14 | */ 15 | @Getter 16 | public class RunningQuery { 17 | private final String id; 18 | private final Query query; 19 | private final String queryString; 20 | private final long startTime; 21 | 22 | /** 23 | * Constructor that takes an id, query, and metadata that contains the query string and start time. 24 | * 25 | * @param id The query id. 26 | * @param query The query object. 27 | * @param metadata The metadata associated with the given query. 28 | */ 29 | public RunningQuery(String id, Query query, Metadata metadata) { 30 | this.id = id; 31 | this.query = query; 32 | this.queryString = (String) metadata.getContent(); 33 | this.startTime = metadata.getCreated(); 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return query.toString(); 39 | } 40 | 41 | /** 42 | * Returns true if this running query has timed out. In other words, it returns whether this has been running 43 | * longer than the query duration. 44 | * 45 | * @return A boolean denoting whether this query has timed out. 46 | */ 47 | public boolean isTimedOut() { 48 | // Never add to query.getDuration() since it can be infinite (Long.MAX_VALUE) 49 | return System.currentTimeMillis() - startTime >= query.getDuration(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/postaggregations/Computation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.postaggregations; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.common.Utilities; 10 | import com.yahoo.bullet.querying.postaggregations.ComputationStrategy; 11 | import com.yahoo.bullet.querying.postaggregations.PostStrategy; 12 | import com.yahoo.bullet.query.Field; 13 | import lombok.Getter; 14 | 15 | import java.util.List; 16 | 17 | @Getter 18 | public class Computation extends PostAggregation { 19 | private static final long serialVersionUID = -1401910210528780976L; 20 | 21 | public static final BulletException COMPUTATION_REQUIRES_FIELDS = 22 | new BulletException("The COMPUTATION post-aggregation requires at least one field.", "Please add at least one field."); 23 | 24 | private List fields; 25 | 26 | /** 27 | * Constructor that creates a Computation post-aggregation. 28 | * 29 | * @param fields The non-null list of fields to compute after aggregation. 30 | */ 31 | public Computation(List fields) { 32 | super(PostAggregationType.COMPUTATION); 33 | Utilities.requireNonNull(fields); 34 | if (fields.isEmpty()) { 35 | throw COMPUTATION_REQUIRES_FIELDS; 36 | } 37 | this.fields = fields; 38 | } 39 | 40 | @Override 41 | public PostStrategy getPostStrategy() { 42 | return new ComputationStrategy(this); 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return "{type: " + type + ", fields: " + fields + "}"; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/aggregations/MockStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.aggregations; 7 | 8 | import com.yahoo.bullet.record.BulletRecord; 9 | import com.yahoo.bullet.result.Clip; 10 | import com.yahoo.bullet.result.Meta; 11 | import lombok.Getter; 12 | 13 | import java.util.List; 14 | 15 | @Getter 16 | public class MockStrategy implements Strategy { 17 | private int consumeCalls = 0; 18 | private int combineCalls = 0; 19 | private int dataCalls = 0; 20 | private int resultCalls = 0; 21 | private int recordCalls = 0; 22 | private int metadataCalls = 0; 23 | private int resetCalls = 0; 24 | private int closedCalls = 0; 25 | 26 | @Override 27 | public void consume(BulletRecord data) { 28 | consumeCalls++; 29 | } 30 | 31 | @Override 32 | public void combine(byte[] data) { 33 | combineCalls++; 34 | } 35 | 36 | @Override 37 | public byte[] getData() { 38 | dataCalls++; 39 | return null; 40 | } 41 | 42 | @Override 43 | public Clip getResult() { 44 | resultCalls++; 45 | return null; 46 | } 47 | 48 | @Override 49 | public List getRecords() { 50 | recordCalls++; 51 | return null; 52 | } 53 | 54 | @Override 55 | public void reset() { 56 | resetCalls++; 57 | } 58 | 59 | @Override 60 | public boolean isClosed() { 61 | closedCalls++; 62 | return false; 63 | } 64 | 65 | @Override 66 | public Meta getMetadata() { 67 | metadataCalls++; 68 | return null; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/result/JSONFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.result; 7 | 8 | import com.google.gson.Gson; 9 | import com.google.gson.GsonBuilder; 10 | import com.google.gson.JsonPrimitive; 11 | import com.google.gson.JsonSerializer; 12 | 13 | public interface JSONFormatter { 14 | JsonSerializer INVALID_DOUBLES = (item, type, context) -> item.isNaN() || item.isInfinite() ? 15 | new JsonPrimitive(item.toString()) : new JsonPrimitive(item); 16 | Gson GSON = new GsonBuilder().serializeNulls().registerTypeAdapter(Double.class, INVALID_DOUBLES).create(); 17 | 18 | /** 19 | * Returns a JSON string representation of object. 20 | * @param object The object to make a JSON out of. 21 | * @return JSON string of the object. 22 | */ 23 | static String asJSON(Object object) { 24 | return GSON.toJson(object); 25 | } 26 | 27 | /** 28 | * Returns a deserialized object from JSON using {@link JSONFormatter#GSON}. 29 | * 30 | * @param json The String json that represents the object. 31 | * @param clazz The class of the object. 32 | * @param The type of the object. It must implement {@link JSONFormatter}. 33 | * @return An instance of the object deserialized from JSON. 34 | */ 35 | static T fromJSON(String json, Class clazz) { 36 | return GSON.fromJson(json, clazz); 37 | } 38 | 39 | /** 40 | * Convert this object to a JSON string. 41 | * @return The JSON representation of this. 42 | */ 43 | String asJSON(); 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/Projection.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query; 7 | 8 | import com.yahoo.bullet.common.Utilities; 9 | import lombok.Getter; 10 | import lombok.extern.slf4j.Slf4j; 11 | 12 | import java.io.Serializable; 13 | import java.util.List; 14 | 15 | @Slf4j @Getter 16 | public class Projection implements Serializable { 17 | private static final long serialVersionUID = -9194169391843941958L; 18 | 19 | /** 20 | * The type of the Projection decides how its fields are projected. 21 | */ 22 | public enum Type { 23 | COPY, // Projects onto a copy of the original record 24 | NO_COPY, // Projects onto a new record 25 | PASS_THROUGH // Passes the original record through 26 | } 27 | 28 | private final List fields; 29 | private final Type type; 30 | 31 | /** 32 | * Default constructor that creates a PASS_THROUGH projection. 33 | */ 34 | public Projection() { 35 | fields = null; 36 | type = Type.PASS_THROUGH; 37 | } 38 | 39 | /** 40 | * Constructor that creates a COPY or NO_COPY projection. 41 | * 42 | * @param fields The list of fields to project. Must not be null or contain null fields. 43 | * @param copy Whether the projection should copy or not copy. 44 | */ 45 | public Projection(List fields, boolean copy) { 46 | this.fields = Utilities.requireNonNull(fields); 47 | this.type = copy ? Type.COPY : Type.NO_COPY; 48 | } 49 | 50 | @Override 51 | public String toString() { 52 | return "{fields: " + fields + ", type: " + type + "}"; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/postaggregations/OrderByTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.postaggregations; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.query.expressions.FieldExpression; 10 | import com.yahoo.bullet.querying.postaggregations.OrderByStrategy; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | import java.util.Arrays; 15 | import java.util.Collections; 16 | 17 | public class OrderByTest { 18 | @Test 19 | public void testOrderBy() { 20 | OrderBy orderBy = new OrderBy(Arrays.asList(new OrderBy.SortItem(new FieldExpression("1"), OrderBy.Direction.ASC), 21 | new OrderBy.SortItem(new FieldExpression("2"), OrderBy.Direction.DESC))); 22 | 23 | Assert.assertEquals(orderBy.getFields().size(), 2); 24 | Assert.assertEquals(orderBy.getType(), PostAggregationType.ORDER_BY); 25 | Assert.assertTrue(orderBy.getPostStrategy() instanceof OrderByStrategy); 26 | Assert.assertEquals(orderBy.toString(), "{type: ORDER_BY, fields: [{expression: {field: 1, type: null}, direction: ASC}, {expression: {field: 2, type: null}, direction: DESC}]}"); 27 | } 28 | 29 | @Test(expectedExceptions = NullPointerException.class) 30 | public void testConstructorNullFields() { 31 | new OrderBy(null); 32 | } 33 | 34 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "The ORDER BY post-aggregation requires at least one field\\.") 35 | public void testConstructorMissingFields() { 36 | new OrderBy(Collections.emptyList()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/pubsub/MockPubSub.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | 10 | import java.util.List; 11 | 12 | import static org.mockito.Mockito.doReturn; 13 | import static org.mockito.Mockito.mock; 14 | 15 | public class MockPubSub extends PubSub { 16 | public static final String MOCK_MESSAGE_NAME = "MOCK_MESSAGE"; 17 | private String mockMessage; 18 | 19 | public MockPubSub(BulletConfig config) throws PubSubException { 20 | super(config); 21 | mockMessage = getRequiredConfig(String.class, MOCK_MESSAGE_NAME); 22 | } 23 | 24 | @Override 25 | public Subscriber getSubscriber() { 26 | Subscriber mockSubscriber = mock(Subscriber.class); 27 | try { 28 | doReturn(new PubSubMessage("", mockMessage)).when(mockSubscriber).receive(); 29 | } catch (Exception e) { 30 | mockSubscriber = null; 31 | } 32 | return mockSubscriber; 33 | } 34 | 35 | @Override 36 | public Publisher getPublisher() { 37 | return mock(Publisher.class); 38 | } 39 | 40 | @Override 41 | public List getSubscribers(int n) { 42 | throw new UnsupportedOperationException(); 43 | } 44 | 45 | @Override 46 | public List getPublishers(int n) { 47 | throw new UnsupportedOperationException(); 48 | } 49 | 50 | @Override 51 | public void switchContext(Context context, BulletConfig config) throws PubSubException { 52 | super.switchContext(context, config); 53 | mockMessage = getRequiredConfig(String.class, MOCK_MESSAGE_NAME); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/LinearDistribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import lombok.Getter; 10 | 11 | @Getter 12 | public class LinearDistribution extends Distribution { 13 | private static final long serialVersionUID = -5320252906943658246L; 14 | private static final BulletException NUMBER_OF_POINTS_MUST_BE_POSITIVE = 15 | new BulletException("If specifying the distribution by the number of points, the number must be positive.", "Please specify a positive number."); 16 | 17 | private final int numberOfPoints; 18 | 19 | /** 20 | * Constructor that creates a DISTRIBUTION aggregation with a specified max size. The number of equidistant 21 | * points in this distribution are specified. 22 | * 23 | * @param field The non-null field. 24 | * @param type The non-null distribution type. 25 | * @param size The max size of the DISTRIBUTION aggregation. Can be null. 26 | * @param numberOfPoints The number of equidistant points for this distribution. 27 | */ 28 | public LinearDistribution(String field, DistributionType type, Integer size, int numberOfPoints) { 29 | super(field, type, size); 30 | if (numberOfPoints <= 0) { 31 | throw NUMBER_OF_POINTS_MUST_BE_POSITIVE; 32 | } 33 | this.numberOfPoints = numberOfPoints; 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return "{size: " + size + ", type: " + type + ", field: " + field + ", distributionType: " + distributionType + ", numberOfPoints: " + numberOfPoints + "}"; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/RateLimitError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying; 7 | 8 | import com.yahoo.bullet.common.BulletError; 9 | import com.yahoo.bullet.result.Meta; 10 | 11 | import java.util.Arrays; 12 | import java.util.Collections; 13 | import java.util.List; 14 | 15 | public class RateLimitError extends BulletError { 16 | private static final long serialVersionUID = 5056840730518175058L; 17 | 18 | public static final String ERROR_FORMAT = "Exceeded the maximum rate limit for the query: %.2f emits per second. " + 19 | "Current rate: %.2f emits per second"; 20 | public static final String NARROW_FILTER = "Try using more filters to reduce the data"; 21 | public static final String TIME_WINDOW = "Try using a time based window instead of a record based window"; 22 | public static final List RESOLUTIONS = Arrays.asList(NARROW_FILTER, TIME_WINDOW); 23 | 24 | /** 25 | /** 26 | * Creates an instance of this from a given absolute exceeded rate and a maximum rate limit. 27 | * 28 | * @param rate The exceeded rate that caused the error. 29 | * @param limit The maximum rate limit for the query. 30 | */ 31 | public RateLimitError(double rate, double limit) { 32 | super(String.format(ERROR_FORMAT, limit * RateLimiter.SECOND, rate * RateLimiter.SECOND), RESOLUTIONS); 33 | } 34 | 35 | /** 36 | * Makes this into a {@link Meta} object. 37 | * 38 | * @return A meta object containing this error. 39 | */ 40 | public Meta makeMeta() { 41 | return new Meta().addErrors(Collections.singletonList(this)); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/FieldTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query; 7 | 8 | import com.yahoo.bullet.query.expressions.ValueExpression; 9 | import org.testng.Assert; 10 | import org.testng.annotations.Test; 11 | 12 | public class FieldTest { 13 | @Test 14 | public void testEquals() { 15 | Field fieldA = new Field("abc", new ValueExpression(5)); 16 | Field fieldB = new Field("def", new ValueExpression(5)); 17 | Field fieldC = new Field("abc", new ValueExpression(1)); 18 | Field fieldD = new Field("abc", new ValueExpression(5)); 19 | 20 | Assert.assertEquals(fieldA, fieldA); 21 | Assert.assertNotEquals(fieldA, fieldB); 22 | Assert.assertNotEquals(fieldA, fieldC); 23 | Assert.assertEquals(fieldA, fieldD); 24 | 25 | // coverage 26 | Assert.assertFalse(fieldA.equals(0)); 27 | } 28 | 29 | @Test 30 | public void testHashCode() { 31 | Field fieldA = new Field("abc", new ValueExpression(5)); 32 | Field fieldB = new Field("def", new ValueExpression(5)); 33 | Field fieldC = new Field("abc", new ValueExpression(1)); 34 | Field fieldD = new Field("abc", new ValueExpression(5)); 35 | 36 | Assert.assertNotEquals(fieldA.hashCode(), fieldB.hashCode()); 37 | Assert.assertNotEquals(fieldA.hashCode(), fieldC.hashCode()); 38 | Assert.assertEquals(fieldA.hashCode(), fieldD.hashCode()); 39 | } 40 | 41 | @Test 42 | public void testToString() { 43 | Field field = new Field("abc", new ValueExpression(5)); 44 | Assert.assertEquals(field.toString(), "{name: abc, value: {value: 5, type: INTEGER}}"); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/evaluators/NAryEvaluatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.evaluators; 7 | 8 | import com.yahoo.bullet.query.expressions.NAryExpression; 9 | import com.yahoo.bullet.query.expressions.Operation; 10 | import com.yahoo.bullet.query.expressions.ValueExpression; 11 | import com.yahoo.bullet.result.RecordBox; 12 | import com.yahoo.bullet.typesystem.Type; 13 | import com.yahoo.bullet.typesystem.TypedObject; 14 | import org.testng.Assert; 15 | import org.testng.annotations.Test; 16 | 17 | import java.util.Arrays; 18 | 19 | import static com.yahoo.bullet.querying.evaluators.NAryOperations.N_ARY_OPERATORS; 20 | 21 | public class NAryEvaluatorTest { 22 | @Test 23 | public void testConstructor() { 24 | NAryExpression expression = new NAryExpression(Arrays.asList(new ValueExpression(false), 25 | new ValueExpression(1), 26 | new ValueExpression(2)), 27 | Operation.IF); 28 | expression.setType(Type.INTEGER); 29 | 30 | NAryEvaluator evaluator = new NAryEvaluator(expression); 31 | Assert.assertTrue(evaluator.operands.get(0) instanceof ValueEvaluator); 32 | Assert.assertTrue(evaluator.operands.get(1) instanceof ValueEvaluator); 33 | Assert.assertTrue(evaluator.operands.get(2) instanceof ValueEvaluator); 34 | Assert.assertEquals(evaluator.op, N_ARY_OPERATORS.get(Operation.IF)); 35 | Assert.assertEquals(evaluator.evaluate(RecordBox.get().getRecord()), new TypedObject(Type.INTEGER, 2)); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/aggregations/CountDistinctTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.querying.aggregations.ThetaSketchingStrategy; 9 | import com.yahoo.bullet.common.BulletConfig; 10 | import com.yahoo.bullet.common.BulletException; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | import java.util.Arrays; 15 | import java.util.Collections; 16 | 17 | public class CountDistinctTest { 18 | private BulletConfig config = new BulletConfig(); 19 | 20 | @Test 21 | public void testCountDistinctAggregation() { 22 | CountDistinct aggregation = new CountDistinct(Arrays.asList("foo", "abc"), "count"); 23 | aggregation.configure(config); 24 | 25 | Assert.assertEquals(aggregation.getType(), AggregationType.COUNT_DISTINCT); 26 | Assert.assertEquals(aggregation.getFields(), Arrays.asList("foo", "abc")); 27 | Assert.assertEquals(aggregation.getName(), "count"); 28 | Assert.assertTrue(aggregation.getStrategy(config) instanceof ThetaSketchingStrategy); 29 | } 30 | 31 | @Test(expectedExceptions = BulletException.class, 32 | expectedExceptionsMessageRegExp = "COUNT DISTINCT requires at least one field\\.") 33 | public void testConstructorMissingFieldsThrows() { 34 | new CountDistinct(Collections.emptyList(), "count"); 35 | } 36 | 37 | @Test 38 | public void testToString() { 39 | CountDistinct aggregation = new CountDistinct(Arrays.asList("foo", "abc"), "count"); 40 | 41 | Assert.assertEquals(aggregation.toString(), "{size: null, type: COUNT_DISTINCT, fields: [foo, abc], name: count}"); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/expressions/ListExpression.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.Utilities; 9 | import com.yahoo.bullet.querying.evaluators.Evaluator; 10 | import com.yahoo.bullet.querying.evaluators.ListEvaluator; 11 | import lombok.Getter; 12 | 13 | import java.util.List; 14 | import java.util.Objects; 15 | 16 | /** 17 | * An expression that holds a list of expressions. 18 | */ 19 | @Getter 20 | public class ListExpression extends Expression { 21 | private static final long serialVersionUID = 311789452858823415L; 22 | 23 | private final List values; 24 | 25 | /** 26 | * Constructor that creates a list expression. 27 | * 28 | * @param values The non-null list of values to be wrapped. 29 | */ 30 | public ListExpression(List values) { 31 | this.values = Utilities.requireNonNull(values); 32 | } 33 | 34 | @Override 35 | public Evaluator getEvaluator() { 36 | return new ListEvaluator(this); 37 | } 38 | 39 | @Override 40 | public boolean equals(Object obj) { 41 | if (obj == this) { 42 | return true; 43 | } 44 | if (!(obj instanceof ListExpression)) { 45 | return false; 46 | } 47 | ListExpression other = (ListExpression) obj; 48 | return Objects.equals(values, other.values) && type == other.type; 49 | } 50 | 51 | @Override 52 | public int hashCode() { 53 | return Objects.hash(values, type); 54 | } 55 | 56 | @Override 57 | public String toString() { 58 | return "{values: " + values + ", " + super.toString() + "}"; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/tablefunctions/LateralView.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.tablefunctions; 7 | 8 | import com.yahoo.bullet.querying.tablefunctors.LateralViewFunctor; 9 | import com.yahoo.bullet.querying.tablefunctors.TableFunctor; 10 | import lombok.Getter; 11 | 12 | import java.util.Collections; 13 | import java.util.List; 14 | import java.util.Objects; 15 | 16 | /** 17 | * A table function that requires another table function. 18 | */ 19 | @Getter 20 | public class LateralView extends TableFunction { 21 | private static final long serialVersionUID = -8238108616312386350L; 22 | 23 | private final List tableFunctions; 24 | 25 | /** 26 | * Constructor that creates a LATERAL VIEW from a {@link List} of {@link TableFunction}. 27 | * 28 | * @param tableFunctions The non-null list of table functions to chain lateral views. 29 | */ 30 | public LateralView(List tableFunctions) { 31 | super(TableFunctionType.LATERAL_VIEW); 32 | this.tableFunctions = Objects.requireNonNull(tableFunctions); 33 | } 34 | 35 | /** 36 | * Constructor that creates a LATERAL VIEW from a {@link TableFunction}. 37 | * 38 | * @param tableFunction The non-null table function to apply a lateral view. 39 | */ 40 | public LateralView(TableFunction tableFunction) { 41 | this(Collections.singletonList(Objects.requireNonNull(tableFunction))); 42 | } 43 | 44 | @Override 45 | public TableFunctor getTableFunctor() { 46 | return new LateralViewFunctor(this); 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return "{type: " + type + ", tableFunctions: " + tableFunctions + "}"; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/ProjectionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query; 7 | 8 | import com.yahoo.bullet.query.expressions.ValueExpression; 9 | import org.testng.Assert; 10 | import org.testng.annotations.Test; 11 | 12 | import java.util.Arrays; 13 | import java.util.Collections; 14 | 15 | @SuppressWarnings("unchecked") 16 | public class ProjectionTest { 17 | @Test 18 | public void testDefault() { 19 | Projection projection = new Projection(); 20 | Assert.assertNull(projection.getFields()); 21 | Assert.assertEquals(projection.getType(), Projection.Type.PASS_THROUGH); 22 | Assert.assertEquals(projection.toString(), "{fields: null, type: PASS_THROUGH}"); 23 | } 24 | 25 | @Test 26 | public void testProjectionCopy() { 27 | Projection projection = new Projection(Arrays.asList(new Field("foo", new ValueExpression(5))), true); 28 | Assert.assertEquals(projection.getFields(), Collections.singletonList(new Field("foo", new ValueExpression(5)))); 29 | Assert.assertEquals(projection.getType(), Projection.Type.COPY); 30 | Assert.assertEquals(projection.toString(), "{fields: [{name: foo, value: {value: 5, type: INTEGER}}], type: COPY}"); 31 | } 32 | 33 | @Test 34 | public void testProjectionNoCopy() { 35 | Projection projection = new Projection(Arrays.asList(new Field("foo", new ValueExpression(5))), false); 36 | Assert.assertEquals(projection.getFields(), Collections.singletonList(new Field("foo", new ValueExpression(5)))); 37 | Assert.assertEquals(projection.getType(), Projection.Type.NO_COPY); 38 | Assert.assertEquals(projection.toString(), "{fields: [{name: foo, value: {value: 5, type: INTEGER}}], type: NO_COPY}"); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/aggregations/LinearDistributionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.querying.aggregations.QuantileSketchingStrategy; 9 | import com.yahoo.bullet.common.BulletConfig; 10 | import com.yahoo.bullet.common.BulletException; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | public class LinearDistributionTest { 15 | private BulletConfig config = new BulletConfig(); 16 | 17 | @Test 18 | public void testLinearDistributionAggregation() { 19 | LinearDistribution aggregation = new LinearDistribution("foo", DistributionType.QUANTILE, 500, 10); 20 | aggregation.configure(config); 21 | 22 | Assert.assertEquals(aggregation.getType(), AggregationType.DISTRIBUTION); 23 | Assert.assertEquals(aggregation.getDistributionType(), DistributionType.QUANTILE); 24 | Assert.assertEquals(aggregation.getNumberOfPoints(), 10); 25 | Assert.assertTrue(aggregation.getStrategy(config) instanceof QuantileSketchingStrategy); 26 | } 27 | 28 | @Test(expectedExceptions = BulletException.class, 29 | expectedExceptionsMessageRegExp = "If specifying the distribution by the number of points, the number must be positive\\.") 30 | public void testConstructorNumberOfPointsNonPositiveThrows() { 31 | new LinearDistribution("foo", DistributionType.QUANTILE, 10, 0); 32 | } 33 | 34 | @Test 35 | public void testToString() { 36 | LinearDistribution aggregation = new LinearDistribution("foo", DistributionType.QUANTILE, null, 10); 37 | 38 | Assert.assertEquals(aggregation.toString(), "{size: null, type: DISTRIBUTION, field: foo, distributionType: QUANTILE, numberOfPoints: 10}"); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/tablefunctions/Explode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.tablefunctions; 7 | 8 | import com.yahoo.bullet.query.expressions.Expression; 9 | import com.yahoo.bullet.querying.tablefunctors.ExplodeFunctor; 10 | import com.yahoo.bullet.querying.tablefunctors.TableFunctor; 11 | import lombok.Getter; 12 | 13 | import java.util.Objects; 14 | 15 | /** 16 | * A table function that requires either a list operand and a key alias or a map operand and both key and value aliases. 17 | */ 18 | @Getter 19 | public class Explode extends OuterableTableFunction { 20 | private static final long serialVersionUID = 6058738006416405818L; 21 | 22 | private final Expression field; 23 | private final String keyAlias; 24 | private final String valueAlias; 25 | 26 | /** 27 | * Constructor that creates an EXPLODE table function. 28 | * 29 | * @param field The non-null field to explode. 30 | * @param keyAlias The non-null alias for the key column of the exploded field. 31 | * @param valueAlias The alias for the value column of the exploded field. 32 | * @param outer The outer option. 33 | */ 34 | public Explode(Expression field, String keyAlias, String valueAlias, boolean outer) { 35 | super(outer, TableFunctionType.EXPLODE); 36 | this.field = Objects.requireNonNull(field); 37 | this.keyAlias = Objects.requireNonNull(keyAlias); 38 | this.valueAlias = valueAlias; 39 | } 40 | 41 | @Override 42 | public TableFunctor getTableFunctor() { 43 | return new ExplodeFunctor(this); 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return "{outer: " + outer + ", type: " + type + ", field: " + field + ", keyAlias: " + keyAlias + ", valueAlias: " + valueAlias + "}"; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/pubsub/rest/RESTQueryPublisher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub.rest; 7 | 8 | import com.yahoo.bullet.pubsub.Metadata; 9 | import com.yahoo.bullet.pubsub.PubSubMessage; 10 | import lombok.AccessLevel; 11 | import lombok.Getter; 12 | import lombok.extern.slf4j.Slf4j; 13 | import org.apache.http.impl.client.CloseableHttpClient; 14 | 15 | @Slf4j 16 | public class RESTQueryPublisher extends RESTPublisher { 17 | @Getter(AccessLevel.PACKAGE) 18 | private String queryURL; 19 | private String resultURL; 20 | 21 | /** 22 | * Create a RESTQueryPublisher from a {@link CloseableHttpClient}, queryURL and resultURL. The BulletConfig must 23 | * contain a valid url in the bullet.pubsub.rest.query.urls field. 24 | * 25 | * @param client The client. 26 | * @param queryURL The URL to which to POST queries. 27 | * @param resultURL The URL that will be added to the Metadata (results will be sent to this URL from the backend). 28 | * @param connectTimeout The minimum time (ms) to wait for a connection to be made. 29 | */ 30 | public RESTQueryPublisher(CloseableHttpClient client, String queryURL, String resultURL, int connectTimeout) { 31 | super(client, connectTimeout); 32 | this.queryURL = queryURL; 33 | this.resultURL = resultURL; 34 | } 35 | 36 | @Override 37 | public PubSubMessage send(PubSubMessage message) { 38 | // Put resultURL in the metadata so the ResponsePublisher knows to which host to send the response 39 | Metadata metadata = message.getMetadata(); 40 | metadata = metadata == null ? new RESTMetadata(resultURL) : new RESTMetadata(resultURL, metadata); 41 | message.setMetadata(metadata); 42 | sendToURL(queryURL, message); 43 | return message; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/GroupAll.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | import com.yahoo.bullet.common.BulletException; 10 | import com.yahoo.bullet.common.Utilities; 11 | import com.yahoo.bullet.querying.aggregations.GroupAllStrategy; 12 | import com.yahoo.bullet.querying.aggregations.Strategy; 13 | import com.yahoo.bullet.querying.aggregations.grouping.GroupOperation; 14 | import lombok.Getter; 15 | 16 | import java.util.Set; 17 | 18 | @Getter 19 | public class GroupAll extends Aggregation { 20 | private static final long serialVersionUID = 5118426551573371428L; 21 | private static final BulletException COUNT_FIELD_INVALID_OPERATION = 22 | new BulletException("COUNT_FIELD is not a valid operation.", "Please remove this operation."); 23 | 24 | private final Set operations; 25 | 26 | /** 27 | * Constructor that creates a GROUP aggregation with a set of group operations. 28 | * 29 | * @param operations The non-null set of group operations. 30 | */ 31 | public GroupAll(Set operations) { 32 | super(null, AggregationType.GROUP); 33 | Utilities.requireNonNull(operations); 34 | if (operations.stream().anyMatch(operation -> operation.getType() == GroupOperation.GroupOperationType.COUNT_FIELD)) { 35 | throw COUNT_FIELD_INVALID_OPERATION; 36 | } 37 | this.operations = operations; 38 | } 39 | 40 | @Override 41 | public Strategy getStrategy(BulletConfig config) { 42 | return new GroupAllStrategy(this, config); 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return "{size: " + size + ", type: " + type + ", operations: " + operations + "}"; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/common/RandomPool.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common; 7 | 8 | import java.util.List; 9 | import java.util.Random; 10 | 11 | public class RandomPool { 12 | private List items; 13 | 14 | private static final Random RANDOM = new Random(); 15 | 16 | /** 17 | * Constructor for the RandomPool that takes a list of items. 18 | * @param items A list of items to form the pool with. 19 | */ 20 | public RandomPool(List items) { 21 | this.items = items; 22 | } 23 | 24 | /** 25 | * Get a random item from the pool. 26 | * 27 | * @return a randomly chosen item from the pool. 28 | */ 29 | public T get() { 30 | if (items == null || items.isEmpty()) { 31 | return null; 32 | } 33 | return items.get(RANDOM.nextInt(items.size())); 34 | } 35 | 36 | /** 37 | * Clear the RandomPool. Gets now return null. 38 | */ 39 | public void clear() { 40 | items = null; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return items == null ? null : items.toString(); 46 | } 47 | 48 | @Override 49 | public boolean equals(Object object) { 50 | if (this == object) { 51 | return true; 52 | } 53 | if (object == null) { 54 | return false; 55 | } 56 | if (!(object instanceof RandomPool)) { 57 | return false; 58 | } 59 | RandomPool asPool = (RandomPool) object; 60 | return items == null ? asPool.items == null : items.equals(asPool.items); 61 | } 62 | 63 | @Override 64 | public int hashCode() { 65 | // Any number would do since we want RandomPools of null to be equal to each other. 66 | return items == null ? 42 : items.hashCode(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/pubsub/BulletPubSubResponder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | import lombok.extern.slf4j.Slf4j; 10 | 11 | /** 12 | * Use this as a {@link PubSubResponder} if async results should be sent to a {@link PubSub} interface. This simply 13 | * creates an instance of a {@link PubSub} from your provided {@link BulletConfig} and tries to create a 14 | * {@link Publisher} from it to respond with results to. 15 | */ 16 | @Slf4j 17 | public class BulletPubSubResponder extends PubSubResponder { 18 | protected PubSub pubSub; 19 | protected Publisher publisher; 20 | 21 | /** 22 | * Constructor. 23 | * 24 | * @param config The {@link BulletConfig} to use. 25 | */ 26 | public BulletPubSubResponder(BulletConfig config) { 27 | super(config); 28 | try { 29 | pubSub = PubSub.from(config); 30 | publisher = pubSub.getPublisher(); 31 | } catch (Exception e) { 32 | throw new RuntimeException("Unable to create a PubSub instance or a Publisher from it", e); 33 | } 34 | } 35 | 36 | @Override 37 | public void respond(String id, PubSubMessage message) { 38 | log.debug("Responding with message {}", id); 39 | log.trace("Responding to {} with payload {}", id, message); 40 | try { 41 | publisher.send(message); 42 | } catch (PubSubException e) { 43 | log.error("Unable to publish message. Ignoring {}: {}", id, message); 44 | log.error("Error", e); 45 | } 46 | } 47 | 48 | @Override 49 | public void close() { 50 | try { 51 | publisher.close(); 52 | } catch (Exception e) { 53 | log.error("Unable to close the publisher", e); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/CountDistinct.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.querying.aggregations.ThetaSketchingStrategy; 9 | import com.yahoo.bullet.querying.aggregations.Strategy; 10 | import com.yahoo.bullet.common.BulletConfig; 11 | import com.yahoo.bullet.common.BulletException; 12 | import com.yahoo.bullet.common.Utilities; 13 | import lombok.Getter; 14 | 15 | import java.util.List; 16 | import java.util.Objects; 17 | 18 | @Getter 19 | public class CountDistinct extends Aggregation { 20 | private static final long serialVersionUID = 3079494553075374672L; 21 | private static final BulletException COUNT_DISTINCT_REQUIRES_FIELDS = 22 | new BulletException("COUNT DISTINCT requires at least one field.", "Please add at least one field."); 23 | 24 | private final List fields; 25 | private final String name; 26 | 27 | /** 28 | * Constructor that creates a COUNT_DISTINCT aggregation. 29 | * 30 | * @param fields The list of fields to count distinct on. 31 | * @param name The name of the count distinct field. 32 | */ 33 | public CountDistinct(List fields, String name) { 34 | super(null, AggregationType.COUNT_DISTINCT); 35 | Utilities.requireNonNull(fields); 36 | if (fields.isEmpty()) { 37 | throw COUNT_DISTINCT_REQUIRES_FIELDS; 38 | } 39 | this.fields = fields; 40 | this.name = Objects.requireNonNull(name); 41 | } 42 | 43 | @Override 44 | public Strategy getStrategy(BulletConfig config) { 45 | return new ThetaSketchingStrategy(this, config); 46 | } 47 | 48 | @Override 49 | public String toString() { 50 | return "{size: " + size + ", type: " + type + ", fields: " + fields + ", name: " + name + "}"; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/aggregations/grouping/GroupDataSummaryFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.aggregations.grouping; 7 | 8 | import com.yahoo.memory.Memory; 9 | import com.yahoo.sketches.tuple.DeserializeResult; 10 | import com.yahoo.sketches.tuple.Summary; 11 | import com.yahoo.sketches.tuple.SummaryFactory; 12 | import com.yahoo.sketches.tuple.SummarySetOperations; 13 | 14 | public class GroupDataSummaryFactory implements SummaryFactory { 15 | public static final int SERIALIZED_SIZE = 1; 16 | public static final byte[] SERIALIZED = new byte[SERIALIZED_SIZE]; 17 | public static final GroupDataSummarySetOperations SUMMARY_OPERATIONS = new GroupDataSummarySetOperations(); 18 | 19 | @Override 20 | public Summary newSummary() { 21 | return new GroupDataSummary(); 22 | } 23 | 24 | @Override 25 | public SummarySetOperations getSummarySetOperations() { 26 | // Stateless so return the static one 27 | return SUMMARY_OPERATIONS; 28 | } 29 | 30 | @Override 31 | public DeserializeResult summaryFromMemory(Memory serializedSummary) { 32 | return GroupDataSummary.fromMemory(serializedSummary); 33 | } 34 | 35 | @Override 36 | public byte[] toByteArray() { 37 | return SERIALIZED; 38 | } 39 | 40 | /** 41 | * Needed to deserialize an instance of this {@link GroupDataSummaryFactory} from a {@link Memory}. 42 | * 43 | * @param summaryFactory The serialized summary factory. 44 | * @return A {@link DeserializeResult} representing the deserialized summary factory. 45 | */ 46 | public static DeserializeResult fromMemory(Memory summaryFactory) { 47 | // This has no state so it does not use the Memory 48 | return new DeserializeResult<>(new GroupDataSummaryFactory(), SERIALIZED_SIZE); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/Aggregation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | import com.yahoo.bullet.common.Configurable; 10 | import com.yahoo.bullet.querying.aggregations.Strategy; 11 | import lombok.AllArgsConstructor; 12 | import lombok.Getter; 13 | 14 | import java.io.Serializable; 15 | import java.util.Collections; 16 | import java.util.List; 17 | 18 | @Getter @AllArgsConstructor 19 | public abstract class Aggregation implements Configurable, Serializable { 20 | private static final long serialVersionUID = -4451469769203362270L; 21 | 22 | protected Integer size; 23 | protected final AggregationType type; 24 | 25 | @Override 26 | public void configure(BulletConfig config) { 27 | int sizeDefault = config.getAs(BulletConfig.AGGREGATION_DEFAULT_SIZE, Integer.class); 28 | int sizeMaximum = config.getAs(BulletConfig.AGGREGATION_MAX_SIZE, Integer.class); 29 | 30 | // Null or not positive, then default, else min of size and max 31 | size = (size == null || size <= 0) ? sizeDefault : Math.min(size, sizeMaximum); 32 | } 33 | 34 | /** 35 | * Returns a new {@link Strategy} instance that handles this aggregation. 36 | * 37 | * @param config The {@link BulletConfig} containing configuration for the strategy. 38 | * @return The created instance of a strategy that can implement this aggregation. 39 | */ 40 | public abstract Strategy getStrategy(BulletConfig config); 41 | 42 | /** 43 | * Gets the aggregation fields. 44 | * 45 | * @return The aggregation fields. 46 | */ 47 | public List getFields() { 48 | return Collections.emptyList(); 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return "{size: " + size + ", type: " + type + "}"; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/storage/NullStorageManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.storage; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | 10 | import java.io.Serializable; 11 | import java.util.Map; 12 | import java.util.Set; 13 | import java.util.concurrent.CompletableFuture; 14 | 15 | /** 16 | * A default implementation that does nothing if you do not want to use a StorageManager. 17 | */ 18 | public class NullStorageManager extends StorageManager implements Serializable { 19 | private static final long serialVersionUID = -1718811448543607136L; 20 | 21 | /** 22 | * Constructor. 23 | * 24 | * @param config The {@link BulletConfig} to create this manager with. 25 | */ 26 | public NullStorageManager(BulletConfig config) { 27 | super(config); 28 | } 29 | 30 | @Override 31 | protected CompletableFuture putRaw(String namespace, String id, byte[] value) { 32 | return SUCCESS; 33 | } 34 | 35 | @Override 36 | protected CompletableFuture getRaw(String namespace, String id) { 37 | return CompletableFuture.completedFuture(null); 38 | } 39 | 40 | @Override 41 | protected CompletableFuture> getAllRaw(String namespace) { 42 | return CompletableFuture.completedFuture(null); 43 | } 44 | 45 | @Override 46 | protected CompletableFuture removeRaw(String namespace, String id) { 47 | return CompletableFuture.completedFuture(null); 48 | } 49 | 50 | @Override 51 | public CompletableFuture wipe() { 52 | return SUCCESS; 53 | } 54 | 55 | @Override 56 | public CompletableFuture clear(String namespace) { 57 | return SUCCESS; 58 | } 59 | 60 | @Override 61 | public CompletableFuture clear(String namespace, Set ids) { 62 | return SUCCESS; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/tablefunctors/OuterableTableFunctor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.tablefunctors; 7 | 8 | import com.yahoo.bullet.record.BulletRecord; 9 | import com.yahoo.bullet.record.BulletRecordProvider; 10 | import lombok.AllArgsConstructor; 11 | 12 | import java.util.Collections; 13 | import java.util.List; 14 | 15 | /** 16 | * The OuterableTableFunctor class adds an outer option. When outer is specified and no records are generated, an empty 17 | * record will be generated. 18 | * 19 | * Classes inheriting from this class must implement {@link OuterableTableFunctor#outerableApply(BulletRecord, BulletRecordProvider)} 20 | * and not {@link TableFunctor#apply(BulletRecord, BulletRecordProvider)}. 21 | */ 22 | @AllArgsConstructor 23 | public abstract class OuterableTableFunctor extends TableFunctor { 24 | private static final long serialVersionUID = 5523194920317863876L; 25 | 26 | protected final boolean outer; 27 | 28 | @Override 29 | public List apply(BulletRecord record, BulletRecordProvider provider) { 30 | List records = outerableApply(record, provider); 31 | if (records.isEmpty() && outer) { 32 | return Collections.singletonList(provider.getInstance()); 33 | } 34 | return records; 35 | } 36 | 37 | /** 38 | * Applies this table functor to the given {@link BulletRecord}. This method is called by the main 39 | * {@link #apply(BulletRecord, BulletRecordProvider)} method which will handle the outer option. 40 | * 41 | * @param record The Bullet record to apply this table functor to. 42 | * @param provider The provider used in generating new Bullet records. 43 | * @return The generated list of records from applying this table functor to the given Bullet record. 44 | */ 45 | protected abstract List outerableApply(BulletRecord record, BulletRecordProvider provider); 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/postaggregations/HavingStrategyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.postaggregations; 7 | 8 | import com.yahoo.bullet.query.expressions.BinaryExpression; 9 | import com.yahoo.bullet.query.expressions.FieldExpression; 10 | import com.yahoo.bullet.query.expressions.Operation; 11 | import com.yahoo.bullet.query.expressions.ValueExpression; 12 | import com.yahoo.bullet.query.postaggregations.Having; 13 | import com.yahoo.bullet.record.BulletRecord; 14 | import com.yahoo.bullet.result.Clip; 15 | import com.yahoo.bullet.result.RecordBox; 16 | import org.testng.Assert; 17 | import org.testng.annotations.Test; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | public class HavingStrategyTest { 23 | @Test 24 | public void testHaving() { 25 | List records = new ArrayList<>(); 26 | records.add(RecordBox.get().add("a", 6).getRecord()); 27 | records.add(RecordBox.get().add("a", 1).getRecord()); 28 | records.add(RecordBox.get().add("a", 8).getRecord()); 29 | records.add(RecordBox.get().add("a", 2).getRecord()); 30 | records.add(RecordBox.get().add("b", 10).getRecord()); 31 | records.add(RecordBox.get().add("a", 7).getRecord()); 32 | 33 | Clip clip = new Clip(); 34 | clip.add(records); 35 | 36 | // a > 5 37 | BinaryExpression expression = new BinaryExpression(new FieldExpression("a"), new ValueExpression(5), Operation.GREATER_THAN); 38 | HavingStrategy strategy = (HavingStrategy) new Having(expression).getPostStrategy(); 39 | Clip result = strategy.execute(clip); 40 | 41 | Assert.assertEquals(result.getRecords().size(), 3); 42 | Assert.assertEquals(result.getRecords().get(0).typedGet("a").getValue(), 6); 43 | Assert.assertEquals(result.getRecords().get(1).typedGet("a").getValue(), 8); 44 | Assert.assertEquals(result.getRecords().get(2).typedGet("a").getValue(), 7); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/common/BulletError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common; 7 | 8 | import com.yahoo.bullet.result.JSONFormatter; 9 | import lombok.Getter; 10 | import lombok.RequiredArgsConstructor; 11 | 12 | import java.io.Serializable; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | import static java.util.Collections.singletonList; 18 | 19 | @Getter @RequiredArgsConstructor 20 | public class BulletError implements JSONFormatter, Serializable { 21 | private static final long serialVersionUID = -8557063189698127685L; 22 | 23 | public static final String ERROR_KEY = "error"; 24 | public static final String RESOLUTIONS_KEY = "resolutions"; 25 | 26 | private final String error; 27 | private final List resolutions; 28 | 29 | /** 30 | * Creates a BulletError with the given error message and resolution. 31 | * 32 | * @param error The message denoting the error. 33 | * @param resolution A resolution message. 34 | */ 35 | public BulletError(String error, String resolution) { 36 | this(error, singletonList(resolution)); 37 | } 38 | 39 | @Override 40 | public String asJSON() { 41 | Map map = new HashMap<>(); 42 | map.put(ERROR_KEY, error); 43 | map.put(RESOLUTIONS_KEY, resolutions); 44 | return JSONFormatter.asJSON(map); 45 | } 46 | 47 | /** 48 | * Creates a BulletError with the given error message and resolution. 49 | * 50 | * @param error The message denoting the error. 51 | * @param resolution A resolution message. 52 | * 53 | * @return A BulletError representing this cause. 54 | */ 55 | public static BulletError makeError(String error, String resolution) { 56 | return new BulletError(error, resolution); 57 | } 58 | 59 | @Override 60 | public String toString() { 61 | return "{error: " + error + ", resolutions: " + resolutions + "}"; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/expressions/UnaryExpressionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.evaluators.UnaryEvaluator; 10 | import com.yahoo.bullet.typesystem.Type; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | public class UnaryExpressionTest { 15 | @Test 16 | public void testConstructor() { 17 | UnaryExpression expression = new UnaryExpression(new ValueExpression(1), Operation.NOT); 18 | Assert.assertEquals(expression.toString(), "{operand: {value: 1, type: INTEGER}, op: NOT, type: null}"); 19 | Assert.assertTrue(expression.getEvaluator() instanceof UnaryEvaluator); 20 | } 21 | 22 | @Test(expectedExceptions = NullPointerException.class) 23 | public void testConstructorNullOperand() { 24 | new UnaryExpression(null, Operation.NOT); 25 | } 26 | 27 | @Test(expectedExceptions = NullPointerException.class) 28 | public void testConstructorNullOp() { 29 | new UnaryExpression(new ValueExpression(1), null); 30 | } 31 | 32 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "Unary expression requires a unary operation\\.") 33 | public void testConstructorNotUnaryOp() { 34 | new UnaryExpression(new ValueExpression(1), Operation.AND); 35 | } 36 | 37 | @Test 38 | public void testEqualsAndHashCode() { 39 | UnaryExpression expression = new UnaryExpression(new ValueExpression(1), Operation.NOT); 40 | expression.setType(Type.BOOLEAN); 41 | 42 | ExpressionUtils.testEqualsAndHashCode(() -> new UnaryExpression(new ValueExpression(1), Operation.NOT), 43 | new UnaryExpression(new ValueExpression(2), Operation.NOT), 44 | new UnaryExpression(new ValueExpression(1), Operation.IS_NOT_NULL), 45 | expression); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/aggregations/GroupAllTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | import com.yahoo.bullet.common.BulletException; 10 | import com.yahoo.bullet.querying.aggregations.GroupAllStrategy; 11 | import com.yahoo.bullet.querying.aggregations.grouping.GroupOperation; 12 | import org.testng.Assert; 13 | import org.testng.annotations.Test; 14 | 15 | import java.util.Collections; 16 | 17 | public class GroupAllTest { 18 | private BulletConfig config = new BulletConfig(); 19 | 20 | @Test 21 | public void testGetType() { 22 | GroupAll aggregation = new GroupAll(Collections.emptySet()); 23 | 24 | Assert.assertEquals(aggregation.getType(), AggregationType.GROUP); 25 | } 26 | 27 | @Test 28 | public void testGetFields() { 29 | GroupAll aggregation = new GroupAll(Collections.emptySet()); 30 | 31 | Assert.assertEquals(aggregation.getFields(), Collections.emptyList()); 32 | } 33 | 34 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "COUNT_FIELD is not a valid operation\\.") 35 | public void testCountFieldInvalid() { 36 | GroupOperation operation = new GroupOperation(GroupOperation.GroupOperationType.COUNT_FIELD, "abc", null); 37 | new GroupAll(Collections.singleton(operation)); 38 | } 39 | 40 | @Test 41 | public void testGetStrategy() { 42 | GroupAll aggregation = new GroupAll(Collections.emptySet()); 43 | aggregation.configure(config); 44 | 45 | Assert.assertTrue(aggregation.getStrategy(new BulletConfig()) instanceof GroupAllStrategy); 46 | } 47 | 48 | @Test 49 | public void testToString() { 50 | GroupOperation operation = new GroupOperation(GroupOperation.GroupOperationType.SUM, "abc", "sum"); 51 | GroupAll aggregation = new GroupAll(Collections.singleton(operation)); 52 | 53 | Assert.assertEquals(aggregation.toString(), "{size: null, type: GROUP, operations: [{type: SUM, field: abc, name: sum}]}"); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/postaggregations/CullingStrategyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.postaggregations; 7 | 8 | import com.yahoo.bullet.query.postaggregations.Culling; 9 | import com.yahoo.bullet.record.BulletRecord; 10 | import com.yahoo.bullet.result.Clip; 11 | import com.yahoo.bullet.result.RecordBox; 12 | import org.testng.Assert; 13 | import org.testng.annotations.Test; 14 | 15 | import java.util.ArrayList; 16 | import java.util.Arrays; 17 | import java.util.HashSet; 18 | import java.util.List; 19 | import java.util.Set; 20 | 21 | public class CullingStrategyTest { 22 | @Test 23 | public void testCulling() { 24 | List records = new ArrayList<>(); 25 | records.add(RecordBox.get().add("a", 1).add("b", 2).add("c", 3).getRecord()); 26 | records.add(RecordBox.get().add("a", 4).add("b", 5).add("c", 6).getRecord()); 27 | records.add(RecordBox.get().add("a", 7).add("b", 8).add("c", 9).getRecord()); 28 | 29 | Clip clip = new Clip(); 30 | clip.add(records); 31 | 32 | Set transientFields = new HashSet<>(Arrays.asList("a", "b")); 33 | CullingStrategy strategy = (CullingStrategy) new Culling(transientFields).getPostStrategy(); 34 | Clip result = strategy.execute(clip); 35 | 36 | Assert.assertEquals(result.getRecords().size(), 3); 37 | Assert.assertTrue(result.getRecords().get(0).typedGet("a").isNull()); 38 | Assert.assertTrue(result.getRecords().get(0).typedGet("b").isNull()); 39 | Assert.assertEquals(result.getRecords().get(0).typedGet("c").getValue(), 3); 40 | Assert.assertTrue(result.getRecords().get(1).typedGet("a").isNull()); 41 | Assert.assertTrue(result.getRecords().get(1).typedGet("b").isNull()); 42 | Assert.assertEquals(result.getRecords().get(1).typedGet("c").getValue(), 6); 43 | Assert.assertTrue(result.getRecords().get(2).typedGet("a").isNull()); 44 | Assert.assertTrue(result.getRecords().get(2).typedGet("b").isNull()); 45 | Assert.assertEquals(result.getRecords().get(2).typedGet("c").getValue(), 9); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/expressions/ValueExpressionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.evaluators.ValueEvaluator; 10 | import com.yahoo.bullet.typesystem.Type; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | import java.util.ArrayList; 15 | 16 | public class ValueExpressionTest { 17 | @Test 18 | public void testConstructor() { 19 | ValueExpression expression = new ValueExpression(1); 20 | Assert.assertEquals(expression.getValue(), 1); 21 | Assert.assertEquals(expression.getType(), Type.INTEGER); 22 | Assert.assertEquals(expression.toString(), "{value: 1, type: INTEGER}"); 23 | 24 | expression = new ValueExpression("1"); 25 | Assert.assertEquals(expression.getValue(), "1"); 26 | Assert.assertEquals(expression.getType(), Type.STRING); 27 | Assert.assertEquals(expression.toString(), "{value: '1', type: STRING}"); 28 | 29 | expression = new ValueExpression(null); 30 | Assert.assertEquals(expression.getValue(), null); 31 | Assert.assertEquals(expression.getType(), Type.NULL); 32 | Assert.assertEquals(expression.toString(), "{value: null, type: NULL}"); 33 | 34 | } 35 | 36 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "Value must be primitive or null\\.") 37 | public void testConstructorThrow() { 38 | new ValueExpression(new ArrayList<>()); 39 | } 40 | 41 | @Test 42 | public void testGetEvaluator() { 43 | Assert.assertTrue(new ValueExpression(1).getEvaluator() instanceof ValueEvaluator); 44 | } 45 | 46 | @Test 47 | public void testEqualsAndHashCode() { 48 | ValueExpression expression = new ValueExpression(1); 49 | expression.setType(Type.LONG); 50 | 51 | ExpressionUtils.testEqualsAndHashCode(() -> new ValueExpression(1), 52 | new ValueExpression(2), 53 | expression); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/tablefunctors/LateralViewFunctor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.tablefunctors; 7 | 8 | import com.yahoo.bullet.query.tablefunctions.LateralView; 9 | import com.yahoo.bullet.query.tablefunctions.TableFunction; 10 | import com.yahoo.bullet.record.BulletRecord; 11 | import com.yahoo.bullet.record.BulletRecordProvider; 12 | 13 | import java.util.List; 14 | import java.util.stream.Collectors; 15 | import java.util.stream.Stream; 16 | 17 | /** 18 | * A table functor that joins the generated records of the nested table functor with the original input record. 19 | */ 20 | public class LateralViewFunctor extends TableFunctor { 21 | private static final long serialVersionUID = 1017033253024183470L; 22 | 23 | final List tableFunctors; 24 | 25 | /** 26 | * Constructor that creates a lateral view table functor from a {@link LateralView}. 27 | * 28 | * @param lateralView The lateral view table function to construct the table functor from. 29 | */ 30 | public LateralViewFunctor(LateralView lateralView) { 31 | tableFunctors = lateralView.getTableFunctions().stream().map(TableFunction::getTableFunctor).collect(Collectors.toList()); 32 | } 33 | 34 | @Override 35 | public List apply(BulletRecord record, BulletRecordProvider provider) { 36 | if (tableFunctors.size() == 1) { 37 | return apply(record, provider, tableFunctors.get(0)); 38 | } 39 | Stream recordsStream = Stream.of(record); 40 | for (TableFunctor tableFunctor : tableFunctors) { 41 | recordsStream = recordsStream.flatMap(r -> apply(r, provider, tableFunctor).stream()); 42 | } 43 | return recordsStream.collect(Collectors.toList()); 44 | } 45 | 46 | private List apply(BulletRecord record, BulletRecordProvider provider, TableFunctor tableFunctor) { 47 | List records = tableFunctor.apply(record, provider); 48 | return records.stream().map(generated -> new LateralViewBulletRecord(record, generated)).collect(Collectors.toList()); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/expressions/NAryExpressionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.evaluators.NAryEvaluator; 10 | import com.yahoo.bullet.typesystem.Type; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | import java.util.ArrayList; 15 | import java.util.Arrays; 16 | 17 | public class NAryExpressionTest { 18 | @Test 19 | public void testConstructor() { 20 | NAryExpression expression = new NAryExpression(Arrays.asList(new ValueExpression(1)), Operation.IF); 21 | Assert.assertEquals(expression.toString(), "{operands: [{value: 1, type: INTEGER}], op: IF, type: null}"); 22 | Assert.assertTrue(expression.getEvaluator() instanceof NAryEvaluator); 23 | } 24 | 25 | @Test(expectedExceptions = NullPointerException.class) 26 | public void testConstructorNullValues() { 27 | new NAryExpression(null, Operation.IF); 28 | } 29 | 30 | @Test(expectedExceptions = NullPointerException.class) 31 | public void testConstructorNullOp() { 32 | new NAryExpression(new ArrayList<>(), null); 33 | } 34 | 35 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "N-ary expression requires an n-ary operation\\.") 36 | public void testConstructorNotNAryOp() { 37 | new NAryExpression(new ArrayList<>(), Operation.ADD); 38 | } 39 | 40 | @Test 41 | public void testEqualsAndHashCode() { 42 | NAryExpression expression = new NAryExpression(Arrays.asList(new ValueExpression(1)), Operation.IF); 43 | expression.setType(Type.INTEGER); 44 | 45 | ExpressionUtils.testEqualsAndHashCode(() -> new NAryExpression(Arrays.asList(new ValueExpression(1)), Operation.IF), 46 | new NAryExpression(Arrays.asList(new ValueExpression(2)), Operation.IF), 47 | new NAryExpression(Arrays.asList(new ValueExpression(1)), Operation.AND), 48 | expression); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/storage/MemoryStorageManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.storage; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | 10 | import java.io.Serializable; 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | import java.util.Set; 14 | import java.util.concurrent.CompletableFuture; 15 | 16 | /** 17 | * A Storage manager that stores everything in-memory and does not support namespaces or partitions. 18 | */ 19 | public class MemoryStorageManager extends StorageManager implements Serializable { 20 | private static final long serialVersionUID = 3815534537510449363L; 21 | 22 | private Map storage; 23 | 24 | /** 25 | * Constructor. 26 | * 27 | * @param config The {@link BulletConfig} to create this manager with. 28 | */ 29 | public MemoryStorageManager(BulletConfig config) { 30 | super(config); 31 | storage = new HashMap<>(); 32 | } 33 | 34 | @Override 35 | protected CompletableFuture putRaw(String namespace, String id, byte[] value) { 36 | storage.put(id, value); 37 | return SUCCESS; 38 | } 39 | 40 | @Override 41 | protected CompletableFuture getRaw(String namespace, String id) { 42 | return CompletableFuture.completedFuture(storage.get(id)); 43 | } 44 | 45 | @Override 46 | protected CompletableFuture> getAllRaw(String namespace) { 47 | return CompletableFuture.completedFuture(new HashMap<>(storage)); 48 | } 49 | 50 | @Override 51 | protected CompletableFuture removeRaw(String namespace, String id) { 52 | return CompletableFuture.completedFuture(storage.remove(id)); 53 | } 54 | 55 | @Override 56 | public CompletableFuture wipe() { 57 | storage.clear(); 58 | return SUCCESS; 59 | } 60 | 61 | @Override 62 | public CompletableFuture clear(String namespace) { 63 | return wipe(); 64 | } 65 | 66 | @Override 67 | public CompletableFuture clear(String namespace, Set ids) { 68 | if (ids != null) { 69 | ids.forEach(storage::remove); 70 | } 71 | return SUCCESS; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/windowing/Tumbling.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.windowing; 7 | 8 | import com.yahoo.bullet.querying.aggregations.Strategy; 9 | import com.yahoo.bullet.common.BulletConfig; 10 | import com.yahoo.bullet.query.Window; 11 | import com.yahoo.bullet.result.Meta; 12 | 13 | import java.util.Map; 14 | 15 | import static com.yahoo.bullet.result.Meta.addIfNonNull; 16 | 17 | public class Tumbling extends Basic { 18 | public static final String NAME = "Tumbling"; 19 | 20 | protected long nextCloseTime; 21 | protected long windowLength; 22 | 23 | /** 24 | * Creates an instance of this windowing scheme with the provided {@link Strategy} and {@link BulletConfig}. 25 | * 26 | * @param aggregation The non-null initialized aggregation strategy that this window will operate. 27 | * @param window The initialized, configured window to use. 28 | * @param config The validated config to use. 29 | */ 30 | public Tumbling(Strategy aggregation, Window window, BulletConfig config) { 31 | super(aggregation, window, config); 32 | windowLength = (long) window.getEmitEvery(); 33 | nextCloseTime = System.currentTimeMillis() + windowLength; 34 | } 35 | 36 | @Override 37 | protected Map getMetadata(Map metadataKeys) { 38 | Map meta = super.getMetadata(metadataKeys); 39 | addIfNonNull(meta, metadataKeys, Meta.Concept.WINDOW_EXPECTED_EMIT_TIME, () -> this.nextCloseTime); 40 | return meta; 41 | } 42 | 43 | @Override 44 | public void reset() { 45 | super.reset(); 46 | nextCloseTime = nextCloseTime + windowLength; 47 | } 48 | 49 | @Override 50 | public boolean isClosed() { 51 | return System.currentTimeMillis() >= nextCloseTime; 52 | } 53 | 54 | @Override 55 | public boolean isClosedForPartition() { 56 | // For tumbling windows, isClosedForPartition is the same as isClosed. 57 | return isClosed(); 58 | } 59 | 60 | @Override 61 | public void start() { 62 | nextCloseTime = System.currentTimeMillis() + windowLength; 63 | } 64 | 65 | @Override 66 | protected String name() { 67 | return NAME; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/expressions/CastExpressionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.evaluators.CastEvaluator; 10 | import com.yahoo.bullet.typesystem.Type; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | public class CastExpressionTest { 15 | @Test 16 | public void testConstructor() { 17 | CastExpression expression = new CastExpression(new ValueExpression(1), Type.LONG); 18 | Assert.assertEquals(expression.toString(), "{value: {value: 1, type: INTEGER}, castType: LONG, type: null}"); 19 | Assert.assertTrue(expression.getEvaluator() instanceof CastEvaluator); 20 | } 21 | 22 | @Test(expectedExceptions = NullPointerException.class) 23 | public void testConstructorNullValue() { 24 | new CastExpression(null, Type.LONG); 25 | } 26 | 27 | @Test(expectedExceptions = NullPointerException.class) 28 | public void testConstructorNullCastType() { 29 | new CastExpression(new ValueExpression(1), null); 30 | } 31 | 32 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "Cast type cannot be null or unknown\\.") 33 | public void testConstructorNullType() { 34 | new CastExpression(new ValueExpression(1), Type.NULL); 35 | } 36 | 37 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "Cast type cannot be null or unknown\\.") 38 | public void testConstructorUnknownType() { 39 | new CastExpression(new ValueExpression(1), Type.UNKNOWN); 40 | } 41 | 42 | @Test 43 | public void testEqualsAndHashCode() { 44 | CastExpression expression = new CastExpression(new ValueExpression(1), Type.LONG); 45 | expression.setType(Type.LONG); 46 | 47 | ExpressionUtils.testEqualsAndHashCode(() -> new CastExpression(new ValueExpression(1), Type.LONG), 48 | new CastExpression(new ValueExpression(2), Type.LONG), 49 | new CastExpression(new ValueExpression(1), Type.DOUBLE), 50 | expression); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/common/SerializerDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.common; 7 | 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | import java.io.ByteArrayInputStream; 11 | import java.io.ByteArrayOutputStream; 12 | import java.io.IOException; 13 | import java.io.ObjectInputStream; 14 | import java.io.ObjectOutputStream; 15 | import java.io.Serializable; 16 | 17 | @Slf4j 18 | public class SerializerDeserializer { 19 | /** 20 | * Convenience method to deserialize an object into a type from raw serialized data produced by the method 21 | * {@link #toBytes(Serializable)}. 22 | * 23 | * @param data The raw serialized byte[] representing the data. 24 | * @param The class to try to get the data as. 25 | * @return A reified object or null if not successful. 26 | */ 27 | public static U fromBytes(byte[] data) { 28 | try ( 29 | ByteArrayInputStream bis = new ByteArrayInputStream(data); 30 | ObjectInputStream ois = new ObjectInputStream(bis) 31 | ) { 32 | return (U) ois.readObject(); 33 | } catch (IOException | ClassNotFoundException | RuntimeException e) { 34 | log.error("Could not reify an Object from raw data {}", data); 35 | log.error("Exception was: ", e); 36 | } 37 | return null; 38 | } 39 | 40 | /** 41 | * Convenience method to serializes the given {@link Serializable} object to a raw byte[]. 42 | * 43 | * @param object The object to serialize. 44 | * @param The subtype of {@link Serializable} to try and serialize. 45 | * @return the serialized byte[] or null if not successful. 46 | */ 47 | public static byte[] toBytes(U object) { 48 | try ( 49 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 50 | ObjectOutputStream oos = new ObjectOutputStream(bos) 51 | ) { 52 | oos.writeObject(object); 53 | return bos.toByteArray(); 54 | } catch (IOException | RuntimeException e) { 55 | log.error("Could not serialize given object {}", object); 56 | log.error("Exception was: ", e); 57 | } 58 | return null; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/expressions/CastExpression.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.evaluators.CastEvaluator; 10 | import com.yahoo.bullet.querying.evaluators.Evaluator; 11 | import com.yahoo.bullet.typesystem.Type; 12 | import lombok.Getter; 13 | 14 | import java.util.Objects; 15 | 16 | /** 17 | * An expression that requires a value and a cast type. 18 | */ 19 | @Getter 20 | public class CastExpression extends Expression { 21 | private static final long serialVersionUID = -1732511101513444236L; 22 | private static final BulletException CAST_EXPRESSION_REQUIRES_VALID_TYPE = 23 | new BulletException("Cast type cannot be null or unknown.", "Please specify a valid cast type."); 24 | 25 | private final Expression value; 26 | private final Type castType; 27 | 28 | /** 29 | * Constructor that creates a cast expression. 30 | * 31 | * @param value The non-null value to cast. 32 | * @param castType The non-null cast type to cast the value to. 33 | */ 34 | public CastExpression(Expression value, Type castType) { 35 | this.value = Objects.requireNonNull(value); 36 | this.castType = Objects.requireNonNull(castType); 37 | if (Type.isNull(castType) || Type.isUnknown(castType)) { 38 | throw CAST_EXPRESSION_REQUIRES_VALID_TYPE; 39 | } 40 | } 41 | 42 | @Override 43 | public Evaluator getEvaluator() { 44 | return new CastEvaluator(this); 45 | } 46 | 47 | @Override 48 | public boolean equals(Object obj) { 49 | if (obj == this) { 50 | return true; 51 | } 52 | if (!(obj instanceof CastExpression)) { 53 | return false; 54 | } 55 | CastExpression other = (CastExpression) obj; 56 | return Objects.equals(value, other.value) && castType == other.castType && type == other.type; 57 | } 58 | 59 | @Override 60 | public int hashCode() { 61 | return Objects.hash(value, castType, type); 62 | } 63 | 64 | @Override 65 | public String toString() { 66 | return "{value: " + value + ", castType: " + castType + ", " + super.toString() + "}"; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/expressions/UnaryExpression.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.evaluators.Evaluator; 10 | import com.yahoo.bullet.querying.evaluators.UnaryEvaluator; 11 | import lombok.Getter; 12 | 13 | import java.util.Objects; 14 | 15 | import static com.yahoo.bullet.query.expressions.Operation.UNARY_OPERATIONS; 16 | 17 | /** 18 | * An expression that requires an operand and a unary operation. 19 | */ 20 | @Getter 21 | public class UnaryExpression extends Expression { 22 | private static final long serialVersionUID = -1893522779659725928L; 23 | private static final BulletException UNARY_EXPRESSION_REQUIRES_UNARY_OPERATION = 24 | new BulletException("Unary expression requires a unary operation.", "Please specify a unary operation."); 25 | 26 | private final Expression operand; 27 | private final Operation op; 28 | 29 | /** 30 | * Constructor that creates a unary expression. 31 | * 32 | * @param operand The non-null operand. 33 | * @param op The non-null unary operation. 34 | */ 35 | public UnaryExpression(Expression operand, Operation op) { 36 | this.operand = Objects.requireNonNull(operand); 37 | this.op = Objects.requireNonNull(op); 38 | if (!UNARY_OPERATIONS.contains(op)) { 39 | throw UNARY_EXPRESSION_REQUIRES_UNARY_OPERATION; 40 | } 41 | } 42 | 43 | @Override 44 | public Evaluator getEvaluator() { 45 | return new UnaryEvaluator(this); 46 | } 47 | 48 | @Override 49 | public boolean equals(Object obj) { 50 | if (obj == this) { 51 | return true; 52 | } 53 | if (!(obj instanceof UnaryExpression)) { 54 | return false; 55 | } 56 | UnaryExpression other = (UnaryExpression) obj; 57 | return Objects.equals(operand, other.operand) && op == other.op && type == other.type; 58 | } 59 | 60 | @Override 61 | public int hashCode() { 62 | return Objects.hash(operand, op, type); 63 | } 64 | 65 | @Override 66 | public String toString() { 67 | return "{operand: " + operand + ", op: " + op + ", " + super.toString() + "}"; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/querying/RunningQueryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | import com.yahoo.bullet.pubsub.Metadata; 10 | import com.yahoo.bullet.query.Projection; 11 | import com.yahoo.bullet.query.Query; 12 | import com.yahoo.bullet.query.Window; 13 | import com.yahoo.bullet.query.aggregations.Raw; 14 | import org.testng.Assert; 15 | import org.testng.annotations.Test; 16 | 17 | public class RunningQueryTest { 18 | @Test 19 | public void testCreatingWithQuery() { 20 | BulletConfig config = new BulletConfig(); 21 | Query query = new Query(new Projection(), null, new Raw(null), null, new Window(), null); 22 | query.configure(config); 23 | 24 | RunningQuery runningQuery = new RunningQuery("foo", query, new Metadata(null, "bar")); 25 | Assert.assertEquals(runningQuery.getId(), "foo"); 26 | Assert.assertNotNull(runningQuery.getQuery()); 27 | Assert.assertEquals(runningQuery.getQueryString(), "bar"); 28 | Assert.assertEquals(runningQuery.toString(), query.toString()); 29 | } 30 | 31 | @Test 32 | public void testStartTime() { 33 | long start = System.currentTimeMillis(); 34 | BulletConfig config = new BulletConfig(); 35 | Query query = new Query(new Projection(), null, new Raw(null), null, new Window(), null); 36 | query.configure(config); 37 | 38 | RunningQuery runningQuery = new RunningQuery("foo", query, new Metadata(null, null)); 39 | 40 | long end = System.currentTimeMillis(); 41 | Assert.assertTrue(runningQuery.getStartTime() >= start); 42 | Assert.assertTrue(runningQuery.getStartTime() <= end); 43 | Assert.assertFalse(runningQuery.isTimedOut()); 44 | } 45 | 46 | @Test 47 | public void testTimingOut() throws Exception { 48 | BulletConfig config = new BulletConfig(); 49 | Query query = new Query(new Projection(), null, new Raw(null), null, new Window(), 1L); 50 | query.configure(config); 51 | 52 | RunningQuery runningQuery = new RunningQuery("foo", query, new Metadata(null, null)); 53 | 54 | // Sleep to make sure it's 1 ms 55 | Thread.sleep(1); 56 | 57 | Assert.assertTrue(runningQuery.isTimedOut()); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/expressions/ValueExpression.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.evaluators.Evaluator; 10 | import com.yahoo.bullet.querying.evaluators.ValueEvaluator; 11 | import com.yahoo.bullet.typesystem.Type; 12 | import lombok.Getter; 13 | 14 | import java.io.Serializable; 15 | import java.util.Objects; 16 | 17 | /** 18 | * An expression that requires a primitive or null value. 19 | */ 20 | @Getter 21 | public class ValueExpression extends Expression { 22 | private static final long serialVersionUID = -6979831483897873098L; 23 | private static final BulletException VALUE_EXPRESSION_REQUIRES_PRIMITIVE_OR_NULL = 24 | new BulletException("Value must be primitive or null.", "Please specify a valid value."); 25 | private static final String SINGLE_QUOTE = "'"; 26 | 27 | private final Serializable value; 28 | 29 | /** 30 | * Constructor that creates a value expression. 31 | * 32 | * @param value The non-null value to be wrapped. 33 | */ 34 | public ValueExpression(Serializable value) { 35 | this.value = value; 36 | this.type = Type.getType(value); 37 | if (!Type.isPrimitive(type) && !Type.isNull(type)) { 38 | throw VALUE_EXPRESSION_REQUIRES_PRIMITIVE_OR_NULL; 39 | } 40 | } 41 | 42 | @Override 43 | public Evaluator getEvaluator() { 44 | return new ValueEvaluator(this); 45 | } 46 | 47 | @Override 48 | public boolean equals(Object obj) { 49 | if (obj == this) { 50 | return true; 51 | } 52 | if (!(obj instanceof ValueExpression)) { 53 | return false; 54 | } 55 | ValueExpression other = (ValueExpression) obj; 56 | return Objects.equals(value, other.value) && type == other.type; 57 | } 58 | 59 | @Override 60 | public int hashCode() { 61 | return Objects.hash(value, type); 62 | } 63 | 64 | private String toFormattedString() { 65 | String asString = Objects.toString(value); 66 | return type == Type.STRING ? SINGLE_QUOTE + asString + SINGLE_QUOTE : asString; 67 | } 68 | 69 | @Override 70 | public String toString() { 71 | return "{value: " + toFormattedString() + ", " + super.toString() + "}"; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/aggregations/KMVStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.aggregations; 7 | 8 | import com.yahoo.bullet.querying.aggregations.sketches.KMVSketch; 9 | import com.yahoo.bullet.common.BulletConfig; 10 | import com.yahoo.bullet.query.aggregations.Aggregation; 11 | import com.yahoo.sketches.ResizeFactor; 12 | 13 | /** 14 | * The parent class for {@link SketchingStrategy} that use the KMV type of Sketch - Theta and Tuple. 15 | */ 16 | abstract class KMVStrategy extends SketchingStrategy { 17 | /** 18 | * Constructor that requires an {@link Aggregation} and a {@link BulletConfig} configuration. 19 | * 20 | * @param aggregation An {@link Aggregation} with valid fields and attributes for this aggregation type. 21 | * @param config The config that has relevant configs for this strategy. 22 | */ 23 | @SuppressWarnings("unchecked") 24 | KMVStrategy(Aggregation aggregation, BulletConfig config) { 25 | super(aggregation, config); 26 | } 27 | 28 | /** 29 | * Converts a integer representing the resizing for Sketches into a {@link ResizeFactor}. 30 | * 31 | * @param config The config that has relevant configs for this strategy. 32 | * @param key The key to get the configured resize factor from the configuration. 33 | * @return A {@link ResizeFactor} represented by the integer or {@link ResizeFactor#X8} otherwise. 34 | */ 35 | @SuppressWarnings("unchecked") 36 | ResizeFactor getResizeFactor(BulletConfig config, String key) { 37 | return getResizeFactor(config.getAs(key, Integer.class)); 38 | } 39 | /** 40 | * Converts a integer representing the resizing for Sketches into a {@link ResizeFactor}. 41 | * 42 | * @param factor An int representing the scaling when the Sketch reaches its threshold. Supports 1, 2, 4 and 8. 43 | * @return A {@link ResizeFactor} represented by the integer or {@link ResizeFactor#X8} otherwise. 44 | */ 45 | public static ResizeFactor getResizeFactor(int factor) { 46 | switch (factor) { 47 | case 1: 48 | return ResizeFactor.X1; 49 | case 2: 50 | return ResizeFactor.X2; 51 | case 4: 52 | return ResizeFactor.X4; 53 | default: 54 | return ResizeFactor.X8; 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/TopK.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.querying.aggregations.FrequentItemsSketchingStrategy; 9 | import com.yahoo.bullet.querying.aggregations.Strategy; 10 | import com.yahoo.bullet.common.BulletConfig; 11 | import com.yahoo.bullet.common.BulletException; 12 | import com.yahoo.bullet.common.Utilities; 13 | import lombok.Getter; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | import java.util.Map; 18 | import java.util.Objects; 19 | 20 | @Getter 21 | public class TopK extends Aggregation { 22 | private static final long serialVersionUID = 2934123789816778466L; 23 | private static final BulletException TOP_K_REQUIRES_FIELDS = 24 | new BulletException("TOP K requires at least one field.", "Please add at least one field."); 25 | 26 | private final Map fieldsToNames; 27 | private final Long threshold; 28 | private final String name; 29 | 30 | /** 31 | * Constructor that creates a TOP_K aggregation with a specified max size. 32 | * 33 | * @param fieldsToNames The mapping of fields to aliases of the Top K aggregation. Must not be empty. 34 | * @param size The max size of the Top K aggregation. Can be null. 35 | * @param threshold The minimum threshold of the Top K aggregation. Can be null. 36 | * @param name The name of the count field. 37 | */ 38 | public TopK(Map fieldsToNames, Integer size, Long threshold, String name) { 39 | super(size, AggregationType.TOP_K); 40 | Utilities.requireNonNull(fieldsToNames); 41 | if (fieldsToNames.isEmpty()) { 42 | throw TOP_K_REQUIRES_FIELDS; 43 | } 44 | this.fieldsToNames = fieldsToNames; 45 | this.threshold = threshold; 46 | this.name = Objects.requireNonNull(name); 47 | } 48 | 49 | @Override 50 | public Strategy getStrategy(BulletConfig config) { 51 | return new FrequentItemsSketchingStrategy(this, config); 52 | } 53 | 54 | @Override 55 | public List getFields() { 56 | return new ArrayList<>(fieldsToNames.keySet()); 57 | } 58 | 59 | @Override 60 | public String toString() { 61 | return "{size: " + size + ", type: " + type + ", fieldsToNames: " + fieldsToNames + ", threshold: " + threshold + ", name: " + getName() + "}"; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/pubsub/IdentityPubSubMessageSerDeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.pubsub; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | import com.yahoo.bullet.query.Projection; 10 | import com.yahoo.bullet.query.Query; 11 | import com.yahoo.bullet.query.Window; 12 | import com.yahoo.bullet.query.aggregations.Raw; 13 | import org.testng.Assert; 14 | import org.testng.annotations.Test; 15 | 16 | public class IdentityPubSubMessageSerDeTest { 17 | @Test 18 | public void testConvertingQuery() { 19 | IdentityPubSubMessageSerDe serDe = new IdentityPubSubMessageSerDe(null); 20 | 21 | Query query = new Query(new Projection(), null, new Raw(1), null, new Window(), 1L); 22 | PubSubMessage actual = serDe.toMessage("id", query, "foo"); 23 | Assert.assertEquals(actual.getId(), "id"); 24 | Assert.assertSame(actual.getContent(), query); 25 | Assert.assertEquals(actual.getMetadata().getContent(), "foo"); 26 | } 27 | 28 | @Test 29 | public void testConverting() { 30 | IdentityPubSubMessageSerDe serDe = new IdentityPubSubMessageSerDe(null); 31 | PubSubMessage expected = new PubSubMessage("foo", new byte[0], new Metadata(Metadata.Signal.CUSTOM, new byte[0])); 32 | Assert.assertSame(serDe.toMessage(expected), expected); 33 | } 34 | 35 | @Test 36 | public void testReverting() { 37 | IdentityPubSubMessageSerDe serDe = new IdentityPubSubMessageSerDe(null); 38 | PubSubMessage expected = new PubSubMessage("foo", new byte[0], new Metadata(Metadata.Signal.CUSTOM, new byte[0])); 39 | Assert.assertSame(serDe.fromMessage(expected), expected); 40 | } 41 | 42 | @Test 43 | public void testFrom() { 44 | BulletConfig config = new BulletConfig(); 45 | config.set(BulletConfig.PUBSUB_MESSAGE_SERDE_CLASS_NAME, IdentityPubSubMessageSerDe.class.getName()); 46 | PubSubMessageSerDe manager = PubSubMessageSerDe.from(config); 47 | Assert.assertTrue(manager instanceof IdentityPubSubMessageSerDe); 48 | } 49 | 50 | @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Cannot create.*") 51 | public void testFromWithABadClass() { 52 | BulletConfig config = new BulletConfig(); 53 | config.set(BulletConfig.PUBSUB_MESSAGE_SERDE_CLASS_NAME, "does.not.exist"); 54 | PubSubMessageSerDe.from(config); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/querying/aggregations/GroupAllStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.querying.aggregations; 7 | 8 | import com.yahoo.bullet.query.aggregations.Aggregation; 9 | import com.yahoo.bullet.query.aggregations.GroupAll; 10 | import com.yahoo.bullet.querying.aggregations.grouping.GroupData; 11 | import com.yahoo.bullet.querying.aggregations.grouping.GroupOperation; 12 | import com.yahoo.bullet.common.BulletConfig; 13 | import com.yahoo.bullet.common.SerializerDeserializer; 14 | import com.yahoo.bullet.record.BulletRecord; 15 | import com.yahoo.bullet.record.BulletRecordProvider; 16 | import com.yahoo.bullet.result.Clip; 17 | import lombok.extern.slf4j.Slf4j; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import java.util.Set; 22 | 23 | @Slf4j 24 | public class GroupAllStrategy implements Strategy { 25 | // We only have a single group. 26 | private GroupData data; 27 | 28 | private Set operations; 29 | private BulletRecordProvider provider; 30 | /** 31 | * Constructor that requires an {@link Aggregation}. 32 | * 33 | * @param aggregation The {@link Aggregation} that specifies how and what this will compute. 34 | * @param config The BulletConfig. 35 | */ 36 | public GroupAllStrategy(GroupAll aggregation, BulletConfig config) { 37 | // GroupOperations is all we care about - size etc. are meaningless for Group All since it's a single result 38 | operations = aggregation.getOperations(); 39 | data = new GroupData(operations); 40 | this.provider = config.getBulletRecordProvider(); 41 | } 42 | 43 | @Override 44 | public void consume(BulletRecord data) { 45 | this.data.consume(data); 46 | } 47 | 48 | @Override 49 | public void combine(byte[] data) { 50 | this.data.combine(data); 51 | } 52 | 53 | @Override 54 | public byte[] getData() { 55 | return SerializerDeserializer.toBytes(data); 56 | } 57 | 58 | @Override 59 | public Clip getResult() { 60 | return Clip.of(getRecords()); 61 | } 62 | 63 | @Override 64 | public List getRecords() { 65 | List list = new ArrayList<>(); 66 | list.add(data.getMetricsAsBulletRecord(provider)); 67 | return list; 68 | } 69 | 70 | @Override 71 | public void reset() { 72 | data = new GroupData(operations); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/expressions/NAryExpression.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.common.Utilities; 10 | import com.yahoo.bullet.querying.evaluators.Evaluator; 11 | import com.yahoo.bullet.querying.evaluators.NAryEvaluator; 12 | import lombok.Getter; 13 | 14 | import java.util.List; 15 | import java.util.Objects; 16 | 17 | import static com.yahoo.bullet.query.expressions.Operation.N_ARY_OPERATIONS; 18 | 19 | /** 20 | * An expression that requires a list of operands and an n-ary operation. 21 | */ 22 | @Getter 23 | public class NAryExpression extends Expression { 24 | private static final long serialVersionUID = -1000391436451418013L; 25 | private static final BulletException N_ARY_EXPRESSION_REQUIRES_N_ARY_OPERATION = 26 | new BulletException("N-ary expression requires an n-ary operation.", "Please specify an n-ary operation."); 27 | private static final String DELIMITER = ", "; 28 | 29 | private final List operands; 30 | private final Operation op; 31 | 32 | /** 33 | * Constructor that creates an n-ary expression. 34 | * 35 | * @param operands The non-null list of operands. 36 | * @param op The non-null n-ary operation. 37 | */ 38 | public NAryExpression(List operands, Operation op) { 39 | this.operands = Utilities.requireNonNull(operands); 40 | this.op = Objects.requireNonNull(op); 41 | if (!N_ARY_OPERATIONS.contains(op)) { 42 | throw N_ARY_EXPRESSION_REQUIRES_N_ARY_OPERATION; 43 | } 44 | } 45 | 46 | @Override 47 | public Evaluator getEvaluator() { 48 | return new NAryEvaluator(this); 49 | } 50 | 51 | @Override 52 | public boolean equals(Object obj) { 53 | if (obj == this) { 54 | return true; 55 | } 56 | if (!(obj instanceof NAryExpression)) { 57 | return false; 58 | } 59 | NAryExpression other = (NAryExpression) obj; 60 | return Objects.equals(operands, other.operands) && op == other.op && type == other.type; 61 | } 62 | 63 | @Override 64 | public int hashCode() { 65 | return Objects.hash(operands, op, type); 66 | } 67 | 68 | @Override 69 | public String toString() { 70 | return "{operands: " + operands + ", op: " + op + ", " + super.toString() + "}"; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/expressions/BinaryExpression.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.evaluators.BinaryEvaluator; 10 | import com.yahoo.bullet.querying.evaluators.Evaluator; 11 | import lombok.Getter; 12 | 13 | import java.util.Objects; 14 | 15 | import static com.yahoo.bullet.query.expressions.Operation.BINARY_OPERATIONS; 16 | 17 | /** 18 | * An expression that requires two operands and a binary operation. 19 | */ 20 | @Getter 21 | public class BinaryExpression extends Expression { 22 | private static final long serialVersionUID = -7911485746578844403L; 23 | private static final BulletException BINARY_EXPRESSION_REQUIRES_BINARY_OPERATION = 24 | new BulletException("Binary expression requires a binary operation.", "Please specify a binary operation."); 25 | 26 | private final Expression left; 27 | private final Expression right; 28 | private final Operation op; 29 | 30 | /** 31 | * Constructor that creates a binary expression. 32 | * 33 | * @param left The non-null left operand. 34 | * @param right The non-null right operand. 35 | * @param op The non-null binary operation. 36 | */ 37 | public BinaryExpression(Expression left, Expression right, Operation op) { 38 | this.left = Objects.requireNonNull(left); 39 | this.right = Objects.requireNonNull(right); 40 | this.op = Objects.requireNonNull(op); 41 | if (!BINARY_OPERATIONS.contains(op)) { 42 | throw BINARY_EXPRESSION_REQUIRES_BINARY_OPERATION; 43 | } 44 | } 45 | 46 | @Override 47 | public Evaluator getEvaluator() { 48 | return new BinaryEvaluator(this); 49 | } 50 | 51 | @Override 52 | public boolean equals(Object obj) { 53 | if (obj == this) { 54 | return true; 55 | } 56 | if (!(obj instanceof BinaryExpression)) { 57 | return false; 58 | } 59 | BinaryExpression other = (BinaryExpression) obj; 60 | return Objects.equals(left, other.left) && 61 | Objects.equals(right, other.right) && 62 | op == other.op && 63 | type == other.type; 64 | } 65 | 66 | @Override 67 | public int hashCode() { 68 | return Objects.hash(left, right, op, type); 69 | } 70 | 71 | @Override 72 | public String toString() { 73 | return "{left: " + left + ", right: " + right + ", op: " + op + ", " + super.toString() + "}"; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/storage/StorageConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.storage; 7 | 8 | import com.yahoo.bullet.common.BulletConfig; 9 | import com.yahoo.bullet.common.Config; 10 | import com.yahoo.bullet.common.Validator; 11 | import lombok.extern.slf4j.Slf4j; 12 | 13 | import java.util.Collections; 14 | import java.util.HashSet; 15 | import java.util.List; 16 | 17 | @Slf4j 18 | public class StorageConfig extends BulletConfig { 19 | private static final long serialVersionUID = 2635594185278740577L; 20 | 21 | // Field names 22 | public static final String PREFIX = "bullet.storage."; 23 | public static final String NAMESPACES = PREFIX + "namespaces"; 24 | public static final String PARTITION_COUNT = PREFIX + "partition.count"; 25 | 26 | // Defaults 27 | public static final int DEFAULT_PARTITION_COUNT = 1; 28 | public static final String DEFAULT_NAMESPACE = ""; 29 | public static final List DEFAULT_NAMESPACES = Collections.singletonList(DEFAULT_NAMESPACE); 30 | 31 | private static final Validator VALIDATOR = new Validator(); 32 | static { 33 | VALIDATOR.define(NAMESPACES) 34 | .defaultTo(DEFAULT_NAMESPACES) 35 | .checkIf(Validator.isListOfType(String.class)) 36 | .checkIf(Validator::isNonEmptyList) 37 | .castTo(StorageConfig::asSet); 38 | VALIDATOR.define(PARTITION_COUNT) 39 | .defaultTo(DEFAULT_PARTITION_COUNT) 40 | .checkIf(Validator::isPositiveInt) 41 | .unless(Validator::isNull) 42 | .castTo(Validator::asInt); 43 | } 44 | 45 | /** 46 | * Constructor that loads specific file augmented with defaults. 47 | * 48 | * @param file YAML file to load. 49 | */ 50 | public StorageConfig(String file) { 51 | this(new BulletConfig(file)); 52 | } 53 | 54 | /** 55 | * Constructor that loads the defaults and augments it with defaults. 56 | * 57 | * @param other The other config to wrap. 58 | */ 59 | public StorageConfig(Config other) { 60 | super(); 61 | merge(other); 62 | log.info("Merged settings:\n {}", this); 63 | } 64 | 65 | @Override 66 | public StorageConfig validate() { 67 | super.validate(); 68 | VALIDATOR.validate(this); 69 | return this; 70 | } 71 | 72 | @SuppressWarnings("unchecked") 73 | private static Object asSet(Object o) { 74 | List asList = (List) o; 75 | return new HashSet<>(asList); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/expressions/BinaryExpressionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.expressions; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.querying.evaluators.BinaryEvaluator; 10 | import com.yahoo.bullet.typesystem.Type; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | public class BinaryExpressionTest { 15 | @Test 16 | public void testConstructor() { 17 | BinaryExpression expression = new BinaryExpression(new ValueExpression(1), new ValueExpression(2), Operation.ADD); 18 | Assert.assertEquals(expression.toString(), "{left: {value: 1, type: INTEGER}, right: {value: 2, type: INTEGER}, op: +, type: null}"); 19 | Assert.assertTrue(expression.getEvaluator() instanceof BinaryEvaluator); 20 | } 21 | 22 | @Test(expectedExceptions = NullPointerException.class) 23 | public void testConstructorNullLeft() { 24 | new BinaryExpression(null, new ValueExpression(2), Operation.ADD); 25 | } 26 | 27 | @Test(expectedExceptions = NullPointerException.class) 28 | public void testConstructorNullRight() { 29 | new BinaryExpression(new ValueExpression(1), null, Operation.ADD); 30 | } 31 | 32 | @Test(expectedExceptions = NullPointerException.class) 33 | public void testConstructorNullOp() { 34 | new BinaryExpression(new ValueExpression(1), new ValueExpression(2), null); 35 | } 36 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "Binary expression requires a binary operation\\.") 37 | public void testConstructorNotBinaryOp() { 38 | new BinaryExpression(new ValueExpression(1), new ValueExpression(2), Operation.SIZE_OF); 39 | } 40 | 41 | @Test 42 | public void testEqualsAndHashCode() { 43 | BinaryExpression expression = new BinaryExpression(new ValueExpression(1), new ValueExpression(2), Operation.ADD); 44 | expression.setType(Type.INTEGER); 45 | 46 | ExpressionUtils.testEqualsAndHashCode(() -> new BinaryExpression(new ValueExpression(1), new ValueExpression(2), Operation.ADD), 47 | new BinaryExpression(new ValueExpression(2), new ValueExpression(2), Operation.ADD), 48 | new BinaryExpression(new ValueExpression(1), new ValueExpression(1), Operation.ADD), 49 | new BinaryExpression(new ValueExpression(1), new ValueExpression(2), Operation.SUB), 50 | expression); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/com/yahoo/bullet/query/aggregations/TopKTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.querying.aggregations.FrequentItemsSketchingStrategy; 9 | import com.yahoo.bullet.common.BulletConfig; 10 | import com.yahoo.bullet.common.BulletException; 11 | import org.testng.Assert; 12 | import org.testng.annotations.Test; 13 | 14 | import java.util.Collections; 15 | 16 | public class TopKTest { 17 | private static final String COUNT_NAME = "count"; 18 | private BulletConfig config = new BulletConfig(); 19 | 20 | @Test 21 | public void testTopKAggregation() { 22 | TopK aggregation = new TopK(Collections.singletonMap("abc", "def"), null, null, COUNT_NAME); 23 | aggregation.configure(config); 24 | 25 | Assert.assertEquals(aggregation.getType(), AggregationType.TOP_K); 26 | Assert.assertEquals(aggregation.getFields(), Collections.singletonList("abc")); 27 | Assert.assertEquals(aggregation.getFieldsToNames(), Collections.singletonMap("abc", "def")); 28 | Assert.assertTrue(aggregation.getStrategy(config) instanceof FrequentItemsSketchingStrategy); 29 | } 30 | 31 | @Test(expectedExceptions = NullPointerException.class) 32 | public void testConstructorNullFieldsThrows() { 33 | new TopK(null, null, null, null); 34 | } 35 | 36 | @Test(expectedExceptions = BulletException.class, expectedExceptionsMessageRegExp = "TOP K requires at least one field\\.") 37 | public void testConstructorEmptyFieldsThrows() { 38 | new TopK(Collections.emptyMap(), null, null, null); 39 | } 40 | 41 | @Test(expectedExceptions = NullPointerException.class) 42 | public void testConstructorNullNameThrows() { 43 | new TopK(Collections.singletonMap("abc", "def"), null, null, null); 44 | } 45 | 46 | @Test 47 | public void testGetThreshold() { 48 | TopK aggregation = new TopK(Collections.singletonMap("abc", "def"), null, 100L, COUNT_NAME); 49 | Assert.assertEquals(aggregation.getThreshold(), (Long) 100L); 50 | } 51 | 52 | @Test 53 | public void testGetName() { 54 | TopK aggregation = new TopK(Collections.singletonMap("abc", "def"), null, null, COUNT_NAME); 55 | Assert.assertEquals(aggregation.getName(), COUNT_NAME); 56 | } 57 | 58 | @Test 59 | public void testGetToString() { 60 | TopK aggregation = new TopK(Collections.singletonMap("abc", "def"), 500, 100L, COUNT_NAME); 61 | Assert.assertEquals(aggregation.toString(), "{size: 500, type: TOP_K, fieldsToNames: {abc=def}, threshold: 100, name: count}"); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/yahoo/bullet/query/aggregations/ManualDistribution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020, Yahoo Inc. 3 | * Licensed under the terms of the Apache License, Version 2.0. 4 | * See the LICENSE file associated with the project for terms. 5 | */ 6 | package com.yahoo.bullet.query.aggregations; 7 | 8 | import com.yahoo.bullet.common.BulletException; 9 | import com.yahoo.bullet.common.Utilities; 10 | import lombok.Getter; 11 | 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.stream.Collectors; 15 | 16 | @Getter 17 | public class ManualDistribution extends Distribution { 18 | private static final long serialVersionUID = 7022392121466809427L; 19 | private static final BulletException MANUAL_DISTRIBUTION_MISSING_POINTS = 20 | new BulletException("If specifying the distribution by a list of points, the list must contain at least one point.", 21 | "Please specify at least one point."); 22 | private static final BulletException QUANTILE_POINTS_INVALID_RANGE = 23 | new BulletException("The QUANTILE distribution requires points to be within the interval [0, 1] inclusive.", 24 | "Please specify a list of valid points."); 25 | 26 | 27 | private final List points; 28 | 29 | /** 30 | * Constructor that creates a DISTRIBUTION aggregation with a specified max size. The points in this distribution 31 | * are specified manually. 32 | * 33 | * @param field The non-null field. 34 | * @param type The non-null distribution type. 35 | * @param size The max size of the DISTRIBUTION aggregation. Can be null. 36 | * @param points The points of the distribution. Must be not be empty. 37 | */ 38 | public ManualDistribution(String field, DistributionType type, Integer size, List points) { 39 | super(field, type, size); 40 | Utilities.requireNonNull(points); 41 | if (points.isEmpty()) { 42 | throw MANUAL_DISTRIBUTION_MISSING_POINTS; 43 | } 44 | if (isListInvalid(points, type)) { 45 | throw QUANTILE_POINTS_INVALID_RANGE; 46 | } 47 | this.points = points.stream().distinct().sorted().collect(Collectors.toCollection(ArrayList::new)); 48 | } 49 | 50 | private static boolean isListInvalid(List points, DistributionType type) { 51 | // if type is QUANTILE, invalid range if start < 0 or end > 1 52 | return type == DistributionType.QUANTILE && (points.get(0) < 0.0 || points.get(points.size() - 1) > 1.0); 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return "{size: " + size + ", type: " + type + ", field: " + field + ", distributionType: " + distributionType + ", points: " + points + "}"; 58 | } 59 | } 60 | --------------------------------------------------------------------------------