├── example ├── example.yml ├── regexp_multibyte.csv ├── regexp_multibyte.yml ├── where.yml ├── or.yml ├── and.yml └── example.csv ├── settings.gradle ├── gradle ├── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties └── dependency-locks │ └── embulkPluginRuntime.lockfile ├── .travis.yml ├── src ├── main │ └── java │ │ └── org │ │ └── embulk │ │ └── filter │ │ └── row │ │ ├── condition │ │ ├── Condition.java │ │ ├── ConditionConfig.java │ │ ├── BooleanCondition.java │ │ ├── StringCondition.java │ │ ├── DoubleCondition.java │ │ ├── LongCondition.java │ │ ├── TimestampCondition.java │ │ └── ConditionFactory.java │ │ ├── where │ │ ├── ParserNode.java │ │ ├── ParserVal.java │ │ ├── _lexer.l │ │ ├── _parser.y │ │ ├── ParserLiteral.java │ │ ├── ParserExp.java │ │ └── Yylex.java │ │ ├── GuardColumnVisitorWhereImpl.java │ │ ├── AbstractGuardColumnVisitor.java │ │ ├── BuilderColumnVisitorImpl.java │ │ ├── GuardColumnVisitorOrImpl.java │ │ ├── GuardColumnVisitorAndImpl.java │ │ └── RowFilterPlugin.java └── test │ └── java │ └── org │ └── embulk │ └── filter │ └── row │ ├── condition │ ├── TestBooleanCondition.java │ ├── TestTimestampCondition.java │ ├── TestStringCondition.java │ ├── TestDoubleCondition.java │ ├── TestLongCondition.java │ └── TestConditionFactory.java │ └── where │ ├── TestYylex.java │ └── TestParser.java ├── .gitignore ├── script └── byaccj.sh ├── LICENSE.txt ├── CHANGELOG.md ├── gradlew.bat ├── config └── checkstyle │ └── checkstyle.xml ├── gradlew └── README.md /example/example.yml: -------------------------------------------------------------------------------- 1 | and.yml -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'embulk-filter-row' 2 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sonots/embulk-filter-row/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk7 4 | - oraclejdk7 5 | - oraclejdk8 6 | script: 7 | - ./gradlew test 8 | 9 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/condition/Condition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | public interface Condition 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/where/ParserNode.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.where; 2 | 3 | // Node of AST (Abstract Syntax Tree) 4 | public abstract class ParserNode 5 | { 6 | } 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | /pkg/ 3 | /tmp/ 4 | *.gemspec 5 | .gradle/ 6 | /classpath/ 7 | build/ 8 | .idea 9 | *.csv 10 | .tags 11 | .ruby-version 12 | *.iml 13 | *.class 14 | script/yacc 15 | script/jflex* 16 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /example/regexp_multibyte.csv: -------------------------------------------------------------------------------- 1 | time,foo,bar,flag,id,name,json,score 2 | 2015-07-13,,bar,true,0,ゼロ,"{\"foo\":\"bar\"}",1370 3 | 2015-07-13,,bar,true,1,イチ,"{\"foo\":\"bar\"}",3962 4 | 2015-07-13,,bar,true,2,ニ,"{\"foo\":\"bar\"}",7323 5 | 2015-07-13,,bar,true,3,サン,"{\"foo\":\"bar\"}",5905 6 | 2015-07-13,,bar,true,4,ヨン,"{\"foo\":\"bar\"}",8378 7 | 2015-07-13,,bar,true,5,ゴ,"{\"foo\":\"bar\"}",275 8 | 2015-07-13,,bar,true,6,ロク,"{\"foo\":\"bar\"}",43 9 | 2015-07-13,,bar,true,7,シチ,"{\"foo\":\"bar\"}",6130 10 | 2015-07-13,,bar,true,8,ハチ,"{\"foo\":\"bar\"}",6652 11 | 2015-07-13,,bar,true,9,キュウ,"{\"foo\":\"bar\"}",6822 12 | 2015-07-13,NULL,bar,false,NULL,NULL,NULL,4170 13 | -------------------------------------------------------------------------------- /example/regexp_multibyte.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: example/regexp_multibyte.csv 4 | parser: 5 | type: csv 6 | charset: UTF-8 7 | newline: CRLF 8 | null_string: "NULL" 9 | skip_header_lines: 1 10 | comment_line_marker: '#' 11 | columns: 12 | - {name: time, type: timestamp, format: "%Y-%m-%d"} 13 | - {name: foo, type: string} 14 | - {name: bar, type: string} 15 | - {name: flag, type: boolean} 16 | - {name: id, type: long} 17 | - {name: name, type: string} 18 | - {name: json, type: json} 19 | - {name: score, type: double} 20 | filters: 21 | - type: row 22 | where: name REGEXP '.*チ' 23 | out: 24 | type: stdout 25 | -------------------------------------------------------------------------------- /example/where.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: example/example.csv 4 | parser: 5 | type: csv 6 | charset: UTF-8 7 | newline: CRLF 8 | null_string: "NULL" 9 | skip_header_lines: 1 10 | comment_line_marker: '#' 11 | columns: 12 | - {name: time, type: timestamp, format: "%Y-%m-%d"} 13 | - {name: foo, type: string} 14 | - {name: bar, type: string} 15 | - {name: flag, type: boolean} 16 | - {name: id, type: long} 17 | - {name: name, type: string} 18 | - {name: json, type: json} 19 | - {name: score, type: double} 20 | filters: 21 | - type: row 22 | where: |- 23 | (time = TIMESTAMP '2015-07-13' AND 24 | foo IS NULL AND 25 | bar = 'bar' AND score>2000 AND 26 | flag=false) OR (id=9) 27 | out: 28 | type: stdout 29 | -------------------------------------------------------------------------------- /gradle/dependency-locks/embulkPluginRuntime.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7 5 | com.fasterxml.jackson.core:jackson-core:2.6.7 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7 8 | javax.validation:validation-api:1.1.0.Final 9 | org.apache.commons:commons-lang3:3.12.0 10 | org.embulk:embulk-util-config:0.3.0 11 | org.embulk:embulk-util-file:0.1.3 12 | org.embulk:embulk-util-json:0.1.1 13 | org.embulk:embulk-util-rubytime:0.3.2 14 | org.embulk:embulk-util-text:0.1.1 15 | org.embulk:embulk-util-timestamp:0.2.1 16 | org.jruby.jcodings:jcodings:1.0.55 17 | org.jruby.joni:joni:2.1.41 18 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/GuardColumnVisitorWhereImpl.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row; 2 | 3 | import org.embulk.filter.row.RowFilterPlugin.PluginTask; 4 | import org.embulk.filter.row.where.ParserExp; 5 | import org.embulk.spi.Exec; 6 | import org.embulk.spi.PageReader; 7 | import org.embulk.spi.Schema; 8 | import org.slf4j.Logger; 9 | 10 | class GuardColumnVisitorWhereImpl 11 | extends AbstractGuardColumnVisitor 12 | { 13 | private static final Logger logger = Exec.getLogger(RowFilterPlugin.class); 14 | ParserExp parserExp; 15 | 16 | GuardColumnVisitorWhereImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, ParserExp parserExp) 17 | { 18 | super(task, inputSchema, outputSchema, pageReader); 19 | this.parserExp = parserExp; 20 | } 21 | 22 | public boolean visitColumns(Schema inputSchema) 23 | { 24 | return parserExp.eval(pageReader); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /example/or.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: example/example.csv 4 | parser: 5 | type: csv 6 | charset: UTF-8 7 | newline: CRLF 8 | null_string: 'NULL' 9 | skip_header_lines: 1 10 | comment_line_marker: '#' 11 | columns: 12 | - {name: time, type: timestamp, format: "%Y-%m-%d"} 13 | - {name: foo, type: string} 14 | - {name: bar, type: string} 15 | - {name: flag, type: boolean} 16 | - {name: id, type: long} 17 | - {name: name, type: string} 18 | - {name: json, type: json} 19 | - {name: score, type: double} 20 | filters: 21 | - type: row 22 | condition: OR 23 | conditions: 24 | - {column: name, operator: ==, argument: "Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY"} 25 | - {column: score, operator: ==, argument: 43} 26 | - {column: id, operator: ==, argument: 9} 27 | - {column: flag, operator: ==, argument: false} 28 | out: 29 | type: stdout 30 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/condition/ConditionConfig.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | import org.embulk.util.config.Config; 4 | import org.embulk.util.config.ConfigDefault; 5 | import org.embulk.util.config.Task; 6 | 7 | import java.util.Optional; 8 | 9 | public interface ConditionConfig extends Task 10 | { 11 | @Config("column") 12 | public String getColumn(); 13 | 14 | @Config("operator") 15 | @ConfigDefault("\"==\"") 16 | public Optional getOperator(); 17 | 18 | @Config("argument") 19 | @ConfigDefault("null") 20 | public Optional getArgument(); 21 | 22 | @Config("not") 23 | @ConfigDefault("false") 24 | public Optional getNot(); 25 | 26 | @Config("format") 27 | @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%N %z\"") 28 | public Optional getFormat(); 29 | 30 | @Config("timezone") 31 | @ConfigDefault("\"UTC\"") 32 | public Optional getTimezone(); 33 | } 34 | -------------------------------------------------------------------------------- /script/byaccj.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | BYACCJ_VERSION="1.15" 3 | JFLEX_VERSION="1.6.1" 4 | 5 | ROOT_PATH=$(cd $(dirname $0); cd ..; pwd) 6 | SRC_DIR="src/main/java/org/embulk/filter/row/where" 7 | 8 | if [ $(uname) = "Darwin" ]; then 9 | platform="macosx" 10 | else 11 | platform="linux" 12 | fi 13 | 14 | if [ ! -f "$ROOT_PATH/script/yacc" ]; then 15 | curl -s -L "http://downloads.sourceforge.net/project/byaccj/byaccj/${BYACCJ_VERSION}/byaccj${BYACCJ_VERSION}_${platform}.tar.gz" | tar xz -C $ROOT_PATH/script/ 16 | mv $ROOT_PATH/script/yacc.${platform} $ROOT_PATH/script/yacc 17 | chmod a+x $ROOT_PATH/script/yacc 18 | fi 19 | 20 | if [ ! -L "$ROOT_PATH/script/jflex" ]; then 21 | curl -s -L "http://jflex.de/release/jflex-${JFLEX_VERSION}.tar.gz" | tar xz -C $ROOT_PATH/script/ 22 | fi 23 | 24 | # yacc 25 | $ROOT_PATH/script/yacc -J -Jclass=Parser -Jpackage=org.embulk.filter.row.where -Jnoconstruct "$SRC_DIR/_parser.y" 26 | mv Parser.java ParserVal.java $SRC_DIR/ 27 | 28 | # flex 29 | $ROOT_PATH/script/jflex/bin/jflex -d $SRC_DIR "$SRC_DIR/_lexer.l" 30 | -------------------------------------------------------------------------------- /example/and.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: example/example.csv 4 | parser: 5 | type: csv 6 | charset: UTF-8 7 | newline: CRLF 8 | null_string: "NULL" 9 | skip_header_lines: 1 10 | comment_line_marker: '#' 11 | columns: 12 | - {name: time, type: timestamp, format: "%Y-%m-%d"} 13 | - {name: foo, type: string} 14 | - {name: bar, type: string} 15 | - {name: flag, type: boolean} 16 | - {name: id, type: long} 17 | - {name: name, type: string} 18 | - {name: json, type: json} 19 | - {name: score, type: double} 20 | filters: 21 | - type: row 22 | conditions: 23 | - {column: time, operator: ==, argument: "2015-07-13", format: "%Y-%m-%d"} 24 | - {column: foo, operator: "IS NULL" } 25 | - {column: bar, operator: ==, argument: "bar"} 26 | - {column: score, operator: ">", argument: 2000} 27 | - {column: score, operator: "<", argument: 6000} 28 | # - {column: missing, operator: "==", argument: 6000} 29 | out: 30 | type: stdout 31 | -------------------------------------------------------------------------------- /example/example.csv: -------------------------------------------------------------------------------- 1 | time,foo,bar,flag,id,name,json,score 2 | 2015-07-13,,bar,true,0,Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY,"{\"foo\":\"bar\"}",1370 3 | 2015-07-13,,bar,true,1,VmjbjAA0tOoSEPv_vKAGMtD_0aXZji0abGe7_VXHmUQ,"{\"foo\":\"bar\"}",3962 4 | 2015-07-13,,bar,true,2,C40P5H1WcBx-aWFDJCI8th6QPEI2DOUgupt_gB8UutE,"{\"foo\":\"bar\"}",7323 5 | 2015-07-13,,bar,true,3,Prr0_u_T1ts4myUofBorOJFpCYcOTLOmNBMuRmKIPJU,"{\"foo\":\"bar\"}",5905 6 | 2015-07-13,,bar,true,4,AEGIhHVW5cV6Xlb62uvx3TVl3kmh3Do8AvvtLDS7MDw,"{\"foo\":\"bar\"}",8378 7 | 2015-07-13,,bar,true,5,eupqWLrnCHr_1UaX4dUInLRxx5Q_cyQ4t0oSJBcw0MA,"{\"foo\":\"bar\"}",275 8 | 2015-07-13,,bar,true,6,BN8cQ47EXRb_oCGOoN96bhBldoiyoCp5O_vGHwg0XCg,"{\"foo\":\"bar\"}",43 9 | 2015-07-13,,bar,true,7,RvV35-6jY6MC9_Wnm4nPsmyyfqcr-hlnBt88sXtn1nU,"{\"foo\":\"bar\"}",6130 10 | 2015-07-13,,bar,true,8,6OZiuPiJKjWNLMPgiEbJarB0F80lTPYkkePP8LMliv0,"{\"foo\":\"bar\"}",6652 11 | 2015-07-13,,bar,true,9,13CgEU_ApAMVE6Ll6Y-mSu-aubskNgHbynj2rj8f8oE,"{\"foo\":\"bar\"}",6822 12 | 2015-07-13,NULL,bar,false,NULL,NULL,NULL,4170 13 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/where/ParserVal.java: -------------------------------------------------------------------------------- 1 | //############################################# 2 | //## file: Parser.java 3 | //## Generated by Byacc/j 4 | //############################################# 5 | package org.embulk.filter.row.where; 6 | 7 | /** 8 | * BYACC/J Semantic Value for parser: Parser 9 | * This class provides some of the functionality 10 | * of the yacc/C 'union' directive 11 | */ 12 | public class ParserVal 13 | { 14 | /** 15 | * integer value of this 'union' 16 | */ 17 | public int ival; 18 | 19 | /** 20 | * double value of this 'union' 21 | */ 22 | public double dval; 23 | 24 | /** 25 | * string value of this 'union' 26 | */ 27 | public String sval; 28 | 29 | /** 30 | * object value of this 'union' 31 | */ 32 | public Object obj; 33 | 34 | //############################################# 35 | //## C O N S T R U C T O R S 36 | //############################################# 37 | /** 38 | * Initialize me without a value 39 | */ 40 | public ParserVal() 41 | { 42 | } 43 | /** 44 | * Initialize me as an int 45 | */ 46 | public ParserVal(int val) 47 | { 48 | ival=val; 49 | } 50 | 51 | /** 52 | * Initialize me as a double 53 | */ 54 | public ParserVal(double val) 55 | { 56 | dval=val; 57 | } 58 | 59 | /** 60 | * Initialize me as a string 61 | */ 62 | public ParserVal(String val) 63 | { 64 | sval=val; 65 | } 66 | 67 | /** 68 | * Initialize me as an Object 69 | */ 70 | public ParserVal(Object val) 71 | { 72 | obj=val; 73 | } 74 | }//end class 75 | 76 | //############################################# 77 | //## E N D O F F I L E 78 | //############################################# 79 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/AbstractGuardColumnVisitor.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row; 2 | 3 | import org.embulk.filter.row.RowFilterPlugin.PluginTask; 4 | import org.embulk.filter.row.condition.Condition; 5 | import org.embulk.filter.row.condition.ConditionConfig; 6 | import org.embulk.filter.row.condition.ConditionFactory; 7 | import org.embulk.spi.Column; 8 | import org.embulk.spi.PageReader; 9 | import org.embulk.spi.Schema; 10 | 11 | import java.util.ArrayList; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | 15 | abstract class AbstractGuardColumnVisitor 16 | { 17 | PluginTask task; 18 | Schema inputSchema; 19 | Schema outputSchema; 20 | PageReader pageReader; 21 | 22 | AbstractGuardColumnVisitor(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader) 23 | { 24 | this.task = task; 25 | this.inputSchema = inputSchema; 26 | this.outputSchema = outputSchema; 27 | this.pageReader = pageReader; 28 | } 29 | 30 | static HashMap> buildConditionMap(PluginTask task, Schema outputSchema) 31 | { 32 | HashMap> conditionMap = new HashMap<>(); 33 | for (Column column : outputSchema.getColumns()) { 34 | String columnName = column.getName(); 35 | conditionMap.put(columnName, new ArrayList()); 36 | } 37 | 38 | for (ConditionConfig conditionConfig : task.getConditions().get()) { 39 | String columnName = conditionConfig.getColumn(); 40 | for (Column column : outputSchema.getColumns()) { 41 | if (columnName.equals(column.getName())) { 42 | ConditionFactory factory = new ConditionFactory(column, conditionConfig); 43 | Condition condition = factory.createCondition(); 44 | conditionMap.get(columnName).add(condition); 45 | break; 46 | } 47 | } 48 | } 49 | return conditionMap; 50 | } 51 | 52 | abstract public boolean visitColumns(Schema inputSchema); 53 | } 54 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.6.1 (2022-10-24) 2 | 3 | Fixes: 4 | 5 | * Avoid NullPointerException (thanks to Tetsuro Sano) 6 | 7 | # 0.6.0 (2021-06-10) 8 | 9 | Changes: 10 | 11 | * Get ready for v0.11 and v1.0 (thanks to hiroyuki-sato) 12 | 13 | # 0.5.0 (2017-08-23) 14 | 15 | Changes 16 | 17 | * Follow new TimestampParser API of embulk >= 0.8.29. 18 | * Note that embulk-filter-row >= 0.5.0 requires embulk >= 0.8.29. 19 | 20 | # 0.4.0 (2017-05-18) 21 | 22 | Enhancements 23 | 24 | * Use joni library for REGEXP operator to improve performance 25 | * Note that This may introduce trival incompatibility changes 26 | 27 | # 0.3.3 (2016-08-09) 28 | 29 | Enhancements 30 | 31 | * Support REGEXP operator 32 | * Allow to omit TIMESTAMP keyword 33 | 34 | # 0.3.2 (2016-08-08) 35 | 36 | Changes 37 | 38 | * `conditions` option is now deprecated. Show deprecation warning messages. 39 | 40 | Fxies: 41 | 42 | * Fix NullPointerException for `where` option 43 | 44 | # 0.3.1 (2016-08-08) 45 | 46 | Changes: 47 | 48 | * Change identifier syntax from `[a-zA-Z$][a-zA-z0-9\.\-_]*` to `[a-zA-Z_][a-zA-z0-9_]*` to allow starting _, disallow staring $, disallow -, disallow . (dot). 49 | 50 | # 0.3.0 (2016-08-06) 51 | 52 | Enhancements: 53 | 54 | * Support SQL-like Syntax 55 | 56 | # 0.2.2 (2016-08-05) 57 | 58 | Enhancements: 59 | 60 | * Support `type: json` (not supporting filtering with json yet, though) 61 | 62 | # 0.2.1 (2016-06-27) 63 | 64 | Enhancements: 65 | 66 | * Add a workaround to treat `0.x` argument as a double which is treated as a String by sanekyaml (thanks to @toyama0919) 67 | 68 | # 0.2.0 (2015-12-05) 69 | 70 | Enhancements: 71 | 72 | * Support OR condition 73 | 74 | # 0.1.4 75 | 76 | Fixes: 77 | 78 | * raise SchemaConfigException if column name is not found 79 | 80 | # 0.1.3 81 | 82 | Fixes: 83 | 84 | * Remove debug print 85 | 86 | # 0.1.2 87 | 88 | Enhancements: 89 | 90 | * Support Java 1.7 91 | 92 | # 0.1.1 93 | 94 | Enhancements: 95 | 96 | * Support `startsWith`, `endsWith`, `contains` string operators 97 | 98 | # 0.1.0 99 | 100 | first version 101 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/condition/BooleanCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | public class BooleanCondition implements Condition 4 | { 5 | private BooleanComparator comparator; 6 | 7 | // @FunctionalInterface 8 | interface BooleanComparator 9 | { 10 | boolean compare(Boolean subject); 11 | } 12 | 13 | public BooleanCondition(final String operator, final Boolean argument, final boolean not) 14 | { 15 | final BooleanComparator comparator; 16 | switch (operator.toUpperCase()) { 17 | case "IS NULL": 18 | comparator = new BooleanComparator() { 19 | public boolean compare(Boolean subject) 20 | { 21 | return subject == null; 22 | } 23 | }; 24 | break; 25 | case "IS NOT NULL": 26 | comparator = new BooleanComparator() { 27 | public boolean compare(Boolean subject) 28 | { 29 | return subject != null; 30 | } 31 | }; 32 | break; 33 | case "!=": 34 | comparator = new BooleanComparator() { 35 | public boolean compare(Boolean subject) 36 | { 37 | return subject == null ? true : !subject.equals(argument); 38 | } 39 | }; 40 | break; 41 | default: // case "==": 42 | comparator = new BooleanComparator() { 43 | public boolean compare(Boolean subject) 44 | { 45 | return subject == null ? false : subject.equals(argument); 46 | } 47 | }; 48 | break; 49 | } 50 | this.comparator = comparator; 51 | if (not) { 52 | this.comparator = new BooleanComparator() { 53 | public boolean compare(Boolean subject) 54 | { 55 | return !comparator.compare(subject); 56 | } 57 | }; 58 | } 59 | } 60 | 61 | public boolean compare(Boolean subject) 62 | { 63 | return this.comparator.compare(subject); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/org/embulk/filter/row/condition/TestBooleanCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertFalse; 6 | import static org.junit.Assert.assertTrue; 7 | 8 | public class TestBooleanCondition 9 | { 10 | @Test 11 | public void testIsNull() 12 | { 13 | BooleanCondition condition = new BooleanCondition("IS NULL", null, false); 14 | assertTrue(condition.compare(null)); 15 | assertFalse(condition.compare(new Boolean(true))); 16 | assertFalse(condition.compare(new Boolean(false))); 17 | } 18 | 19 | @Test 20 | public void testIsNotNull() 21 | { 22 | BooleanCondition condition = new BooleanCondition("IS NOT NULL", null, false); 23 | assertFalse(condition.compare(null)); 24 | assertTrue(condition.compare(new Boolean(true))); 25 | assertTrue(condition.compare(new Boolean(false))); 26 | } 27 | 28 | @Test 29 | public void testNot() 30 | { 31 | BooleanCondition condition = new BooleanCondition("IS NOT NULL", null, true); 32 | assertTrue(condition.compare(null)); 33 | assertFalse(condition.compare(new Boolean(true))); 34 | assertFalse(condition.compare(new Boolean(false))); 35 | } 36 | 37 | @Test 38 | public void testEquals() 39 | { 40 | BooleanCondition condition = new BooleanCondition("==", new Boolean(true), false); 41 | assertTrue(condition.compare(new Boolean(true))); 42 | assertFalse(condition.compare(new Boolean(false))); 43 | assertFalse(condition.compare(null)); 44 | 45 | condition = new BooleanCondition("==", new Boolean(false), false); 46 | assertTrue(condition.compare(new Boolean(false))); 47 | assertFalse(condition.compare(new Boolean(true))); 48 | assertFalse(condition.compare(null)); 49 | 50 | // Prohibited by Factory 51 | // BooleanCondition condition = new BooleanCondition("==", null, false); 52 | } 53 | 54 | @Test 55 | public void testNotEquals() 56 | { 57 | BooleanCondition condition = new BooleanCondition("!=", new Boolean(true), false); 58 | assertFalse(condition.compare(new Boolean(true))); 59 | assertTrue(condition.compare(new Boolean(false))); 60 | assertTrue(condition.compare(null)); 61 | 62 | condition = new BooleanCondition("!=", new Boolean(false), false); 63 | assertFalse(condition.compare(new Boolean(false))); 64 | assertTrue(condition.compare(new Boolean(true))); 65 | assertTrue(condition.compare(null)); 66 | 67 | // Prohibited by Factory 68 | // BooleanCondition condition = new BooleanCondition("!=", null, false); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/BuilderColumnVisitorImpl.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row; 2 | 3 | import org.embulk.filter.row.RowFilterPlugin.PluginTask; 4 | import org.embulk.spi.Column; 5 | import org.embulk.spi.ColumnVisitor; 6 | import org.embulk.spi.Exec; 7 | import org.embulk.spi.PageBuilder; 8 | import org.embulk.spi.PageReader; 9 | import org.embulk.spi.Schema; 10 | import org.slf4j.Logger; 11 | 12 | public class BuilderColumnVisitorImpl 13 | implements ColumnVisitor 14 | { 15 | private static final Logger logger = Exec.getLogger(RowFilterPlugin.class); 16 | private final PluginTask task; 17 | private final Schema inputSchema; 18 | private final Schema outputSchema; 19 | private final PageReader pageReader; 20 | private final PageBuilder pageBuilder; 21 | 22 | BuilderColumnVisitorImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder) 23 | { 24 | this.task = task; 25 | this.inputSchema = inputSchema; 26 | this.outputSchema = outputSchema; 27 | this.pageReader = pageReader; 28 | this.pageBuilder = pageBuilder; 29 | } 30 | 31 | @Override 32 | public void booleanColumn(Column column) 33 | { 34 | if (pageReader.isNull(column)) { 35 | pageBuilder.setNull(column); 36 | } 37 | else { 38 | pageBuilder.setBoolean(column, pageReader.getBoolean(column)); 39 | } 40 | } 41 | 42 | @Override 43 | public void longColumn(Column column) 44 | { 45 | if (pageReader.isNull(column)) { 46 | pageBuilder.setNull(column); 47 | } 48 | else { 49 | pageBuilder.setLong(column, pageReader.getLong(column)); 50 | } 51 | } 52 | 53 | @Override 54 | public void doubleColumn(Column column) 55 | { 56 | if (pageReader.isNull(column)) { 57 | pageBuilder.setNull(column); 58 | } 59 | else { 60 | pageBuilder.setDouble(column, pageReader.getDouble(column)); 61 | } 62 | } 63 | 64 | @Override 65 | public void stringColumn(Column column) 66 | { 67 | if (pageReader.isNull(column)) { 68 | pageBuilder.setNull(column); 69 | } 70 | else { 71 | pageBuilder.setString(column, pageReader.getString(column)); 72 | } 73 | } 74 | 75 | @Override 76 | public void timestampColumn(Column column) 77 | { 78 | if (pageReader.isNull(column)) { 79 | pageBuilder.setNull(column); 80 | } 81 | else { 82 | pageBuilder.setTimestamp(column, pageReader.getTimestamp(column)); 83 | } 84 | } 85 | 86 | @Override 87 | public void jsonColumn(Column column) 88 | { 89 | if (pageReader.isNull(column)) { 90 | pageBuilder.setNull(column); 91 | } 92 | else { 93 | pageBuilder.setJson(column, pageReader.getJson(column)); 94 | } 95 | } 96 | } -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/condition/StringCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | public class StringCondition implements Condition 4 | { 5 | private StringComparator comparator; 6 | 7 | // @FunctionalInterface 8 | interface StringComparator 9 | { 10 | boolean compare(String subject); 11 | } 12 | 13 | public StringCondition(final String operator, final String argument, final boolean not) 14 | { 15 | final StringComparator comparator; 16 | switch (operator.toUpperCase()) { 17 | case "START_WITH": 18 | case "STARTSWITH": 19 | comparator = new StringComparator() { 20 | public boolean compare(String subject) 21 | { 22 | return subject == null ? false : subject.startsWith(argument); 23 | } 24 | }; 25 | break; 26 | case "END_WITH": 27 | case "ENDSWITH": 28 | comparator = new StringComparator() { 29 | public boolean compare(String subject) 30 | { 31 | return subject == null ? false : subject.endsWith(argument); 32 | } 33 | }; 34 | break; 35 | case "INCLUDE": 36 | case "CONTAINS": 37 | comparator = new StringComparator() { 38 | public boolean compare(String subject) 39 | { 40 | return subject == null ? false : subject.contains(argument); 41 | } 42 | }; 43 | break; 44 | case "IS NULL": 45 | comparator = new StringComparator() { 46 | public boolean compare(String subject) 47 | { 48 | return subject == null; 49 | } 50 | }; 51 | break; 52 | case "IS NOT NULL": 53 | comparator = new StringComparator() { 54 | public boolean compare(String subject) 55 | { 56 | return subject != null; 57 | } 58 | }; 59 | break; 60 | case "!=": 61 | comparator = new StringComparator() { 62 | public boolean compare(String subject) 63 | { 64 | return subject == null ? true : !subject.equals(argument); 65 | } 66 | }; 67 | break; 68 | default: // case "==": 69 | comparator = new StringComparator() { 70 | public boolean compare(String subject) 71 | { 72 | return subject == null ? false : subject.equals(argument); 73 | } 74 | }; 75 | break; 76 | } 77 | this.comparator = comparator; 78 | if (not) { 79 | this.comparator = new StringComparator() { 80 | public boolean compare(String subject) 81 | { 82 | return !comparator.compare(subject); 83 | } 84 | }; 85 | } 86 | } 87 | 88 | public boolean compare(String subject) 89 | { 90 | return this.comparator.compare(subject); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/test/java/org/embulk/filter/row/condition/TestTimestampCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | import org.junit.Test; 4 | 5 | import java.time.Instant; 6 | 7 | import static org.junit.Assert.assertFalse; 8 | import static org.junit.Assert.assertTrue; 9 | 10 | public class TestTimestampCondition 11 | { 12 | @Test 13 | public void testIsNull() 14 | { 15 | TimestampCondition condition = new TimestampCondition("IS NULL", null, false); 16 | assertTrue(condition.compare(null)); 17 | assertFalse(condition.compare(Instant.ofEpochSecond(10))); 18 | } 19 | 20 | @Test 21 | public void testIsNotNull() 22 | { 23 | TimestampCondition condition = new TimestampCondition("IS NOT NULL", null, false); 24 | assertFalse(condition.compare(null)); 25 | assertTrue(condition.compare(Instant.ofEpochSecond(10))); 26 | } 27 | 28 | @Test 29 | public void testEquals() 30 | { 31 | TimestampCondition condition = new TimestampCondition("==", Instant.ofEpochSecond(10), false); 32 | assertFalse(condition.compare(null)); 33 | assertTrue(condition.compare(Instant.ofEpochSecond(10))); 34 | assertFalse(condition.compare(Instant.ofEpochSecond(11))); 35 | 36 | // Prohibited by Factory 37 | // TimestampCondition condition = new TimestampCondition("==", null, false); 38 | } 39 | 40 | @Test 41 | public void testNotEquals() 42 | { 43 | TimestampCondition condition = new TimestampCondition("!=", Instant.ofEpochSecond(10), false); 44 | assertTrue(condition.compare(null)); 45 | assertFalse(condition.compare(Instant.ofEpochSecond(10))); 46 | assertTrue(condition.compare(Instant.ofEpochSecond(11))); 47 | } 48 | 49 | @Test 50 | public void testGreaterThan() 51 | { 52 | TimestampCondition condition = new TimestampCondition(">", Instant.ofEpochSecond(10), false); 53 | assertFalse(condition.compare(null)); 54 | assertFalse(condition.compare(Instant.ofEpochSecond(10))); 55 | assertTrue(condition.compare(Instant.ofEpochSecond(11))); 56 | } 57 | 58 | @Test 59 | public void testGreaterEqual() 60 | { 61 | TimestampCondition condition = new TimestampCondition(">=", Instant.ofEpochSecond(11), false); 62 | assertFalse(condition.compare(null)); 63 | assertFalse(condition.compare(Instant.ofEpochSecond(10))); 64 | assertTrue(condition.compare(Instant.ofEpochSecond(11))); 65 | } 66 | 67 | @Test 68 | public void testLessThan() 69 | { 70 | TimestampCondition condition = new TimestampCondition("<", Instant.ofEpochSecond(11), false); 71 | assertFalse(condition.compare(null)); 72 | assertFalse(condition.compare(Instant.ofEpochSecond(11))); 73 | assertTrue(condition.compare(Instant.ofEpochSecond(10))); 74 | } 75 | 76 | @Test 77 | public void testLessEqual() 78 | { 79 | TimestampCondition condition = new TimestampCondition("<=", Instant.ofEpochSecond(11), false); 80 | assertFalse(condition.compare(null)); 81 | assertFalse(condition.compare(Instant.ofEpochSecond(12))); 82 | assertTrue(condition.compare(Instant.ofEpochSecond(11))); 83 | } 84 | 85 | @Test 86 | public void testNot() 87 | { 88 | TimestampCondition condition = new TimestampCondition("<=", Instant.ofEpochSecond(11), true); 89 | assertTrue(condition.compare(null)); 90 | assertTrue(condition.compare(Instant.ofEpochSecond(12))); 91 | assertFalse(condition.compare(Instant.ofEpochSecond(11))); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/condition/DoubleCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | public class DoubleCondition implements Condition 4 | { 5 | private DoubleComparator comparator; 6 | 7 | // @FunctionalInterface 8 | interface DoubleComparator 9 | { 10 | boolean compare(Double subject); 11 | } 12 | 13 | public DoubleCondition(final String operator, final Double argument, final boolean not) 14 | { 15 | final DoubleComparator comparator; 16 | switch (operator.toUpperCase()) { 17 | case "IS NULL": 18 | comparator = new DoubleComparator() { 19 | public boolean compare(Double subject) 20 | { 21 | return subject == null; 22 | } 23 | }; 24 | break; 25 | case "IS NOT NULL": 26 | comparator = new DoubleComparator() { 27 | public boolean compare(Double subject) 28 | { 29 | return subject != null; 30 | } 31 | }; 32 | break; 33 | case ">": 34 | comparator = new DoubleComparator() { 35 | public boolean compare(Double subject) 36 | { 37 | return subject == null ? false : subject.compareTo(argument) > 0; 38 | } 39 | }; 40 | break; 41 | case ">=": 42 | comparator = new DoubleComparator() { 43 | public boolean compare(Double subject) 44 | { 45 | return subject == null ? false : subject.compareTo(argument) >= 0; 46 | } 47 | }; 48 | break; 49 | case "<": 50 | comparator = new DoubleComparator() { 51 | public boolean compare(Double subject) 52 | { 53 | return subject == null ? false : subject.compareTo(argument) < 0; 54 | } 55 | }; 56 | break; 57 | case "<=": 58 | comparator = new DoubleComparator() { 59 | public boolean compare(Double subject) 60 | { 61 | return subject == null ? false : subject.compareTo(argument) <= 0; 62 | } 63 | }; 64 | break; 65 | case "!=": 66 | comparator = new DoubleComparator() { 67 | public boolean compare(Double subject) 68 | { 69 | return subject == null ? true : !subject.equals(argument); 70 | } 71 | }; 72 | break; 73 | default: // case "==": 74 | comparator = new DoubleComparator() { 75 | public boolean compare(Double subject) 76 | { 77 | return subject == null ? false : subject.equals(argument); 78 | } 79 | }; 80 | break; 81 | } 82 | this.comparator = comparator; 83 | if (not) { 84 | this.comparator = new DoubleComparator() { 85 | public boolean compare(Double subject) 86 | { 87 | return !comparator.compare(subject); 88 | } 89 | }; 90 | } 91 | } 92 | 93 | public boolean compare(Double subject) 94 | { 95 | return this.comparator.compare(subject); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/condition/LongCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | 4 | public class LongCondition implements Condition 5 | { 6 | private LongComparator comparator; 7 | 8 | // @FunctionalInterface 9 | interface LongComparator 10 | { 11 | boolean compare(Long subject); 12 | } 13 | 14 | public LongCondition(final String operator, final Double argument, final boolean not) 15 | { 16 | final LongComparator comparator; 17 | switch (operator.toUpperCase()) { 18 | case "IS NULL": 19 | comparator = new LongComparator() { 20 | public boolean compare(Long subject) 21 | { 22 | return subject == null; 23 | } 24 | }; 25 | break; 26 | case "IS NOT NULL": 27 | comparator = new LongComparator() { 28 | public boolean compare(Long subject) 29 | { 30 | return subject != null; 31 | } 32 | }; 33 | break; 34 | case ">": 35 | comparator = new LongComparator() { 36 | public boolean compare(Long subject) 37 | { 38 | return subject == null ? false : new Double(subject).compareTo(argument) > 0; 39 | } 40 | }; 41 | break; 42 | case ">=": 43 | comparator = new LongComparator() { 44 | public boolean compare(Long subject) 45 | { 46 | return subject == null ? false : new Double(subject).compareTo(argument) >= 0; 47 | } 48 | }; 49 | break; 50 | case "<": 51 | comparator = new LongComparator() { 52 | public boolean compare(Long subject) 53 | { 54 | return subject == null ? false : new Double(subject).compareTo(argument) < 0; 55 | } 56 | }; 57 | break; 58 | case "<=": 59 | comparator = new LongComparator() { 60 | public boolean compare(Long subject) 61 | { 62 | return subject == null ? false : new Double(subject).compareTo(argument) <= 0; 63 | } 64 | }; 65 | break; 66 | case "!=": 67 | comparator = new LongComparator() { 68 | public boolean compare(Long subject) 69 | { 70 | return subject == null ? true : !new Double(subject).equals(argument); 71 | } 72 | }; 73 | break; 74 | default: // case "==": 75 | comparator = new LongComparator() { 76 | public boolean compare(Long subject) 77 | { 78 | return subject == null ? false : new Double(subject).equals(argument); 79 | } 80 | }; 81 | break; 82 | } 83 | this.comparator = comparator; 84 | if (not) { 85 | this.comparator = new LongComparator() { 86 | public boolean compare(Long subject) 87 | { 88 | return !comparator.compare(subject); 89 | } 90 | }; 91 | } 92 | } 93 | 94 | public boolean compare(Long subject) 95 | { 96 | return this.comparator.compare(subject); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/test/java/org/embulk/filter/row/condition/TestStringCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertFalse; 6 | import static org.junit.Assert.assertTrue; 7 | 8 | public class TestStringCondition 9 | { 10 | @Test 11 | public void testIsNull() 12 | { 13 | StringCondition condition = new StringCondition("IS NULL", null, false); 14 | assertTrue(condition.compare(null)); 15 | assertFalse(condition.compare("foo")); 16 | } 17 | 18 | @Test 19 | public void testIsNotNull() 20 | { 21 | StringCondition condition = new StringCondition("IS NOT NULL", null, false); 22 | assertFalse(condition.compare(null)); 23 | assertTrue(condition.compare("foo")); 24 | } 25 | 26 | @Test 27 | public void testEquals() 28 | { 29 | StringCondition condition = new StringCondition("==", "foo", false); 30 | assertFalse(condition.compare(null)); 31 | assertTrue(condition.compare("foo")); 32 | assertFalse(condition.compare("bar")); 33 | 34 | // Prohibited by Factory 35 | // StringCondition condition = new StringCondition("==", null, false); 36 | } 37 | 38 | @Test 39 | public void testNotEquals() 40 | { 41 | StringCondition condition = new StringCondition("!=", "foo", false); 42 | assertTrue(condition.compare(null)); 43 | assertFalse(condition.compare("foo")); 44 | assertTrue(condition.compare("bar")); 45 | } 46 | 47 | @Test 48 | public void testStartWith() 49 | { 50 | StringCondition condition = new StringCondition("start_with", "f", false); 51 | assertFalse(condition.compare(null)); 52 | assertTrue(condition.compare("foo")); 53 | assertFalse(condition.compare("bar")); 54 | } 55 | 56 | @Test 57 | public void testStartsWith() 58 | { 59 | StringCondition condition = new StringCondition("startsWith", "f", false); 60 | assertFalse(condition.compare(null)); 61 | assertTrue(condition.compare("foo")); 62 | assertFalse(condition.compare("bar")); 63 | } 64 | 65 | @Test 66 | public void testEndWith() 67 | { 68 | StringCondition condition = new StringCondition("end_with", "o", false); 69 | assertFalse(condition.compare(null)); 70 | assertTrue(condition.compare("foo")); 71 | assertFalse(condition.compare("bar")); 72 | } 73 | 74 | @Test 75 | public void testEndsWith() 76 | { 77 | StringCondition condition = new StringCondition("endsWith", "o", false); 78 | assertFalse(condition.compare(null)); 79 | assertTrue(condition.compare("foo")); 80 | assertFalse(condition.compare("bar")); 81 | } 82 | 83 | @Test 84 | public void testInclude() 85 | { 86 | StringCondition condition = new StringCondition("include", "o", false); 87 | assertFalse(condition.compare(null)); 88 | assertTrue(condition.compare("foo")); 89 | assertFalse(condition.compare("bar")); 90 | } 91 | 92 | @Test 93 | public void testContains() 94 | { 95 | StringCondition condition = new StringCondition("contains", "o", false); 96 | assertFalse(condition.compare(null)); 97 | assertTrue(condition.compare("foo")); 98 | assertFalse(condition.compare("bar")); 99 | } 100 | 101 | @Test 102 | public void testNot() 103 | { 104 | StringCondition condition = new StringCondition("include", "o", true); 105 | assertTrue(condition.compare(null)); 106 | assertFalse(condition.compare("foo")); 107 | assertTrue(condition.compare("bar")); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/condition/TimestampCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | import java.time.Instant; 4 | 5 | public class TimestampCondition implements Condition 6 | { 7 | private TimestampComparator comparator; 8 | 9 | // @FunctionalInterface 10 | interface TimestampComparator 11 | { 12 | boolean compare(Instant subject); 13 | } 14 | 15 | public TimestampCondition(final String operator, final Instant argument, final boolean not) 16 | { 17 | final TimestampComparator comparator; 18 | switch (operator.toUpperCase()) { 19 | case ">": 20 | comparator = new TimestampComparator() { 21 | public boolean compare(Instant subject) 22 | { 23 | return subject == null ? false : subject.compareTo(argument) > 0; 24 | } 25 | }; 26 | break; 27 | case ">=": 28 | comparator = new TimestampComparator() { 29 | public boolean compare(Instant subject) 30 | { 31 | return subject == null ? false : subject.compareTo(argument) >= 0; 32 | } 33 | }; 34 | break; 35 | case "<": 36 | comparator = new TimestampComparator() { 37 | public boolean compare(Instant subject) 38 | { 39 | return subject == null ? false : subject.compareTo(argument) < 0; 40 | } 41 | }; 42 | break; 43 | case "<=": 44 | comparator = new TimestampComparator() { 45 | public boolean compare(Instant subject) 46 | { 47 | return subject == null ? false : subject.compareTo(argument) <= 0; 48 | } 49 | }; 50 | break; 51 | case "IS NULL": 52 | comparator = new TimestampComparator() { 53 | public boolean compare(Instant subject) 54 | { 55 | return subject == null; 56 | } 57 | }; 58 | break; 59 | case "IS NOT NULL": 60 | comparator = new TimestampComparator() { 61 | public boolean compare(Instant subject) 62 | { 63 | return subject != null; 64 | } 65 | }; 66 | break; 67 | case "!=": 68 | comparator = new TimestampComparator() { 69 | public boolean compare(Instant subject) 70 | { 71 | return subject == null ? true : !subject.equals(argument); 72 | } 73 | }; 74 | break; 75 | default: // case "==": 76 | comparator = new TimestampComparator() { 77 | public boolean compare(Instant subject) 78 | { 79 | return subject == null ? false : subject.equals(argument); 80 | } 81 | }; 82 | break; 83 | } 84 | this.comparator = comparator; 85 | if (not) { 86 | this.comparator = new TimestampComparator() { 87 | public boolean compare(Instant subject) 88 | { 89 | return !comparator.compare(subject); 90 | } 91 | }; 92 | } 93 | } 94 | 95 | public boolean compare(Instant subject) 96 | { 97 | return this.comparator.compare(subject); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/test/java/org/embulk/filter/row/condition/TestDoubleCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertFalse; 6 | import static org.junit.Assert.assertTrue; 7 | 8 | public class TestDoubleCondition 9 | { 10 | @Test 11 | public void testIsNull() 12 | { 13 | DoubleCondition condition = new DoubleCondition("IS NULL", null, false); 14 | assertTrue(condition.compare(null)); 15 | assertFalse(condition.compare(new Double(10))); 16 | } 17 | 18 | @Test 19 | public void testIsNotNull() 20 | { 21 | DoubleCondition condition = new DoubleCondition("IS NOT NULL", null, false); 22 | assertFalse(condition.compare(null)); 23 | assertTrue(condition.compare(new Double(10))); 24 | } 25 | 26 | @Test 27 | public void testEquals() 28 | { 29 | DoubleCondition condition = new DoubleCondition("==", new Double(10), false); 30 | assertFalse(condition.compare(null)); 31 | assertTrue(condition.compare(new Double(10))); 32 | assertFalse(condition.compare(new Double(11))); 33 | assertTrue(condition.compare(new Double(10.0))); 34 | assertFalse(condition.compare(new Double(11.0))); 35 | 36 | // Prohibited by Factory 37 | // DoubleCondition condition = new DoubleCondition("==", null, false); 38 | } 39 | 40 | @Test 41 | public void testNotEquals() 42 | { 43 | DoubleCondition condition = new DoubleCondition("!=", new Double(10), false); 44 | assertTrue(condition.compare(null)); 45 | assertFalse(condition.compare(new Double(10))); 46 | assertTrue(condition.compare(new Double(11))); 47 | assertFalse(condition.compare(new Double(10.0))); 48 | assertTrue(condition.compare(new Double(11.0))); 49 | } 50 | 51 | @Test 52 | public void testGreaterThan() 53 | { 54 | DoubleCondition condition = new DoubleCondition(">", new Double(10), false); 55 | assertFalse(condition.compare(null)); 56 | assertFalse(condition.compare(new Double(10))); 57 | assertTrue(condition.compare(new Double(11))); 58 | assertFalse(condition.compare(new Double(10.0))); 59 | assertTrue(condition.compare(new Double(11.0))); 60 | } 61 | 62 | @Test 63 | public void testGreaterEqual() 64 | { 65 | DoubleCondition condition = new DoubleCondition(">=", new Double(11), false); 66 | assertFalse(condition.compare(null)); 67 | assertFalse(condition.compare(new Double(10))); 68 | assertTrue(condition.compare(new Double(11))); 69 | assertFalse(condition.compare(new Double(10.0))); 70 | assertTrue(condition.compare(new Double(11.0))); 71 | } 72 | 73 | @Test 74 | public void testLessThan() 75 | { 76 | DoubleCondition condition = new DoubleCondition("<", new Double(11), false); 77 | assertFalse(condition.compare(null)); 78 | assertFalse(condition.compare(new Double(11))); 79 | assertTrue(condition.compare(new Double(10))); 80 | assertFalse(condition.compare(new Double(11.0))); 81 | assertTrue(condition.compare(new Double(10.0))); 82 | } 83 | 84 | @Test 85 | public void testLessEqual() 86 | { 87 | DoubleCondition condition = new DoubleCondition("<=", new Double(11), false); 88 | assertFalse(condition.compare(null)); 89 | assertFalse(condition.compare(new Double(12))); 90 | assertTrue(condition.compare(new Double(11))); 91 | assertFalse(condition.compare(new Double(12.0))); 92 | assertTrue(condition.compare(new Double(11.0))); 93 | } 94 | 95 | @Test 96 | public void testNot() 97 | { 98 | DoubleCondition condition = new DoubleCondition("<=", new Double(11), true); 99 | assertTrue(condition.compare(null)); 100 | assertTrue(condition.compare(new Double(12))); 101 | assertFalse(condition.compare(new Double(11))); 102 | assertTrue(condition.compare(new Double(12.0))); 103 | assertFalse(condition.compare(new Double(11.0))); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/where/_lexer.l: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.where; 2 | 3 | import org.embulk.config.ConfigException; 4 | import org.embulk.spi.Schema; 5 | %% 6 | 7 | %byaccj 8 | 9 | %{ 10 | private StringBuffer string = new StringBuffer(); 11 | 12 | protected Parser yyparser; 13 | protected Schema schema; 14 | 15 | public Yylex(String str, Parser yyparser) { 16 | this(new java.io.StringReader(str)); 17 | this.yyparser = yyparser; 18 | this.schema = yyparser.schema; 19 | } 20 | %} 21 | 22 | %state STRING 23 | %state IDENTIFIER 24 | 25 | Number = -?[0-9]+(\.[0-9]+)? 26 | QuotedIdentifierChar = [^\r\n\"\\] 27 | NonQuotedIdentifier = [a-zA-Z_][a-zA-z0-9_]* 28 | StringChar = [^\r\n\'\\]+ 29 | Newline = \n|\r|\r\n 30 | 31 | %% 32 | 33 | { 34 | /* operators */ 35 | "(" { return (int) yycharat(0); } 36 | ")" { return (int) yycharat(0); } 37 | "AND" { return Parser.AND; } 38 | "OR" { return Parser.OR; } 39 | "=" { return Parser.EQ; } 40 | "<>" | "!=" { return Parser.NEQ; } 41 | "!=" { return Parser.NEQ; } 42 | ">" { return Parser.GT; } 43 | ">=" { return Parser.GE; } 44 | "<" { return Parser.LT; } 45 | "<=" { return Parser.LE; } 46 | "START_WITH" { return Parser.START_WITH; } 47 | "END_WITH" { return Parser.END_WITH; } 48 | "INCLUDE" { return Parser.INCLUDE; } 49 | "REGEXP" { return Parser.REGEXP; } 50 | "IS" { return Parser.IS; } 51 | "NOT" { return Parser.NOT; } 52 | "NULL" { return Parser.NULL; } 53 | "TIMESTAMP" { return Parser.TIMESTAMP; } 54 | 55 | /* boolean literal */ 56 | "true" | "TRUE" { yyparser.yylval = new ParserVal(new BooleanLiteral(yytext())); return Parser.BOOLEAN; } 57 | "false" | "FALSE" { yyparser.yylval = new ParserVal(new BooleanLiteral(yytext())); return Parser.BOOLEAN; } 58 | 59 | /* number literal */ 60 | {Number} { yyparser.yylval = new ParserVal(new NumberLiteral(yytext())); return Parser.NUMBER; } 61 | 62 | /* identifier literal */ 63 | {NonQuotedIdentifier} { yyparser.yylval = new ParserVal(new IdentifierLiteral(yytext(), schema)); return Parser.IDENTIFIER; } 64 | \" { yybegin(IDENTIFIER); string.setLength(0); } 65 | 66 | /* string literal */ 67 | \' { yybegin(STRING); string.setLength(0); } 68 | 69 | /* whitespace */ 70 | [ \t]+ { } 71 | 72 | /* newline */ 73 | {Newline} { } 74 | } 75 | 76 | { 77 | \" { yybegin(YYINITIAL); yyparser.yylval = new ParserVal(new IdentifierLiteral(string.toString(), schema)); return Parser.IDENTIFIER; } 78 | 79 | {QuotedIdentifierChar}+ { string.append( yytext() ); } 80 | 81 | /* escape sequences */ 82 | "\\\"" { string.append( '\"' ); } 83 | "\\'" { string.append( '\'' ); } 84 | "\\\\" { string.append( '\\' ); } 85 | 86 | /* error cases */ 87 | \\. { throw new ConfigException("yylex: Illegal escape sequence \""+yytext()+"\""); } 88 | } 89 | 90 | { 91 | \' { yybegin(YYINITIAL); yyparser.yylval = new ParserVal(new StringLiteral(string.toString())); return Parser.STRING; } 92 | 93 | {StringChar}+ { string.append( yytext() ); } 94 | 95 | /* escape sequences */ 96 | "\\b" { string.append( '\b' ); } 97 | "\\t" { string.append( '\t' ); } 98 | "\\n" { string.append( '\n' ); } 99 | "\\f" { string.append( '\f' ); } 100 | "\\r" { string.append( '\r' ); } 101 | "\\\"" { string.append( '\"' ); } 102 | "\\'" { string.append( '\'' ); } 103 | "\\\\" { string.append( '\\' ); } 104 | 105 | /* error cases */ 106 | \\. { throw new ConfigException("yylex: Illegal escape sequence \""+yytext()+"\""); } 107 | } 108 | 109 | /* error fallback */ 110 | [^] { throw new ConfigException("yylex: Unexpected character '"+yytext()+"'"); } 111 | -------------------------------------------------------------------------------- /src/test/java/org/embulk/filter/row/condition/TestLongCondition.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertFalse; 6 | import static org.junit.Assert.assertTrue; 7 | 8 | public class TestLongCondition 9 | { 10 | @Test 11 | public void testIsNull() 12 | { 13 | LongCondition condition = new LongCondition("IS NULL", null, false); 14 | assertTrue(condition.compare(null)); 15 | assertFalse(condition.compare(new Long(10))); 16 | } 17 | 18 | @Test 19 | public void testIsNotNull() 20 | { 21 | LongCondition condition = new LongCondition("IS NOT NULL", null, false); 22 | assertFalse(condition.compare(null)); 23 | assertTrue(condition.compare(new Long(10))); 24 | } 25 | 26 | @Test 27 | public void testEquals() 28 | { 29 | LongCondition condition = new LongCondition("==", new Double(10), false); 30 | assertFalse(condition.compare(null)); 31 | assertTrue(condition.compare(new Long(10))); 32 | assertFalse(condition.compare(new Long(11))); 33 | 34 | // Prohibited by Factory 35 | // LongCondition condition = new LongCondition("==", null, false); 36 | } 37 | 38 | @Test 39 | public void testNotEquals() 40 | { 41 | LongCondition condition = new LongCondition("!=", new Double(10), false); 42 | assertTrue(condition.compare(null)); 43 | assertFalse(condition.compare(new Long(10))); 44 | assertTrue(condition.compare(new Long(11))); 45 | } 46 | 47 | @Test 48 | public void testGreaterThan() 49 | { 50 | LongCondition condition = new LongCondition(">", new Double(10), false); 51 | assertFalse(condition.compare(null)); 52 | assertFalse(condition.compare(new Long(10))); 53 | assertTrue(condition.compare(new Long(11))); 54 | } 55 | 56 | @Test 57 | public void testGreaterEqual() 58 | { 59 | LongCondition condition = new LongCondition(">=", new Double(11), false); 60 | assertFalse(condition.compare(null)); 61 | assertFalse(condition.compare(new Long(10))); 62 | assertTrue(condition.compare(new Long(11))); 63 | } 64 | 65 | @Test 66 | public void testLessThan() 67 | { 68 | LongCondition condition = new LongCondition("<", new Double(11), false); 69 | assertFalse(condition.compare(null)); 70 | assertFalse(condition.compare(new Long(11))); 71 | assertTrue(condition.compare(new Long(10))); 72 | } 73 | 74 | @Test 75 | public void testLessEqual() 76 | { 77 | LongCondition condition = new LongCondition("<=", new Double(11), false); 78 | assertFalse(condition.compare(null)); 79 | assertFalse(condition.compare(new Long(12))); 80 | assertTrue(condition.compare(new Long(11))); 81 | } 82 | 83 | @Test 84 | public void testNot() 85 | { 86 | LongCondition condition = new LongCondition("<=", new Double(11), true); 87 | assertTrue(condition.compare(null)); 88 | assertTrue(condition.compare(new Long(12))); 89 | assertFalse(condition.compare(new Long(11))); 90 | } 91 | 92 | @Test 93 | public void testDoubleOperatorEquals() 94 | { 95 | LongCondition condition = new LongCondition("==", new Double(10.5), false); 96 | assertFalse(condition.compare(null)); 97 | assertFalse(condition.compare(new Long(10))); 98 | assertFalse(condition.compare(new Long(11))); 99 | } 100 | 101 | @Test 102 | public void testDoubleOperatorNotEquals() 103 | { 104 | LongCondition condition = new LongCondition("!=", new Double(10.5), false); 105 | assertTrue(condition.compare(null)); 106 | assertTrue(condition.compare(new Long(10))); 107 | assertTrue(condition.compare(new Long(11))); 108 | } 109 | 110 | @Test 111 | public void testDoubleOperatorGreaterThan() 112 | { 113 | LongCondition condition = new LongCondition(">", new Double(10.5), false); 114 | assertFalse(condition.compare(null)); 115 | assertFalse(condition.compare(new Long(10))); 116 | assertTrue(condition.compare(new Long(11))); 117 | } 118 | 119 | @Test 120 | public void testDoubleOperatorGreaterEqual() 121 | { 122 | LongCondition condition = new LongCondition(">=", new Double(10.5), false); 123 | assertFalse(condition.compare(null)); 124 | assertFalse(condition.compare(new Long(10))); 125 | assertTrue(condition.compare(new Long(11))); 126 | } 127 | 128 | @Test 129 | public void testDoubleOperatorLessThan() 130 | { 131 | LongCondition condition = new LongCondition("<", new Double(10.5), false); 132 | assertFalse(condition.compare(null)); 133 | assertFalse(condition.compare(new Long(11))); 134 | assertTrue(condition.compare(new Long(10))); 135 | } 136 | 137 | @Test 138 | public void testDoubleOperatorLessEqual() 139 | { 140 | LongCondition condition = new LongCondition("<=", new Double(11.5), false); 141 | assertFalse(condition.compare(null)); 142 | assertFalse(condition.compare(new Long(12))); 143 | assertTrue(condition.compare(new Long(11))); 144 | } 145 | 146 | @Test 147 | public void testDoubleOperatorNot() 148 | { 149 | LongCondition condition = new LongCondition("<=", new Double(11.5), true); 150 | assertTrue(condition.compare(null)); 151 | assertTrue(condition.compare(new Long(12))); 152 | assertFalse(condition.compare(new Long(11))); 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /config/checkstyle/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/GuardColumnVisitorOrImpl.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row; 2 | 3 | import org.embulk.filter.row.RowFilterPlugin.PluginTask; 4 | import org.embulk.filter.row.condition.BooleanCondition; 5 | import org.embulk.filter.row.condition.Condition; 6 | import org.embulk.filter.row.condition.DoubleCondition; 7 | import org.embulk.filter.row.condition.LongCondition; 8 | import org.embulk.filter.row.condition.StringCondition; 9 | import org.embulk.filter.row.condition.TimestampCondition; 10 | import org.embulk.spi.Column; 11 | import org.embulk.spi.ColumnVisitor; 12 | import org.embulk.spi.Exec; 13 | import org.embulk.spi.PageReader; 14 | import org.embulk.spi.Schema; 15 | import org.slf4j.Logger; 16 | 17 | import java.time.Instant; 18 | import java.util.HashMap; 19 | import java.util.List; 20 | 21 | class GuardColumnVisitorOrImpl 22 | extends AbstractGuardColumnVisitor 23 | implements ColumnVisitor 24 | { 25 | private static final Logger logger = Exec.getLogger(RowFilterPlugin.class); 26 | private boolean shouldAddRecord; 27 | private HashMap> conditionMap; 28 | 29 | GuardColumnVisitorOrImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader) 30 | { 31 | super(task, inputSchema, outputSchema, pageReader); 32 | this.conditionMap = buildConditionMap(task, outputSchema); 33 | } 34 | 35 | public boolean visitColumns(Schema inputSchema) 36 | { 37 | //Visitor objects are created for each thread :) 38 | //System.out.println(String.format("thread_id:%d object_id:%d", Thread.currentThread().getId(), this.hashCode())); 39 | shouldAddRecord = false; 40 | for (Column column : inputSchema.getColumns()) { 41 | column.visit(this); 42 | } 43 | return shouldAddRecord; 44 | } 45 | 46 | @Override 47 | public void booleanColumn(Column column) 48 | { 49 | if (shouldAddRecord) { 50 | return; 51 | } 52 | List conditionList = conditionMap.get(column.getName()); 53 | for (Condition tempCondition : conditionList) { 54 | BooleanCondition condition = (BooleanCondition) tempCondition; 55 | if (pageReader.isNull(column)) { 56 | if (condition.compare(null)) { 57 | shouldAddRecord = true; 58 | break; 59 | } 60 | } 61 | else { 62 | boolean subject = pageReader.getBoolean(column); 63 | if (condition.compare(subject)) { 64 | shouldAddRecord = true; 65 | break; 66 | } 67 | } 68 | } 69 | } 70 | 71 | @Override 72 | public void longColumn(Column column) 73 | { 74 | if (shouldAddRecord) { 75 | return; 76 | } 77 | List conditionList = conditionMap.get(column.getName()); 78 | for (Condition tempCondition : conditionList) { 79 | LongCondition condition = (LongCondition) tempCondition; 80 | if (pageReader.isNull(column)) { 81 | if (condition.compare(null)) { 82 | shouldAddRecord = true; 83 | break; 84 | } 85 | } 86 | else { 87 | long subject = pageReader.getLong(column); 88 | if (condition.compare(subject)) { 89 | shouldAddRecord = true; 90 | break; 91 | } 92 | } 93 | } 94 | } 95 | 96 | @Override 97 | public void doubleColumn(Column column) 98 | { 99 | if (shouldAddRecord) { 100 | return; 101 | } 102 | List conditionList = conditionMap.get(column.getName()); 103 | for (Condition tempCondition : conditionList) { 104 | DoubleCondition condition = (DoubleCondition) tempCondition; 105 | if (pageReader.isNull(column)) { 106 | if (condition.compare(null)) { 107 | shouldAddRecord = true; 108 | break; 109 | } 110 | } 111 | else { 112 | double subject = pageReader.getDouble(column); 113 | if (condition.compare(subject)) { 114 | shouldAddRecord = true; 115 | break; 116 | } 117 | } 118 | } 119 | } 120 | 121 | @Override 122 | public void stringColumn(Column column) 123 | { 124 | if (shouldAddRecord) { 125 | return; 126 | } 127 | List conditionList = conditionMap.get(column.getName()); 128 | for (Condition tempCondition : conditionList) { 129 | StringCondition condition = (StringCondition) tempCondition; 130 | if (pageReader.isNull(column)) { 131 | if (condition.compare(null)) { 132 | shouldAddRecord = true; 133 | break; 134 | } 135 | } 136 | else { 137 | String subject = pageReader.getString(column); 138 | if (condition.compare(subject)) { 139 | shouldAddRecord = true; 140 | break; 141 | } 142 | } 143 | } 144 | } 145 | 146 | @Override 147 | public void timestampColumn(Column column) 148 | { 149 | if (shouldAddRecord) { 150 | return; 151 | } 152 | List conditionList = conditionMap.get(column.getName()); 153 | for (Condition tempCondition : conditionList) { 154 | TimestampCondition condition = (TimestampCondition) tempCondition; 155 | if (pageReader.isNull(column)) { 156 | if (condition.compare(null)) { 157 | shouldAddRecord = true; 158 | break; 159 | } 160 | } 161 | else { 162 | // TODO: use getTimstampInstant after dropping v0.9 163 | Instant subject = pageReader.getTimestamp(column).getInstant(); 164 | if (condition.compare(subject)) { 165 | shouldAddRecord = true; 166 | break; 167 | } 168 | } 169 | } 170 | } 171 | 172 | @Override 173 | public void jsonColumn(Column column) 174 | { 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/GuardColumnVisitorAndImpl.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row; 2 | 3 | import org.embulk.filter.row.RowFilterPlugin.PluginTask; 4 | import org.embulk.filter.row.condition.BooleanCondition; 5 | import org.embulk.filter.row.condition.Condition; 6 | import org.embulk.filter.row.condition.DoubleCondition; 7 | import org.embulk.filter.row.condition.LongCondition; 8 | import org.embulk.filter.row.condition.StringCondition; 9 | import org.embulk.filter.row.condition.TimestampCondition; 10 | import org.embulk.spi.Column; 11 | import org.embulk.spi.ColumnVisitor; 12 | import org.embulk.spi.Exec; 13 | import org.embulk.spi.PageReader; 14 | import org.embulk.spi.Schema; 15 | import org.slf4j.Logger; 16 | 17 | import java.time.Instant; 18 | import java.util.HashMap; 19 | import java.util.List; 20 | 21 | class GuardColumnVisitorAndImpl 22 | extends AbstractGuardColumnVisitor 23 | implements ColumnVisitor 24 | { 25 | private static final Logger logger = Exec.getLogger(RowFilterPlugin.class); 26 | private boolean shouldAddRecord; 27 | private HashMap> conditionMap; 28 | 29 | GuardColumnVisitorAndImpl(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader) 30 | { 31 | super(task, inputSchema, outputSchema, pageReader); 32 | this.conditionMap = buildConditionMap(task, outputSchema); 33 | } 34 | 35 | public boolean visitColumns(Schema inputSchema) 36 | { 37 | //Visitor objects are created for each thread :) 38 | //System.out.println(String.format("thread_id:%d object_id:%d", Thread.currentThread().getId(), this.hashCode())); 39 | shouldAddRecord = true; 40 | for (Column column : inputSchema.getColumns()) { 41 | column.visit(this); 42 | } 43 | return shouldAddRecord; 44 | } 45 | 46 | @Override 47 | public void booleanColumn(Column column) 48 | { 49 | if (!shouldAddRecord) { 50 | return; 51 | } 52 | List conditionList = conditionMap.get(column.getName()); 53 | for (Condition tempCondition : conditionList) { 54 | BooleanCondition condition = (BooleanCondition) tempCondition; 55 | if (pageReader.isNull(column)) { 56 | if (!condition.compare(null)) { 57 | shouldAddRecord = false; 58 | break; 59 | } 60 | } 61 | else { 62 | boolean subject = pageReader.getBoolean(column); 63 | if (!condition.compare(subject)) { 64 | shouldAddRecord = false; 65 | break; 66 | } 67 | } 68 | } 69 | } 70 | 71 | @Override 72 | public void longColumn(Column column) 73 | { 74 | if (!shouldAddRecord) { 75 | return; 76 | } 77 | List conditionList = conditionMap.get(column.getName()); 78 | for (Condition tempCondition : conditionList) { 79 | LongCondition condition = (LongCondition) tempCondition; 80 | if (pageReader.isNull(column)) { 81 | if (!condition.compare(null)) { 82 | shouldAddRecord = false; 83 | break; 84 | } 85 | } 86 | else { 87 | long subject = pageReader.getLong(column); 88 | if (!condition.compare(subject)) { 89 | shouldAddRecord = false; 90 | break; 91 | } 92 | } 93 | } 94 | } 95 | 96 | @Override 97 | public void doubleColumn(Column column) 98 | { 99 | if (!shouldAddRecord) { 100 | return; 101 | } 102 | List conditionList = conditionMap.get(column.getName()); 103 | for (Condition tempCondition : conditionList) { 104 | DoubleCondition condition = (DoubleCondition) tempCondition; 105 | if (pageReader.isNull(column)) { 106 | if (!condition.compare(null)) { 107 | shouldAddRecord = false; 108 | break; 109 | } 110 | } 111 | else { 112 | double subject = pageReader.getDouble(column); 113 | if (!condition.compare(subject)) { 114 | shouldAddRecord = false; 115 | break; 116 | } 117 | } 118 | } 119 | } 120 | 121 | @Override 122 | public void stringColumn(Column column) 123 | { 124 | if (!shouldAddRecord) { 125 | return; 126 | } 127 | List conditionList = conditionMap.get(column.getName()); 128 | for (Condition tempCondition : conditionList) { 129 | StringCondition condition = (StringCondition) tempCondition; 130 | if (pageReader.isNull(column)) { 131 | if (!condition.compare(null)) { 132 | shouldAddRecord = false; 133 | break; 134 | } 135 | } 136 | else { 137 | String subject = pageReader.getString(column); 138 | if (!condition.compare(subject)) { 139 | shouldAddRecord = false; 140 | break; 141 | } 142 | } 143 | } 144 | } 145 | 146 | @Override 147 | public void timestampColumn(Column column) 148 | { 149 | if (!shouldAddRecord) { 150 | return; 151 | } 152 | List conditionList = conditionMap.get(column.getName()); 153 | for (Condition tempCondition : conditionList) { 154 | TimestampCondition condition = (TimestampCondition) tempCondition; 155 | if (pageReader.isNull(column)) { 156 | if (!condition.compare(null)) { 157 | shouldAddRecord = false; 158 | break; 159 | } 160 | } 161 | else { 162 | // TODO: use getTimstampInstant after dropping v0.9 163 | Instant subject = pageReader.getTimestamp(column).getInstant(); 164 | if (!condition.compare(subject)) { 165 | shouldAddRecord = false; 166 | break; 167 | } 168 | } 169 | } 170 | } 171 | 172 | @Override 173 | public void jsonColumn(Column column) 174 | { 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/where/_parser.y: -------------------------------------------------------------------------------- 1 | %{ 2 | import org.embulk.config.ConfigException; 3 | import org.embulk.spi.Column; 4 | import org.embulk.spi.Schema; 5 | %} 6 | 7 | /* YACC Declarations */ 8 | %token EQ /* = */ 9 | %token NEQ /* <> != */ 10 | %token GT /* > */ 11 | %token GE /* >= */ 12 | %token LT /* < */ 13 | %token LE /* <= */ 14 | 15 | %token START_WITH 16 | %token END_WITH 17 | %token INCLUDE 18 | %token IS 19 | %token NOT 20 | 21 | %token AND 22 | %token OR 23 | 24 | %token NULL 25 | %token BOOLEAN 26 | %token STRING 27 | %token NUMBER 28 | %token IDENTIFIER 29 | 30 | %token TIMESTAMP 31 | 32 | %left OR 33 | %left AND 34 | %right TIMESTAMP 35 | %right NOT 36 | 37 | /* Grammar follows */ 38 | %% 39 | input: /* empty string */ 40 | | exp { root = $1; } 41 | ; 42 | 43 | timestamp: TIMESTAMP STRING { $$ = new ParserVal(new TimestampLiteral($2)); } 44 | | TIMESTAMP NUMBER { $$ = new ParserVal(new TimestampLiteral($2)); } 45 | 46 | exp: IDENTIFIER EQ BOOLEAN { $$ = new ParserVal(BinaryOpExp.create($1, $3, EQ)); } 47 | | IDENTIFIER NEQ BOOLEAN { $$ = new ParserVal(BinaryOpExp.create($1, $3, NEQ)); } 48 | | BOOLEAN EQ IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, EQ)); } 49 | | BOOLEAN NEQ IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, NEQ)); } 50 | | IDENTIFIER EQ NUMBER { $$ = new ParserVal(BinaryOpExp.create($1, $3, EQ)); } 51 | | IDENTIFIER NEQ NUMBER { $$ = new ParserVal(BinaryOpExp.create($1, $3, NEQ)); } 52 | | IDENTIFIER GT NUMBER { $$ = new ParserVal(BinaryOpExp.create($1, $3, GT)); } 53 | | IDENTIFIER GE NUMBER { $$ = new ParserVal(BinaryOpExp.create($1, $3, GE)); } 54 | | IDENTIFIER LT NUMBER { $$ = new ParserVal(BinaryOpExp.create($1, $3, LT)); } 55 | | IDENTIFIER LE NUMBER { $$ = new ParserVal(BinaryOpExp.create($1, $3, LE)); } 56 | | NUMBER EQ IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, EQ)); } 57 | | NUMBER NEQ IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, NEQ)); } 58 | | NUMBER GT IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, GT)); } 59 | | NUMBER GE IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, GE)); } 60 | | NUMBER LT IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, LT)); } 61 | | NUMBER LE IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, LE)); } 62 | | IDENTIFIER EQ STRING { $$ = new ParserVal(BinaryOpExp.create($1, $3, EQ)); } 63 | | IDENTIFIER NEQ STRING { $$ = new ParserVal(BinaryOpExp.create($1, $3, NEQ)); } 64 | | IDENTIFIER GT STRING { $$ = new ParserVal(BinaryOpExp.create($1, $3, GT)); } 65 | | IDENTIFIER GE STRING { $$ = new ParserVal(BinaryOpExp.create($1, $3, GE)); } 66 | | IDENTIFIER LT STRING { $$ = new ParserVal(BinaryOpExp.create($1, $3, LT)); } 67 | | IDENTIFIER LE STRING { $$ = new ParserVal(BinaryOpExp.create($1, $3, LE)); } 68 | | IDENTIFIER START_WITH STRING { $$ = new ParserVal(BinaryOpExp.create($1, $3, START_WITH)); } 69 | | IDENTIFIER END_WITH STRING { $$ = new ParserVal(BinaryOpExp.create($1, $3, END_WITH)); } 70 | | IDENTIFIER INCLUDE STRING { $$ = new ParserVal(BinaryOpExp.create($1, $3, INCLUDE)); } 71 | | STRING EQ IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, EQ)); } 72 | | STRING NEQ IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, NEQ)); } 73 | | STRING GT IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, GT)); } 74 | | STRING GE IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, GE)); } 75 | | STRING LT IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, LT)); } 76 | | STRING LE IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, LE)); } 77 | | STRING START_WITH IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, START_WITH)); } 78 | | STRING END_WITH IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, END_WITH)); } 79 | | STRING INCLUDE IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, INCLUDE)); } 80 | | IDENTIFIER EQ timestamp { $$ = new ParserVal(BinaryOpExp.create($1, $3, EQ)); } 81 | | IDENTIFIER NEQ timestamp { $$ = new ParserVal(BinaryOpExp.create($1, $3, NEQ)); } 82 | | IDENTIFIER GT timestamp { $$ = new ParserVal(BinaryOpExp.create($1, $3, GT)); } 83 | | IDENTIFIER GE timestamp { $$ = new ParserVal(BinaryOpExp.create($1, $3, GE)); } 84 | | IDENTIFIER LT timestamp { $$ = new ParserVal(BinaryOpExp.create($1, $3, LT)); } 85 | | IDENTIFIER LE timestamp { $$ = new ParserVal(BinaryOpExp.create($1, $3, LE)); } 86 | | timestamp EQ IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, EQ)); } 87 | | timestamp NEQ IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, NEQ)); } 88 | | timestamp GT IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, GT)); } 89 | | timestamp GE IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, GE)); } 90 | | timestamp LT IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, LT)); } 91 | | timestamp LE IDENTIFIER { $$ = new ParserVal(BinaryOpExp.create($1, $3, LE)); } 92 | | IDENTIFIER REGEXP STRING { $$ = new ParserVal(new RegexpOpExp($1, $3, REGEXP)); } 93 | | IDENTIFIER IS NULL { $$ = new ParserVal(new NullOpExp($1, EQ)); } 94 | | IDENTIFIER IS NOT NULL { $$ = new ParserVal(new NullOpExp($1, NEQ)); } 95 | | exp OR exp { $$ = new ParserVal(new LogicalOpExp($1, $3, OR)); } 96 | | exp AND exp { $$ = new ParserVal(new LogicalOpExp($1, $3, AND)); } 97 | | NOT exp { $$ = new ParserVal(new NegateOpExp($2)); } 98 | | '(' exp ')' { $$ = $2; } 99 | ; 100 | %% 101 | 102 | protected Schema schema; 103 | protected Yylex lexer; 104 | protected ParserVal root; 105 | 106 | public Parser(final Schema schema) 107 | { 108 | this.schema = schema; 109 | } 110 | 111 | public Parser(final Schema schema, boolean yydebug) 112 | { 113 | this.schema = schema; 114 | this.yydebug = yydebug; 115 | } 116 | 117 | public ParserExp parse(String str) 118 | { 119 | lexer = new Yylex(str, this); 120 | yyparse(); 121 | return ((ParserExp)(root.obj)); 122 | } 123 | 124 | private int yylex () { 125 | int token = -1; 126 | try { 127 | token = lexer.yylex(); // next token 128 | } 129 | catch (java.io.IOException e) { 130 | e.printStackTrace(); // should not happen 131 | } 132 | return token; 133 | } 134 | 135 | void yyerror(String s) 136 | { 137 | throw new ConfigException("yyerror: " + s); 138 | } 139 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/condition/ConditionFactory.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | import org.apache.commons.lang3.math.NumberUtils; 4 | import org.embulk.config.ConfigException; 5 | import org.embulk.spi.Column; 6 | import org.embulk.spi.type.BooleanType; 7 | import org.embulk.spi.type.DoubleType; 8 | import org.embulk.spi.type.LongType; 9 | import org.embulk.spi.type.StringType; 10 | import org.embulk.spi.type.TimestampType; 11 | import org.embulk.spi.type.Type; 12 | import org.embulk.util.timestamp.TimestampFormatter; 13 | 14 | import java.time.Instant; 15 | import java.time.format.DateTimeParseException; 16 | 17 | public class ConditionFactory 18 | { 19 | private Column column; 20 | private String columnName; 21 | private Type columnType; 22 | private ConditionConfig conditionConfig; 23 | private String operator; 24 | private boolean not; 25 | 26 | public ConditionFactory(Column column, ConditionConfig conditionConfig) 27 | { 28 | this.column = column; 29 | this.columnName = column.getName(); 30 | this.columnType = column.getType(); 31 | this.conditionConfig = conditionConfig; 32 | this.operator = conditionConfig.getOperator().get().toUpperCase(); // default: == 33 | this.not = conditionConfig.getNot().get().booleanValue(); // default: false 34 | } 35 | 36 | public Condition createCondition() 37 | { 38 | if (columnType instanceof BooleanType) { 39 | return createBooleanCondition(); 40 | } 41 | else if (columnType instanceof LongType) { 42 | return createLongCondition(); 43 | } 44 | else if (columnType instanceof DoubleType) { 45 | return createDoubleCondition(); 46 | } 47 | else if (columnType instanceof StringType) { 48 | return createStringCondition(); 49 | } 50 | else if (columnType instanceof TimestampType) { 51 | return createTimestampCondition(); 52 | } 53 | assert false; 54 | return null; 55 | } 56 | 57 | public BooleanCondition createBooleanCondition() 58 | { 59 | if (operator.equals("IS NULL") || operator.equals("IS NOT NULL")) { 60 | return new BooleanCondition(operator, null, not); 61 | } 62 | else if (!conditionConfig.getArgument().isPresent()) { 63 | throw new ConfigException(String.format("RowFilterPlugin: Argument is missing on column: %s", columnName)); 64 | } 65 | else if (conditionConfig.getArgument().get() instanceof Boolean) { 66 | Boolean argument = (Boolean) conditionConfig.getArgument().get(); 67 | return new BooleanCondition(operator, argument, not); 68 | } 69 | else { 70 | throw new ConfigException(String.format("RowFilterPlugin: Type mismatch on column: %s", columnName)); 71 | } 72 | } 73 | 74 | public LongCondition createLongCondition() 75 | { 76 | if (operator.equals("IS NULL") || operator.equals("IS NOT NULL")) { 77 | return new LongCondition(operator, null, not); 78 | } 79 | else if (!conditionConfig.getArgument().isPresent()) { 80 | throw new ConfigException(String.format("RowFilterPlugin: Argument is missing on column: %s", columnName)); 81 | } 82 | else if (NumberUtils.isNumber(conditionConfig.getArgument().get().toString())) { 83 | Double argument = new Double(conditionConfig.getArgument().get().toString()); 84 | return new LongCondition(operator, argument, not); 85 | } 86 | else { 87 | throw new ConfigException(String.format("RowFilterPlugin: Type mismatch on column: %s", columnName)); 88 | } 89 | } 90 | 91 | public DoubleCondition createDoubleCondition() 92 | { 93 | if (operator.equals("IS NULL") || operator.equals("IS NOT NULL")) { 94 | return new DoubleCondition(operator, null, not); 95 | } 96 | else if (!conditionConfig.getArgument().isPresent()) { 97 | throw new ConfigException(String.format("RowFilterPlugin: Argument is missing on column: %s", columnName)); 98 | } 99 | else if (NumberUtils.isNumber(conditionConfig.getArgument().get().toString())) { 100 | Double argument = new Double(conditionConfig.getArgument().get().toString()); 101 | return new DoubleCondition(operator, argument, not); 102 | } 103 | else { 104 | throw new ConfigException(String.format("RowFilterPlugin: Type mismatch on column: %s", columnName)); 105 | } 106 | } 107 | 108 | public StringCondition createStringCondition() 109 | { 110 | if (operator.equals("IS NULL") || operator.equals("IS NOT NULL")) { 111 | return new StringCondition(operator, null, not); 112 | } 113 | else if (!conditionConfig.getArgument().isPresent()) { 114 | throw new ConfigException(String.format("RowFilterPlugin: Argument is missing on column: %s", columnName)); 115 | } 116 | else if (conditionConfig.getArgument().get() instanceof String) { 117 | String argument = (String) conditionConfig.getArgument().get(); 118 | return new StringCondition(operator, argument, not); 119 | } 120 | else { 121 | throw new ConfigException(String.format("RowFilterPlugin: Type mismatch on column: %s", columnName)); 122 | } 123 | } 124 | 125 | public TimestampCondition createTimestampCondition() 126 | { 127 | if (operator.equals("IS NULL") || operator.equals("IS NOT NULL")) { 128 | return new TimestampCondition(operator, null, not); 129 | } 130 | else if (!conditionConfig.getArgument().isPresent()) { 131 | throw new ConfigException(String.format("RowFilterPlugin: Argument is missing on column: %s", columnName)); 132 | } 133 | else if (conditionConfig.getArgument().get() instanceof String) { 134 | String argument = (String) conditionConfig.getArgument().get(); 135 | String format = (String) conditionConfig.getFormat().get(); 136 | String timezone = conditionConfig.getTimezone().get(); 137 | 138 | TimestampFormatter parser = createTimestampParser(format, timezone); 139 | try { 140 | Instant timestamp = parser.parse(argument); 141 | return new TimestampCondition(operator, timestamp, not); 142 | } 143 | catch (DateTimeParseException ex) { 144 | throw new RuntimeException(ex); 145 | } 146 | } 147 | else { 148 | throw new ConfigException(String.format("RowFilterPlugin: Type mismatch on column: %s", columnName)); 149 | } 150 | } 151 | 152 | private TimestampFormatter createTimestampParser(String format, String timezone) 153 | { 154 | return TimestampFormatter.builder(format,true) 155 | .setDefaultDateFromString("1970-01-01") 156 | .setDefaultZoneFromString(timezone) 157 | .build(); 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/RowFilterPlugin.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row; 2 | 3 | import org.embulk.config.ConfigException; 4 | import org.embulk.config.ConfigSource; 5 | import org.embulk.config.TaskSource; 6 | import org.embulk.filter.row.condition.ConditionConfig; 7 | import org.embulk.filter.row.where.Parser; 8 | import org.embulk.filter.row.where.ParserExp; 9 | import org.embulk.spi.Exec; 10 | import org.embulk.spi.FilterPlugin; 11 | import org.embulk.spi.Page; 12 | import org.embulk.spi.PageBuilder; 13 | import org.embulk.spi.PageOutput; 14 | import org.embulk.spi.PageReader; 15 | import org.embulk.spi.Schema; 16 | import org.embulk.util.config.Config; 17 | import org.embulk.util.config.ConfigDefault; 18 | import org.embulk.util.config.ConfigMapper; 19 | import org.embulk.util.config.ConfigMapperFactory; 20 | import org.embulk.util.config.Task; 21 | import org.embulk.util.config.TaskMapper; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.util.List; 26 | import java.util.Optional; 27 | 28 | public class RowFilterPlugin implements FilterPlugin 29 | { 30 | private static final Logger logger = LoggerFactory.getLogger(RowFilterPlugin.class); 31 | private static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = ConfigMapperFactory 32 | .builder() 33 | .addDefaultModules() 34 | .build(); 35 | private static final ConfigMapper CONFIG_MAPPER = CONFIG_MAPPER_FACTORY.createConfigMapper(); 36 | public RowFilterPlugin() {} 37 | 38 | public interface PluginTask extends Task 39 | { 40 | @Config("condition") 41 | @ConfigDefault("\"AND\"") 42 | @Deprecated 43 | public String getCondition(); 44 | 45 | @Config("conditions") 46 | @ConfigDefault("null") 47 | @Deprecated 48 | public Optional> getConditions(); 49 | 50 | @Config("where") 51 | @ConfigDefault("null") 52 | public Optional getWhere(); 53 | 54 | // From org.embulk.spi.time.TimestampParser.Task. 55 | @Config("default_timezone") 56 | @ConfigDefault("\"UTC\"") 57 | String getDefaultTimeZoneId(); 58 | 59 | // From org.embulk.spi.time.TimestampParser.Task. 60 | @Config("default_timestamp_format") 61 | @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%N %z\"") 62 | String getDefaultTimestampFormat(); 63 | 64 | // From org.embulk.spi.time.TimestampParser.Task. 65 | @Config("default_date") 66 | @ConfigDefault("\"1970-01-01\"") 67 | String getDefaultDate(); 68 | } 69 | 70 | public interface TimestampColumnOption extends Task 71 | { 72 | // From org.embulk.spi.time.TimestampParser.TimestampColumnOption. 73 | @Config("timezone") 74 | @ConfigDefault("null") 75 | public Optional getTimeZoneId(); 76 | 77 | // From org.embulk.spi.time.TimestampParser.TimestampColumnOption. 78 | @Config("format") 79 | @ConfigDefault("null") 80 | public Optional getFormat(); 81 | 82 | // From org.embulk.spi.time.TimestampParser.TimestampColumnOption. 83 | @Config("date") 84 | @ConfigDefault("null") 85 | public Optional getDate(); 86 | } 87 | 88 | @Override 89 | public void transaction(ConfigSource config, Schema inputSchema, 90 | FilterPlugin.Control control) 91 | { 92 | final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class); 93 | 94 | configure(task, inputSchema); 95 | Schema outputSchema = inputSchema; 96 | 97 | control.run(task.dump(), outputSchema); 98 | } 99 | 100 | void configure(PluginTask task, Schema inputSchema) throws ConfigException 101 | { 102 | if (task.getConditions().isPresent()) { 103 | logger.warn("embulk-filter-row: \"conditions\" is deprecated, use \"where\" instead."); 104 | for (ConditionConfig conditionConfig : task.getConditions().get()) { 105 | String columnName = conditionConfig.getColumn(); 106 | inputSchema.lookupColumn(columnName); // throw SchemaConfigException if not found 107 | } 108 | 109 | String condition = task.getCondition().toLowerCase(); 110 | if (!condition.equals("or") && !condition.equals("and")) { 111 | throw new ConfigException("condition must be either of \"or\" or \"and\"."); 112 | } 113 | } 114 | else if (task.getWhere().isPresent()) { 115 | String where = task.getWhere().get(); 116 | Parser parser = new Parser(inputSchema); 117 | // objects must be created in open since plugin instances in transaction and open are different 118 | parser.parse(where); // throws ConfigException if something wrong 119 | } 120 | else { 121 | throw new ConfigException("Either of `conditions` or `where` must be set."); 122 | } 123 | } 124 | 125 | @Override 126 | public PageOutput open(final TaskSource taskSource, final Schema inputSchema, 127 | final Schema outputSchema, final PageOutput output) 128 | { 129 | final TaskMapper taskMapper = CONFIG_MAPPER_FACTORY.createTaskMapper(); 130 | final PluginTask task = taskMapper.map(taskSource, PluginTask.class); 131 | final boolean orCondition = task.getCondition().toLowerCase().equals("or"); 132 | final PageReader pageReader = new PageReader(inputSchema); 133 | final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output); 134 | 135 | final AbstractGuardColumnVisitor guradVisitor; 136 | if (task.getWhere().isPresent()) { 137 | ParserExp parserExp = new Parser(inputSchema).parse(task.getWhere().get()); 138 | guradVisitor = new GuardColumnVisitorWhereImpl(task, inputSchema, outputSchema, pageReader, parserExp); 139 | } 140 | else if (orCondition) { 141 | guradVisitor = new GuardColumnVisitorOrImpl(task, inputSchema, outputSchema, pageReader); 142 | } 143 | else { 144 | guradVisitor = new GuardColumnVisitorAndImpl(task, inputSchema, outputSchema, pageReader); 145 | } 146 | 147 | final BuilderColumnVisitorImpl builderVisitor; 148 | builderVisitor = new BuilderColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder); 149 | 150 | return new PageOutput() { 151 | @Override 152 | public void finish() 153 | { 154 | pageBuilder.finish(); 155 | } 156 | 157 | @Override 158 | public void close() 159 | { 160 | pageBuilder.close(); 161 | } 162 | 163 | @Override 164 | public void add(Page page) 165 | { 166 | pageReader.setPage(page); 167 | 168 | while (pageReader.nextRecord()) { 169 | if (guradVisitor.visitColumns(inputSchema)) { 170 | // output.add(page); did not work, double release() error occurred. We need to copy from reader to builder... 171 | outputSchema.visitColumns(builderVisitor); 172 | pageBuilder.addRecord(); 173 | } 174 | } 175 | } 176 | }; 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Row filter plugin for Embulk 2 | 3 | [![Build Status](https://secure.travis-ci.org/sonots/embulk-filter-row.png?branch=master)](http://travis-ci.org/sonots/embulk-filter-row) 4 | 5 | A filter plugin for Embulk to filter out rows 6 | 7 | ## Configuration 8 | 9 | Requirement: version >= 0.3.0 10 | 11 | * **where**: Select only rows which match with conditions written in SQL-like syntax. See [SQL-like Syntax](#sql-like-syntax) 12 | 13 | ## Example 14 | 15 | ```yaml 16 | filters: 17 | - type: row 18 | where: column1 = 'str' 19 | ``` 20 | 21 | ```yaml 22 | filters: 23 | - type: row 24 | where: |- 25 | ( 26 | string_column START_WITH 'str' AND 27 | number_column > 1.0 28 | ) 29 | OR 30 | ( 31 | time_column = TIMESTAMP '2016-01-01 +0900' AND 32 | "true_column" = true 33 | ) 34 | ``` 35 | 36 | See [SQL-like Syntax](#sql-like-syntax) for more details 37 | 38 | # SQL-like Syntax 39 | 40 | This syntax must be similar with a standard SQL syntax. 41 | 42 | ```sql 43 | where: column1 = 'str' 44 | ``` 45 | 46 | ```sql 47 | where: |- 48 | ( 49 | string_column START_WITH 'str' AND 50 | number_column > 1.0 51 | 52 | ) 53 | OR 54 | ( 55 | time_column = TIMESTAMP '2016-01-01 +0900' AND 56 | "true_column" = true AND 57 | string_column REGEXP '^reg' 58 | ) 59 | ``` 60 | 61 | ## Literals 62 | 63 | ### Boolean Literal 64 | 65 | `true` or `TRUE` or `false` or `FALSE` are considered as a boolean literal 66 | 67 | ### Number Literal 68 | 69 | Characters matching with a regular expression `-?[0-9]+(\.[0-9]+)?` is considered as a number literal 70 | 71 | ### String Literal 72 | 73 | Characters surrounded by `'` such as `'foo'` is considered as a string literal 74 | 75 | ### Timestamp Literal 76 | 77 | NOTE: It became possible to omit `TIMESTAMP` keyword on comparing with `timestamp` identifier (column) from version >= 0.3.3. 78 | 79 | `TIMESTAMP ( NumberLiteral | StringLiteral )` such as `TIMESTAMP 1470433087.747123` or `TIMESTAMP '2016-08-06 06:38:07.747123 +0900'` is considered as a timestamp literal 80 | 81 | Number is a epoch time since 1970-01-01 UTC with nano time resolution. 82 | 83 | String is a timestamp string which matches with one of following format: 84 | 85 | * `%Y-%m-%d %H:%M:%S.%N %z` 86 | * `%Y-%m-%d %H:%M:%S.%N` 87 | * `%Y-%m-%d %H:%M:%S %z` 88 | * `%Y-%m-%d %H:%M:%S` 89 | * `%Y-%m-%d %z` 90 | * `%Y-%m-%d` 91 | 92 | The time zone for formats without `%z` is UTC, and the time resolution is micro second (caused by limitation of Embulk TimestampParser). 93 | 94 | ### Json Literal 95 | 96 | Not supported yet 97 | 98 | ### Identifier Literal 99 | 100 | Characters matching with a regular expression `[a-zA-Z_][a-zA-z0-9_]*` such as `foobar`, and characters surrounded by `"` such as `"foo-bar"`, `"foo.bar"`, and `"foo\"bar"` are considred as an identifier literal, that is, embulk's column name. 101 | 102 | ## Operators 103 | 104 | ### Boolean Operator 105 | 106 | * `=` 107 | * `!=` 108 | 109 | ### Number Operator (Long and Double) 110 | 111 | * `=` 112 | * `!=` 113 | * `>` 114 | * `>=` 115 | * `<=` 116 | * `<` 117 | 118 | ### String Operator 119 | 120 | * `=` 121 | * `!=` 122 | * `START_WITH` 123 | * `END_WITH` 124 | * `INCLUDE` 125 | * `REGEXP` 126 | 127 | ### Timestamp Operator 128 | 129 | * `=` 130 | * `!=` 131 | * `>` 132 | * `>=` 133 | * `<=` 134 | * `<` 135 | 136 | ### Json Operator 137 | 138 | Not supported yet 139 | 140 | ### unary operator 141 | 142 | * "xxx IS NULL" 143 | * "xxx IS NOT NULL" 144 | * "NOT xxx" 145 | 146 | ## Old Configuration 147 | 148 | Versions >= 0.3.0 has `where` option to supports SQL-like syntax. I recommend to use it. 149 | 150 | Following options are **deprecated**, and **will be removed someday**. 151 | 152 | * **condition**: AND or OR (string, default: AND). 153 | * **conditions**: select only rows which matches with conditions. 154 | * **column**: column name (string, required) 155 | * **operator** operator (string, optional, default: ==) 156 | * boolean operator 157 | * `==` 158 | * `!=` 159 | * numeric operator (long, double, Timestamp) 160 | * `==` 161 | * `!=` 162 | * `>` 163 | * `>=` 164 | * `<=` 165 | * `<` 166 | * string operator 167 | * `==` 168 | * `!=` 169 | * `start_with` (or `startsWith`) 170 | * `end_with` (or `endsWith`) 171 | * `include` (or `contains`) 172 | * unary operator 173 | * `IS NULL` 174 | * `IS NOT NULL` 175 | * **argument**: argument for the operation (string, required for non-unary operators) 176 | * **not**: not (boolean, optional, default: false) 177 | * **format**: special option for timestamp column, specify the format of timestamp argument, parsed argument is compared with the column value as Timestamp object (string, default is `%Y-%m-%d %H:%M:%S.%N %z`) 178 | * **timezone**: special option for timestamp column, specify the timezone of timestamp argument (string, default is `UTC`) 179 | 180 | NOTE: column type is automatically retrieved from input data (inputSchema) 181 | 182 | ## Example (AND) 183 | 184 | **Deprecated** 185 | 186 | ```yaml 187 | filters: 188 | - type: row 189 | condition: AND 190 | conditions: 191 | - {column: foo, operator: "IS NOT NULL"} 192 | - {column: id, operator: ">=", argument: 10} 193 | - {column: id, operator: "<", argument: 20} 194 | - {column: name, opeartor: "include", argument: foo, not: true} 195 | - {column: time, operator: "==", argument: "2015-07-13", format: "%Y-%m-%d"} 196 | ``` 197 | 198 | ## Example (OR) 199 | 200 | **Deprecated** 201 | 202 | ```yaml 203 | filters: 204 | - type: row 205 | condition: OR 206 | conditions: 207 | - {column: a, operator: "IS NOT NULL"} 208 | - {column: b, operator: "IS NOT NULL"} 209 | ``` 210 | 211 | ## Example (AND of OR) 212 | 213 | **Deprecated** 214 | 215 | You can express a condition such as `(A OR B) AND (C OR D)` by combining multiple filters like 216 | 217 | ```yaml 218 | filters: 219 | - type: row 220 | condition: OR 221 | conditions: 222 | - {column: a, operator: "IS NOT NULL"} 223 | - {column: b, operator: "IS NOT NULL"} 224 | - type: row 225 | condition: OR 226 | conditions: 227 | - {column: c, operator: "IS NOT NULL"} 228 | - {column: d, operator: "IS NOT NULL"} 229 | ``` 230 | 231 | ## Comparisions 232 | 233 | * [embulk-filter-calcite](https://github.com/muga/embulk-filter-calcite) 234 | * embulk-filter-calcite is a pretty nice plugin which enables us to write SQL query to filter embulk records, not only `WHERE` but also `SELECT`. 235 | * Based on [my benchmark (Japanese)](http://qiita.com/sonots/items/a70482d29862de87624d), embulk-filter-row was faster than embulk-filter-calcite. 236 | * Choose which to use as your demand. 237 | 238 | ## ToDo 239 | 240 | * Support filtering by values of `type: json` with JSONPath 241 | * Support IN operator 242 | 243 | ## ChangeLog 244 | 245 | [CHANGELOG.md](./CHANGELOG.md) 246 | 247 | ## Development 248 | 249 | Run example: 250 | 251 | ``` 252 | $ ./gradlew classpath 253 | $ embulk preview -I lib example/example.yml 254 | ``` 255 | 256 | Run test: 257 | 258 | ``` 259 | $ ./gradlew test 260 | ``` 261 | 262 | Run checkstyle: 263 | 264 | ``` 265 | $ ./gradlew check 266 | ``` 267 | 268 | Release gem: 269 | 270 | ``` 271 | $ ./gradlew gemPush 272 | ``` 273 | 274 | ## Development of SQL-like Syntax 275 | 276 | Read the article [Supported SQL-like Syntax with embulk-filter-row using BYACC/J and JFlex](http://blog.livedoor.jp/sonots/archives/48172830.html). 277 | 278 | To download BYACC/J and JFlex and run them, you can use: 279 | 280 | ``` 281 | $ script/byaccj.sh 282 | ``` 283 | 284 | or 285 | 286 | ``` 287 | $ ./gradlew byaccj # this runs script/byaccj.sh internally 288 | ``` 289 | 290 | This generates `src/main/java/org/embulk/filter/row/where/{Parser,ParserVal,Yylex}.java`. 291 | 292 | The `byaccj` task of gradle is ran before `compileJava` task (which means to be ran before `classpath` or `test` task also) automatically. 293 | -------------------------------------------------------------------------------- /src/test/java/org/embulk/filter/row/where/TestYylex.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.where; 2 | 3 | import org.embulk.config.ConfigException; 4 | import org.embulk.spi.Schema; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | 8 | import java.io.IOException; 9 | 10 | import static org.embulk.spi.type.Types.BOOLEAN; 11 | import static org.junit.Assert.assertEquals; 12 | import static org.junit.Assert.assertTrue; 13 | 14 | public class TestYylex 15 | { 16 | Parser yyparser; 17 | Yylex lexer; 18 | 19 | @Before 20 | public void setUp() 21 | { 22 | yyparser = new Parser(Schema.builder().build()); 23 | } 24 | 25 | void assertNextToken(int token) 26 | { 27 | try { 28 | assertEquals(token, lexer.yylex()); 29 | } 30 | catch (IOException e) { 31 | e.printStackTrace(); 32 | } 33 | } 34 | 35 | ParserNode currentNode() 36 | { 37 | return (ParserNode)(yyparser.yylval.obj); 38 | } 39 | 40 | @Test 41 | public void testOperator() 42 | { 43 | lexer = new Yylex("START_WITH", yyparser); 44 | assertNextToken(Parser.START_WITH); 45 | 46 | lexer = new Yylex("(", yyparser); 47 | assertNextToken('('); 48 | 49 | lexer = new Yylex(" = ", yyparser); 50 | assertNextToken(Parser.EQ); 51 | } 52 | 53 | void assertIdentifier(String name) 54 | { 55 | assertEquals("IdentifierLiteral", currentNode().getClass().getSimpleName()); 56 | assertEquals(name, ((IdentifierLiteral)currentNode()).name); 57 | } 58 | 59 | @Test 60 | public void testIdentifier() 61 | { 62 | Schema schema = Schema.builder() 63 | .add("foobar", BOOLEAN) 64 | .add("_foobar", BOOLEAN) 65 | .add("foo bar", BOOLEAN) 66 | .add("foo\"bar", BOOLEAN) 67 | .add("$.foo.bar", BOOLEAN) 68 | .add("$['foo'][0:4][*]", BOOLEAN) 69 | .build(); 70 | yyparser = new Parser(schema); 71 | 72 | lexer = new Yylex("foobar", yyparser); 73 | assertNextToken(Parser.IDENTIFIER); 74 | assertIdentifier("foobar"); 75 | 76 | lexer = new Yylex(" foobar ", yyparser); 77 | assertNextToken(Parser.IDENTIFIER); 78 | assertIdentifier("foobar"); 79 | 80 | lexer = new Yylex("\"foobar\"", yyparser); 81 | assertNextToken(Parser.IDENTIFIER); 82 | assertIdentifier("foobar"); 83 | 84 | lexer = new Yylex(" _foobar ", yyparser); 85 | assertNextToken(Parser.IDENTIFIER); 86 | assertIdentifier("_foobar"); 87 | 88 | lexer = new Yylex(" \"foo bar\" ", yyparser); 89 | assertNextToken(Parser.IDENTIFIER); 90 | assertIdentifier("foo bar"); 91 | 92 | lexer = new Yylex(" \"foo\\\"bar\" ", yyparser); 93 | assertNextToken(Parser.IDENTIFIER); 94 | assertIdentifier("foo\"bar"); 95 | 96 | lexer = new Yylex("\"$.foo.bar\"", yyparser); 97 | assertNextToken(Parser.IDENTIFIER); 98 | assertIdentifier("$.foo.bar"); 99 | 100 | lexer = new Yylex("\"$['foo'][0:4][*]\"", yyparser); 101 | assertNextToken(Parser.IDENTIFIER); 102 | assertIdentifier("$['foo'][0:4][*]"); 103 | 104 | try { 105 | lexer = new Yylex("foo-bar", yyparser); 106 | assertNextToken(Parser.IDENTIFIER); 107 | assertTrue(false); 108 | } 109 | catch (ConfigException e) { 110 | } 111 | 112 | try { 113 | lexer = new Yylex("$['foo'][0:4][*]", yyparser); 114 | assertNextToken(Parser.IDENTIFIER); 115 | assertTrue(false); 116 | } 117 | catch (ConfigException e) { 118 | } 119 | 120 | try { 121 | lexer = new Yylex("unknown", yyparser); 122 | assertNextToken(Parser.IDENTIFIER); 123 | assertTrue(false); 124 | } 125 | catch (ConfigException e) { 126 | } 127 | } 128 | 129 | void assertBoolean(boolean val) 130 | { 131 | assertEquals("BooleanLiteral", currentNode().getClass().getSimpleName()); 132 | assertEquals(val, ((BooleanLiteral)currentNode()).val); 133 | } 134 | 135 | @Test 136 | public void testBoolean() 137 | { 138 | lexer = new Yylex("true", yyparser); 139 | assertNextToken(Parser.BOOLEAN); 140 | assertBoolean(true); 141 | 142 | lexer = new Yylex("false", yyparser); 143 | assertNextToken(Parser.BOOLEAN); 144 | assertBoolean(false); 145 | 146 | lexer = new Yylex("TRUE", yyparser); 147 | assertNextToken(Parser.BOOLEAN); 148 | assertBoolean(true); 149 | 150 | lexer = new Yylex("FALSE", yyparser); 151 | assertNextToken(Parser.BOOLEAN); 152 | assertBoolean(false); 153 | 154 | lexer = new Yylex(" true ", yyparser); 155 | assertNextToken(Parser.BOOLEAN); 156 | assertBoolean(true); 157 | } 158 | 159 | void assertNumber(double val) 160 | { 161 | assertEquals("NumberLiteral", currentNode().getClass().getSimpleName()); 162 | assertEquals(val, ((NumberLiteral)currentNode()).val, 0.0); 163 | } 164 | 165 | @Test 166 | public void testNumber() 167 | { 168 | lexer = new Yylex("1.5", yyparser); 169 | assertNextToken(Parser.NUMBER); 170 | assertNumber(1.5); 171 | 172 | lexer = new Yylex("1", yyparser); 173 | assertNextToken(Parser.NUMBER); 174 | assertNumber(1); 175 | 176 | lexer = new Yylex("-1.5", yyparser); 177 | assertNextToken(Parser.NUMBER); 178 | assertNumber(-1.5); 179 | 180 | lexer = new Yylex(" -1.5 ", yyparser); 181 | assertNextToken(Parser.NUMBER); 182 | assertNumber(-1.5); 183 | } 184 | 185 | void assertString(String val) 186 | { 187 | assertEquals("StringLiteral", currentNode().getClass().getSimpleName()); 188 | assertEquals(val, ((StringLiteral)currentNode()).val); 189 | } 190 | 191 | @Test 192 | public void testString() 193 | { 194 | lexer = new Yylex("'foobar'", yyparser); 195 | assertNextToken(Parser.STRING); 196 | assertString("foobar"); 197 | 198 | lexer = new Yylex("'foo bar'", yyparser); 199 | assertNextToken(Parser.STRING); 200 | assertString("foo bar"); 201 | 202 | lexer = new Yylex("'foo\\'bar'", yyparser); 203 | assertNextToken(Parser.STRING); 204 | assertString("foo\'bar"); 205 | } 206 | 207 | @Test 208 | public void testTimestamp() 209 | { 210 | lexer = new Yylex("TIMESTAMP 1.5", yyparser); 211 | assertNextToken(Parser.TIMESTAMP); 212 | assertNextToken(Parser.NUMBER); 213 | assertNumber(1.5); 214 | 215 | lexer = new Yylex("TIMESTAMP '2015-01-01'", yyparser); 216 | assertNextToken(Parser.TIMESTAMP); 217 | assertNextToken(Parser.STRING); 218 | assertString("2015-01-01"); 219 | } 220 | 221 | @Test 222 | public void testComplex() 223 | { 224 | Schema schema = Schema.builder() 225 | .add("true", BOOLEAN) 226 | .add("foobar", BOOLEAN) 227 | .build(); 228 | yyparser = new Parser(schema); 229 | 230 | lexer = new Yylex("( \"true\" = true OR \"true\" = false ) AND foobar = false", yyparser); 231 | assertNextToken('('); 232 | assertNextToken(Parser.IDENTIFIER); 233 | assertNextToken(Parser.EQ); 234 | assertNextToken(Parser.BOOLEAN); 235 | assertNextToken(Parser.OR); 236 | assertNextToken(Parser.IDENTIFIER); 237 | assertNextToken(Parser.EQ); 238 | assertNextToken(Parser.BOOLEAN); 239 | assertNextToken(')'); 240 | assertNextToken(Parser.AND); 241 | assertNextToken(Parser.IDENTIFIER); 242 | assertNextToken(Parser.EQ); 243 | assertNextToken(Parser.BOOLEAN); 244 | 245 | lexer = new Yylex("(\"true\"=true OR\"true\"=false)AND foobar=false", yyparser); 246 | assertNextToken('('); 247 | assertNextToken(Parser.IDENTIFIER); 248 | assertNextToken(Parser.EQ); 249 | assertNextToken(Parser.BOOLEAN); 250 | assertNextToken(Parser.OR); 251 | assertNextToken(Parser.IDENTIFIER); 252 | assertNextToken(Parser.EQ); 253 | assertNextToken(Parser.BOOLEAN); 254 | assertNextToken(')'); 255 | assertNextToken(Parser.AND); 256 | assertNextToken(Parser.IDENTIFIER); 257 | assertNextToken(Parser.EQ); 258 | assertNextToken(Parser.BOOLEAN); 259 | 260 | lexer = new Yylex("NOT(true=\"true\")", yyparser); 261 | assertNextToken(Parser.NOT); 262 | assertNextToken('('); 263 | assertNextToken(Parser.BOOLEAN); 264 | assertNextToken(Parser.EQ); 265 | assertNextToken(Parser.IDENTIFIER); 266 | assertNextToken(')'); 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/where/ParserLiteral.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.where; 2 | 3 | import org.embulk.config.ConfigException; 4 | import org.embulk.spi.Column; 5 | import org.embulk.spi.PageReader; 6 | import org.embulk.spi.Schema; 7 | import org.embulk.spi.type.BooleanType; 8 | import org.embulk.spi.type.DoubleType; 9 | import org.embulk.spi.type.JsonType; 10 | import org.embulk.spi.type.LongType; 11 | import org.embulk.spi.type.StringType; 12 | import org.embulk.spi.type.TimestampType; 13 | import org.embulk.util.timestamp.TimestampFormatter; 14 | import org.msgpack.value.Value; 15 | 16 | import java.time.Instant; 17 | import java.time.format.DateTimeParseException; 18 | 19 | // Literal Node of AST (Abstract Syntax Tree) 20 | public abstract class ParserLiteral extends ParserNode 21 | { 22 | protected String yytext; 23 | 24 | public boolean isBoolean() 25 | { 26 | return false; 27 | } 28 | public boolean isNumber() 29 | { 30 | return false; 31 | } 32 | public boolean isString() 33 | { 34 | return false; 35 | } 36 | public boolean isTimestamp() 37 | { 38 | return false; 39 | } 40 | public boolean isJson() 41 | { 42 | return false; 43 | } 44 | public boolean isIdentifier() 45 | { 46 | return false; 47 | } 48 | 49 | public boolean isNull(PageReader pageReader) 50 | { 51 | throw new RuntimeException(); 52 | } 53 | public boolean getBoolean(PageReader pageReader) 54 | { 55 | throw new RuntimeException(); 56 | } 57 | public double getNumber(PageReader pageReader) 58 | { 59 | throw new RuntimeException(); 60 | } 61 | public String getString(PageReader pageReader) 62 | { 63 | throw new RuntimeException(); 64 | } 65 | public Instant getTimestamp(PageReader pageReader) 66 | { 67 | throw new RuntimeException(); 68 | } 69 | public Value getJson(PageReader pageReader) 70 | { 71 | throw new RuntimeException(); 72 | } 73 | 74 | } 75 | 76 | class BooleanLiteral extends ParserLiteral 77 | { 78 | protected boolean val; 79 | 80 | public BooleanLiteral(String yytext) 81 | { 82 | this.yytext = yytext; 83 | this.val = Boolean.parseBoolean(yytext); // but, only true|TRUE|false|FALSE are allowed in lexer 84 | } 85 | 86 | public boolean isBoolean() 87 | { 88 | return true; 89 | } 90 | 91 | public boolean getBoolean(PageReader pageReader) 92 | { 93 | return val; 94 | } 95 | } 96 | 97 | class NumberLiteral extends ParserLiteral 98 | { 99 | protected double val; 100 | 101 | public NumberLiteral(String yytext) 102 | { 103 | this.yytext = yytext; 104 | this.val = Double.parseDouble(yytext); 105 | } 106 | 107 | public boolean isNumber() 108 | { 109 | return true; 110 | } 111 | 112 | public double getNumber(PageReader pageReader) 113 | { 114 | return val; 115 | } 116 | } 117 | 118 | class StringLiteral extends ParserLiteral 119 | { 120 | protected String val; 121 | 122 | public StringLiteral(String yytext) 123 | { 124 | this.yytext = yytext; 125 | this.val = yytext; 126 | } 127 | 128 | public boolean isString() 129 | { 130 | return true; 131 | } 132 | 133 | public String getString(PageReader pageReader) 134 | { 135 | return val; 136 | } 137 | } 138 | 139 | class TimestampLiteral extends ParserLiteral 140 | { 141 | protected Instant val; 142 | private static final String defaultTimeZone = "UTC"; 143 | 144 | public TimestampLiteral(ParserLiteral literal) 145 | { 146 | if (literal.getClass() == StringLiteral.class) { 147 | initTimestampLiteral((StringLiteral)literal); 148 | } 149 | else if (literal.getClass() == NumberLiteral.class) { 150 | initTimestampLiteral((NumberLiteral)(literal)); 151 | } 152 | else if (literal.getClass() == TimestampLiteral.class) { 153 | initTimestampLiteral((TimestampLiteral)literal); 154 | } 155 | else { 156 | throw new ConfigException(String.format("\"%s\" is not a Timestamp literal", literal.yytext)); 157 | } 158 | } 159 | 160 | public TimestampLiteral(ParserVal val) 161 | { 162 | this((ParserLiteral)(val.obj)); 163 | } 164 | 165 | void initTimestampLiteral(StringLiteral literal) 166 | { 167 | this.yytext = literal.yytext; 168 | String[] formats = { 169 | "%Y-%m-%d %H:%M:%S.%N %z", 170 | "%Y-%m-%d %H:%M:%S.%N", 171 | "%Y-%m-%d %H:%M:%S %z", 172 | "%Y-%m-%d %H:%M:%S", 173 | "%Y-%m-%d %z", 174 | "%Y-%m-%d", 175 | }; 176 | Instant val = null; 177 | DateTimeParseException ex = null; 178 | for (String format : formats) { 179 | try { 180 | TimestampFormatter timestampParser = createTimestampParser(format, defaultTimeZone); 181 | this.val = timestampParser.parse(literal.val); 182 | break; 183 | } 184 | catch (DateTimeParseException e) { 185 | ex = e; 186 | } 187 | } 188 | if (this.val == null) { 189 | if ( ex instanceof RuntimeException ) { 190 | throw (RuntimeException)ex; 191 | } 192 | throw new RuntimeException(ex); 193 | } 194 | } 195 | 196 | void initTimestampLiteral(NumberLiteral literal) 197 | { 198 | this.yytext = literal.yytext; 199 | double epoch = literal.val; 200 | int epochSecond = (int) epoch; 201 | long nanoAdjustment = (long) ((epoch - epochSecond) * 1000000000); 202 | this.val = Instant.ofEpochSecond(epochSecond, nanoAdjustment); 203 | } 204 | 205 | void initTimestampLiteral(TimestampLiteral literal) 206 | { 207 | this.yytext = literal.yytext; 208 | this.val = literal.val; 209 | } 210 | 211 | public boolean isTimestamp() 212 | { 213 | return true; 214 | } 215 | 216 | public Instant getTimestamp(PageReader pageReader) 217 | { 218 | return val; 219 | } 220 | 221 | private TimestampFormatter createTimestampParser(String format, String timezone) 222 | { 223 | TimestampFormatter parser; 224 | 225 | return TimestampFormatter.builder(format,true) 226 | .setDefaultZoneFromString("UTC") 227 | .setDefaultDateFromString("1970-01-01").build(); 228 | } 229 | } 230 | 231 | class IdentifierLiteral extends ParserLiteral 232 | { 233 | protected String name; 234 | protected Column column; 235 | 236 | public IdentifierLiteral(String name, Schema schema) 237 | { 238 | this.name = name; 239 | this.column = schema.lookupColumn(name); // throw SchemaConfigException 240 | // ToDo: Support filtering value with type: json 241 | if (column.getType() instanceof JsonType) { 242 | throw new ConfigException(String.format("Identifier for a json column '%s' is not supported", name)); 243 | } 244 | } 245 | 246 | public boolean isBoolean() 247 | { 248 | return (column.getType() instanceof BooleanType); 249 | } 250 | public boolean isNumber() 251 | { 252 | return (column.getType() instanceof LongType) || (column.getType() instanceof DoubleType); 253 | } 254 | public boolean isString() 255 | { 256 | return (column.getType() instanceof StringType); 257 | } 258 | public boolean isTimestamp() 259 | { 260 | return (column.getType() instanceof TimestampType); 261 | } 262 | public boolean isJson() 263 | { 264 | return (column.getType() instanceof JsonType); 265 | } 266 | public boolean isIdentifier() 267 | { 268 | return true; 269 | } 270 | 271 | public boolean isNull(PageReader pageReader) 272 | { 273 | return pageReader.isNull(column); 274 | } 275 | 276 | public boolean getBoolean(PageReader pageReader) 277 | { 278 | return pageReader.getBoolean(column); 279 | } 280 | 281 | public double getNumber(PageReader pageReader) 282 | { 283 | if (column.getType() instanceof LongType) { 284 | return (double) pageReader.getLong(column); 285 | } 286 | else { 287 | return pageReader.getDouble(column); 288 | } 289 | } 290 | 291 | public String getString(PageReader pageReader) 292 | { 293 | return pageReader.getString(column); 294 | } 295 | 296 | public Instant getTimestamp(PageReader pageReader) 297 | { 298 | // TODO: use getTimstampInstant after dropping v0.9 299 | return pageReader.getTimestamp(column).getInstant(); 300 | } 301 | 302 | public Value getJson(PageReader pageReader) 303 | { 304 | return pageReader.getJson(column); 305 | } 306 | } 307 | -------------------------------------------------------------------------------- /src/test/java/org/embulk/filter/row/condition/TestConditionFactory.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.condition; 2 | 3 | import com.fasterxml.jackson.databind.node.ObjectNode; 4 | import org.embulk.config.ConfigException; 5 | import org.embulk.config.TaskSource; 6 | import org.embulk.spi.Column; 7 | import org.junit.Test; 8 | 9 | import java.util.Optional; 10 | 11 | import static org.embulk.spi.type.Types.BOOLEAN; 12 | import static org.embulk.spi.type.Types.DOUBLE; 13 | import static org.embulk.spi.type.Types.LONG; 14 | import static org.embulk.spi.type.Types.STRING; 15 | import static org.embulk.spi.type.Types.TIMESTAMP; 16 | import static org.junit.Assert.assertTrue; 17 | import static org.junit.Assert.fail; 18 | 19 | public class TestConditionFactory 20 | { 21 | public class DefaultConditionConfig implements ConditionConfig 22 | { 23 | public String getColumn() 24 | { 25 | return "column"; 26 | } 27 | 28 | public Optional getOperator() 29 | { 30 | return Optional.of("IS NULL"); 31 | } 32 | 33 | public Optional getArgument() 34 | { 35 | return Optional.empty(); 36 | } 37 | 38 | public Optional getNot() 39 | { 40 | return Optional.of(false); 41 | } 42 | 43 | public Optional getFormat() 44 | { 45 | return Optional.of("%Y-%m-%d"); 46 | } 47 | 48 | public Optional getTimezone() 49 | { 50 | return Optional.of("UTC"); 51 | } 52 | 53 | public TaskSource dump() 54 | { 55 | return null; 56 | } 57 | 58 | public void validate() 59 | { 60 | } 61 | public TaskSource toTaskSource() 62 | { 63 | return null; 64 | } 65 | public ObjectNode toObjectNode() 66 | { 67 | return null; 68 | } 69 | 70 | } 71 | 72 | public TestConditionFactory() 73 | { 74 | } 75 | 76 | @Test 77 | public void testCreateBooleanCondition() 78 | { 79 | Column column = new Column(0, "column", BOOLEAN); 80 | ConditionConfig config; 81 | BooleanCondition condition; 82 | 83 | config = new DefaultConditionConfig() { 84 | public Optional getOperator() 85 | { 86 | return Optional.of("IS NULL"); 87 | } 88 | }; 89 | condition = (BooleanCondition) new ConditionFactory(column, config).createCondition(); 90 | assertTrue(condition.compare(null)); 91 | 92 | config = new DefaultConditionConfig() { 93 | public Optional getOperator() 94 | { 95 | return Optional.of("=="); 96 | } 97 | public Optional getArgument() 98 | { 99 | return Optional.empty(); 100 | } 101 | }; 102 | try { 103 | condition = (BooleanCondition) new ConditionFactory(column, config).createCondition(); 104 | fail("Argument is required"); 105 | } 106 | catch (ConfigException e) { 107 | } 108 | 109 | config = new DefaultConditionConfig() { 110 | public Optional getOperator() 111 | { 112 | return Optional.of("=="); 113 | } 114 | public Optional getArgument() 115 | { 116 | return Optional.of((Object) new Boolean(true)); 117 | } 118 | }; 119 | condition = (BooleanCondition) new ConditionFactory(column, config).createCondition(); 120 | assertTrue(condition.compare(new Boolean(true))); 121 | 122 | config = new DefaultConditionConfig() { 123 | public Optional getOperator() 124 | { 125 | return Optional.of("=="); 126 | } 127 | public Optional getArgument() 128 | { 129 | return Optional.of((Object) new Long(10)); 130 | } 131 | }; 132 | try { 133 | condition = (BooleanCondition) new ConditionFactory(column, config).createCondition(); 134 | fail("Argument type mismatch"); 135 | } 136 | catch (ConfigException e) { 137 | } 138 | } 139 | 140 | @Test 141 | public void testCreateDoubleCondition() 142 | { 143 | Column column = new Column(0, "column", DOUBLE); 144 | ConditionConfig config; 145 | DoubleCondition condition; 146 | 147 | config = new DefaultConditionConfig() { 148 | public Optional getOperator() 149 | { 150 | return Optional.of("IS NULL"); 151 | } 152 | }; 153 | condition = (DoubleCondition) new ConditionFactory(column, config).createCondition(); 154 | assertTrue(condition.compare(null)); 155 | 156 | config = new DefaultConditionConfig() { 157 | public Optional getOperator() 158 | { 159 | return Optional.of("=="); 160 | } 161 | public Optional getArgument() 162 | { 163 | return Optional.empty(); 164 | } 165 | }; 166 | try { 167 | condition = (DoubleCondition) new ConditionFactory(column, config).createCondition(); 168 | fail("Argument is required"); 169 | } 170 | catch (ConfigException e) { 171 | } 172 | 173 | config = new DefaultConditionConfig() { 174 | public Optional getOperator() 175 | { 176 | return Optional.of("=="); 177 | } 178 | public Optional getArgument() 179 | { 180 | return Optional.of((Object) new Double(10)); 181 | } 182 | }; 183 | condition = (DoubleCondition) new ConditionFactory(column, config).createCondition(); 184 | assertTrue(condition.compare(new Double(10))); 185 | 186 | config = new DefaultConditionConfig() { 187 | public Optional getOperator() 188 | { 189 | return Optional.of("=="); 190 | } 191 | public Optional getArgument() 192 | { 193 | return Optional.of((Object) new Boolean(true)); 194 | } 195 | }; 196 | try { 197 | condition = (DoubleCondition) new ConditionFactory(column, config).createCondition(); 198 | fail("Argument type mismatch"); 199 | } 200 | catch (ConfigException e) { 201 | } 202 | } 203 | 204 | @Test 205 | public void testCreateLongCondition() 206 | { 207 | Column column = new Column(0, "column", LONG); 208 | ConditionConfig config; 209 | LongCondition condition; 210 | 211 | config = new DefaultConditionConfig() { 212 | public Optional getOperator() 213 | { 214 | return Optional.of("IS NULL"); 215 | } 216 | }; 217 | condition = (LongCondition) new ConditionFactory(column, config).createCondition(); 218 | assertTrue(condition.compare(null)); 219 | 220 | config = new DefaultConditionConfig() { 221 | public Optional getOperator() 222 | { 223 | return Optional.of("=="); 224 | } 225 | public Optional getArgument() 226 | { 227 | return Optional.empty(); 228 | } 229 | }; 230 | try { 231 | condition = (LongCondition) new ConditionFactory(column, config).createCondition(); 232 | fail("Argument is required"); 233 | } 234 | catch (ConfigException e) { 235 | } 236 | 237 | config = new DefaultConditionConfig() { 238 | public Optional getOperator() 239 | { 240 | return Optional.of("=="); 241 | } 242 | public Optional getArgument() 243 | { 244 | return Optional.of((Object) new Long(10)); 245 | } 246 | }; 247 | condition = (LongCondition) new ConditionFactory(column, config).createCondition(); 248 | assertTrue(condition.compare(new Long(10))); 249 | 250 | config = new DefaultConditionConfig() { 251 | public Optional getOperator() 252 | { 253 | return Optional.of("=="); 254 | } 255 | public Optional getArgument() 256 | { 257 | return Optional.of((Object) new Boolean(true)); 258 | } 259 | }; 260 | try { 261 | condition = (LongCondition) new ConditionFactory(column, config).createCondition(); 262 | fail("Argument type mismatch"); 263 | } 264 | catch (ConfigException e) { 265 | } 266 | } 267 | 268 | @Test 269 | public void testCreateStringCondition() 270 | { 271 | Column column = new Column(0, "column", STRING); 272 | ConditionConfig config; 273 | StringCondition condition; 274 | 275 | config = new DefaultConditionConfig() { 276 | public Optional getOperator() 277 | { 278 | return Optional.of("IS NULL"); 279 | } 280 | }; 281 | condition = (StringCondition) new ConditionFactory(column, config).createCondition(); 282 | assertTrue(condition.compare(null)); 283 | 284 | config = new DefaultConditionConfig() { 285 | public Optional getOperator() 286 | { 287 | return Optional.of("=="); 288 | } 289 | public Optional getArgument() 290 | { 291 | return Optional.empty(); 292 | } 293 | }; 294 | try { 295 | condition = (StringCondition) new ConditionFactory(column, config).createCondition(); 296 | fail("Argument is required"); 297 | } 298 | catch (ConfigException e) { 299 | } 300 | 301 | config = new DefaultConditionConfig() { 302 | public Optional getOperator() 303 | { 304 | return Optional.of("=="); 305 | } 306 | public Optional getArgument() 307 | { 308 | return Optional.of((Object) "foo"); 309 | } 310 | }; 311 | condition = (StringCondition) new ConditionFactory(column, config).createCondition(); 312 | assertTrue(condition.compare("foo")); 313 | 314 | config = new DefaultConditionConfig() { 315 | public Optional getOperator() 316 | { 317 | return Optional.of("=="); 318 | } 319 | public Optional getArgument() 320 | { 321 | return Optional.of((Object) new Boolean(true)); 322 | } 323 | }; 324 | try { 325 | condition = (StringCondition) new ConditionFactory(column, config).createCondition(); 326 | fail("Argument type mismatch"); 327 | } 328 | catch (ConfigException e) { 329 | } 330 | } 331 | 332 | @Test 333 | public void testCreateTimestampCondition() 334 | { 335 | Column column = new Column(0, "column", TIMESTAMP); 336 | ConditionConfig config; 337 | TimestampCondition condition; 338 | 339 | config = new DefaultConditionConfig() { 340 | public Optional getOperator() 341 | { 342 | return Optional.of("IS NULL"); 343 | } 344 | }; 345 | condition = (TimestampCondition) new ConditionFactory(column, config).createCondition(); 346 | assertTrue(condition.compare(null)); 347 | 348 | config = new DefaultConditionConfig() { 349 | public Optional getOperator() 350 | { 351 | return Optional.of("=="); 352 | } 353 | public Optional getArgument() 354 | { 355 | return Optional.empty(); 356 | } 357 | }; 358 | try { 359 | condition = (TimestampCondition) new ConditionFactory(column, config).createCondition(); 360 | fail("Argument is required"); 361 | } 362 | catch (ConfigException e) { 363 | } 364 | 365 | config = new DefaultConditionConfig() { 366 | public Optional getOperator() 367 | { 368 | return Optional.of("=="); 369 | } 370 | public Optional getArgument() 371 | { 372 | return Optional.of((Object) new Boolean(true)); 373 | } 374 | }; 375 | try { 376 | condition = (TimestampCondition) new ConditionFactory(column, config).createCondition(); 377 | fail("Argument type mismatch"); 378 | } 379 | catch (ConfigException e) { 380 | } 381 | } 382 | } 383 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/where/ParserExp.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.where; 2 | 3 | import org.embulk.config.ConfigException; 4 | import org.embulk.spi.PageReader; 5 | import org.jcodings.specific.UTF8Encoding; 6 | import org.joni.Matcher; 7 | import org.joni.Option; 8 | import org.joni.Regex; 9 | 10 | import java.nio.charset.StandardCharsets; 11 | import java.time.Instant; 12 | 13 | // Operation Node of AST (Abstract Syntax Tree) 14 | public abstract class ParserExp extends ParserNode 15 | { 16 | public abstract boolean eval(PageReader pageReader); 17 | } 18 | 19 | abstract class BinaryOpExp extends ParserExp 20 | { 21 | ParserLiteral left; 22 | ParserLiteral right; 23 | int operator; 24 | 25 | public BinaryOpExp() {} 26 | 27 | public BinaryOpExp(ParserLiteral left, ParserLiteral right, int operator) 28 | { 29 | this.left = left; 30 | this.right = right; 31 | this.operator = operator; 32 | } 33 | 34 | public BinaryOpExp(ParserVal left, ParserVal right, int operator) 35 | { 36 | this((ParserLiteral)(left.obj), (ParserLiteral)(right.obj), operator); 37 | } 38 | 39 | static BinaryOpExp create(ParserLiteral left, ParserLiteral right, int operator) 40 | { 41 | if (left.isTimestamp() || right.isTimestamp()) { 42 | // Either of left or right would be a string or a number 43 | return new TimestampOpExp(left, right, operator); 44 | } 45 | else if (left.isString() || right.isString()) { 46 | return new StringOpExp(left, right, operator); 47 | } 48 | else if (left.isNumber() || right.isNumber()) { 49 | return new NumberOpExp(left, right, operator); 50 | } 51 | else if (left.isBoolean() || right.isBoolean()) { 52 | return new BooleanOpExp(left, right, operator); 53 | } 54 | else { 55 | throw new RuntimeException(); 56 | } 57 | } 58 | 59 | static BinaryOpExp create(ParserVal left, ParserVal right, int operator) 60 | { 61 | return BinaryOpExp.create(((ParserLiteral)left.obj), ((ParserLiteral)right.obj), operator); 62 | } 63 | 64 | static boolean isOperatorAllowed(int[] operators, int operator) 65 | { 66 | for (int o : operators) { 67 | if (operator == o) { 68 | return true; 69 | } 70 | } 71 | return false; 72 | } 73 | } 74 | 75 | class BooleanOpExp extends BinaryOpExp 76 | { 77 | public static int[] operators = {Parser.EQ, Parser.NEQ, Parser.GT, Parser.GE, Parser.LT, Parser.LE}; 78 | 79 | public BooleanOpExp(ParserLiteral left, ParserLiteral right, int operator) 80 | { 81 | super(left, right, operator); 82 | if (! left.isBoolean()) { 83 | throw new ConfigException(String.format("\"%s\" is not a Boolean column", ((IdentifierLiteral)left).name)); 84 | } 85 | if (! right.isBoolean()) { 86 | throw new ConfigException(String.format("\"%s\" is not a Boolean column", ((IdentifierLiteral)right).name)); 87 | } 88 | if (! isOperatorAllowed(operators, operator)) { 89 | throw new ConfigException(String.format("\"%s\" is not an allowed operator for BooleanOpExp", Parser.yyname[operator])); 90 | } 91 | } 92 | 93 | public BooleanOpExp(ParserVal left, ParserVal right, int operator) 94 | { 95 | this((ParserLiteral)(left.obj), (ParserLiteral)(right.obj), operator); 96 | } 97 | 98 | public boolean eval(PageReader pageReader) 99 | { 100 | boolean l = left.getBoolean(pageReader); 101 | boolean r = right.getBoolean(pageReader); 102 | if (operator == Parser.EQ) { 103 | return l == r; 104 | } 105 | else if (operator == Parser.NEQ) { 106 | return l != r; 107 | } 108 | else { 109 | assert(false); 110 | return false; 111 | } 112 | } 113 | } 114 | 115 | class NumberOpExp extends BinaryOpExp 116 | { 117 | public static int[] operators = {Parser.EQ, Parser.NEQ, Parser.GT, Parser.GE, Parser.LT, Parser.LE}; 118 | 119 | public NumberOpExp(ParserLiteral left, ParserLiteral right, int operator) 120 | { 121 | super(left, right, operator); 122 | if (! left.isNumber()) { 123 | throw new ConfigException(String.format("\"%s\" is not a Number column", ((IdentifierLiteral)left).name)); 124 | } 125 | if (! right.isNumber()) { 126 | throw new ConfigException(String.format("\"%s\" is not a Number column", ((IdentifierLiteral)right).name)); 127 | } 128 | if (! isOperatorAllowed(operators, operator)) { 129 | throw new ConfigException(String.format("\"%s\" is not an allowed operator for NumberOpExp", Parser.yyname[operator])); 130 | } 131 | } 132 | 133 | public NumberOpExp(ParserVal left, ParserVal right, int operator) 134 | { 135 | this((ParserLiteral)(left.obj), (ParserLiteral)(right.obj), operator); 136 | } 137 | 138 | public boolean eval(PageReader pageReader) 139 | { 140 | double l = left.getNumber(pageReader); 141 | double r = right.getNumber(pageReader); 142 | if (operator == Parser.EQ) { 143 | return l == r; 144 | } 145 | else if (operator == Parser.NEQ) { 146 | return l != r; 147 | } 148 | else if (operator == Parser.GT) { 149 | return l > r; 150 | } 151 | else if (operator == Parser.GE) { 152 | return l >= r; 153 | } 154 | else if (operator == Parser.LT) { 155 | return l < r; 156 | } 157 | else if (operator == Parser.LE) { 158 | return l <= r; 159 | } 160 | else { 161 | assert(false); 162 | return false; 163 | } 164 | } 165 | } 166 | 167 | class TimestampOpExp extends BinaryOpExp 168 | { 169 | public static int[] operators = {Parser.EQ, Parser.NEQ, Parser.GT, Parser.GE, Parser.LT, Parser.LE}; 170 | 171 | public TimestampOpExp(ParserLiteral left, ParserLiteral right, int operator) 172 | { 173 | this.left = left.isIdentifier() ? left : new TimestampLiteral(left); 174 | this.right = right.isIdentifier() ? right : new TimestampLiteral(right); 175 | this.operator = operator; 176 | 177 | if (! this.left.isTimestamp()) { 178 | throw new ConfigException(String.format("\"%s\" is not a Timestamp column", ((IdentifierLiteral)this.left).name)); 179 | } 180 | if (! this.right.isTimestamp()) { 181 | throw new ConfigException(String.format("\"%s\" is not a Timestamp column", ((IdentifierLiteral)this.right).name)); 182 | } 183 | if (! isOperatorAllowed(operators, operator)) { 184 | throw new ConfigException(String.format("\"%s\" is not an allowed operator for TimestampOpExp", Parser.yyname[operator])); 185 | } 186 | } 187 | 188 | public TimestampOpExp(ParserVal left, ParserVal right, int operator) 189 | { 190 | this((ParserLiteral)(left.obj), (ParserLiteral)(right.obj), operator); 191 | } 192 | 193 | public boolean eval(PageReader pageReader) 194 | { 195 | boolean lIsNull = left.isIdentifier() && left.isNull(pageReader); 196 | boolean rIsNull = right.isIdentifier() && right.isNull(pageReader); 197 | if (lIsNull && rIsNull && operator == Parser.EQ) { 198 | return true; // Both of left and right are equals as null, but it will never happen 199 | } 200 | else if (lIsNull != rIsNull && operator == Parser.NEQ) { 201 | return true; // Either of left or right is null and both are not equals 202 | } 203 | else if (lIsNull || rIsNull) { 204 | return false; // Can't be evaluated by any other operator if either of left or right is null 205 | } 206 | Instant l = left.getTimestamp(pageReader); 207 | Instant r = right.getTimestamp(pageReader); 208 | if (operator == Parser.EQ) { 209 | return l.equals(r); 210 | } 211 | else if (operator == Parser.NEQ) { 212 | return ! l.equals(r); 213 | } 214 | else if (operator == Parser.GT) { 215 | return l.compareTo(r) > 0; 216 | } 217 | else if (operator == Parser.GE) { 218 | return l.compareTo(r) >= 0; 219 | } 220 | else if (operator == Parser.LT) { 221 | return l.compareTo(r) < 0; 222 | } 223 | else if (operator == Parser.LE) { 224 | return l.compareTo(r) <= 0; 225 | } 226 | else { 227 | assert(false); 228 | return false; 229 | } 230 | } 231 | } 232 | 233 | class StringOpExp extends BinaryOpExp 234 | { 235 | public static int[] operators = { 236 | Parser.EQ, Parser.NEQ, Parser.GT, Parser.GE, Parser.LT, Parser.LE, 237 | Parser.START_WITH, Parser.END_WITH, Parser.INCLUDE, Parser.REGEXP 238 | }; 239 | 240 | public StringOpExp(ParserLiteral left, ParserLiteral right, int operator) 241 | { 242 | super(left, right, operator); 243 | if (! left.isString()) { 244 | throw new ConfigException(String.format("\"%s\" is not a String column", ((IdentifierLiteral)left).name)); 245 | } 246 | if (! right.isString()) { 247 | throw new ConfigException(String.format("\"%s\" is not a String column", ((IdentifierLiteral)right).name)); 248 | } 249 | if (! isOperatorAllowed(operators, operator)) { 250 | throw new ConfigException(String.format("\"%s\" is not an allowed operator for StringOpExp", Parser.yyname[operator])); 251 | } 252 | } 253 | 254 | public StringOpExp(ParserVal left, ParserVal right, int operator) 255 | { 256 | this((ParserLiteral)(left.obj), (ParserLiteral)(right.obj), operator); 257 | } 258 | 259 | public boolean eval(PageReader pageReader) 260 | { 261 | boolean lIsNull = left.isIdentifier() && left.isNull(pageReader); 262 | boolean rIsNull = right.isIdentifier() && right.isNull(pageReader); 263 | if (lIsNull && rIsNull && operator == Parser.EQ) { 264 | return true; // Both of left and right are equals as null, but it will never happen 265 | } 266 | else if (lIsNull != rIsNull && operator == Parser.NEQ) { 267 | return true; // Either of left or right is null and both are not equals 268 | } 269 | else if (lIsNull || rIsNull) { 270 | return false; // Can't be evaluated by any other operator if either of left or right is null 271 | } 272 | String l = left.getString(pageReader); 273 | String r = right.getString(pageReader); 274 | if (operator == Parser.EQ) { 275 | return l.equals(r); 276 | } 277 | else if (operator == Parser.NEQ) { 278 | return ! l.equals(r); 279 | } 280 | else if (operator == Parser.GT) { 281 | return l.compareTo(r) > 0; 282 | } 283 | else if (operator == Parser.GE) { 284 | return l.compareTo(r) >= 0; 285 | } 286 | else if (operator == Parser.LT) { 287 | return l.compareTo(r) < 0; 288 | } 289 | else if (operator == Parser.LE) { 290 | return l.compareTo(r) <= 0; 291 | } 292 | else if (operator == Parser.START_WITH) { 293 | return l.startsWith(r); 294 | } 295 | else if (operator == Parser.END_WITH) { 296 | return l.endsWith(r); 297 | } 298 | else if (operator == Parser.INCLUDE) { 299 | return l.contains(r); 300 | } 301 | else { 302 | assert(false); 303 | return false; 304 | } 305 | } 306 | } 307 | 308 | class RegexpOpExp extends BinaryOpExp 309 | { 310 | Regex regex; 311 | 312 | public RegexpOpExp(ParserLiteral left, ParserLiteral right, int operator) 313 | { 314 | super(left, right, operator); 315 | 316 | byte[] pattern = (((StringLiteral)right).val).getBytes(StandardCharsets.UTF_8); 317 | this.regex = new Regex(pattern, 0, pattern.length, Option.NONE, UTF8Encoding.INSTANCE); 318 | 319 | if (! left.isString()) { 320 | throw new ConfigException(String.format("\"%s\" is not a String column", ((IdentifierLiteral)left).name)); 321 | } 322 | } 323 | 324 | public RegexpOpExp(ParserVal left, ParserVal right, int operator) 325 | { 326 | this((ParserLiteral)(left.obj), (ParserLiteral)(right.obj), operator); 327 | } 328 | 329 | public boolean eval(PageReader pageReader) 330 | { 331 | if (left.isNull(pageReader)) { 332 | return false; 333 | } 334 | byte[] l = left.getString(pageReader).getBytes(StandardCharsets.UTF_8); 335 | Matcher matcher = regex.matcher(l); 336 | int result = matcher.search(0, l.length, Option.DEFAULT); 337 | return result != -1; 338 | } 339 | } 340 | 341 | class NullOpExp extends ParserExp 342 | { 343 | protected ParserLiteral val; 344 | protected int operator; 345 | 346 | public NullOpExp(ParserLiteral val, int operator) 347 | { 348 | this.val = val; 349 | this.operator = operator; 350 | } 351 | 352 | public NullOpExp(ParserVal val, int operator) 353 | { 354 | this((ParserLiteral)(val.obj), operator); 355 | } 356 | 357 | public boolean eval(PageReader pageReader) 358 | { 359 | boolean isNull = val.isNull(pageReader); 360 | if (operator == Parser.EQ) { 361 | return isNull; 362 | } 363 | else if (operator == Parser.NEQ) { 364 | return ! isNull; 365 | } 366 | else { 367 | assert(false); 368 | return false; 369 | } 370 | } 371 | } 372 | 373 | class LogicalOpExp extends ParserExp 374 | { 375 | protected ParserExp left; 376 | protected ParserExp right; 377 | protected int operator; 378 | 379 | public LogicalOpExp(ParserExp left, ParserExp right, int operator) 380 | { 381 | this.left = left; 382 | this.right = right; 383 | this.operator = operator; 384 | } 385 | 386 | public LogicalOpExp(ParserVal left, ParserVal right, int operator) 387 | { 388 | this((ParserExp)(left.obj), (ParserExp)(right.obj), operator); 389 | } 390 | 391 | public boolean eval(PageReader pageReader) 392 | { 393 | boolean l = left.eval(pageReader); 394 | boolean r = right.eval(pageReader); 395 | if (operator == Parser.OR) { 396 | return l || r; 397 | } 398 | else if (operator == Parser.AND) { 399 | return l && r; 400 | } 401 | else { 402 | assert(false); 403 | return false; 404 | } 405 | } 406 | } 407 | 408 | class NegateOpExp extends ParserExp 409 | { 410 | protected ParserExp exp; 411 | 412 | public NegateOpExp(ParserExp exp) 413 | { 414 | this.exp = exp; 415 | } 416 | 417 | public NegateOpExp(ParserVal exp) 418 | { 419 | this((ParserExp)(exp.obj)); 420 | } 421 | 422 | public boolean eval(PageReader pageReader) 423 | { 424 | return ! exp.eval(pageReader); 425 | } 426 | } 427 | -------------------------------------------------------------------------------- /src/test/java/org/embulk/filter/row/where/TestParser.java: -------------------------------------------------------------------------------- 1 | package org.embulk.filter.row.where; 2 | 3 | import org.embulk.EmbulkTestRuntime; 4 | import org.embulk.config.ConfigException; 5 | import org.embulk.spi.Page; 6 | import org.embulk.spi.PageReader; 7 | import org.embulk.spi.PageTestUtils; 8 | import org.embulk.spi.Schema; 9 | import org.embulk.spi.SchemaConfigException; 10 | import org.embulk.spi.time.Timestamp; 11 | import org.junit.BeforeClass; 12 | import org.junit.Test; 13 | import org.msgpack.value.Value; 14 | import org.msgpack.value.ValueFactory; 15 | 16 | import java.time.format.DateTimeParseException; 17 | import java.util.List; 18 | 19 | import static org.embulk.spi.type.Types.BOOLEAN; 20 | import static org.embulk.spi.type.Types.DOUBLE; 21 | import static org.embulk.spi.type.Types.JSON; 22 | import static org.embulk.spi.type.Types.LONG; 23 | import static org.embulk.spi.type.Types.STRING; 24 | import static org.embulk.spi.type.Types.TIMESTAMP; 25 | import static org.junit.Assert.assertFalse; 26 | import static org.junit.Assert.assertTrue; 27 | 28 | public class TestParser 29 | { 30 | private static EmbulkTestRuntime runtime = new EmbulkTestRuntime(); // very slow 31 | 32 | private static PageReader buildPageReader(Schema schema, final Object... objects) 33 | { 34 | PageReader pageReader = new PageReader(schema); 35 | List pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, objects); 36 | for (Page page : pages) { 37 | pageReader.setPage(page); 38 | } 39 | pageReader.nextRecord(); 40 | return pageReader; 41 | } 42 | 43 | private static PageReader reader; 44 | private static Schema schema; 45 | 46 | @BeforeClass 47 | public static void setupBeforeClass() 48 | { 49 | // {"k1":{"k1":"v"},"k2":{"k2":"v"}} 50 | Value k1 = ValueFactory.newString("k1"); 51 | Value k2 = ValueFactory.newString("k2"); 52 | Value v = ValueFactory.newString("v"); 53 | Value map = ValueFactory.newMap( 54 | k1, ValueFactory.newMap(k1, v), 55 | k2, ValueFactory.newMap(k2, v)); 56 | 57 | schema = Schema.builder() 58 | .add("timestamp", TIMESTAMP) 59 | .add("string", STRING) 60 | .add("boolean", BOOLEAN) 61 | .add("long", LONG) 62 | .add("double", DOUBLE) 63 | .add("true", BOOLEAN) 64 | .add("null_timestamp", TIMESTAMP) 65 | .add("null_string", STRING) 66 | .add("null", BOOLEAN) 67 | .add("json", JSON) 68 | .build(); 69 | 70 | reader = buildPageReader(schema, 71 | Timestamp.ofEpochSecond(1, 500000000), 72 | "string", 73 | true, 74 | 1L, 75 | 1.5, 76 | true, 77 | null, 78 | null, 79 | null, 80 | map 81 | ); 82 | } 83 | 84 | @Test 85 | public void testIdentifierLiteral() 86 | { 87 | Parser parser = new Parser(schema); 88 | ParserExp exp; 89 | 90 | exp = parser.parse("boolean = true"); 91 | assertTrue(exp.eval(reader)); 92 | 93 | exp = parser.parse("\"true\" = true"); 94 | assertTrue(exp.eval(reader)); 95 | 96 | exp = parser.parse("null_timestamp IS NULL"); 97 | assertTrue(exp.eval(reader)); 98 | exp = parser.parse("null_timestamp IS NOT NULL"); 99 | assertFalse(exp.eval(reader)); 100 | exp = parser.parse("null_timestamp = TIMESTAMP '1970-01-01 09:00:01.5 +0900'"); 101 | assertFalse(exp.eval(reader)); 102 | exp = parser.parse("null_timestamp != TIMESTAMP '1970-01-01 09:00:01.5 +0900'"); 103 | assertTrue(exp.eval(reader)); 104 | 105 | exp = parser.parse("null_string IS NULL"); 106 | assertTrue(exp.eval(reader)); 107 | exp = parser.parse("null_string IS NOT NULL"); 108 | assertFalse(exp.eval(reader)); 109 | exp = parser.parse("null_string = 'string'"); 110 | assertFalse(exp.eval(reader)); 111 | exp = parser.parse("null_string != 'string'"); 112 | assertTrue(exp.eval(reader)); 113 | 114 | try { 115 | parser.parse("null_timestamp = null_string"); // Both of left and right are an identifier 116 | assertTrue(false); 117 | } 118 | catch (ConfigException e) { 119 | } 120 | 121 | try { 122 | parser.parse("\"unknown\" IS NULL"); 123 | assertTrue(false); 124 | } 125 | catch (SchemaConfigException e) { 126 | } 127 | } 128 | 129 | @Test 130 | public void testBooleanOpExp() 131 | { 132 | Parser parser = new Parser(schema); 133 | ParserExp exp; 134 | 135 | exp = parser.parse("boolean = true"); 136 | assertTrue(exp.eval(reader)); 137 | exp = parser.parse("boolean = false"); 138 | assertFalse(exp.eval(reader)); 139 | 140 | exp = parser.parse("boolean != false"); 141 | assertTrue(exp.eval(reader)); 142 | exp = parser.parse("boolean != true"); 143 | assertFalse(exp.eval(reader)); 144 | 145 | exp = parser.parse("true = boolean"); 146 | assertTrue(exp.eval(reader)); 147 | 148 | try { 149 | parser.parse("timestamp = true"); 150 | assertTrue(false); 151 | } 152 | catch (ConfigException e) { 153 | } 154 | 155 | try { 156 | parser.parse("boolean > true"); 157 | assertTrue(false); 158 | } 159 | catch (ConfigException e) { 160 | } 161 | } 162 | 163 | @Test 164 | public void testNumberOpExp() 165 | { 166 | Parser parser = new Parser(schema); 167 | ParserExp exp; 168 | 169 | exp = parser.parse("double = 1.5"); 170 | assertTrue(exp.eval(reader)); 171 | exp = parser.parse("double = 1.0"); 172 | assertFalse(exp.eval(reader)); 173 | 174 | exp = parser.parse("double != 1.0"); 175 | assertTrue(exp.eval(reader)); 176 | exp = parser.parse("double != 1.5"); 177 | assertFalse(exp.eval(reader)); 178 | 179 | exp = parser.parse("double > 1.0"); 180 | assertTrue(exp.eval(reader)); 181 | exp = parser.parse("double > 1.5"); 182 | assertFalse(exp.eval(reader)); 183 | 184 | exp = parser.parse("double >= 1.5"); 185 | assertTrue(exp.eval(reader)); 186 | exp = parser.parse("double >= 2.0"); 187 | assertFalse(exp.eval(reader)); 188 | 189 | exp = parser.parse("double < 2.0"); 190 | assertTrue(exp.eval(reader)); 191 | exp = parser.parse("double < 1.5"); 192 | assertFalse(exp.eval(reader)); 193 | 194 | exp = parser.parse("double <= 1.5"); 195 | assertTrue(exp.eval(reader)); 196 | exp = parser.parse("double <= 1.0"); 197 | assertFalse(exp.eval(reader)); 198 | 199 | exp = parser.parse("1.5 = double"); 200 | assertTrue(exp.eval(reader)); 201 | 202 | try { 203 | parser.parse("boolean = 1.5"); 204 | assertTrue(false); 205 | } 206 | catch (ConfigException e) { 207 | } 208 | 209 | try { 210 | parser.parse("double START_WITH 1.5"); 211 | assertTrue(false); 212 | } 213 | catch (ConfigException e) { 214 | } 215 | } 216 | 217 | @Test 218 | public void testStringOpExp() 219 | { 220 | Parser parser = new Parser(schema); 221 | ParserExp exp; 222 | 223 | exp = parser.parse("string = 'string'"); 224 | assertTrue(exp.eval(reader)); 225 | exp = parser.parse("string = 'foobar'"); 226 | assertFalse(exp.eval(reader)); 227 | 228 | exp = parser.parse("string != 'foobar'"); 229 | assertTrue(exp.eval(reader)); 230 | exp = parser.parse("string != 'string'"); 231 | assertFalse(exp.eval(reader)); 232 | 233 | exp = parser.parse("string <> 'foobar'"); 234 | assertTrue(exp.eval(reader)); 235 | exp = parser.parse("string <> 'string'"); 236 | assertFalse(exp.eval(reader)); 237 | 238 | exp = parser.parse("string > 's'"); 239 | assertTrue(exp.eval(reader)); 240 | exp = parser.parse("string > 't'"); 241 | assertFalse(exp.eval(reader)); 242 | 243 | exp = parser.parse("string >= 's'"); 244 | assertTrue(exp.eval(reader)); 245 | exp = parser.parse("string >= 't'"); 246 | assertFalse(exp.eval(reader)); 247 | 248 | exp = parser.parse("string < 't'"); 249 | assertTrue(exp.eval(reader)); 250 | exp = parser.parse("string < 's'"); 251 | assertFalse(exp.eval(reader)); 252 | 253 | exp = parser.parse("string <= 't'"); 254 | assertTrue(exp.eval(reader)); 255 | exp = parser.parse("string <= 's'"); 256 | assertFalse(exp.eval(reader)); 257 | 258 | exp = parser.parse("string START_WITH 's'"); 259 | assertTrue(exp.eval(reader)); 260 | exp = parser.parse("string START_WITH 'f'"); 261 | assertFalse(exp.eval(reader)); 262 | 263 | exp = parser.parse("string END_WITH 'g'"); 264 | assertTrue(exp.eval(reader)); 265 | exp = parser.parse("string END_WITH 'r'"); 266 | assertFalse(exp.eval(reader)); 267 | 268 | exp = parser.parse("string INCLUDE 'tr'"); 269 | assertTrue(exp.eval(reader)); 270 | exp = parser.parse("string INCLUDE 'oo'"); 271 | assertFalse(exp.eval(reader)); 272 | 273 | exp = parser.parse("'string' = string"); 274 | assertTrue(exp.eval(reader)); 275 | 276 | try { 277 | parser.parse("boolean = 'string'"); 278 | assertTrue(false); 279 | } 280 | catch (ConfigException e) { 281 | } 282 | 283 | try { 284 | parser.parse("string AND 'string'"); 285 | assertTrue(false); 286 | } 287 | catch (ConfigException e) { 288 | } 289 | } 290 | 291 | @Test 292 | public void testTimestampOpExpWithNumber() 293 | { 294 | Parser parser = new Parser(schema); 295 | ParserExp exp; 296 | 297 | exp = parser.parse("timestamp = TIMESTAMP 1.5"); 298 | assertTrue(exp.eval(reader)); 299 | exp = parser.parse("timestamp = TIMESTAMP 1.0"); 300 | assertFalse(exp.eval(reader)); 301 | 302 | exp = parser.parse("timestamp != TIMESTAMP 1.0"); 303 | assertTrue(exp.eval(reader)); 304 | exp = parser.parse("timestamp != TIMESTAMP 1.5"); 305 | assertFalse(exp.eval(reader)); 306 | 307 | exp = parser.parse("timestamp > TIMESTAMP 1.0"); 308 | assertTrue(exp.eval(reader)); 309 | exp = parser.parse("timestamp > TIMESTAMP 1.5"); 310 | assertFalse(exp.eval(reader)); 311 | 312 | exp = parser.parse("timestamp >= TIMESTAMP 1.5"); 313 | assertTrue(exp.eval(reader)); 314 | exp = parser.parse("timestamp >= TIMESTAMP 2.0"); 315 | assertFalse(exp.eval(reader)); 316 | 317 | exp = parser.parse("timestamp < TIMESTAMP 2.0"); 318 | assertTrue(exp.eval(reader)); 319 | exp = parser.parse("timestamp < TIMESTAMP 1.5"); 320 | assertFalse(exp.eval(reader)); 321 | 322 | exp = parser.parse("timestamp <= TIMESTAMP 1.5"); 323 | assertTrue(exp.eval(reader)); 324 | exp = parser.parse("timestamp <= TIMESTAMP 1.0"); 325 | assertFalse(exp.eval(reader)); 326 | 327 | exp = parser.parse("TIMESTAMP 1.5 = timestamp"); 328 | assertTrue(exp.eval(reader)); 329 | 330 | // auto guess of TIMESTAMP 331 | exp = parser.parse("timestamp = 1.5"); 332 | assertTrue(exp.eval(reader)); 333 | exp = parser.parse("1.5 = timestamp"); 334 | assertTrue(exp.eval(reader)); 335 | 336 | try { 337 | parser.parse("timestamp START_WITH 1.5"); 338 | assertTrue(false); 339 | } 340 | catch (ConfigException e) { 341 | } 342 | } 343 | 344 | @Test 345 | public void testTimestampOpExpWithString() 346 | { 347 | Parser parser = new Parser(schema); 348 | ParserExp exp; 349 | 350 | exp = parser.parse("timestamp = TIMESTAMP '1970-01-01 09:00:01.5 +0900'"); 351 | assertTrue(exp.eval(reader)); 352 | exp = parser.parse("timestamp = TIMESTAMP '1970-01-01 09:00:01.0 +09:00'"); 353 | assertFalse(exp.eval(reader)); 354 | 355 | exp = parser.parse("timestamp != TIMESTAMP '1970-01-01 09:00:01.0 +0900'"); 356 | assertTrue(exp.eval(reader)); 357 | exp = parser.parse("timestamp != TIMESTAMP '1970-01-01 09:00:01.5 +0900'"); 358 | assertFalse(exp.eval(reader)); 359 | 360 | exp = parser.parse("timestamp > TIMESTAMP '1970-01-01 09:00:01.0 +0900'"); 361 | assertTrue(exp.eval(reader)); 362 | exp = parser.parse("timestamp > TIMESTAMP '1970-01-01 09:00:01.5 +0900'"); 363 | assertFalse(exp.eval(reader)); 364 | 365 | exp = parser.parse("timestamp >= TIMESTAMP '1970-01-01 09:00:01.5 +09:00'"); 366 | assertTrue(exp.eval(reader)); 367 | exp = parser.parse("timestamp >= TIMESTAMP '1970-01-01 09:00:02.0 +09:00'"); 368 | assertFalse(exp.eval(reader)); 369 | 370 | exp = parser.parse("timestamp < TIMESTAMP '1970-01-01 09:00:02.0 +09:00'"); 371 | assertTrue(exp.eval(reader)); 372 | exp = parser.parse("timestamp < TIMESTAMP '1970-01-01 09:00:01.5 +09:00'"); 373 | assertFalse(exp.eval(reader)); 374 | 375 | exp = parser.parse("timestamp <= TIMESTAMP '1970-01-01 09:00:01.5 +09:00'"); 376 | assertTrue(exp.eval(reader)); 377 | exp = parser.parse("timestamp <= TIMESTAMP '1970-01-01 09:00:01.0 +09:00'"); 378 | assertFalse(exp.eval(reader)); 379 | 380 | exp = parser.parse("TIMESTAMP '1970-01-01 09:00:01.5 +09:00' = timestamp"); 381 | assertTrue(exp.eval(reader)); 382 | 383 | exp = parser.parse("timestamp = TIMESTAMP '1970-01-01 09:00:01.5 +0900'"); 384 | assertTrue(exp.eval(reader)); 385 | exp = parser.parse("timestamp = TIMESTAMP '1970-01-01 00:00:01.5'"); 386 | assertTrue(exp.eval(reader)); 387 | exp = parser.parse("timestamp = TIMESTAMP '1970-01-01 09:00:01 +09:00'"); 388 | assertFalse(exp.eval(reader)); 389 | exp = parser.parse("timestamp = TIMESTAMP '1970-01-01 00:00:01'"); 390 | assertFalse(exp.eval(reader)); 391 | exp = parser.parse("timestamp = TIMESTAMP '1970-01-01 +09:00'"); 392 | assertFalse(exp.eval(reader)); 393 | exp = parser.parse("timestamp = TIMESTAMP '1970-01-01'"); 394 | assertFalse(exp.eval(reader)); 395 | 396 | // auto guess of TIMESTAMP 397 | exp = parser.parse("timestamp = '1970-01-01 09:00:01.5 +0900'"); 398 | assertTrue(exp.eval(reader)); 399 | exp = parser.parse("'1970-01-01 09:00:01.5 +0900' = timestamp"); 400 | assertTrue(exp.eval(reader)); 401 | 402 | try { 403 | parser.parse("timestamp = '1970:01:01'"); 404 | assertTrue(false); 405 | } 406 | catch (DateTimeParseException ex) { 407 | } 408 | 409 | try { 410 | parser.parse("timestamp START_WITH '1970-01-01 09:00:01.5 +0900'"); 411 | assertTrue(false); 412 | } 413 | catch (ConfigException e) { 414 | } 415 | } 416 | 417 | @Test 418 | public void testRegexpOpExp() 419 | { 420 | Parser parser = new Parser(schema); 421 | ParserExp exp; 422 | 423 | exp = parser.parse("string REGEXP '^st'"); 424 | assertTrue(exp.eval(reader)); 425 | exp = parser.parse("string REGEXP 'st$'"); 426 | assertFalse(exp.eval(reader)); 427 | 428 | exp = parser.parse("null_string REGEXP '^st'"); 429 | assertFalse(exp.eval(reader)); 430 | 431 | try { 432 | // right-side identifier is not allowed 433 | parser.parse("'string' REGEXP string"); 434 | assertTrue(false); 435 | } 436 | catch (ConfigException e) { 437 | } 438 | 439 | try { 440 | parser.parse("string REGEXP 1.5"); 441 | assertTrue(false); 442 | } 443 | catch (ConfigException e) { 444 | } 445 | 446 | try { 447 | parser.parse("boolean REGEXP '^st'"); 448 | assertTrue(false); 449 | } 450 | catch (ConfigException e) { 451 | } 452 | } 453 | 454 | @Test 455 | public void testNullOpExp() 456 | { 457 | Parser parser = new Parser(schema); 458 | ParserExp exp; 459 | 460 | exp = parser.parse("null IS NULL"); 461 | assertTrue(exp.eval(reader)); 462 | exp = parser.parse("null IS NOT NULL"); 463 | assertFalse(exp.eval(reader)); 464 | 465 | exp = parser.parse("string IS NOT NULL"); 466 | assertTrue(exp.eval(reader)); 467 | exp = parser.parse("string IS NULL"); 468 | assertFalse(exp.eval(reader)); 469 | } 470 | 471 | @Test 472 | public void testLogicalOpExp() 473 | { 474 | Parser parser = new Parser(schema); 475 | ParserExp exp; 476 | 477 | exp = parser.parse("\"true\" = true AND \"true\" = true"); 478 | assertTrue(exp.eval(reader)); 479 | exp = parser.parse("\"true\" = true AND \"true\" = false"); 480 | assertFalse(exp.eval(reader)); 481 | 482 | exp = parser.parse("\"true\" = false OR \"true\" = true"); 483 | assertTrue(exp.eval(reader)); 484 | exp = parser.parse("\"true\" = false OR \"true\" = false"); 485 | assertFalse(exp.eval(reader)); 486 | 487 | // a AND b OR c #=> (a AND b) OR c 488 | // a OR b AND c #=> a OR (b AND c) 489 | exp = parser.parse("\"true\" = true OR \"true\" = false AND \"true\" = false"); 490 | assertTrue(exp.eval(reader)); 491 | exp = parser.parse("( \"true\" = true OR \"true\" = false ) AND \"true\" = false"); 492 | assertFalse(exp.eval(reader)); 493 | } 494 | 495 | @Test 496 | public void testNegateOpExp() 497 | { 498 | Parser parser = new Parser(schema); 499 | ParserExp exp; 500 | 501 | exp = parser.parse("NOT \"true\" = false"); 502 | assertTrue(exp.eval(reader)); 503 | exp = parser.parse("NOT ( \"true\" = false )"); 504 | assertTrue(exp.eval(reader)); 505 | 506 | // NOT a AND B #=> (NOT a) AND B 507 | exp = parser.parse("NOT \"true\" = false AND \"true\" = true"); 508 | assertTrue(exp.eval(reader)); 509 | } 510 | } 511 | -------------------------------------------------------------------------------- /src/main/java/org/embulk/filter/row/where/Yylex.java: -------------------------------------------------------------------------------- 1 | /* The following code was generated by JFlex 1.6.1 */ 2 | 3 | package org.embulk.filter.row.where; 4 | 5 | import org.embulk.config.ConfigException; 6 | import org.embulk.spi.Schema; 7 | 8 | /** 9 | * This class is a scanner generated by 10 | * JFlex 1.6.1 11 | * from the specification file src/main/java/org/embulk/filter/row/where/_lexer.l 12 | */ 13 | class Yylex { 14 | 15 | /** This character denotes the end of file */ 16 | public static final int YYEOF = -1; 17 | 18 | /** initial size of the lookahead buffer */ 19 | private static final int ZZ_BUFFERSIZE = 16384; 20 | 21 | /** lexical states */ 22 | public static final int YYINITIAL = 0; 23 | public static final int STRING = 2; 24 | public static final int IDENTIFIER = 4; 25 | 26 | /** 27 | * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l 28 | * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l 29 | * at the beginning of a line 30 | * l is of the form l = 2*k, k a non negative integer 31 | */ 32 | private static final int ZZ_LEXSTATE[] = { 33 | 0, 0, 1, 1, 2, 2 34 | }; 35 | 36 | /** 37 | * Translates characters to character classes 38 | */ 39 | private static final String ZZ_CMAP_PACKED = 40 | "\11\0\1\54\1\12\1\55\1\55\1\11\22\0\1\54\1\24\1\4"+ 41 | "\4\0\1\10\1\13\1\13\3\0\1\1\1\3\1\0\12\2\2\0"+ 42 | "\1\22\1\21\1\23\2\0\1\14\1\5\1\34\1\16\1\33\1\53"+ 43 | "\1\37\1\32\1\31\2\5\1\35\1\42\1\15\1\17\1\41\1\5"+ 44 | "\1\20\1\25\1\26\1\36\1\5\1\30\1\40\2\5\1\6\1\7"+ 45 | "\2\6\1\27\1\6\1\50\1\56\2\5\1\46\1\47\5\5\1\51"+ 46 | "\1\5\1\57\3\5\1\44\1\52\1\43\1\45\5\5\12\0\1\55"+ 47 | "\u1fa2\0\1\55\1\55\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\udfe6\0"; 48 | 49 | /** 50 | * Translates characters to character classes 51 | */ 52 | private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED); 53 | 54 | /** 55 | * Translates DFA states to action switch labels. 56 | */ 57 | private static final int [] ZZ_ACTION = zzUnpackAction(); 58 | 59 | private static final String ZZ_ACTION_PACKED_0 = 60 | "\3\0\2\1\1\2\1\3\1\4\1\5\2\6\1\7"+ 61 | "\4\4\1\10\1\11\1\12\1\1\7\4\1\6\1\13"+ 62 | "\1\1\1\14\1\13\1\15\1\1\1\0\3\4\1\16"+ 63 | "\1\4\1\17\1\20\1\21\4\4\1\22\4\4\1\23"+ 64 | "\1\24\1\25\1\26\1\27\1\30\1\31\1\32\1\33"+ 65 | "\1\2\1\34\1\35\12\4\1\36\2\4\1\37\10\4"+ 66 | "\1\40\6\4\1\41\3\4\1\42\1\4\1\43\1\44"; 67 | 68 | private static int [] zzUnpackAction() { 69 | int [] result = new int[101]; 70 | int offset = 0; 71 | offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); 72 | return result; 73 | } 74 | 75 | private static int zzUnpackAction(String packed, int offset, int [] result) { 76 | int i = 0; /* index in packed string */ 77 | int j = offset; /* index in unpacked array */ 78 | int l = packed.length(); 79 | while (i < l) { 80 | int count = packed.charAt(i++); 81 | int value = packed.charAt(i++); 82 | do result[j++] = value; while (--count > 0); 83 | } 84 | return j; 85 | } 86 | 87 | 88 | /** 89 | * Translates a state to a row index in the transition table 90 | */ 91 | private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); 92 | 93 | private static final String ZZ_ROWMAP_PACKED_0 = 94 | "\0\0\0\60\0\140\0\220\0\300\0\360\0\220\0\u0120"+ 95 | "\0\220\0\u0150\0\220\0\220\0\u0180\0\u01b0\0\u01e0\0\u0210"+ 96 | "\0\220\0\u0240\0\u0270\0\u02a0\0\u02d0\0\u0300\0\u0330\0\u0360"+ 97 | "\0\u0390\0\u03c0\0\u03f0\0\u0420\0\u0450\0\u0480\0\220\0\u04b0"+ 98 | "\0\220\0\u04e0\0\u0510\0\u0540\0\u0570\0\u05a0\0\u0120\0\u05d0"+ 99 | "\0\220\0\220\0\220\0\u0600\0\u0630\0\u0660\0\u0690\0\u0120"+ 100 | "\0\u06c0\0\u06f0\0\u0720\0\u0750\0\220\0\220\0\220\0\220"+ 101 | "\0\220\0\220\0\220\0\220\0\220\0\u0510\0\u0120\0\u0120"+ 102 | "\0\u0780\0\u07b0\0\u07e0\0\u0810\0\u0840\0\u0870\0\u08a0\0\u08d0"+ 103 | "\0\u0900\0\u0930\0\u0120\0\u0960\0\u0990\0\u0120\0\u09c0\0\u09f0"+ 104 | "\0\u0a20\0\u0a50\0\u0a80\0\u0ab0\0\u0ae0\0\u0b10\0\u0120\0\u0b40"+ 105 | "\0\u0b70\0\u0ba0\0\u0bd0\0\u0c00\0\u0c30\0\u0120\0\u0c60\0\u0c90"+ 106 | "\0\u0cc0\0\u0120\0\u0cf0\0\u0120\0\u0120"; 107 | 108 | private static int [] zzUnpackRowMap() { 109 | int [] result = new int[101]; 110 | int offset = 0; 111 | offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); 112 | return result; 113 | } 114 | 115 | private static int zzUnpackRowMap(String packed, int offset, int [] result) { 116 | int i = 0; /* index in packed string */ 117 | int j = offset; /* index in unpacked array */ 118 | int l = packed.length(); 119 | while (i < l) { 120 | int high = packed.charAt(i++) << 16; 121 | result[j++] = high | packed.charAt(i++); 122 | } 123 | return j; 124 | } 125 | 126 | /** 127 | * The transition table of the DFA 128 | */ 129 | private static final int [] ZZ_TRANS = zzUnpackTrans(); 130 | 131 | private static final String ZZ_TRANS_PACKED_0 = 132 | "\1\4\1\5\1\6\1\4\1\7\1\10\2\4\1\11"+ 133 | "\1\12\1\13\1\14\1\15\1\16\1\10\1\17\1\20"+ 134 | "\1\21\1\22\1\23\1\24\1\25\1\26\2\10\1\27"+ 135 | "\1\10\1\30\7\10\1\31\3\10\1\32\3\10\1\33"+ 136 | "\1\34\1\4\2\10\7\35\1\36\1\37\2\4\45\35"+ 137 | "\4\40\1\41\2\40\1\42\1\40\2\4\45\40\62\0"+ 138 | "\1\6\57\0\1\6\1\43\56\0\1\10\2\0\3\10"+ 139 | "\4\0\5\10\4\0\27\10\2\0\2\10\12\0\1\13"+ 140 | "\47\0\1\10\2\0\3\10\4\0\1\10\1\44\3\10"+ 141 | "\4\0\27\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 142 | "\4\0\3\10\1\45\1\10\4\0\11\10\1\46\15\10"+ 143 | "\2\0\2\10\2\0\1\10\2\0\3\10\4\0\4\10"+ 144 | "\1\47\4\0\27\10\2\0\2\10\2\0\1\10\2\0"+ 145 | "\3\10\4\0\5\10\4\0\6\10\1\50\20\10\2\0"+ 146 | "\2\10\21\0\1\51\1\0\1\52\55\0\1\53\57\0"+ 147 | "\1\52\40\0\1\10\2\0\3\10\4\0\5\10\4\0"+ 148 | "\1\10\1\54\25\10\2\0\2\10\2\0\1\10\2\0"+ 149 | "\3\10\4\0\4\10\1\55\4\0\4\10\1\56\22\10"+ 150 | "\2\0\2\10\2\0\1\10\2\0\3\10\4\0\1\10"+ 151 | "\1\57\3\10\4\0\1\60\26\10\2\0\2\10\2\0"+ 152 | "\1\10\2\0\3\10\4\0\1\10\1\61\3\10\4\0"+ 153 | "\27\10\2\0\2\10\2\0\1\10\2\0\3\10\4\0"+ 154 | "\5\10\4\0\17\10\1\62\7\10\2\0\2\10\2\0"+ 155 | "\1\10\2\0\3\10\4\0\5\10\4\0\23\10\1\63"+ 156 | "\3\10\2\0\2\10\2\0\1\10\2\0\3\10\4\0"+ 157 | "\1\64\4\10\4\0\27\10\2\0\2\10\54\0\1\34"+ 158 | "\3\0\7\35\4\0\45\35\4\65\1\66\2\65\1\67"+ 159 | "\1\70\2\0\30\65\1\71\1\72\2\65\1\73\5\65"+ 160 | "\1\0\1\74\1\75\4\40\1\0\2\40\1\0\1\40"+ 161 | "\2\0\45\40\4\65\1\66\2\65\1\67\1\70\2\0"+ 162 | "\42\65\1\0\2\65\2\0\1\76\57\0\1\10\2\0"+ 163 | "\3\10\4\0\2\10\1\77\2\10\4\0\27\10\2\0"+ 164 | "\2\10\2\0\1\10\2\0\3\10\4\0\5\10\4\0"+ 165 | "\1\10\1\100\25\10\2\0\2\10\2\0\1\10\2\0"+ 166 | "\3\10\4\0\5\10\4\0\10\10\1\101\16\10\2\0"+ 167 | "\2\10\2\0\1\10\2\0\3\10\4\0\5\10\4\0"+ 168 | "\12\10\1\102\14\10\2\0\2\10\2\0\1\10\2\0"+ 169 | "\3\10\4\0\1\103\4\10\4\0\27\10\2\0\2\10"+ 170 | "\2\0\1\10\2\0\3\10\4\0\5\10\4\0\11\10"+ 171 | "\1\104\15\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 172 | "\4\0\5\10\4\0\15\10\1\105\11\10\2\0\2\10"+ 173 | "\2\0\1\10\2\0\3\10\4\0\5\10\4\0\7\10"+ 174 | "\1\106\17\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 175 | "\4\0\2\10\1\107\2\10\4\0\27\10\2\0\2\10"+ 176 | "\2\0\1\10\2\0\3\10\4\0\5\10\4\0\20\10"+ 177 | "\1\110\6\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 178 | "\4\0\5\10\4\0\24\10\1\111\2\10\2\0\2\10"+ 179 | "\2\0\1\10\2\0\3\10\4\0\5\10\4\0\10\10"+ 180 | "\1\112\16\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 181 | "\4\0\5\10\4\0\10\10\1\113\16\10\2\0\2\10"+ 182 | "\2\0\1\10\2\0\3\10\4\0\5\10\4\0\6\10"+ 183 | "\1\114\20\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 184 | "\4\0\4\10\1\115\4\0\27\10\2\0\2\10\2\0"+ 185 | "\1\10\2\0\3\10\4\0\5\10\4\0\6\10\1\116"+ 186 | "\20\10\2\0\2\10\2\0\1\10\2\0\3\10\4\0"+ 187 | "\5\10\4\0\6\10\1\117\20\10\2\0\2\10\2\0"+ 188 | "\1\10\2\0\3\10\4\0\5\10\4\0\10\10\1\120"+ 189 | "\16\10\2\0\2\10\2\0\1\10\2\0\3\10\4\0"+ 190 | "\5\10\4\0\2\10\1\121\24\10\2\0\2\10\2\0"+ 191 | "\1\10\2\0\3\10\4\0\5\10\4\0\21\10\1\116"+ 192 | "\5\10\2\0\2\10\2\0\1\10\2\0\3\10\4\0"+ 193 | "\5\10\4\0\25\10\1\110\1\10\2\0\2\10\2\0"+ 194 | "\1\10\2\0\3\10\4\0\5\10\4\0\1\104\26\10"+ 195 | "\2\0\2\10\2\0\1\10\2\0\3\10\4\0\5\10"+ 196 | "\4\0\13\10\1\122\13\10\2\0\2\10\2\0\1\10"+ 197 | "\2\0\3\10\4\0\5\10\4\0\1\10\1\123\25\10"+ 198 | "\2\0\2\10\2\0\1\10\2\0\3\10\4\0\5\10"+ 199 | "\4\0\1\124\26\10\2\0\2\10\2\0\1\10\2\0"+ 200 | "\3\10\4\0\5\10\4\0\11\10\1\125\15\10\2\0"+ 201 | "\2\10\2\0\1\10\2\0\3\10\4\0\5\10\4\0"+ 202 | "\3\10\1\126\23\10\2\0\2\10\2\0\1\10\2\0"+ 203 | "\3\10\4\0\5\10\4\0\14\10\1\127\12\10\2\0"+ 204 | "\2\10\2\0\1\10\2\0\3\10\4\0\5\10\4\0"+ 205 | "\2\10\1\130\24\10\2\0\2\10\2\0\1\10\2\0"+ 206 | "\3\10\4\0\5\10\4\0\1\10\1\131\25\10\2\0"+ 207 | "\2\10\2\0\1\10\2\0\3\10\4\0\2\10\1\132"+ 208 | "\2\10\4\0\27\10\2\0\2\10\2\0\1\10\2\0"+ 209 | "\3\10\4\0\5\10\4\0\4\10\1\133\22\10\2\0"+ 210 | "\2\10\2\0\1\10\2\0\3\10\4\0\5\10\4\0"+ 211 | "\3\10\1\134\23\10\2\0\2\10\2\0\1\10\2\0"+ 212 | "\3\10\4\0\1\135\4\10\4\0\27\10\2\0\2\10"+ 213 | "\2\0\1\10\2\0\3\10\4\0\5\10\4\0\6\10"+ 214 | "\1\136\20\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 215 | "\4\0\5\10\4\0\1\10\1\137\25\10\2\0\2\10"+ 216 | "\2\0\1\10\2\0\3\10\4\0\5\10\4\0\4\10"+ 217 | "\1\140\22\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 218 | "\4\0\5\10\4\0\15\10\1\141\11\10\2\0\2\10"+ 219 | "\2\0\1\10\2\0\3\10\4\0\5\10\4\0\5\10"+ 220 | "\1\142\21\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 221 | "\4\0\5\10\4\0\1\10\1\143\25\10\2\0\2\10"+ 222 | "\2\0\1\10\2\0\3\10\4\0\5\10\4\0\14\10"+ 223 | "\1\144\12\10\2\0\2\10\2\0\1\10\2\0\3\10"+ 224 | "\4\0\5\10\4\0\5\10\1\145\21\10\2\0\2\10"; 225 | 226 | private static int [] zzUnpackTrans() { 227 | int [] result = new int[3360]; 228 | int offset = 0; 229 | offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); 230 | return result; 231 | } 232 | 233 | private static int zzUnpackTrans(String packed, int offset, int [] result) { 234 | int i = 0; /* index in packed string */ 235 | int j = offset; /* index in unpacked array */ 236 | int l = packed.length(); 237 | while (i < l) { 238 | int count = packed.charAt(i++); 239 | int value = packed.charAt(i++); 240 | value--; 241 | do result[j++] = value; while (--count > 0); 242 | } 243 | return j; 244 | } 245 | 246 | 247 | /* error codes */ 248 | private static final int ZZ_UNKNOWN_ERROR = 0; 249 | private static final int ZZ_NO_MATCH = 1; 250 | private static final int ZZ_PUSHBACK_2BIG = 2; 251 | 252 | /* error messages for the codes above */ 253 | private static final String ZZ_ERROR_MSG[] = { 254 | "Unknown internal scanner error", 255 | "Error: could not match input", 256 | "Error: pushback value was too large" 257 | }; 258 | 259 | /** 260 | * ZZ_ATTRIBUTE[aState] contains the attributes of state aState 261 | */ 262 | private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute(); 263 | 264 | private static final String ZZ_ATTRIBUTE_PACKED_0 = 265 | "\3\0\1\11\2\1\1\11\1\1\1\11\1\1\2\11"+ 266 | "\4\1\1\11\15\1\1\11\1\1\1\11\1\1\1\0"+ 267 | "\5\1\3\11\11\1\11\11\50\1"; 268 | 269 | private static int [] zzUnpackAttribute() { 270 | int [] result = new int[101]; 271 | int offset = 0; 272 | offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); 273 | return result; 274 | } 275 | 276 | private static int zzUnpackAttribute(String packed, int offset, int [] result) { 277 | int i = 0; /* index in packed string */ 278 | int j = offset; /* index in unpacked array */ 279 | int l = packed.length(); 280 | while (i < l) { 281 | int count = packed.charAt(i++); 282 | int value = packed.charAt(i++); 283 | do result[j++] = value; while (--count > 0); 284 | } 285 | return j; 286 | } 287 | 288 | /** the input device */ 289 | private java.io.Reader zzReader; 290 | 291 | /** the current state of the DFA */ 292 | private int zzState; 293 | 294 | /** the current lexical state */ 295 | private int zzLexicalState = YYINITIAL; 296 | 297 | /** this buffer contains the current text to be matched and is 298 | the source of the yytext() string */ 299 | private char zzBuffer[] = new char[ZZ_BUFFERSIZE]; 300 | 301 | /** the textposition at the last accepting state */ 302 | private int zzMarkedPos; 303 | 304 | /** the current text position in the buffer */ 305 | private int zzCurrentPos; 306 | 307 | /** startRead marks the beginning of the yytext() string in the buffer */ 308 | private int zzStartRead; 309 | 310 | /** endRead marks the last character in the buffer, that has been read 311 | from input */ 312 | private int zzEndRead; 313 | 314 | /** number of newlines encountered up to the start of the matched text */ 315 | private int yyline; 316 | 317 | /** the number of characters up to the start of the matched text */ 318 | private int yychar; 319 | 320 | /** 321 | * the number of characters from the last newline up to the start of the 322 | * matched text 323 | */ 324 | private int yycolumn; 325 | 326 | /** 327 | * zzAtBOL == true <=> the scanner is currently at the beginning of a line 328 | */ 329 | private boolean zzAtBOL = true; 330 | 331 | /** zzAtEOF == true <=> the scanner is at the EOF */ 332 | private boolean zzAtEOF; 333 | 334 | /** denotes if the user-EOF-code has already been executed */ 335 | private boolean zzEOFDone; 336 | 337 | /** 338 | * The number of occupied positions in zzBuffer beyond zzEndRead. 339 | * When a lead/high surrogate has been read from the input stream 340 | * into the final zzBuffer position, this will have a value of 1; 341 | * otherwise, it will have a value of 0. 342 | */ 343 | private int zzFinalHighSurrogate = 0; 344 | 345 | /* user code: */ 346 | private StringBuffer string = new StringBuffer(); 347 | 348 | protected Parser yyparser; 349 | protected Schema schema; 350 | 351 | public Yylex(String str, Parser yyparser) { 352 | this(new java.io.StringReader(str)); 353 | this.yyparser = yyparser; 354 | this.schema = yyparser.schema; 355 | } 356 | 357 | 358 | /** 359 | * Creates a new scanner 360 | * 361 | * @param in the java.io.Reader to read input from. 362 | */ 363 | Yylex(java.io.Reader in) { 364 | this.zzReader = in; 365 | } 366 | 367 | 368 | /** 369 | * Unpacks the compressed character translation table. 370 | * 371 | * @param packed the packed character translation table 372 | * @return the unpacked character translation table 373 | */ 374 | private static char [] zzUnpackCMap(String packed) { 375 | char [] map = new char[0x110000]; 376 | int i = 0; /* index in packed string */ 377 | int j = 0; /* index in unpacked array */ 378 | while (i < 180) { 379 | int count = packed.charAt(i++); 380 | char value = packed.charAt(i++); 381 | do map[j++] = value; while (--count > 0); 382 | } 383 | return map; 384 | } 385 | 386 | 387 | /** 388 | * Refills the input buffer. 389 | * 390 | * @return false, iff there was new input. 391 | * 392 | * @exception java.io.IOException if any I/O-Error occurs 393 | */ 394 | private boolean zzRefill() throws java.io.IOException { 395 | 396 | /* first: make room (if you can) */ 397 | if (zzStartRead > 0) { 398 | zzEndRead += zzFinalHighSurrogate; 399 | zzFinalHighSurrogate = 0; 400 | System.arraycopy(zzBuffer, zzStartRead, 401 | zzBuffer, 0, 402 | zzEndRead-zzStartRead); 403 | 404 | /* translate stored positions */ 405 | zzEndRead-= zzStartRead; 406 | zzCurrentPos-= zzStartRead; 407 | zzMarkedPos-= zzStartRead; 408 | zzStartRead = 0; 409 | } 410 | 411 | /* is the buffer big enough? */ 412 | if (zzCurrentPos >= zzBuffer.length - zzFinalHighSurrogate) { 413 | /* if not: blow it up */ 414 | char newBuffer[] = new char[zzBuffer.length*2]; 415 | System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length); 416 | zzBuffer = newBuffer; 417 | zzEndRead += zzFinalHighSurrogate; 418 | zzFinalHighSurrogate = 0; 419 | } 420 | 421 | /* fill the buffer with new input */ 422 | int requested = zzBuffer.length - zzEndRead; 423 | int numRead = zzReader.read(zzBuffer, zzEndRead, requested); 424 | 425 | /* not supposed to occur according to specification of java.io.Reader */ 426 | if (numRead == 0) { 427 | throw new java.io.IOException("Reader returned 0 characters. See JFlex examples for workaround."); 428 | } 429 | if (numRead > 0) { 430 | zzEndRead += numRead; 431 | /* If numRead == requested, we might have requested to few chars to 432 | encode a full Unicode character. We assume that a Reader would 433 | otherwise never return half characters. */ 434 | if (numRead == requested) { 435 | if (Character.isHighSurrogate(zzBuffer[zzEndRead - 1])) { 436 | --zzEndRead; 437 | zzFinalHighSurrogate = 1; 438 | } 439 | } 440 | /* potentially more input available */ 441 | return false; 442 | } 443 | 444 | /* numRead < 0 ==> end of stream */ 445 | return true; 446 | } 447 | 448 | 449 | /** 450 | * Closes the input stream. 451 | */ 452 | public final void yyclose() throws java.io.IOException { 453 | zzAtEOF = true; /* indicate end of file */ 454 | zzEndRead = zzStartRead; /* invalidate buffer */ 455 | 456 | if (zzReader != null) 457 | zzReader.close(); 458 | } 459 | 460 | 461 | /** 462 | * Resets the scanner to read from a new input stream. 463 | * Does not close the old reader. 464 | * 465 | * All internal variables are reset, the old input stream 466 | * cannot be reused (internal buffer is discarded and lost). 467 | * Lexical state is set to ZZ_INITIAL. 468 | * 469 | * Internal scan buffer is resized down to its initial length, if it has grown. 470 | * 471 | * @param reader the new input stream 472 | */ 473 | public final void yyreset(java.io.Reader reader) { 474 | zzReader = reader; 475 | zzAtBOL = true; 476 | zzAtEOF = false; 477 | zzEOFDone = false; 478 | zzEndRead = zzStartRead = 0; 479 | zzCurrentPos = zzMarkedPos = 0; 480 | zzFinalHighSurrogate = 0; 481 | yyline = yychar = yycolumn = 0; 482 | zzLexicalState = YYINITIAL; 483 | if (zzBuffer.length > ZZ_BUFFERSIZE) 484 | zzBuffer = new char[ZZ_BUFFERSIZE]; 485 | } 486 | 487 | 488 | /** 489 | * Returns the current lexical state. 490 | */ 491 | public final int yystate() { 492 | return zzLexicalState; 493 | } 494 | 495 | 496 | /** 497 | * Enters a new lexical state 498 | * 499 | * @param newState the new lexical state 500 | */ 501 | public final void yybegin(int newState) { 502 | zzLexicalState = newState; 503 | } 504 | 505 | 506 | /** 507 | * Returns the text matched by the current regular expression. 508 | */ 509 | public final String yytext() { 510 | return new String( zzBuffer, zzStartRead, zzMarkedPos-zzStartRead ); 511 | } 512 | 513 | 514 | /** 515 | * Returns the character at position pos from the 516 | * matched text. 517 | * 518 | * It is equivalent to yytext().charAt(pos), but faster 519 | * 520 | * @param pos the position of the character to fetch. 521 | * A value from 0 to yylength()-1. 522 | * 523 | * @return the character at position pos 524 | */ 525 | public final char yycharat(int pos) { 526 | return zzBuffer[zzStartRead+pos]; 527 | } 528 | 529 | 530 | /** 531 | * Returns the length of the matched text region. 532 | */ 533 | public final int yylength() { 534 | return zzMarkedPos-zzStartRead; 535 | } 536 | 537 | 538 | /** 539 | * Reports an error that occured while scanning. 540 | * 541 | * In a wellformed scanner (no or only correct usage of 542 | * yypushback(int) and a match-all fallback rule) this method 543 | * will only be called with things that "Can't Possibly Happen". 544 | * If this method is called, something is seriously wrong 545 | * (e.g. a JFlex bug producing a faulty scanner etc.). 546 | * 547 | * Usual syntax/scanner level error handling should be done 548 | * in error fallback rules. 549 | * 550 | * @param errorCode the code of the errormessage to display 551 | */ 552 | private void zzScanError(int errorCode) { 553 | String message; 554 | try { 555 | message = ZZ_ERROR_MSG[errorCode]; 556 | } 557 | catch (ArrayIndexOutOfBoundsException e) { 558 | message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; 559 | } 560 | 561 | throw new Error(message); 562 | } 563 | 564 | 565 | /** 566 | * Pushes the specified amount of characters back into the input stream. 567 | * 568 | * They will be read again by then next call of the scanning method 569 | * 570 | * @param number the number of characters to be read again. 571 | * This number must not be greater than yylength()! 572 | */ 573 | public void yypushback(int number) { 574 | if ( number > yylength() ) 575 | zzScanError(ZZ_PUSHBACK_2BIG); 576 | 577 | zzMarkedPos -= number; 578 | } 579 | 580 | 581 | /** 582 | * Contains user EOF-code, which will be executed exactly once, 583 | * when the end of file is reached 584 | */ 585 | private void zzDoEOF() throws java.io.IOException { 586 | if (!zzEOFDone) { 587 | zzEOFDone = true; 588 | yyclose(); 589 | } 590 | } 591 | 592 | 593 | /** 594 | * Resumes scanning until the next regular expression is matched, 595 | * the end of input is encountered or an I/O-Error occurs. 596 | * 597 | * @return the next token 598 | * @exception java.io.IOException if any I/O-Error occurs 599 | */ 600 | public int yylex() throws java.io.IOException { 601 | int zzInput; 602 | int zzAction; 603 | 604 | // cached fields: 605 | int zzCurrentPosL; 606 | int zzMarkedPosL; 607 | int zzEndReadL = zzEndRead; 608 | char [] zzBufferL = zzBuffer; 609 | char [] zzCMapL = ZZ_CMAP; 610 | 611 | int [] zzTransL = ZZ_TRANS; 612 | int [] zzRowMapL = ZZ_ROWMAP; 613 | int [] zzAttrL = ZZ_ATTRIBUTE; 614 | 615 | while (true) { 616 | zzMarkedPosL = zzMarkedPos; 617 | 618 | zzAction = -1; 619 | 620 | zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL; 621 | 622 | zzState = ZZ_LEXSTATE[zzLexicalState]; 623 | 624 | // set up zzAction for empty match case: 625 | int zzAttributes = zzAttrL[zzState]; 626 | if ( (zzAttributes & 1) == 1 ) { 627 | zzAction = zzState; 628 | } 629 | 630 | 631 | zzForAction: { 632 | while (true) { 633 | 634 | if (zzCurrentPosL < zzEndReadL) { 635 | zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL); 636 | zzCurrentPosL += Character.charCount(zzInput); 637 | } 638 | else if (zzAtEOF) { 639 | zzInput = YYEOF; 640 | break zzForAction; 641 | } 642 | else { 643 | // store back cached positions 644 | zzCurrentPos = zzCurrentPosL; 645 | zzMarkedPos = zzMarkedPosL; 646 | boolean eof = zzRefill(); 647 | // get translated positions and possibly new buffer 648 | zzCurrentPosL = zzCurrentPos; 649 | zzMarkedPosL = zzMarkedPos; 650 | zzBufferL = zzBuffer; 651 | zzEndReadL = zzEndRead; 652 | if (eof) { 653 | zzInput = YYEOF; 654 | break zzForAction; 655 | } 656 | else { 657 | zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL); 658 | zzCurrentPosL += Character.charCount(zzInput); 659 | } 660 | } 661 | int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ]; 662 | if (zzNext == -1) break zzForAction; 663 | zzState = zzNext; 664 | 665 | zzAttributes = zzAttrL[zzState]; 666 | if ( (zzAttributes & 1) == 1 ) { 667 | zzAction = zzState; 668 | zzMarkedPosL = zzCurrentPosL; 669 | if ( (zzAttributes & 8) == 8 ) break zzForAction; 670 | } 671 | 672 | } 673 | } 674 | 675 | // store back cached position 676 | zzMarkedPos = zzMarkedPosL; 677 | 678 | if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { 679 | zzAtEOF = true; 680 | zzDoEOF(); 681 | { return 0; } 682 | } 683 | else { 684 | switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) { 685 | case 1: 686 | { throw new ConfigException("yylex: Unexpected character '"+yytext()+"'"); 687 | } 688 | case 37: break; 689 | case 2: 690 | { yyparser.yylval = new ParserVal(new NumberLiteral(yytext())); return Parser.NUMBER; 691 | } 692 | case 38: break; 693 | case 3: 694 | { yybegin(IDENTIFIER); string.setLength(0); 695 | } 696 | case 39: break; 697 | case 4: 698 | { yyparser.yylval = new ParserVal(new IdentifierLiteral(yytext(), schema)); return Parser.IDENTIFIER; 699 | } 700 | case 40: break; 701 | case 5: 702 | { yybegin(STRING); string.setLength(0); 703 | } 704 | case 41: break; 705 | case 6: 706 | { 707 | } 708 | case 42: break; 709 | case 7: 710 | { return (int) yycharat(0); 711 | } 712 | case 43: break; 713 | case 8: 714 | { return Parser.EQ; 715 | } 716 | case 44: break; 717 | case 9: 718 | { return Parser.LT; 719 | } 720 | case 45: break; 721 | case 10: 722 | { return Parser.GT; 723 | } 724 | case 46: break; 725 | case 11: 726 | { string.append( yytext() ); 727 | } 728 | case 47: break; 729 | case 12: 730 | { yybegin(YYINITIAL); yyparser.yylval = new ParserVal(new StringLiteral(string.toString())); return Parser.STRING; 731 | } 732 | case 48: break; 733 | case 13: 734 | { yybegin(YYINITIAL); yyparser.yylval = new ParserVal(new IdentifierLiteral(string.toString(), schema)); return Parser.IDENTIFIER; 735 | } 736 | case 49: break; 737 | case 14: 738 | { return Parser.OR; 739 | } 740 | case 50: break; 741 | case 15: 742 | { return Parser.LE; 743 | } 744 | case 51: break; 745 | case 16: 746 | { return Parser.NEQ; 747 | } 748 | case 52: break; 749 | case 17: 750 | { return Parser.GE; 751 | } 752 | case 53: break; 753 | case 18: 754 | { return Parser.IS; 755 | } 756 | case 54: break; 757 | case 19: 758 | { throw new ConfigException("yylex: Illegal escape sequence \""+yytext()+"\""); 759 | } 760 | case 55: break; 761 | case 20: 762 | { string.append( '\"' ); 763 | } 764 | case 56: break; 765 | case 21: 766 | { string.append( '\\' ); 767 | } 768 | case 57: break; 769 | case 22: 770 | { string.append( '\'' ); 771 | } 772 | case 58: break; 773 | case 23: 774 | { string.append( '\t' ); 775 | } 776 | case 59: break; 777 | case 24: 778 | { string.append( '\r' ); 779 | } 780 | case 60: break; 781 | case 25: 782 | { string.append( '\f' ); 783 | } 784 | case 61: break; 785 | case 26: 786 | { string.append( '\b' ); 787 | } 788 | case 62: break; 789 | case 27: 790 | { string.append( '\n' ); 791 | } 792 | case 63: break; 793 | case 28: 794 | { return Parser.AND; 795 | } 796 | case 64: break; 797 | case 29: 798 | { return Parser.NOT; 799 | } 800 | case 65: break; 801 | case 30: 802 | { return Parser.NULL; 803 | } 804 | case 66: break; 805 | case 31: 806 | { yyparser.yylval = new ParserVal(new BooleanLiteral(yytext())); return Parser.BOOLEAN; 807 | } 808 | case 67: break; 809 | case 32: 810 | { return Parser.REGEXP; 811 | } 812 | case 68: break; 813 | case 33: 814 | { return Parser.INCLUDE; 815 | } 816 | case 69: break; 817 | case 34: 818 | { return Parser.END_WITH; 819 | } 820 | case 70: break; 821 | case 35: 822 | { return Parser.TIMESTAMP; 823 | } 824 | case 71: break; 825 | case 36: 826 | { return Parser.START_WITH; 827 | } 828 | case 72: break; 829 | default: 830 | zzScanError(ZZ_NO_MATCH); 831 | } 832 | } 833 | } 834 | } 835 | 836 | 837 | } 838 | --------------------------------------------------------------------------------