├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── comparison.go ├── constant.go ├── constant_test.go ├── field.go ├── field_test.go ├── lexer.go ├── lexer_test.go ├── operand.go ├── operand_test.go ├── operators.go ├── parser.go ├── parser_test.go ├── query.go ├── query_test.go ├── record ├── csv.go ├── csv_test.go ├── json.go └── json_test.go ├── samples ├── csv │ ├── csv.go │ └── population.csv └── json │ ├── json.go │ └── people.jsons ├── tests └── grammar │ ├── grammar.abnf │ └── grammartester.go ├── token.go └── token_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | /samples/json/json 2 | /samples/csv/csv 3 | /tests/lexer/lexer 4 | /tests/grammar/grammar 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.2 4 | - 1.3 5 | - 1.4 6 | - 1.5 7 | - 1.6 8 | - 1.7 9 | - tip 10 | script: 11 | - go get -t ./... && go test ./... 12 | notifications: 13 | email: false 14 | slack: 15 | secure: uV4GgqS7Op/8+vSlOnFbhu+h6cHrgv7OqfpGUZlSXsl/zOiLX7Gvhi9frWOOPWk9urZcXCAPfTV9TVpByZI17vEVAt584sJAOGNW77s2VrXOYuQtmrcWyprFk6FIcgjuvXUvP1jvR6sosoy0rn5veXdPTqG98zTLPBhea2mhxHYDhI9Rt5UHttH/ZjWvKjkR/rlEOL/7ovzoKYGVXvFYnx8KAbVqxCXFJzFf+VTkFjzOcctXnUyuY6gyglS3XGlyUSVAS25Wz1gCQtgIIawFU8YjH4pFVDwf//AIHB6GlqoAQpmrbGMydfEzAmE2hCrC1mETxpu+dyGaBN8oGh1YfWOICqH+B4vZtvj6D910+chWSA9wg6dHlguHrhC8Ske157/tRuRgl3YO60OMVZ7lr32cL775v6snOCt7/g8cbdg64Q/H0CpoIZ+K4y/fIYvYtHTLoAuBNZXU43Cn0pJfvmuDlpNbfZvGtHVZXnhl2PiWgpxLppKQBAly3tNAv3HGb7PuK/BAzgzRSJ6hENbi5ztZlXvFQW1+TWR5XOz0Wyce9Amd/t5gpgBYif9NAC+UujpzKfgqA8i/EFpM8b02QEZo8JaMZIzPGBFqpTjW+iVPar1cMbYkyp4eO6kj+bJM2vk3ovjv4dsyk4RhB49OPjY2SmP0bWlX1IhFKPCBy3I= 16 | sudo: false 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2015 Batch.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Charlatan 2 | 3 | [![Build Status](https://travis-ci.org/BatchLabs/charlatan.svg?branch=master)](https://travis-ci.org/BatchLabs/charlatan) 4 | 5 | **Charlatan** is a query engine for lists or streams of records in different 6 | formats. It natively supports CSV and JSON formats but can easily be extended 7 | to others. 8 | 9 | It supports an SQL-like query language that is defined below. Queries are 10 | applied to records to extract values depending on zero or more criteria. 11 | 12 | ## Query Syntax 13 | 14 | ``` 15 | SELECT FROM [ WHERE ] [ STARTING AT ] [ LIMIT [,] ] 16 | ``` 17 | 18 | - `` is a list of comma-separated field names. Each field name must 19 | exist in the source. When reading CSV files, the field names are the column 20 | names, while when reading JSON they represent keys. 21 | - `` is the filename from which the data is read. The API is agnostique 22 | on this and one can implement support for any source type. 23 | - `` is a SQL-like value, which can be either a constant (e.g. 24 | `WHERE 1`), a field (e.g. `WHERE archived`) or any operation using comparison 25 | operators (`=`, `!=`, `<`, `<=`, `>`, `>=`, `AND`, `OR`) and optionally 26 | parentheses (e.g. `WHERE (foo > 2) AND (bar = "yo")`). The parser allows to 27 | use `&&` instead of `AND` and `||` instead of `OR`. It also support inclusive 28 | range tests, like `WHERE age BETWEEN 20 AND 30`. 29 | - `LIMIT N` can be used to keep only the first N matched records. It also 30 | support the MySQL way to specify offsets: `LIMIT M, N` can be used to get the 31 | first N matched records after the M-th. 32 | - `STARTING AT ` can be used to skip the first N records. It’s 33 | equivalent to the `` field of the `LIMIT` clause, and if both clauses 34 | are used in a query, the last one will be used. 35 | 36 | Constant values include strings, integers, floats, booleans and the `null` 37 | value. 38 | 39 | ### Examples 40 | 41 | ```sql 42 | SELECT CountryName FROM sample/csv/population.csv WHERE Year = 2010 AND Value > 50000000 AND Value < 70000000 43 | SELECT name, age FROM sample/json/people.jsons WHERE stats.walking > 30 AND stats.biking < 300 44 | SELECT name, age FROM sample/json/people.jsons WHERE stats.walking BETWEEN 20 AND 100 LIMIT 10, 5 45 | ``` 46 | 47 | ### Type Coercion Rules 48 | 49 | * int: same value if the constant is an integer. Truncated value if it’s a 50 | float. `1` if it’s a `true` boolean. `0` for everything else. 51 | * float: same value if the constant is an integer or a float. `1.0` if it’s a 52 | `true` boolean. `0.0` for everything else. 53 | * boolean: `true` if it’s a string (even if it’s empty), a `true` boolean, a 54 | non-zero integer or float. `false` for everything else. 55 | * string: the string representation of the constant. `null` becomes `"null"` 56 | 57 | These rules mean that e.g. `WHERE 0` is equivalent to `WHERE false` and 58 | `WHERE ""` is equivalent to `WHERE true`. 59 | 60 | ## API 61 | 62 | The library is responsible for parsing the query and executing against records. 63 | Everything else is up to you, including how fields are retrieved from records. 64 | 65 | Note: code examples below don’t include error handling for clarity purposes. 66 | 67 | ```go 68 | // parse the query 69 | query, _ := charlatan.QueryFromString("SELECT foo FROM myfile.json WHERE foo > 2") 70 | 71 | // open the source file 72 | reader, _ := os.Open(query.From()) 73 | 74 | defer reader.Close() 75 | 76 | // skip lines if the query contains "STARTING AT " 77 | skip := query.StartingAt() 78 | 79 | decoder := json.NewDecoder(reader) 80 | 81 | for { 82 | // here we use STARTING AT to skip all lines, not only the ones that match 83 | // the query. This is not the usual behavior, but we can do whatever we 84 | // want here. 85 | skip-- 86 | if skip >= 0 { 87 | continue 88 | } 89 | 90 | // get a new JSON record 91 | r, err := record.NewJSONRecordFromDecoder(decoder) 92 | 93 | if err == io.EOF { 94 | break 95 | } 96 | 97 | // evaluate the query against the record to test if it matches 98 | if match, _ := query.Evaluate(r); !match { 99 | continue 100 | } 101 | 102 | // extract the values and print them 103 | values, _ := query.FieldsValues(r) 104 | fmt.Printf("%v\n", values) 105 | } 106 | ``` 107 | 108 | Two record types are included: `JSONRecord` and `CSVRecord`. Implementing a 109 | record only requires one method: `Find(*Field) (*Const, error)`, which takes a 110 | field and return its value. 111 | 112 | As an example, let’s implement a `LineRecord` that’ll be used to get specific 113 | characters on each line of a file, `c0` being the first character: 114 | 115 | ```go 116 | type LineRecord struct { Line string } 117 | 118 | func (r *LineRecord) Find(f *charlatan.Field) (*charlatan.Const, error) { 119 | 120 | // this is the field value we must return 121 | name := f.Name() 122 | 123 | // we reject fields that doesn't start with 'c' 124 | if len(name) < 2 || name[0] != 'c' { 125 | return nil, fmt.Errorf("Unknown field '%s'", name) 126 | } 127 | 128 | // we extract the character index from the field name. 129 | index, err := strconv.ParseInt(name[1:], 10, 64) 130 | if err != nil { 131 | return nil, err 132 | } 133 | 134 | // let's not be too strict and accept out-of-range indexes 135 | if index < 0 || index >= int64(len(r.Line)) { 136 | return charlatan.StringConst(""), nil 137 | } 138 | 139 | return charlatan.StringConst(fmt.Sprintf("%c", r.Line[index])), nil 140 | } 141 | ``` 142 | 143 | One can now loop over a file’s content, construct `LineRecord`s from its lines 144 | and evaluate queries against them: 145 | 146 | ```go 147 | query, _ := charlatan.QueryFromString("SELECT c1 FROM myfile WHERE c0 = 'a'") 148 | 149 | f, _ := os.Open(query.From()) 150 | defer f.Close() 151 | 152 | s := bufio.NewScanner(f) 153 | for s.Scan() { 154 | r := &LineRecord{Line: s.Text()} 155 | 156 | if m, _ := query.Evaluate(r); !m { 157 | continue 158 | } 159 | 160 | values, _ := query.FieldsValues(r) 161 | fmt.Printf("%v\n", values) 162 | } 163 | ``` 164 | 165 | ### Examples 166 | 167 | Two examples are included in the repository under `sample/csv/` and 168 | `sample/json/`. 169 | 170 | ### Authors 171 | 172 | - [Nicolas DOUILLET](https://github.com/minimarcel) 173 | - [Vincent RISCHMANN](https://github.com/vrischmann) 174 | - [Baptiste FONTAINE](https://github.com/bfontaine) 175 | 176 | -------------------------------------------------------------------------------- /comparison.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import "fmt" 4 | 5 | // newComparison creates a new comparison from the given operands 6 | func newComparison(left operand, operator operatorType, right operand) (*comparison, error) { 7 | 8 | if left == nil { 9 | return nil, fmt.Errorf("Can't creates a new comparison with a nil left operand") 10 | } 11 | 12 | if right == nil { 13 | return nil, fmt.Errorf("Can't creates a new comparison with a nil right operand") 14 | } 15 | 16 | if !operator.isComparison() { 17 | return nil, fmt.Errorf("The operator should be a comparison operator") 18 | } 19 | 20 | return &comparison{left, operator, right}, nil 21 | } 22 | 23 | // Evaluate evaluates the comparison against a given record and return the 24 | // resulting value 25 | func (c *comparison) Evaluate(record Record) (*Const, error) { 26 | var err error 27 | var leftValue, rightValue *Const 28 | 29 | leftValue, err = c.left.Evaluate(record) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | rightValue, err = c.right.Evaluate(record) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | r, err := leftValue.CompareTo(rightValue) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | switch c.operator { 45 | case operatorEq: 46 | return BoolConst(r == 0), nil 47 | case operatorNeq: 48 | return BoolConst(r != 0), nil 49 | case operatorLt: 50 | return BoolConst(r < 0), nil 51 | case operatorLte: 52 | return BoolConst(r <= 0), nil 53 | case operatorGt: 54 | return BoolConst(r > 0), nil 55 | case operatorGte: 56 | return BoolConst(r >= 0), nil 57 | } 58 | 59 | return nil, fmt.Errorf("Unknown operator %s", c.operator) 60 | } 61 | 62 | func (c *comparison) String() string { 63 | return fmt.Sprintf("%s %s %s", c.left, c.operator, c.right) 64 | } 65 | -------------------------------------------------------------------------------- /constant.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | // constType represents the constant types 10 | type constType int 11 | 12 | // types are either null, int, float, bool or string 13 | const ( 14 | constNull constType = iota 15 | constInt 16 | constFloat 17 | constBool 18 | constString 19 | ) 20 | 21 | // Const represents a Constant 22 | type Const struct { 23 | // the type of this constant 24 | constType constType 25 | // the values, stored into the right var 26 | intValue int64 27 | floatValue float64 28 | boolValue bool 29 | stringValue string 30 | } 31 | 32 | // NewConst creates a new constant whatever the type is 33 | func NewConst(value interface{}) (*Const, error) { 34 | if value == nil { 35 | return NullConst(), nil 36 | } 37 | switch value := value.(type) { 38 | case int: 39 | return IntConst(int64(value)), nil 40 | case int8: 41 | return IntConst(int64(value)), nil 42 | case int16: 43 | return IntConst(int64(value)), nil 44 | case int32: 45 | return IntConst(int64(value)), nil 46 | case int64: 47 | return IntConst(value), nil 48 | case float64: 49 | return FloatConst(float64(value)), nil 50 | case float32: 51 | return FloatConst(float64(value)), nil 52 | case bool: 53 | return BoolConst(value), nil 54 | case string: 55 | return StringConst(value), nil 56 | 57 | case *int: 58 | return IntConst(int64(*value)), nil 59 | case *int8: 60 | return IntConst(int64(*value)), nil 61 | case *int16: 62 | return IntConst(int64(*value)), nil 63 | case *int32: 64 | return IntConst(int64(*value)), nil 65 | case *int64: 66 | return IntConst(*value), nil 67 | case *float32: 68 | return FloatConst(float64(*value)), nil 69 | case *float64: 70 | return FloatConst(*value), nil 71 | case *bool: 72 | return BoolConst(*value), nil 73 | case *string: 74 | return StringConst(*value), nil 75 | default: 76 | return nil, fmt.Errorf("unexpected constant type %T", value) 77 | } 78 | } 79 | 80 | // NullConst returns a new const of type null 81 | func NullConst() *Const { 82 | return &Const{constType: constNull} 83 | } 84 | 85 | // IntConst returns a new Const of type int 86 | func IntConst(value int64) *Const { 87 | return &Const{intValue: value, constType: constInt} 88 | } 89 | 90 | // FloatConst returns a new Const of type float 91 | func FloatConst(value float64) *Const { 92 | return &Const{floatValue: value, constType: constFloat} 93 | } 94 | 95 | // BoolConst returns a new Const of type Bool 96 | func BoolConst(value bool) *Const { 97 | return &Const{boolValue: value, constType: constBool} 98 | } 99 | 100 | // StringConst returns a new Const of type String 101 | func StringConst(value string) *Const { 102 | return &Const{stringValue: value, constType: constString} 103 | } 104 | 105 | // ConstFromString parses a Const from a string 106 | func ConstFromString(s string) *Const { 107 | 108 | if i, err := strconv.ParseInt(s, 10, 64); err == nil { 109 | return IntConst(i) 110 | } 111 | 112 | if f, err := strconv.ParseFloat(s, 64); err == nil { 113 | return FloatConst(f) 114 | } 115 | 116 | if b, err := parseBool(s); err == nil { 117 | return BoolConst(b) 118 | } 119 | 120 | return StringConst(s) 121 | } 122 | 123 | // Evaluate evaluates a const against a record. In practice it always returns a 124 | // pointer on itself 125 | func (c Const) Evaluate(r Record) (*Const, error) { 126 | return &c, nil 127 | } 128 | 129 | func parseBool(s string) (bool, error) { 130 | switch strings.ToUpper(s) { 131 | case "TRUE": 132 | return true, nil 133 | case "FALSE": 134 | return false, nil 135 | default: 136 | return false, fmt.Errorf("Unrecognized boolean: %s", s) 137 | } 138 | } 139 | 140 | // IsNumeric tests if a const has a numeric type (int or float) 141 | func (c Const) IsNumeric() bool { 142 | return c.constType == constInt || c.constType == constFloat 143 | } 144 | 145 | // IsBool tests if a const is a bool 146 | func (c Const) IsBool() bool { 147 | return c.constType == constBool 148 | } 149 | 150 | // IsString tests if a const is a string 151 | func (c Const) IsString() bool { 152 | return c.constType == constString 153 | } 154 | 155 | // IsNull tests if a const is null 156 | func (c Const) IsNull() bool { 157 | return c.constType == constNull 158 | } 159 | 160 | // Value returns the value of a const 161 | func (c Const) Value() interface{} { 162 | switch c.constType { 163 | case constInt: 164 | return c.intValue 165 | case constFloat: 166 | return c.floatValue 167 | case constBool: 168 | return c.boolValue 169 | case constString: 170 | return c.stringValue 171 | } 172 | return nil 173 | } 174 | 175 | func (c Const) String() string { 176 | switch c.constType { 177 | case constString: 178 | return "\"" + c.stringValue + "\"" 179 | default: 180 | return c.AsString() 181 | } 182 | } 183 | 184 | // AsFloat converts into a float64 185 | // Returns 0 if the const is a string or null 186 | func (c Const) AsFloat() float64 { 187 | switch c.constType { 188 | case constInt: 189 | return float64(c.intValue) 190 | case constFloat: 191 | return c.floatValue 192 | case constBool: 193 | if c.boolValue { 194 | return 1.0 195 | } 196 | return 0.0 197 | } 198 | return 0 199 | } 200 | 201 | // AsInt converts into an int64 202 | // Returns 0 if the const is a string or null 203 | func (c Const) AsInt() int64 { 204 | switch c.constType { 205 | case constInt: 206 | return c.intValue 207 | case constFloat: 208 | return int64(c.floatValue) 209 | case constBool: 210 | if c.boolValue { 211 | return 1 212 | } 213 | return 0 214 | } 215 | return 0 216 | } 217 | 218 | // AsBool converts into a bool 219 | // - for bool, returns the value 220 | // - for null, returns false 221 | // - for numeric, returns true if not 0 222 | // - for strings, return true (test existence) 223 | func (c Const) AsBool() bool { 224 | switch c.constType { 225 | case constNull: 226 | return false 227 | case constInt: 228 | return c.intValue != 0 229 | case constFloat: 230 | return c.floatValue != 0 231 | case constBool: 232 | return c.boolValue 233 | case constString: 234 | return true 235 | } 236 | return false 237 | } 238 | 239 | // AsString converts into a string 240 | func (c Const) AsString() string { 241 | switch c.constType { 242 | case constNull: 243 | return "null" 244 | case constInt: 245 | return strconv.FormatInt(c.intValue, 10) 246 | case constFloat: 247 | return strconv.FormatFloat(c.floatValue, 'f', 2, 64) 248 | case constBool: 249 | return strconv.FormatBool(c.boolValue) 250 | case constString: 251 | return c.stringValue 252 | } 253 | 254 | // fallback to sprintf .... should never append 255 | return fmt.Sprintf("%v", c.Value()) 256 | } 257 | 258 | // CompareTo returns: 259 | // - a positive integer if this Constant is greater than the given one 260 | // - a negative integer if this Constant is lower than the given one 261 | // - zero, is this constant is equals to the given one 262 | // 263 | // If the comparison is not possible (incompatible types), an error will be 264 | // returned 265 | func (c Const) CompareTo(c2 *Const) (int, error) { 266 | 267 | if c.constType == c2.constType { 268 | switch c.constType { 269 | case constNull: 270 | return 0, nil 271 | case constInt: 272 | return int(c.intValue - c2.intValue), nil 273 | case constFloat: 274 | return int(c.floatValue - c2.floatValue), nil 275 | case constBool: 276 | return cmpBools(c.boolValue, c2.boolValue), nil 277 | case constString: 278 | return cmpStrings(c.stringValue, c2.stringValue), nil 279 | default: 280 | return 0, fmt.Errorf("Unknown const type: %v", c.constType) 281 | } 282 | 283 | } 284 | if c.IsNull() { 285 | return -1, nil 286 | } 287 | if c2.IsNull() { 288 | return 1, nil 289 | } 290 | if c.IsNumeric() && c2.IsNumeric() { 291 | return int(c.AsFloat() - c2.AsFloat()), nil 292 | 293 | } 294 | if c.IsBool() || c2.IsBool() { 295 | return cmpBools(c.AsBool(), c2.AsBool()), nil 296 | 297 | } 298 | if c.IsString() || c2.IsString() { 299 | return cmpStrings(c.AsString(), c2.AsString()), nil 300 | } 301 | 302 | return 0, fmt.Errorf("Can't compare the two constants %s(%v), %s(%v)", 303 | c.constType, c.Value(), c2.constType, c2.Value()) 304 | } 305 | 306 | func cmpBools(b1, b2 bool) int { 307 | if b1 == b2 { 308 | return 0 309 | } 310 | if b1 { 311 | return 1 312 | } 313 | return -1 314 | } 315 | 316 | func cmpStrings(s1, s2 string) int { 317 | if s1 == s2 { 318 | return 0 319 | } 320 | if s1 > s2 { 321 | return 1 322 | } 323 | return -1 324 | } 325 | 326 | func (t constType) String() string { 327 | switch t { 328 | case constNull: 329 | return "NULL" 330 | case constInt: 331 | return "INT" 332 | case constFloat: 333 | return "FLOAT" 334 | case constBool: 335 | return "BOOL" 336 | case constString: 337 | return "STRING" 338 | default: 339 | return "UNDEFINED" 340 | } 341 | } 342 | -------------------------------------------------------------------------------- /constant_test.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func makeTestConst(t *testing.T, v interface{}) *Const { 11 | c, err := NewConst(v) 12 | assert.Nil(t, err) 13 | require.NotNil(t, c) 14 | return c 15 | } 16 | 17 | func TestNewConstNull(t *testing.T) { 18 | assert.True(t, makeTestConst(t, nil).IsNull()) 19 | } 20 | 21 | func TestNewConstInt(t *testing.T) { 22 | c := makeTestConst(t, 42) 23 | assert.True(t, c.IsNumeric()) 24 | assert.Equal(t, int64(42), c.AsInt()) 25 | 26 | c = makeTestConst(t, int8(42)) 27 | assert.True(t, c.IsNumeric()) 28 | assert.Equal(t, int64(42), c.AsInt()) 29 | 30 | c = makeTestConst(t, int16(42)) 31 | assert.True(t, c.IsNumeric()) 32 | assert.Equal(t, int64(42), c.AsInt()) 33 | 34 | c = makeTestConst(t, int32(42)) 35 | assert.True(t, c.IsNumeric()) 36 | assert.Equal(t, int64(42), c.AsInt()) 37 | 38 | c = makeTestConst(t, int64(42)) 39 | assert.True(t, c.IsNumeric()) 40 | assert.Equal(t, int64(42), c.AsInt()) 41 | } 42 | 43 | func TestNewConstIntPtr(t *testing.T) { 44 | i := 42 45 | i8 := int8(42) 46 | i16 := int16(42) 47 | i32 := int32(42) 48 | i64 := int64(42) 49 | 50 | c := makeTestConst(t, &i) 51 | assert.True(t, c.IsNumeric()) 52 | assert.Equal(t, i64, c.AsInt()) 53 | 54 | c = makeTestConst(t, &i8) 55 | assert.True(t, c.IsNumeric()) 56 | assert.Equal(t, i64, c.AsInt()) 57 | 58 | c = makeTestConst(t, &i16) 59 | assert.True(t, c.IsNumeric()) 60 | assert.Equal(t, i64, c.AsInt()) 61 | 62 | c = makeTestConst(t, &i32) 63 | assert.True(t, c.IsNumeric()) 64 | assert.Equal(t, i64, c.AsInt()) 65 | 66 | c = makeTestConst(t, &i64) 67 | assert.True(t, c.IsNumeric()) 68 | assert.Equal(t, i64, c.AsInt()) 69 | } 70 | 71 | func TestNewConstFloat(t *testing.T) { 72 | c := makeTestConst(t, float32(42.0)) 73 | assert.True(t, c.IsNumeric()) 74 | assert.Equal(t, float64(42.0), c.AsFloat()) 75 | 76 | c = makeTestConst(t, float64(42.0)) 77 | assert.True(t, c.IsNumeric()) 78 | assert.Equal(t, float64(42.0), c.AsFloat()) 79 | } 80 | 81 | func TestNewConstFloatPtr(t *testing.T) { 82 | f32 := float32(42.0) 83 | f64 := float64(42.0) 84 | 85 | c := makeTestConst(t, &f32) 86 | assert.True(t, c.IsNumeric()) 87 | assert.Equal(t, f64, c.AsFloat()) 88 | 89 | c = makeTestConst(t, &f64) 90 | assert.True(t, c.IsNumeric()) 91 | assert.Equal(t, f64, c.AsFloat()) 92 | } 93 | 94 | func TestNewConstBool(t *testing.T) { 95 | c := makeTestConst(t, true) 96 | assert.True(t, c.IsBool()) 97 | assert.Equal(t, true, c.AsBool()) 98 | 99 | c = makeTestConst(t, false) 100 | assert.True(t, c.IsBool()) 101 | assert.Equal(t, false, c.AsBool()) 102 | } 103 | 104 | func TestNewConstBoolPtr(t *testing.T) { 105 | btrue := true 106 | bfalse := false 107 | 108 | c := makeTestConst(t, &btrue) 109 | assert.True(t, c.IsBool()) 110 | assert.Equal(t, true, c.AsBool()) 111 | 112 | c = makeTestConst(t, &bfalse) 113 | assert.True(t, c.IsBool()) 114 | assert.Equal(t, false, c.AsBool()) 115 | } 116 | 117 | func TestNewConstString(t *testing.T) { 118 | c := makeTestConst(t, "yolo") 119 | assert.True(t, c.IsString()) 120 | assert.Equal(t, "yolo", c.AsString()) 121 | } 122 | 123 | func TestNewConstStringPtr(t *testing.T) { 124 | s := "yolo" 125 | c := makeTestConst(t, &s) 126 | assert.True(t, c.IsString()) 127 | assert.Equal(t, s, c.AsString()) 128 | } 129 | 130 | func TestNullConst(t *testing.T) { 131 | assert.True(t, NullConst().IsNull()) 132 | } 133 | 134 | func TestIntConst(t *testing.T) { 135 | c := IntConst(42) 136 | assert.True(t, c.IsNumeric()) 137 | assert.Equal(t, int64(42), c.AsInt()) 138 | } 139 | 140 | func TestFloatConst(t *testing.T) { 141 | c := FloatConst(42) 142 | assert.True(t, c.IsNumeric()) 143 | assert.Equal(t, float64(42), c.AsFloat()) 144 | } 145 | 146 | func TestBoolConst(t *testing.T) { 147 | c := BoolConst(true) 148 | assert.True(t, c.IsBool()) 149 | assert.Equal(t, true, c.AsBool()) 150 | } 151 | 152 | func TestStringConst(t *testing.T) { 153 | c := StringConst("yo") 154 | assert.True(t, c.IsString()) 155 | assert.Equal(t, "yo", c.AsString()) 156 | } 157 | 158 | func TestConstFromStringInt(t *testing.T) { 159 | c := ConstFromString("42") 160 | assert.True(t, c.IsNumeric()) 161 | assert.Equal(t, int64(42), c.AsInt()) 162 | } 163 | 164 | func TestConstFromStringFloat(t *testing.T) { 165 | c := ConstFromString("42.0") 166 | assert.True(t, c.IsNumeric()) 167 | assert.Equal(t, float64(42.0), c.AsFloat()) 168 | } 169 | 170 | func TestConstFromStringBool(t *testing.T) { 171 | c := ConstFromString("true") 172 | assert.True(t, c.IsBool()) 173 | assert.Equal(t, true, c.AsBool()) 174 | } 175 | 176 | func TestConstFromStringString(t *testing.T) { 177 | c := ConstFromString("[]") 178 | assert.True(t, c.IsString()) 179 | assert.Equal(t, "[]", c.AsString()) 180 | } 181 | 182 | func TestConstValue(t *testing.T) { 183 | assert.Nil(t, NullConst().Value()) 184 | 185 | { 186 | v, ok := IntConst(41).Value().(int64) 187 | assert.True(t, ok) 188 | assert.Equal(t, int64(41), v) 189 | } 190 | 191 | { 192 | v, ok := FloatConst(41).Value().(float64) 193 | assert.True(t, ok) 194 | assert.Equal(t, float64(41), v) 195 | } 196 | 197 | { 198 | v, ok := BoolConst(true).Value().(bool) 199 | assert.True(t, ok) 200 | assert.Equal(t, true, v) 201 | } 202 | 203 | { 204 | v, ok := StringConst("ya").Value().(string) 205 | assert.True(t, ok) 206 | assert.Equal(t, "ya", v) 207 | } 208 | } 209 | 210 | func TestConstString(t *testing.T) { 211 | assert.Equal(t, "42", IntConst(42).String()) 212 | assert.Equal(t, "\"foobar\"", StringConst("foobar").String()) 213 | } 214 | 215 | func TestConstAsFloat(t *testing.T) { 216 | assert.Equal(t, float64(3.14), FloatConst(3.14).AsFloat()) 217 | assert.Equal(t, float64(3), IntConst(3).AsFloat()) 218 | assert.Equal(t, float64(1), BoolConst(true).AsFloat()) 219 | assert.Equal(t, float64(0), BoolConst(false).AsFloat()) 220 | assert.Equal(t, float64(0), StringConst("yo").AsFloat()) 221 | assert.Equal(t, float64(0), NullConst().AsFloat()) 222 | } 223 | 224 | func TestConstAsInt(t *testing.T) { 225 | assert.Equal(t, int64(3), FloatConst(3.14).AsInt()) 226 | assert.Equal(t, int64(3), IntConst(3).AsInt()) 227 | assert.Equal(t, int64(1), BoolConst(true).AsInt()) 228 | assert.Equal(t, int64(0), BoolConst(false).AsInt()) 229 | assert.Equal(t, int64(0), StringConst("yo").AsInt()) 230 | assert.Equal(t, int64(0), NullConst().AsInt()) 231 | } 232 | 233 | func TestConstAsBool(t *testing.T) { 234 | assert.Equal(t, true, FloatConst(3.14).AsBool()) 235 | assert.Equal(t, false, FloatConst(0).AsBool()) 236 | assert.Equal(t, true, IntConst(3).AsBool()) 237 | assert.Equal(t, false, IntConst(0).AsBool()) 238 | assert.Equal(t, true, BoolConst(true).AsBool()) 239 | assert.Equal(t, false, BoolConst(false).AsBool()) 240 | assert.Equal(t, true, StringConst("yo").AsBool()) 241 | assert.Equal(t, true, StringConst("").AsBool()) 242 | assert.Equal(t, false, NullConst().AsBool()) 243 | } 244 | 245 | func TestConstAsString(t *testing.T) { 246 | assert.Equal(t, "3.14", FloatConst(3.14).AsString()) 247 | assert.Equal(t, "3", IntConst(3).AsString()) 248 | assert.Equal(t, "0", IntConst(0).AsString()) 249 | assert.Equal(t, "true", BoolConst(true).AsString()) 250 | assert.Equal(t, "false", BoolConst(false).AsString()) 251 | assert.Equal(t, "yo", StringConst("yo").AsString()) 252 | assert.Equal(t, "", StringConst("").AsString()) 253 | assert.Equal(t, "null", NullConst().AsString()) 254 | } 255 | 256 | func testCmpConsts(t *testing.T, c1, c2 *Const) int { 257 | i, err := c1.CompareTo(c2) 258 | assert.Nil(t, err) 259 | return i 260 | } 261 | 262 | func TestConstCompareToSameTypes(t *testing.T) { 263 | assert.Equal(t, 0, testCmpConsts(t, NullConst(), NullConst())) 264 | assert.Equal(t, 0, testCmpConsts(t, IntConst(42), IntConst(42))) 265 | assert.Equal(t, 0, testCmpConsts(t, FloatConst(42), FloatConst(42))) 266 | assert.Equal(t, 0, testCmpConsts(t, BoolConst(true), BoolConst(true))) 267 | assert.Equal(t, 0, testCmpConsts(t, BoolConst(false), BoolConst(false))) 268 | assert.Equal(t, 0, testCmpConsts(t, StringConst("tutu"), StringConst("tutu"))) 269 | assert.Equal(t, 0, testCmpConsts(t, StringConst(""), StringConst(""))) 270 | 271 | // is it inversed? 272 | assert.True(t, 0 > testCmpConsts(t, IntConst(41), IntConst(42))) 273 | assert.True(t, 0 > testCmpConsts(t, FloatConst(41), FloatConst(42))) 274 | assert.True(t, 0 > testCmpConsts(t, BoolConst(false), BoolConst(true))) 275 | assert.True(t, 0 > testCmpConsts(t, StringConst("tatu"), StringConst("tutu"))) 276 | 277 | assert.True(t, 0 < testCmpConsts(t, IntConst(43), IntConst(42))) 278 | assert.True(t, 0 < testCmpConsts(t, FloatConst(100), FloatConst(42))) 279 | assert.True(t, 0 < testCmpConsts(t, BoolConst(true), BoolConst(false))) 280 | assert.True(t, 0 < testCmpConsts(t, StringConst("tytu"), StringConst("tutu"))) 281 | } 282 | 283 | func TestConstCompareToDifferentTypes(t *testing.T) { 284 | assert.Equal(t, 0, testCmpConsts(t, FloatConst(42.0), IntConst(42))) 285 | assert.Equal(t, 0, testCmpConsts(t, IntConst(42), FloatConst(42.0))) 286 | 287 | assert.True(t, 0 < testCmpConsts(t, FloatConst(42.0), IntConst(18))) 288 | assert.True(t, 0 < testCmpConsts(t, IntConst(42), FloatConst(18.0))) 289 | 290 | assert.True(t, 0 > testCmpConsts(t, FloatConst(2.0), IntConst(18))) 291 | assert.True(t, 0 > testCmpConsts(t, IntConst(2), FloatConst(18.0))) 292 | } 293 | 294 | func TestConstTypeString(t *testing.T) { 295 | for _, ty := range []constType{ 296 | constNull, constInt, constFloat, constBool, constString, 297 | } { 298 | assert.NotEqual(t, "", ty.String()) 299 | assert.NotEqual(t, "UNDEFINED", ty.String()) 300 | } 301 | } 302 | -------------------------------------------------------------------------------- /field.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | // Field is a field, contained into the SELECT part and the condition. 4 | // A field is an operand, it can return the value extracted into the Record. 5 | type Field struct { 6 | name string 7 | } 8 | 9 | // NewField returns a new field from the given string 10 | func NewField(name string) *Field { 11 | return &Field{name} 12 | } 13 | 14 | // Evaluate evaluates the field on a record 15 | func (f Field) Evaluate(record Record) (*Const, error) { 16 | return record.Find(&f) 17 | } 18 | 19 | // Name returns the field's name 20 | func (f Field) Name() string { 21 | return f.name 22 | } 23 | 24 | func (f Field) String() string { 25 | return f.name 26 | } 27 | -------------------------------------------------------------------------------- /field_test.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestFieldName(t *testing.T) { 10 | assert.Equal(t, "yo", NewField("yo").Name()) 11 | } 12 | -------------------------------------------------------------------------------- /lexer.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "strconv" 9 | "strings" 10 | "unicode" 11 | ) 12 | 13 | // lexer is a lexer 14 | type lexer struct { 15 | r *bufio.Reader 16 | index int 17 | } 18 | 19 | // lexerFromString creates a new lexer from the given string 20 | func lexerFromString(s string) *lexer { 21 | return &lexer{r: bufio.NewReader(strings.NewReader(s))} 22 | } 23 | 24 | func (l *lexer) readRune() (rune, error) { 25 | l.index++ 26 | r, _, err := l.r.ReadRune() 27 | return r, err 28 | } 29 | 30 | func (l *lexer) unread() error { 31 | l.index-- 32 | return l.r.UnreadRune() 33 | } 34 | 35 | func (l *lexer) skipWhiteSpaces() error { 36 | for { 37 | r, err := l.readRune() 38 | if err != nil { 39 | return err 40 | } 41 | if !unicode.IsSpace(r) { 42 | l.unread() 43 | break 44 | } 45 | } 46 | return nil 47 | } 48 | 49 | // NextToken reads the next token and returns it 50 | func (l *lexer) NextToken() (*token, error) { 51 | if err := l.skipWhiteSpaces(); err != nil { 52 | if err == io.EOF { 53 | return l.eof() 54 | } 55 | 56 | return nil, err 57 | } 58 | 59 | r, err := l.readRune() 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | index := l.index 65 | 66 | // delimiters: `, ", ' 67 | switch r { 68 | case '`', '"', '\'': 69 | v, err := l.readUntil(r) 70 | if err != nil { 71 | return nil, err 72 | } 73 | // consume the trailing delimiter 74 | if _, err := l.readRune(); err != nil { 75 | return nil, err 76 | } 77 | // `foo` 78 | if r == '`' { 79 | return l.field(v, index) 80 | } 81 | // "foo" or 'foo' 82 | return l.str(v, index) 83 | case '(': 84 | return l.simpleToken(tokLeftParenthesis, index) 85 | case ')': 86 | return l.simpleToken(tokRightParenthesis, index) 87 | case ',': 88 | return l.simpleToken(tokComma, index) 89 | } 90 | 91 | if err := l.unread(); err != nil { 92 | return nil, err 93 | } 94 | 95 | // one char is no longer enough, read the next word instead 96 | w, err := l.readWord() 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | // keywords 102 | switch k := strings.ToUpper(w); k { 103 | case "SELECT": 104 | return l.token(tokSelect, k, index) 105 | case "FROM": 106 | return l.token(tokFrom, k, index) 107 | case "WHERE": 108 | return l.token(tokWhere, k, index) 109 | case "STARTING": 110 | return l.token(tokStarting, k, index) 111 | case "AT": 112 | return l.token(tokAt, k, index) 113 | case "AND": 114 | return l.token(tokAnd, k, index) 115 | case "OR": 116 | return l.token(tokOr, k, index) 117 | case "BETWEEN": 118 | return l.token(tokBetween, k, index) 119 | case "LIMIT": 120 | return l.token(tokLimit, k, index) 121 | } 122 | 123 | // special values 124 | switch w { 125 | case "true": 126 | return l.token(tokTrue, "true", index) 127 | case "false": 128 | return l.token(tokFalse, "false", index) 129 | case "null", "NULL": 130 | return l.token(tokNull, "null", index) 131 | } 132 | 133 | if _, err := strconv.ParseInt(w, 10, 64); err == nil { 134 | return l.token(tokInt, w, index) 135 | } 136 | 137 | if _, err := strconv.ParseFloat(w, 10); err == nil { 138 | return l.token(tokFloat, w, index) 139 | } 140 | 141 | if w != "" { 142 | return l.token(tokField, w, index) 143 | } 144 | 145 | // operators 146 | 147 | op, err := l.readOperator() 148 | if err != nil { 149 | return nil, err 150 | } 151 | 152 | switch op { 153 | case "=": 154 | return l.token(tokEq, op, index) 155 | case "!=": 156 | return l.token(tokNeq, op, index) 157 | case "<": 158 | return l.token(tokLt, op, index) 159 | case ">": 160 | return l.token(tokGt, op, index) 161 | case "<=": 162 | return l.token(tokLte, op, index) 163 | case ">=": 164 | return l.token(tokGte, op, index) 165 | case "&&": 166 | return l.token(tokAnd, op, index) 167 | case "||": 168 | return l.token(tokOr, op, index) 169 | } 170 | 171 | if op != "" { 172 | return nil, fmt.Errorf("Invalid operator '%s'", op) 173 | } 174 | 175 | return nil, fmt.Errorf("No known alternative at input %d", index) 176 | } 177 | 178 | func (l *lexer) token(typ tokenType, v string, index int) (*token, error) { 179 | return &token{Type: typ, Value: v, Pos: index}, nil 180 | } 181 | 182 | func (l *lexer) eof() (*token, error) { 183 | return l.token(tokEnd, "", l.index) 184 | } 185 | 186 | func (l *lexer) field(v string, index int) (*token, error) { 187 | return l.token(tokField, v, index) 188 | } 189 | 190 | func (l *lexer) str(v string, index int) (*token, error) { 191 | return l.token(tokString, v, index) 192 | } 193 | 194 | func (l *lexer) simpleToken(typ tokenType, index int) (*token, error) { 195 | return l.token(typ, "", index) 196 | } 197 | 198 | func (l *lexer) readUntil(delim rune) (string, error) { 199 | var buf bytes.Buffer 200 | 201 | for { 202 | r, err := l.readRune() 203 | if err != nil { 204 | return "", err 205 | } 206 | 207 | if r == delim { 208 | l.unread() 209 | break 210 | } 211 | 212 | buf.WriteRune(r) 213 | } 214 | 215 | return buf.String(), nil 216 | } 217 | 218 | func (l *lexer) readWord() (string, error) { return l.readWhile(isWordRune) } 219 | func (l *lexer) readOperator() (string, error) { return l.readWhile(isOperatorRune) } 220 | 221 | func (l *lexer) readWhile(cond func(rune) bool) (string, error) { 222 | var buf bytes.Buffer 223 | 224 | for { 225 | r, err := l.readRune() 226 | if err == io.EOF { 227 | break 228 | } 229 | if err != nil { 230 | return "", err 231 | } 232 | if !cond(r) { 233 | l.unread() 234 | break 235 | } 236 | buf.WriteRune(r) 237 | } 238 | 239 | return buf.String(), nil 240 | } 241 | 242 | func isWordRune(r rune) bool { 243 | return !unicode.IsSpace(r) && strings.IndexRune("(),`'\"|&=!<>[]", r) == -1 244 | } 245 | 246 | func isOperatorRune(r rune) bool { 247 | return strings.IndexRune("<>=!&|", r) > -1 248 | } 249 | -------------------------------------------------------------------------------- /lexer_test.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func assertNextToken(t *testing.T, l *lexer, expected tokenType) { 11 | tok, err := l.NextToken() 12 | assert.Nil(t, err) 13 | assert.Equal(t, expected, tok.Type, 14 | fmt.Sprintf("Expected %s, got %s", expected, tok.Type)) 15 | } 16 | 17 | func assertNextTokens(t *testing.T, l *lexer, expected ...tokenType) { 18 | for _, e := range expected { 19 | assertNextToken(t, l, e) 20 | } 21 | } 22 | 23 | func TestLexerEOF(t *testing.T) { 24 | l := lexerFromString("") 25 | assertNextToken(t, l, tokEnd) 26 | } 27 | 28 | func TestLexerSimpleSelectFrom(t *testing.T) { 29 | l := lexerFromString("SELECT foo FROM bar") 30 | assertNextTokens(t, l, tokSelect, tokField, tokFrom, tokField, tokEnd) 31 | } 32 | 33 | func TestLexerSimpleSelectFromWhere(t *testing.T) { 34 | l := lexerFromString("SELECT foo FROM bar WHERE 1 > 2") 35 | assertNextTokens(t, l, tokSelect, tokField, tokFrom, tokField, tokWhere, 36 | tokInt, tokGt, tokInt, tokEnd) 37 | } 38 | 39 | func TestLexerOperators(t *testing.T) { 40 | for s, tokType := range map[string]tokenType{ 41 | "1 = 2": tokEq, 42 | "1 != 2": tokNeq, 43 | "1 < 2": tokLt, 44 | "1 > 2": tokGt, 45 | "1 <= 2": tokLte, 46 | "1 >= 2": tokGte, 47 | "1 OR 2": tokOr, 48 | "1 || 2": tokOr, 49 | "1 AND 2": tokAnd, 50 | "1 && 2": tokAnd, 51 | } { 52 | l := lexerFromString(s) 53 | assertNextTokens(t, l, tokInt, tokType, tokInt, tokEnd) 54 | } 55 | 56 | l := lexerFromString("1 BETWEEN 2 AND 3") 57 | assertNextTokens(t, l, tokInt, tokBetween, tokInt, tokAnd, tokInt, tokEnd) 58 | } 59 | 60 | func TestLexerStringDoubleQuotes(t *testing.T) { 61 | l := lexerFromString(`"some string"`) 62 | assertNextTokens(t, l, tokString, tokEnd) 63 | } 64 | 65 | func TestLexerStringSingleQuotes(t *testing.T) { 66 | l := lexerFromString(`'some string'`) 67 | assertNextTokens(t, l, tokString, tokEnd) 68 | } 69 | -------------------------------------------------------------------------------- /operand.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // operand is an operand, can be evaluated and have to return a constant. 9 | // Returns a error, if the evaluation is not possible 10 | type operand interface { 11 | Evaluate(Record) (*Const, error) 12 | String() string 13 | } 14 | 15 | var _ operand = Const{} 16 | var _ operand = &Field{} 17 | 18 | // comparison is the comparison operation 19 | type comparison struct { 20 | left operand 21 | operator operatorType 22 | right operand 23 | } 24 | 25 | // logicalOperation is the logical operation 26 | type logicalOperation struct { 27 | left operand 28 | operator operatorType 29 | right operand 30 | } 31 | 32 | type rangeTestOperation struct { 33 | test, min, max operand 34 | } 35 | 36 | // groupOperand is the group operand 37 | // Just keep in mind that there was () surrounding this operation 38 | type groupOperand struct { 39 | operand operand 40 | } 41 | 42 | // A constructor with the left operand only 43 | func newLeftLogicalOperation(left operand) (*logicalOperation, error) { 44 | 45 | if left == nil { 46 | return nil, fmt.Errorf("Can't creates a new comparison with the left operand nil") 47 | } 48 | 49 | return &logicalOperation{left, -1, nil}, nil 50 | } 51 | 52 | // Chain a right operand with the given operator 53 | func (o *logicalOperation) chain(operator operatorType, right operand) error { 54 | 55 | if right == nil { 56 | return fmt.Errorf("Can't creates a new comparison with the right operand nil") 57 | } 58 | 59 | if !operator.IsLogical() { 60 | return fmt.Errorf("The operator should be a logical operator") 61 | } 62 | 63 | o.operator = operator 64 | o.right = right 65 | 66 | return nil 67 | } 68 | 69 | // Simplify this operation 70 | // In case of the right operand is missing, just return the left one 71 | func (o *logicalOperation) simplify() operand { 72 | 73 | if left, ok := o.left.(*logicalOperation); ok { 74 | o.left = left.simplify() 75 | } 76 | 77 | if o.right == nil { 78 | return o.left 79 | } 80 | 81 | if right, ok := o.right.(*logicalOperation); ok { 82 | o.right = right.simplify() 83 | } 84 | 85 | return o 86 | } 87 | 88 | // Evaluate evaluates the logical operation against the given record 89 | func (o *logicalOperation) Evaluate(record Record) (*Const, error) { 90 | 91 | var err error 92 | var leftValue, rightValue *Const 93 | 94 | leftValue, err = o.left.Evaluate(record) 95 | if err != nil { 96 | return nil, err 97 | } 98 | 99 | leftBool := leftValue.AsBool() 100 | 101 | // AND 102 | if !leftBool && o.operator == operatorAnd { 103 | return BoolConst(false), nil 104 | } 105 | 106 | // OR 107 | if leftBool && o.operator == operatorOr { 108 | return BoolConst(true), nil 109 | } 110 | 111 | rightValue, err = o.right.Evaluate(record) 112 | if err != nil { 113 | return nil, err 114 | } 115 | 116 | return BoolConst(rightValue.AsBool()), nil 117 | } 118 | 119 | func (o *logicalOperation) String() string { 120 | switch o.operator { 121 | case operatorAnd: 122 | return fmt.Sprintf("%s AND %s", o.left, o.right) 123 | case operatorOr: 124 | return fmt.Sprintf("%s OR %s", o.left, o.right) 125 | default: 126 | return "Unknown operator" 127 | } 128 | } 129 | 130 | // newGroupOperand returns a new group operand from the given operand 131 | func newGroupOperand(operand operand) (*groupOperand, error) { 132 | if operand == nil { 133 | return nil, errors.New("Can't creates a new group with the an operand nil") 134 | } 135 | 136 | return &groupOperand{operand}, nil 137 | } 138 | 139 | // Evaluate evaluates the group operand against the given record 140 | func (o *groupOperand) Evaluate(record Record) (*Const, error) { 141 | return o.operand.Evaluate(record) 142 | } 143 | 144 | func (o *groupOperand) String() string { 145 | return fmt.Sprintf("(%s)", o.operand) 146 | } 147 | 148 | func (rg *rangeTestOperation) Evaluate(record Record) (*Const, error) { 149 | test, err := rg.test.Evaluate(record) 150 | if err != nil { 151 | return nil, err 152 | } 153 | 154 | min, err := rg.min.Evaluate(record) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | max, err := rg.max.Evaluate(record) 160 | if err != nil { 161 | return nil, err 162 | } 163 | 164 | minComp, err := min.CompareTo(test) 165 | if err != nil { 166 | return nil, err 167 | } 168 | if minComp > 0 { 169 | return BoolConst(false), nil 170 | } 171 | 172 | maxComp, err := max.CompareTo(test) 173 | if err != nil { 174 | return nil, err 175 | } 176 | if maxComp < 0 { 177 | return BoolConst(false), nil 178 | } 179 | 180 | return BoolConst(true), nil 181 | } 182 | 183 | func (rg *rangeTestOperation) String() string { 184 | return fmt.Sprintf("%s BETWEEN %s AND %s", rg.test, rg.min, rg.max) 185 | } 186 | -------------------------------------------------------------------------------- /operand_test.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | -------------------------------------------------------------------------------- /operators.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | // operatorType is the type of an operator 4 | type operatorType int 5 | 6 | // operators can be either logical or comparison-al 7 | const ( 8 | operatorInvalid operatorType = iota 9 | 10 | operatorAnd 11 | operatorOr 12 | 13 | operatorEq 14 | operatorNeq 15 | operatorLt 16 | operatorLte 17 | operatorGt 18 | operatorGte 19 | ) 20 | 21 | // operatorTypeFromTokenType converts a TokenType to an operatorType 22 | func operatorTypeFromTokenType(ty tokenType) operatorType { 23 | switch ty { 24 | case tokAnd: 25 | return operatorAnd 26 | case tokOr: 27 | return operatorOr 28 | case tokEq: 29 | return operatorEq 30 | case tokNeq: 31 | return operatorNeq 32 | case tokLt: 33 | return operatorLt 34 | case tokLte: 35 | return operatorLte 36 | case tokGt: 37 | return operatorGt 38 | case tokGte: 39 | return operatorGte 40 | default: 41 | return operatorInvalid 42 | } 43 | } 44 | 45 | // IsLogical tests if an operator is logical 46 | func (o operatorType) IsLogical() bool { 47 | return o == operatorAnd || o == operatorOr 48 | } 49 | 50 | // isComparison tests if an operator is a comparison 51 | func (o operatorType) isComparison() bool { 52 | return !o.IsLogical() && o != operatorInvalid 53 | } 54 | 55 | func (o operatorType) String() string { 56 | switch o { 57 | case operatorAnd: 58 | return "&&" 59 | case operatorOr: 60 | return "||" 61 | case operatorEq: 62 | return "=" 63 | case operatorNeq: 64 | return "!=" 65 | case operatorLt: 66 | return "<" 67 | case operatorLte: 68 | return "<=" 69 | case operatorGt: 70 | return ">" 71 | case operatorGte: 72 | return ">=" 73 | default: 74 | return "" 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /parser.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // the automate state 9 | type state int 10 | 11 | const ( 12 | invalidState state = -1 13 | 14 | initial state = iota 15 | 16 | selectInitial 17 | selectField 18 | 19 | fromInitial 20 | 21 | operationInitial 22 | leftOperand 23 | operator 24 | rightOperand 25 | 26 | startingInitial 27 | startingAt 28 | 29 | limitInitial 30 | limitSep 31 | limitMax 32 | 33 | rangeMin 34 | rangeAnd 35 | rangeMax 36 | 37 | clauseEnd 38 | end 39 | ) 40 | 41 | // parser is the parser itself 42 | type parser struct { 43 | // the lexer to read tokens 44 | lexer *lexer 45 | // the current state 46 | state state 47 | 48 | // the array of fields, 49 | // before the query is initialized 50 | fields []*Field 51 | 52 | // the query 53 | query *Query 54 | 55 | // the operation context stack 56 | stack []*context 57 | // and the current context 58 | current *context 59 | } 60 | 61 | // the context, that contains informations 62 | // about the current operations 63 | type context struct { 64 | // the current operation 65 | left operand 66 | operator tokenType 67 | right operand 68 | 69 | // the logical operation nodes 70 | first *logicalOperation 71 | last *logicalOperation 72 | 73 | // the current range test, if there's one 74 | rangeTest *rangeTestOperation 75 | 76 | // the current logical operator 77 | logicalOperator tokenType 78 | 79 | // if stacked, in which state 80 | // this context should be restored 81 | // only leftOperand or rightOperand 82 | stackState state 83 | } 84 | 85 | // parserFromString creates a new parser from the given string 86 | func parserFromString(s string) *parser { 87 | return &parser{ 88 | lexer: lexerFromString(s), 89 | state: initial, 90 | fields: make([]*Field, 0), 91 | stack: make([]*context, 0), 92 | current: newContext(), 93 | } 94 | } 95 | 96 | // Parse parses and returns the query 97 | func (p *parser) Parse() (*Query, error) { 98 | 99 | // Read tokens, move step by step 100 | // until the next state is the end 101 | 102 | for p.state != end { 103 | tok, err := p.lexer.NextToken() 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | switch p.state { 109 | 110 | // the very begining 111 | case initial: 112 | p.state, err = p.initialState(tok) 113 | 114 | // SELECT 115 | case selectInitial: 116 | p.state, err = p.selectState(tok) 117 | case selectField: 118 | p.state, err = p.selectFieldState(tok) 119 | 120 | // FROM 121 | case fromInitial: 122 | p.state, err = p.fromState(tok) 123 | 124 | // WHERE 125 | case operationInitial: 126 | p.state, err = p.operationState(tok) 127 | case leftOperand: 128 | p.state, err = p.operandLeftState(tok) 129 | case operator: 130 | p.state, err = p.operatorState(tok) 131 | case rightOperand: 132 | p.state, err = p.operandRightState(tok) 133 | 134 | // range 135 | case rangeMin: 136 | p.state, err = p.rangeMin(tok) 137 | case rangeAnd: 138 | p.state, err = p.expect(tokAnd, tok, rangeMax) 139 | case rangeMax: 140 | p.state, err = p.rangeMax(tok) 141 | 142 | // STARTING 143 | case startingInitial: 144 | p.state, err = p.expect(tokAt, tok, startingAt) 145 | // AT 146 | case startingAt: 147 | p.state, err = p.startingAt(tok) 148 | 149 | // LIMIT N 150 | // ^ 151 | case limitInitial: 152 | p.state, err = p.limit(tok) 153 | 154 | // LIMIT N, M 155 | // ^ 156 | case limitSep: 157 | p.state, err = p.limitSep(tok) 158 | 159 | // LIMIT N, M 160 | // ^ 161 | case limitMax: 162 | p.state, err = p.limitMax(tok) 163 | 164 | case clauseEnd: 165 | p.state, err = p.clauseEnd(tok) 166 | 167 | // unknown 168 | default: 169 | err = fmt.Errorf("Unknown state %d", p.state) 170 | } 171 | 172 | // an error occured during handling the state 173 | if err != nil { 174 | return nil, err 175 | } 176 | } 177 | 178 | // Handle the operation context 179 | 180 | if len(p.stack) > 0 { 181 | return nil, fmt.Errorf("Unbalanced parenthesis") 182 | } 183 | 184 | if p.current.first != nil { 185 | // affect the logical node as the expression 186 | // FIXME should we finish the operation ??? 187 | p.query.setWhere(p.current.first.simplify()) 188 | } 189 | 190 | return p.query, nil 191 | } 192 | 193 | // We’re only waiting for the SELECT keyword 194 | func (p *parser) initialState(tok *token) (state, error) { 195 | if tok.Type != tokSelect { 196 | return unexpected(tok, tokSelect) 197 | } 198 | 199 | return selectInitial, nil 200 | } 201 | 202 | // We’re waiting for a field 203 | func (p *parser) selectState(tok *token) (state, error) { 204 | if tok.Type != tokField { 205 | return unexpected(tok, tokField) 206 | } 207 | 208 | p.fields = append(p.fields, NewField(tok.Value)) 209 | 210 | return selectField, nil 211 | } 212 | 213 | // We’re waiting for a comma, or the FROM keyword 214 | func (p *parser) selectFieldState(tok *token) (state, error) { 215 | if tok.Type == tokComma { 216 | // we just jump to the next field 217 | return selectInitial, nil 218 | } 219 | 220 | if tok.Type != tokFrom { 221 | return unexpected(tok, tokFrom) 222 | } 223 | 224 | return fromInitial, nil 225 | } 226 | 227 | // We’re waiting for a name of the from 228 | func (p *parser) fromState(tok *token) (state, error) { 229 | if tok.Type != tokField { 230 | return unexpected(tok, tokField) 231 | } 232 | 233 | p.query = NewQuery(tok.Value) 234 | p.query.AddFields(p.fields) 235 | p.fields = nil 236 | 237 | return clauseEnd, nil 238 | } 239 | 240 | // We’re waiting for any of the WHERE, STARTING, or LIMIT keywords, or the end 241 | func (p *parser) clauseEnd(tok *token) (state, error) { 242 | switch tok.Type { 243 | case tokEnd: 244 | return end, nil 245 | case tokWhere: 246 | return operationInitial, nil 247 | case tokStarting: 248 | return startingInitial, nil 249 | case tokLimit: 250 | return limitInitial, nil 251 | default: 252 | if p.query != nil && p.query.expression != nil { 253 | if p.query.startingAt == 0 { 254 | return unexpected(tok, tokStarting) 255 | } 256 | 257 | if !p.query.HasLimit() { 258 | return unexpected(tok, tokLimit) 259 | } 260 | } 261 | 262 | return unexpected(tok, tokWhere) 263 | } 264 | } 265 | 266 | func tok2operand(tok *token) (operand, error) { 267 | if tok.isField() { 268 | return NewField(tok.Value), nil 269 | } 270 | if tok.isConst() { 271 | return tok.Const() 272 | } 273 | return nil, fmt.Errorf("Invalid token: not an operand: %v", tok) 274 | } 275 | 276 | // We’re waiting for a left operand, or a ( 277 | func (p *parser) operationState(tok *token) (state, error) { 278 | if tok.Type == tokLeftParenthesis { 279 | // push the context and start a new operation 280 | p.pushContext(leftOperand) 281 | return operationInitial, nil 282 | } 283 | 284 | c, err := tok2operand(tok) 285 | if err != nil { 286 | return invalidState, err 287 | } 288 | p.current.left = c 289 | 290 | // the left operand has been setted jump to the left operand state 291 | return leftOperand, nil 292 | } 293 | 294 | // We’re waiting for an operator: 295 | // - logical operator, we step to the next operation 296 | // - comparison operator, we continue to the operator state 297 | // - "BETWEEN", we continue to the range operator state 298 | // 299 | // We can encounter a ), or the end 300 | func (p *parser) operandLeftState(tok *token) (state, error) { 301 | 302 | if tok.isEnd() { 303 | // end the previous operation 304 | if err := p.current.endOperation(); err != nil { 305 | return invalidState, err 306 | } 307 | 308 | return end, nil 309 | } 310 | 311 | switch tok.Type { 312 | case tokStarting: 313 | return startingInitial, nil 314 | case tokLimit: 315 | return limitInitial, nil 316 | } 317 | 318 | if tok.isLogicalOperator() { 319 | 320 | // this is the end of the previous operation 321 | if err := p.current.endOperation(); err != nil { 322 | return invalidState, err 323 | } 324 | 325 | // set this operator and jump to a new operation 326 | p.current.logicalOperator = tok.Type 327 | return operationInitial, nil 328 | 329 | } 330 | 331 | if tok.isComparisonOperator() { 332 | 333 | // set the operator and jump to the operator state 334 | p.current.operator = tok.Type 335 | return operator, nil 336 | } 337 | 338 | // we close a context, pop it 339 | if tok.Type == tokRightParenthesis { 340 | return p.popContext() 341 | } 342 | 343 | if tok.Type == tokBetween { 344 | p.current.rangeTest = &rangeTestOperation{ 345 | test: p.current.left, 346 | } 347 | p.current.left = p.current.rangeTest 348 | 349 | return rangeMin, nil 350 | } 351 | 352 | return unexpected(tok, tokInvalid) 353 | } 354 | 355 | func (p *parser) rangeMin(tok *token) (state, error) { 356 | c, err := tok2operand(tok) 357 | if err != nil { 358 | return invalidState, err 359 | } 360 | 361 | if p.current.rangeTest == nil { 362 | return invalidState, errors.New("Nil range test") 363 | } 364 | 365 | p.current.rangeTest.min = c 366 | 367 | return rangeAnd, nil 368 | } 369 | 370 | func (p *parser) rangeMax(tok *token) (state, error) { 371 | c, err := tok2operand(tok) 372 | if err != nil { 373 | return invalidState, err 374 | } 375 | 376 | if p.current.rangeTest == nil { 377 | return invalidState, errors.New("Nil range test") 378 | } 379 | 380 | p.current.rangeTest.max = c 381 | 382 | return rightOperand, nil 383 | } 384 | 385 | // We're waiting for a right operand, nothing else, or a ( 386 | func (p *parser) operatorState(tok *token) (state, error) { 387 | 388 | // handle the ( 389 | if tok.Type == tokLeftParenthesis { 390 | // we push the state 391 | p.pushContext(rightOperand) 392 | // jump the the start of an operation 393 | return operationInitial, nil 394 | } 395 | 396 | if tok.isField() { 397 | p.current.right = NewField(tok.Value) 398 | } else if tok.isConst() { 399 | c, err := tok.Const() 400 | if err != nil { 401 | return invalidState, err 402 | } 403 | p.current.right = c 404 | } else { 405 | return unexpected(tok, tokInvalid) 406 | } 407 | 408 | // the right operand has been setted jump to the left operand state 409 | return rightOperand, nil 410 | } 411 | 412 | // We're waiting for a logical operator, ), or the end 413 | func (p *parser) operandRightState(tok *token) (state, error) { 414 | 415 | // end of a context 416 | // we pop, and continue to the correct state 417 | // (the pop will handle the end of the operation) 418 | if tok.Type == tokRightParenthesis { 419 | return p.popContext() 420 | } 421 | 422 | // in all following case we have to end the current operation 423 | if err := p.current.endOperation(); err != nil { 424 | return invalidState, err 425 | } 426 | 427 | switch tok.Type { 428 | case tokStarting: 429 | return startingInitial, nil 430 | case tokLimit: 431 | return limitInitial, nil 432 | } 433 | 434 | // the end of the expression 435 | if tok.isEnd() { 436 | return end, nil 437 | } 438 | 439 | // we continue to chain 440 | if tok.isLogicalOperator() { 441 | 442 | // set this operator and jump to a new operation 443 | p.current.logicalOperator = tok.Type 444 | return operationInitial, nil 445 | } 446 | 447 | return unexpected(tok, tokInvalid) 448 | } 449 | 450 | func (p *parser) startingInitial(tok *token) (state, error) { 451 | return p.expect(tokAt, tok, startingAt) 452 | } 453 | 454 | func (p *parser) startingAt(tok *token) (state, error) { 455 | 456 | if tok.isNumeric() { 457 | c, err := tok.Const() 458 | if err != nil { 459 | return invalidState, err 460 | } 461 | 462 | p.query.startingAt = c.AsInt() 463 | 464 | return clauseEnd, nil 465 | } 466 | 467 | return unexpected(tok, tokInt) 468 | } 469 | 470 | func (p *parser) limit(tok *token) (state, error) { 471 | if tok.isNumeric() { 472 | c, err := tok.Const() 473 | if err != nil { 474 | return invalidState, err 475 | } 476 | 477 | p.query.setLimit(c.AsInt()) 478 | 479 | return limitSep, nil 480 | } 481 | 482 | return unexpected(tok, tokInt) 483 | } 484 | 485 | func (p *parser) limitSep(tok *token) (state, error) { 486 | if tok.Type == tokComma { 487 | return limitMax, nil 488 | } 489 | 490 | return p.clauseEnd(tok) 491 | } 492 | 493 | func (p *parser) limitMax(tok *token) (state, error) { 494 | if tok.isNumeric() { 495 | c, err := tok.Const() 496 | if err != nil { 497 | return invalidState, err 498 | } 499 | 500 | p.query.startingAt = p.query.Limit() 501 | p.query.setLimit(c.AsInt()) 502 | 503 | return limitSep, nil 504 | } 505 | 506 | return unexpected(tok, tokInt) 507 | } 508 | 509 | func (p *parser) expect(expected tokenType, tok *token, state state) (state, error) { 510 | if tok.Type != expected { 511 | return unexpected(tok, expected) 512 | } 513 | return state, nil 514 | } 515 | 516 | // Push the context and create a new one 517 | func (p *parser) pushContext(state state) { 518 | // stack the context 519 | p.current.stackState = state 520 | p.stack = append(p.stack, p.current) 521 | 522 | // creates a new one and reset it 523 | p.current = newContext() 524 | } 525 | 526 | // End the current operation, and continue to the correct state (given on push) 527 | func (p *parser) popContext() (state, error) { 528 | 529 | // first we end the operation 530 | if err := p.current.endOperation(); err != nil { 531 | return invalidState, err 532 | } 533 | 534 | // creates a group operand 535 | g, err := newGroupOperand(p.current.first.simplify()) 536 | if err != nil { 537 | return invalidState, err 538 | } 539 | 540 | // pop 541 | l := len(p.stack) 542 | p.current = p.stack[l-1] 543 | p.stack = p.stack[:l-1] 544 | 545 | // according the stack state, put the group it on the correct side 546 | switch p.current.stackState { 547 | case leftOperand: 548 | p.current.left = g 549 | case rightOperand: 550 | p.current.right = g 551 | } 552 | 553 | // clear the current stackState 554 | s := p.current.stackState 555 | p.current.stackState = -1 556 | 557 | // return the next state 558 | return s, nil 559 | } 560 | 561 | // Creates a new and fresh context 562 | func newContext() *context { 563 | return &context{ 564 | operator: tokInvalid, 565 | logicalOperator: tokInvalid, 566 | stackState: invalidState, 567 | } 568 | } 569 | 570 | // End the current operation 571 | // wrap the current operation into a simple LogicalOperation, and chain it 572 | func (c *context) endOperation() error { 573 | 574 | if c.left == nil { 575 | return errors.New("Empty operation") 576 | } 577 | 578 | // Creates the operand 579 | 580 | var op operand 581 | var err error 582 | 583 | if c.rangeTest != nil { 584 | op = c.rangeTest 585 | } 586 | 587 | if c.operator != tokInvalid { 588 | op, err = newComparison(c.left, operatorTypeFromTokenType(c.operator), c.right) 589 | if err != nil { 590 | return err 591 | } 592 | } else { 593 | op = c.left 594 | } 595 | 596 | // reset the operation 597 | c.left = nil 598 | c.right = nil 599 | c.operator = tokInvalid 600 | 601 | // Creates a simple logical operation and chain it 602 | 603 | lo, err := newLeftLogicalOperation(op) 604 | if err != nil { 605 | return err 606 | } 607 | 608 | if c.logicalOperator == tokInvalid { 609 | 610 | if c.first != nil || c.last != nil { 611 | return errors.New("Unexpected state, the logical operations should be not initialized") 612 | } 613 | 614 | c.first = lo 615 | c.last = lo 616 | 617 | return nil 618 | } 619 | 620 | if c.first == nil || c.last == nil { 621 | return errors.New("Unexpected state, no logical operation initialized") 622 | } 623 | 624 | // define how to chain according the logical operator 625 | switch c.logicalOperator { 626 | case tokAnd: 627 | c.last.chain(operatorTypeFromTokenType(c.logicalOperator), lo) 628 | c.last = lo 629 | case tokOr: 630 | c.first, err = newLeftLogicalOperation(c.first) 631 | if err != nil { 632 | return err 633 | } 634 | 635 | c.first.chain(operatorTypeFromTokenType(c.logicalOperator), lo) 636 | c.last = lo 637 | } 638 | 639 | // reset the logical operator 640 | c.logicalOperator = tokInvalid 641 | 642 | return nil 643 | } 644 | 645 | // Helper to creates the unexpected error 646 | func unexpected(tok *token, expected tokenType) (state, error) { 647 | 648 | if expected != tokInvalid { 649 | 650 | if tok.isEnd() { 651 | return invalidState, fmt.Errorf( 652 | "Expected '%s' at position %d, got end of stream", 653 | expected, tok.Pos) 654 | } 655 | 656 | return invalidState, fmt.Errorf( 657 | "Expected '%s' at position %d, got '%s'", 658 | expected, tok.Pos, tok.Value) 659 | } 660 | 661 | if tok.isEnd() { 662 | return invalidState, fmt.Errorf( 663 | "Unexpected end of stream at pos %d", tok.Pos) 664 | } 665 | 666 | return invalidState, fmt.Errorf( 667 | "Unexpected '%s' at pos %d", tok.Value, tok.Pos) 668 | } 669 | -------------------------------------------------------------------------------- /parser_test.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func okQuery(t *testing.T, q string) { 10 | qry, err := parserFromString(q).Parse() 11 | require.Nil(t, err, "There should be no error parsing '%s'", q) 12 | require.NotNil(t, qry) 13 | } 14 | 15 | func TestParserParseLimit(t *testing.T) { 16 | for _, s := range []string{ 17 | "SELECT x FROM y limit 42", 18 | "SELECT x FROM y starting at 3 limit 42", 19 | "SELECT x FROM y limit 42 starting at 3", 20 | "SELECT x FROM y WHERE z limit 42", 21 | "SELECT x FROM y WHERE z = 2 limit 42", 22 | "SELECT x FROM y WHERE (z = 2) limit 42", 23 | "SELECT x FROM y WHERE (z = 2 && 43) limit 42", 24 | "SELECT x FROM y WHERE z starting at 2 limit 42", 25 | "SELECT x FROM y WHERE z limit 42 starting at 2", 26 | } { 27 | okQuery(t, s) 28 | } 29 | } 30 | 31 | func TestParserParseLimitWithOffset(t *testing.T) { 32 | for _, s := range []string{ 33 | "SELECT x FROM y limit 0, 42", 34 | "SELECT x FROM y starting at 3 limit 17, 42", 35 | "SELECT x FROM y limit 42, 78 starting at 3", 36 | "SELECT x FROM y WHERE z limit 1, 42", 37 | "SELECT x FROM y WHERE z = 2 limit 4, 42", 38 | "SELECT x FROM y WHERE (z = 2) limit 45, 42", 39 | "SELECT x FROM y WHERE (z = 2 && 43) limit 2, 42", 40 | "SELECT x FROM y WHERE z starting at 2 limit 3, 42", 41 | "SELECT x FROM y WHERE z limit 44,2 starting at 45", 42 | } { 43 | okQuery(t, s) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /query.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import "bytes" 4 | 5 | // Query is a query 6 | type Query struct { 7 | // the fields to select if condition match the object 8 | fields []*Field 9 | // the resource from wich we want to evaluate and select fields 10 | from string 11 | // the expression to evaluate on each record. The resulting constant will 12 | // always be converted as a bool 13 | expression operand 14 | // the record index to start from 15 | startingAt int64 16 | // the record index to stop at 17 | limit *int64 18 | } 19 | 20 | // A Record is a record 21 | type Record interface { 22 | Find(*Field) (*Const, error) 23 | } 24 | 25 | // QueryFromString creates a query from the given string 26 | func QueryFromString(s string) (*Query, error) { 27 | return parserFromString(s).Parse() 28 | } 29 | 30 | // NewQuery creates a new query with the given from part 31 | func NewQuery(from string) *Query { 32 | return &Query{from: from} 33 | } 34 | 35 | // From returns the FROM part of this query 36 | func (q *Query) From() string { 37 | return q.from 38 | } 39 | 40 | // StartingAt returns the 'STARTING AT' part of the query, or 0 if it's not 41 | // present 42 | func (q *Query) StartingAt() int64 { 43 | return q.startingAt 44 | } 45 | 46 | // HasLimit tests if the query has a limit 47 | func (q *Query) HasLimit() bool { 48 | return q.limit != nil 49 | } 50 | 51 | // Limit returns the 'LIMIT' part of the query, or 0 if it's not present 52 | func (q *Query) Limit() int64 { 53 | if q.limit == nil { 54 | return 0 55 | } 56 | return *q.limit 57 | } 58 | 59 | // AddField adds one field 60 | func (q *Query) AddField(field *Field) { 61 | if field != nil { 62 | q.fields = append(q.fields, field) 63 | } 64 | } 65 | 66 | // AddFields adds multiple fields 67 | func (q *Query) AddFields(fields []*Field) { 68 | for _, field := range fields { 69 | q.AddField(field) 70 | } 71 | } 72 | 73 | // Fields returns the fields 74 | func (q *Query) Fields() []*Field { 75 | return q.fields 76 | } 77 | 78 | // setWhere sets the where condition 79 | func (q *Query) setWhere(op operand) { 80 | 81 | if op == nil { 82 | return 83 | } 84 | 85 | q.expression = op 86 | } 87 | 88 | func (q *Query) setLimit(limit int64) { 89 | q.limit = &limit 90 | } 91 | 92 | // FieldsValues extracts the values of each fields into the given record 93 | // Note that you should evaluate the query first 94 | func (q *Query) FieldsValues(record Record) ([]*Const, error) { 95 | values := make([]*Const, len(q.fields)) 96 | 97 | for i, field := range q.fields { 98 | 99 | value, err := field.Evaluate(record) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | values[i] = value 105 | } 106 | 107 | return values, nil 108 | } 109 | 110 | // Evaluate evaluates the query against the given record 111 | func (q *Query) Evaluate(record Record) (bool, error) { 112 | 113 | // no expression, always valid 114 | if q.expression == nil { 115 | return true, nil 116 | } 117 | 118 | constant, err := q.expression.Evaluate(record) 119 | if err != nil { 120 | return false, err 121 | } 122 | 123 | return constant.AsBool(), nil 124 | } 125 | 126 | // String returns a string representation of this query 127 | func (q *Query) String() string { 128 | var buffer bytes.Buffer 129 | 130 | buffer.WriteString("SELECT ") 131 | 132 | for i, field := range q.fields { 133 | if i > 0 { 134 | buffer.WriteString(", ") 135 | } 136 | buffer.WriteString(field.Name()) 137 | } 138 | 139 | buffer.WriteString(" FROM ") 140 | buffer.WriteString(q.from) 141 | 142 | if q.expression != nil { 143 | buffer.WriteString(" WHERE ") 144 | buffer.WriteString(q.expression.String()) 145 | } 146 | 147 | return buffer.String() 148 | } 149 | -------------------------------------------------------------------------------- /query_test.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "errors" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | type dummyPerson struct { 13 | name string 14 | age int 15 | } 16 | 17 | func (d *dummyPerson) Find(f *Field) (*Const, error) { 18 | switch f.Name() { 19 | case "name": 20 | return StringConst(d.name), nil 21 | case "age": 22 | return IntConst(int64(d.age)), nil 23 | default: 24 | return nil, errors.New("wrong field") 25 | } 26 | } 27 | 28 | var _ Record = &dummyPerson{} 29 | 30 | func TestQueryInRange(t *testing.T) { 31 | q := &Query{ 32 | fields: []*Field{ 33 | &Field{"name"}, 34 | }, 35 | expression: &rangeTestOperation{ 36 | test: Field{"age"}, 37 | min: IntConst(10), 38 | max: IntConst(20), 39 | }, 40 | } 41 | 42 | m, err := q.Evaluate(&dummyPerson{name: "A", age: 15}) 43 | assert.Nil(t, err) 44 | assert.True(t, m) 45 | 46 | m, err = q.Evaluate(&dummyPerson{name: "A", age: 25}) 47 | assert.Nil(t, err) 48 | assert.False(t, m) 49 | 50 | vals, err := q.FieldsValues(&dummyPerson{name: "A", age: 12}) 51 | assert.Nil(t, err) 52 | assert.Equal(t, 1, len(vals)) 53 | assert.Equal(t, "A", vals[0].AsString()) 54 | } 55 | 56 | func TestQueryFromStringInRange(t *testing.T) { 57 | q, err := QueryFromString("SELECT a FROM b WHERE a BETWEEN 1 AND 2") 58 | require.Nil(t, err) 59 | assert.NotNil(t, q) 60 | assert.NotNil(t, q.fields) 61 | assert.NotNil(t, q.expression) 62 | 63 | assert.Equal(t, "b", q.from) 64 | assert.Equal(t, int64(0), q.startingAt) 65 | 66 | assert.Equal(t, 1, len(q.fields)) 67 | assert.NotNil(t, q.fields[0]) 68 | assert.Equal(t, "a", q.fields[0].name) 69 | 70 | expr, ok := q.expression.(*rangeTestOperation) 71 | require.True(t, ok, "%v should be a range test", q.expression) 72 | 73 | assert.NotNil(t, expr.test) 74 | assert.NotNil(t, expr.min) 75 | assert.NotNil(t, expr.max) 76 | 77 | f, ok := expr.test.(*Field) 78 | require.True(t, ok) 79 | assert.Equal(t, "a", f.name) 80 | 81 | min, ok := expr.min.(*Const) 82 | require.True(t, ok, "%v should be a const (not %s)", 83 | expr.min, reflect.TypeOf(expr.min)) 84 | assert.True(t, min.IsNumeric()) 85 | assert.Equal(t, int64(1), min.AsInt()) 86 | 87 | max, ok := expr.max.(*Const) 88 | require.True(t, ok, "%v should be a const (not %s)", 89 | expr.min, reflect.TypeOf(expr.max)) 90 | assert.True(t, max.IsNumeric()) 91 | assert.Equal(t, int64(2), max.AsInt()) 92 | } 93 | 94 | func TestQueryHasLimit(t *testing.T) { 95 | q := NewQuery("foo") 96 | assert.False(t, q.HasLimit()) 97 | assert.Equal(t, int64(0), q.Limit()) 98 | 99 | q.setLimit(42) 100 | assert.True(t, q.HasLimit()) 101 | assert.Equal(t, int64(42), q.Limit()) 102 | } 103 | 104 | func TestQueryFromStringLimitOffset(t *testing.T) { 105 | q, err := QueryFromString("SELECT a FROM b WHERE a LIMIT 40, 100") 106 | require.Nil(t, err) 107 | assert.NotNil(t, q) 108 | 109 | assert.Equal(t, int64(40), q.startingAt) 110 | assert.True(t, q.HasLimit()) 111 | assert.Equal(t, int64(100), q.Limit()) 112 | } 113 | -------------------------------------------------------------------------------- /record/csv.go: -------------------------------------------------------------------------------- 1 | // Package record provides records for the charlatan package 2 | package record 3 | 4 | import ( 5 | "fmt" 6 | "strconv" 7 | 8 | ch "github.com/BatchLabs/charlatan" 9 | ) 10 | 11 | // CSVRecord is a CSV record, i.e. a line from a CSV file 12 | // 13 | // If created with a header its fields can be retrieved with their column name. 14 | // In any case, one can use a "$N" field name, where N is the column index, 15 | // starting at 0. 16 | // 17 | // All values are retrieved as strings, and the special field "*" can be used 18 | // to get a string representation of the record values. 19 | type CSVRecord struct { 20 | header, record []string 21 | } 22 | 23 | var _ ch.Record = &CSVRecord{} 24 | 25 | // NewCSVRecord returns a new CSVRecord 26 | func NewCSVRecord(record []string) *CSVRecord { 27 | return &CSVRecord{record: record} 28 | } 29 | 30 | // NewCSVRecordWithHeader returns a new CSVRecord with the given header 31 | func NewCSVRecordWithHeader(record []string, header []string) *CSVRecord { 32 | return &CSVRecord{header: header, record: record} 33 | } 34 | 35 | // Find implements the charlatan.Record interface 36 | func (r *CSVRecord) Find(field *ch.Field) (*ch.Const, error) { 37 | 38 | name := field.Name() 39 | 40 | if name == "*" { 41 | return ch.StringConst(fmt.Sprintf("%v", r.record)), nil 42 | } 43 | 44 | // Column index 45 | 46 | if name[0] == '$' { 47 | index, err := strconv.ParseInt(name[1:], 10, 64) 48 | if err != nil { 49 | return nil, fmt.Errorf("Invalid column index %s: %s", name, err) 50 | } 51 | 52 | return r.AtIndex(int(index)) 53 | } 54 | 55 | // Column name 56 | 57 | index := r.ColumnNameIndex(name) 58 | if index < 0 { 59 | return nil, fmt.Errorf("Can't find field name: %s", name) 60 | } 61 | 62 | return r.AtIndex(index) 63 | } 64 | 65 | // AtIndex gets the value at the given index 66 | func (r *CSVRecord) AtIndex(index int) (*ch.Const, error) { 67 | 68 | if index < 0 || index > len(r.record) { 69 | return nil, fmt.Errorf("index out of bounds %d", index) 70 | } 71 | 72 | // FIXME should we accept NULL values? 73 | value := r.record[index] 74 | if value == "NULL" { 75 | return ch.NullConst(), nil 76 | } 77 | 78 | return ch.ConstFromString(value), nil 79 | } 80 | 81 | // ColumnNameIndex searches the index of the column name in this record’s 82 | // header. If it doesn’t have a header or if the column wasn’t found, the 83 | // method returns -1. The column name match is case-sensitive, the first 84 | // matching one is used. 85 | func (r *CSVRecord) ColumnNameIndex(name string) int { 86 | for index, element := range r.header { 87 | if element == name { 88 | return index 89 | } 90 | } 91 | 92 | return -1 93 | } 94 | -------------------------------------------------------------------------------- /record/csv_test.go: -------------------------------------------------------------------------------- 1 | package record 2 | 3 | import ( 4 | "testing" 5 | 6 | ch "github.com/BatchLabs/charlatan" 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestNewCSVRecordWithoutHeader(t *testing.T) { 12 | c := NewCSVRecord([]string{"a", "b", "c"}) 13 | require.NotNil(t, c) 14 | assert.Nil(t, c.header) 15 | } 16 | 17 | func TestFindNameWithoutHeader(t *testing.T) { 18 | c := NewCSVRecord([]string{"a", "b", "c"}) 19 | require.NotNil(t, c) 20 | 21 | _, err := c.Find(ch.NewField("foo")) 22 | assert.NotNil(t, err) 23 | } 24 | 25 | func TestFindColumnIndex(t *testing.T) { 26 | c := NewCSVRecord([]string{"a", "b", "c"}) 27 | require.NotNil(t, c) 28 | 29 | _, err := c.Find(ch.NewField("$-1")) 30 | assert.NotNil(t, err) 31 | 32 | _, err = c.Find(ch.NewField("$42")) 33 | assert.NotNil(t, err) 34 | 35 | v, err := c.Find(ch.NewField("$1")) 36 | assert.Nil(t, err) 37 | assert.True(t, v.IsString()) 38 | assert.Equal(t, "b", v.AsString()) 39 | } 40 | 41 | func TestFindColumnName(t *testing.T) { 42 | c := NewCSVRecordWithHeader([]string{"a", "b", "c"}, []string{"id", "x", "y"}) 43 | require.NotNil(t, c) 44 | 45 | _, err := c.Find(ch.NewField("yo")) 46 | assert.NotNil(t, err) 47 | 48 | _, err = c.Find(ch.NewField("xy")) 49 | assert.NotNil(t, err) 50 | 51 | v, err := c.Find(ch.NewField("y")) 52 | assert.Nil(t, err) 53 | assert.Equal(t, "c", v.AsString()) 54 | } 55 | 56 | func TestFindStar(t *testing.T) { 57 | c := NewCSVRecord([]string{"x", "y", "z"}) 58 | require.NotNil(t, c) 59 | 60 | v, err := c.Find(ch.NewField("*")) 61 | assert.Nil(t, err) 62 | assert.True(t, v.IsString()) 63 | assert.Equal(t, "[x y z]", v.AsString()) 64 | } 65 | -------------------------------------------------------------------------------- /record/json.go: -------------------------------------------------------------------------------- 1 | package record 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "strings" 8 | 9 | ch "github.com/BatchLabs/charlatan" 10 | ) 11 | 12 | // JSONRecord is a record for JSON objects. 13 | // 14 | // It supports the special field "*", as in "SELECT * FROM x WHERE y", which 15 | // returns the JSON as-is, except that the keys order is not garanteed. 16 | // 17 | // If the SoftMatching attribute is set to true, non-existing fields are 18 | // returned as null contants instead of failing with an error. 19 | type JSONRecord struct { 20 | attrs map[string]*json.RawMessage 21 | SoftMatching bool 22 | } 23 | 24 | var _ ch.Record = &JSONRecord{} 25 | 26 | var errEmptyField = errors.New("Empty field name") 27 | 28 | // NewJSONRecordFromDecoder creates a new JSONRecord from a JSON decoder 29 | func NewJSONRecordFromDecoder(dec *json.Decoder) (*JSONRecord, error) { 30 | attrs := make(map[string]*json.RawMessage) 31 | 32 | if err := dec.Decode(&attrs); err != nil { 33 | return nil, err 34 | } 35 | 36 | return &JSONRecord{attrs: attrs}, nil 37 | } 38 | 39 | // Find implements the charlatan.Record interface 40 | func (r *JSONRecord) Find(field *ch.Field) (*ch.Const, error) { 41 | var ok bool 42 | var partial *json.RawMessage 43 | var name string 44 | 45 | if name = field.Name(); len(name) == 0 { 46 | return nil, errEmptyField 47 | } 48 | 49 | // support for "SELECT *" 50 | if name == "*" { 51 | b, err := json.Marshal(r.attrs) 52 | if err != nil { 53 | return nil, err 54 | } 55 | return ch.StringConst(string(b)), nil 56 | } 57 | 58 | attrs := r.attrs 59 | parts := strings.Split(name, ".") 60 | 61 | for i, k := range parts { 62 | partial, ok = attrs[k] 63 | 64 | if !ok { 65 | if r.SoftMatching { 66 | return ch.NullConst(), nil 67 | } 68 | 69 | return nil, fmt.Errorf("Unknown '%s' field (in '%s')", k, field.Name()) 70 | } 71 | 72 | // update the attrs if we need to go deeper 73 | if i < len(parts)-1 { 74 | attrs = make(map[string]*json.RawMessage) 75 | if err := json.Unmarshal(*partial, &attrs); err != nil { 76 | if r.SoftMatching { 77 | return ch.NullConst(), nil 78 | } 79 | return nil, err 80 | } 81 | } 82 | } 83 | 84 | return jsonToConst(partial) 85 | } 86 | 87 | func jsonToConst(partial *json.RawMessage) (*ch.Const, error) { 88 | var value string 89 | 90 | if partial == nil { 91 | return ch.NullConst(), nil 92 | } 93 | 94 | asString := string(*partial) 95 | 96 | if asString == "null" { 97 | return ch.NullConst(), nil 98 | } 99 | 100 | if err := json.Unmarshal(*partial, &value); err != nil { 101 | if err, ok := err.(*json.UnmarshalTypeError); ok { 102 | // we failed to unmarshal into a string, let's try the other types 103 | switch err.Value { 104 | case "number": 105 | var n json.Number 106 | if err := json.Unmarshal(*partial, &n); err != nil { 107 | return nil, err 108 | } 109 | 110 | value = n.String() 111 | 112 | case "bool", "object", "array": 113 | value = asString 114 | 115 | default: 116 | return nil, err 117 | } 118 | } 119 | } 120 | 121 | return ch.ConstFromString(value), nil 122 | } 123 | -------------------------------------------------------------------------------- /record/json_test.go: -------------------------------------------------------------------------------- 1 | package record 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "strings" 7 | "testing" 8 | 9 | ch "github.com/BatchLabs/charlatan" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func testJSONDecoder() *json.Decoder { 16 | return json.NewDecoder(strings.NewReader(` 17 | { 18 | "name": "Michel", 19 | "b": true, 20 | "age": 92, 21 | "n": null, 22 | "a": [], 23 | "we":{"need": {"to": {"go": {"deeper": 1, "a": "d"}}}} 24 | } 25 | `)) 26 | } 27 | 28 | func TestFindUnexistingField(t *testing.T) { 29 | r, err := NewJSONRecordFromDecoder(testJSONDecoder()) 30 | require.Nil(t, err) 31 | require.NotNil(t, r) 32 | 33 | _, err = r.Find(ch.NewField("yolo")) 34 | assert.NotNil(t, err) 35 | } 36 | 37 | func TestFindArrayField(t *testing.T) { 38 | r, err := NewJSONRecordFromDecoder(testJSONDecoder()) 39 | require.Nil(t, err) 40 | require.NotNil(t, r) 41 | 42 | c, err := r.Find(ch.NewField("a")) 43 | require.Nil(t, err) 44 | require.NotNil(t, c) 45 | 46 | require.True(t, c.IsString()) 47 | require.Equal(t, `[]`, c.AsString()) 48 | } 49 | 50 | func TestFindObjectField(t *testing.T) { 51 | r, err := NewJSONRecordFromDecoder(testJSONDecoder()) 52 | require.Nil(t, err) 53 | require.NotNil(t, r) 54 | 55 | c, err := r.Find(ch.NewField("we")) 56 | require.Nil(t, err) 57 | require.NotNil(t, c) 58 | 59 | require.True(t, c.IsString()) 60 | require.Equal(t, `{"need": {"to": {"go": {"deeper": 1, "a": "d"}}}}`, c.AsString()) 61 | } 62 | 63 | func TestFindTopLevelStringField(t *testing.T) { 64 | r, err := NewJSONRecordFromDecoder(testJSONDecoder()) 65 | require.Nil(t, err) 66 | require.NotNil(t, r) 67 | 68 | c, err := r.Find(ch.NewField("name")) 69 | require.Nil(t, err) 70 | require.NotNil(t, c) 71 | 72 | assert.True(t, c.IsString()) 73 | assert.Equal(t, "Michel", c.AsString()) 74 | } 75 | 76 | func TestFindTopLevelIntField(t *testing.T) { 77 | r, err := NewJSONRecordFromDecoder(testJSONDecoder()) 78 | require.Nil(t, err) 79 | require.NotNil(t, r) 80 | 81 | c, err := r.Find(ch.NewField("age")) 82 | require.Nil(t, err) 83 | require.NotNil(t, c) 84 | 85 | assert.True(t, c.IsNumeric()) 86 | assert.Equal(t, int64(92), c.AsInt()) 87 | } 88 | 89 | func TestFindTopLevelBoolField(t *testing.T) { 90 | r, err := NewJSONRecordFromDecoder(testJSONDecoder()) 91 | require.Nil(t, err) 92 | require.NotNil(t, r) 93 | 94 | c, err := r.Find(ch.NewField("b")) 95 | require.Nil(t, err) 96 | require.NotNil(t, c) 97 | 98 | assert.True(t, c.IsBool()) 99 | assert.Equal(t, true, c.AsBool()) 100 | } 101 | 102 | func TestFindTopLevelNullField(t *testing.T) { 103 | r, err := NewJSONRecordFromDecoder(testJSONDecoder()) 104 | require.Nil(t, err) 105 | require.NotNil(t, r) 106 | 107 | c, err := r.Find(ch.NewField("n")) 108 | require.Nil(t, err) 109 | require.NotNil(t, c) 110 | 111 | assert.True(t, c.IsNull()) 112 | } 113 | 114 | func TestFindTopLevelEmptyStringField(t *testing.T) { 115 | r, err := NewJSONRecordFromDecoder(json.NewDecoder(strings.NewReader(`{"foo": ""}`))) 116 | require.Nil(t, err) 117 | require.NotNil(t, r) 118 | 119 | c, err := r.Find(ch.NewField("foo")) 120 | require.Nil(t, err) 121 | require.NotNil(t, c) 122 | 123 | assert.False(t, c.IsNull()) 124 | assert.True(t, c.IsString()) 125 | } 126 | 127 | func TestFindDeepStringField(t *testing.T) { 128 | r, err := NewJSONRecordFromDecoder(testJSONDecoder()) 129 | require.Nil(t, err) 130 | require.NotNil(t, r) 131 | 132 | c, err := r.Find(ch.NewField("we.need.to.go.deeper")) 133 | require.Nil(t, err) 134 | require.NotNil(t, c) 135 | 136 | assert.True(t, c.IsNumeric()) 137 | assert.Equal(t, int64(1), c.AsInt()) 138 | } 139 | 140 | func TestJSONDecoderMultipleRecords(t *testing.T) { 141 | r := json.NewDecoder(strings.NewReader(` 142 | {"age": 42} 143 | {"age": 19} 144 | `)) 145 | require.NotNil(t, r) 146 | 147 | _, err := NewJSONRecordFromDecoder(r) 148 | require.Nil(t, err) 149 | 150 | _, err = NewJSONRecordFromDecoder(r) 151 | require.Nil(t, err) 152 | 153 | _, err = NewJSONRecordFromDecoder(r) 154 | assert.Equal(t, io.EOF, err) 155 | } 156 | 157 | func TestJSONRecordSelectStar(t *testing.T) { 158 | r := json.NewDecoder(strings.NewReader(`{"foo": 42}`)) 159 | require.NotNil(t, r) 160 | 161 | rec, err := NewJSONRecordFromDecoder(r) 162 | require.Nil(t, err) 163 | require.NotNil(t, rec) 164 | 165 | all, err := rec.Find(ch.NewField("*")) 166 | require.Nil(t, err) 167 | require.NotNil(t, all) 168 | 169 | require.True(t, all.IsString()) 170 | require.Equal(t, `{"foo":42}`, all.AsString()) 171 | } 172 | 173 | func TestJSONRecordTopLevelSoftFind(t *testing.T) { 174 | rec, err := NewJSONRecordFromDecoder(json.NewDecoder(strings.NewReader(`{"foo":42}`))) 175 | require.Nil(t, err) 176 | require.NotNil(t, rec) 177 | 178 | _, err = rec.Find(ch.NewField("yo")) 179 | assert.NotNil(t, err) 180 | 181 | rec.SoftMatching = true 182 | v, err := rec.Find(ch.NewField("yo")) 183 | assert.Nil(t, err) 184 | assert.True(t, v.IsNull()) 185 | } 186 | 187 | func TestJSONRecordSoftFindNotAnObject(t *testing.T) { 188 | rec, err := NewJSONRecordFromDecoder(json.NewDecoder(strings.NewReader(`{"foo":42}`))) 189 | require.Nil(t, err) 190 | require.NotNil(t, rec) 191 | 192 | _, err = rec.Find(ch.NewField("foo.bar.qux")) 193 | assert.NotNil(t, err) 194 | 195 | rec.SoftMatching = true 196 | v, err := rec.Find(ch.NewField("foo.bar.qux")) 197 | assert.Nil(t, err) 198 | require.NotNil(t, v) 199 | assert.True(t, v.IsNull()) 200 | } 201 | 202 | func TestJSONRecordSoftFind(t *testing.T) { 203 | rec, err := NewJSONRecordFromDecoder(json.NewDecoder(strings.NewReader(` 204 | {"foo": {"a": 2}} 205 | `))) 206 | require.Nil(t, err) 207 | require.NotNil(t, rec) 208 | 209 | _, err = rec.Find(ch.NewField("foo.bar.qux")) 210 | assert.NotNil(t, err) 211 | 212 | rec.SoftMatching = true 213 | v, err := rec.Find(ch.NewField("foo.bar.qux")) 214 | assert.Nil(t, err) 215 | require.NotNil(t, v) 216 | assert.True(t, v.IsNull()) 217 | } 218 | -------------------------------------------------------------------------------- /samples/csv/csv.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "io" 7 | "os" 8 | 9 | "github.com/BatchLabs/charlatan" 10 | "github.com/BatchLabs/charlatan/record" 11 | ) 12 | 13 | func main() { 14 | 15 | if len(os.Args) != 2 { 16 | usage() 17 | return 18 | } 19 | 20 | s := os.Args[1] 21 | 22 | // Creates the query 23 | 24 | query, err := charlatan.QueryFromString(s) 25 | if err != nil { 26 | fmt.Println(">>> ", err) 27 | return 28 | } 29 | 30 | reader, err := os.Open(query.From()) 31 | if err != nil { 32 | fmt.Printf(">>> Error opening %s: %v\n", query.From(), err) 33 | return 34 | } 35 | 36 | executeRequest(csv.NewReader(reader), query) 37 | } 38 | 39 | func executeRequest(reader *csv.Reader, query *charlatan.Query) { 40 | 41 | fmt.Println("$ ", query) 42 | fmt.Println("$") 43 | 44 | var header []string 45 | var limit int64 46 | 47 | hasLimit := query.HasLimit() 48 | 49 | if hasLimit { 50 | limit = query.Limit() 51 | 52 | if limit <= 0 { 53 | return 54 | } 55 | } 56 | 57 | line := 0 58 | offset := query.StartingAt() 59 | 60 | for { 61 | 62 | records, err := reader.Read() 63 | 64 | // end of file 65 | if err == io.EOF { 66 | return 67 | } 68 | // unknown error 69 | if err != nil { 70 | fmt.Println(err) 71 | return 72 | } 73 | 74 | // set the header 75 | if header == nil { 76 | header = records 77 | continue 78 | } 79 | 80 | // creates the record 81 | r := record.NewCSVRecordWithHeader(records, header) 82 | // evaluate the query 83 | match, err := query.Evaluate(r) 84 | if err != nil { 85 | fmt.Println(">>> Error while evaluating the query at line", line, ":", err) 86 | return 87 | } 88 | 89 | if !match { 90 | continue 91 | } 92 | 93 | offset-- 94 | 95 | if offset > 0 { 96 | continue 97 | } 98 | 99 | line++ 100 | 101 | // extract the fields 102 | values, err := query.FieldsValues(r) 103 | if err != nil { 104 | fmt.Println(">>> Error while extracting the fields ", err) 105 | return 106 | } 107 | 108 | fmt.Println("# ", values) 109 | 110 | if hasLimit { 111 | limit-- 112 | if limit <= 0 { 113 | break 114 | } 115 | } 116 | } 117 | } 118 | 119 | func usage() { 120 | fmt.Printf("Usage of %s\n", os.Args[0]) 121 | fmt.Printf("%s query\n", os.Args[0]) 122 | fmt.Printf(" query : the query to execute on a file (from being the file name)\n") 123 | } 124 | -------------------------------------------------------------------------------- /samples/json/json.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "os" 8 | 9 | "github.com/BatchLabs/charlatan" 10 | "github.com/BatchLabs/charlatan/record" 11 | ) 12 | 13 | func usage() { 14 | fmt.Printf("Usage:\n\n\t%s \n", os.Args[0]) 15 | os.Exit(1) 16 | } 17 | 18 | func fatalf(format string, args ...interface{}) { 19 | fmt.Fprintf(os.Stderr, format, args...) 20 | os.Exit(1) 21 | } 22 | 23 | func main() { 24 | var limit int64 25 | 26 | if len(os.Args) != 2 { 27 | usage() 28 | } 29 | 30 | query, err := charlatan.QueryFromString(os.Args[1]) 31 | if err != nil { 32 | fatalf("Error: %v\n", err) 33 | } 34 | 35 | reader, err := os.Open(query.From()) 36 | if err != nil { 37 | fatalf("Error: %v\n", err) 38 | } 39 | 40 | defer reader.Close() 41 | 42 | decoder := json.NewDecoder(reader) 43 | 44 | hasLimit := query.HasLimit() 45 | 46 | if hasLimit { 47 | limit = query.Limit() 48 | 49 | if limit <= 0 { 50 | return 51 | } 52 | } 53 | 54 | line := 0 55 | offset := query.StartingAt() 56 | 57 | for { 58 | r, err := record.NewJSONRecordFromDecoder(decoder) 59 | 60 | // end of file 61 | if err == io.EOF { 62 | return 63 | } 64 | 65 | // unknown error 66 | if err != nil { 67 | fatalf("Error at line %d: %v\n", line, err) 68 | } 69 | 70 | match, err := query.Evaluate(r) 71 | if err != nil { 72 | fatalf("Error while evaluating the query at line %d: %v\n", line, err) 73 | } 74 | 75 | if !match { 76 | continue 77 | } 78 | 79 | offset-- 80 | 81 | if offset > 0 { 82 | continue 83 | } 84 | 85 | line++ 86 | 87 | // extract the fields 88 | values, err := query.FieldsValues(r) 89 | if err != nil { 90 | fatalf("Error while extracting the fields at line %d: %v\n", line, err) 91 | } 92 | 93 | fmt.Println("# ", values) 94 | if hasLimit { 95 | limit-- 96 | if limit <= 0 { 97 | break 98 | } 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /samples/json/people.jsons: -------------------------------------------------------------------------------- 1 | {"name": "Nico", "age": 54, "stats": {"walking": 42, "running": 104, "biking": 13}} 2 | {"name": "Vincent", "age": 32, "stats": {"walking": 100, "running": 2, "biking": 3}} 3 | {"name": "Cedric", "age": 52, "stats": {"walking": 2, "running": 2, "biking": 1000}} 4 | {"name": "Michel", "age": 83, "stats": {"walking": 0, "running": 0, "biking": 0}} 5 | {"name": "Gerard", "age": 55, "stats": {"walking": 54, "running": 12, "biking": 33}} 6 | -------------------------------------------------------------------------------- /tests/grammar/grammar.abnf: -------------------------------------------------------------------------------- 1 | query = "SELECT" 1*SP select 1*SP 2 | "FROM" 1*SP field *1( 1*SP "WHERE" 1*SP expressions ) 3 | *1( 1*SP "STARTING" 1*SP "AT" 1*SP int ) 4 | *1( 1*SP "LIMIT" 1*SP int ( *SP "," *SP int ) ) 5 | 6 | select = field *( *SP "," *SP field ) 7 | 8 | field = fieldtoken *( "." fieldtoken ) 9 | / "`" 1*( 10 | alphanumeric / DIGIT / DQUOTE / LWSP / CR / LF 11 | / punctuation / miscchars / "(" / ")" / "'" 12 | ) "`" 13 | 14 | fieldtoken = ALPHA *( alphanumeric ) 15 | 16 | expressions = top-expression *( 1*SP bool-operator 1*SP top-expression ) 17 | 18 | top-expression = expression / range-test 19 | 20 | value = field / constant 21 | 22 | expression = value 23 | / "(" *SP expression *SP ")" 24 | / "(" *SP expression operator expression *SP ")" 25 | 26 | range-test = value 1*SP "BETWEEN" 1*SP value 1*SP "AND" 1*SP value 27 | 28 | operator = *SP comp-operator *SP 29 | / 1*SP bool-operator 1*SP 30 | 31 | comp-operator = "=" / "!=" / "<" / "<=" / ">" / ">=" 32 | 33 | bool-operator = "&&" / "||" 34 | / "AND" / "OR" 35 | 36 | constant = string 37 | / int 38 | / float 39 | / bool 40 | / null 41 | 42 | 43 | string = DQUOTE anycharexceptdoublequote DQUOTE 44 | / "'" anycharexceptsinglequote "'" 45 | 46 | 47 | int = *1( *1( "-" ) ) DIGIT *( DIGIT ) 48 | 49 | float = int *1( "." DIGIT *( DIGIT ) ) 50 | 51 | bool = "true" / "false" 52 | 53 | null = "null" / "NULL" 54 | 55 | alphanumeric = DIGIT / ALPHA 56 | 57 | punctuation = "." / "," / "?" / ";" / ":" / "!" 58 | 59 | miscchars = "=" / "+" / "/" / "*" / "&" / "@" / "#" 60 | 61 | anycharexceptdoublequote = alphanumeric / punctuation / miscchars / "'" 62 | anycharexceptsinglequote = alphanumeric / punctuation / miscchars / DQUOTE 63 | -------------------------------------------------------------------------------- /tests/grammar/grammartester.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "os/exec" 7 | "strconv" 8 | 9 | "github.com/BatchLabs/charlatan" 10 | ) 11 | 12 | const testsCount = 10 13 | 14 | func generateQueryString() (string, error) { 15 | c := exec.Command("abnfgen", "-l", "-s", "query", "grammar.abnf") 16 | c.Stderr = nil 17 | 18 | out, err := c.Output() 19 | return string(out), err 20 | } 21 | 22 | func runTest() (string, error) { 23 | s, err := generateQueryString() 24 | if err != nil { 25 | log.Fatalf("Query generation error: %v", err) 26 | } 27 | 28 | _, err = charlatan.QueryFromString(s) 29 | return s, err 30 | } 31 | 32 | func main() { 33 | var err error 34 | var good, count, tc int64 35 | 36 | if len(os.Args) == 2 { 37 | if count, err = strconv.ParseInt(os.Args[1], 10, 64); err != nil { 38 | count = testsCount 39 | } 40 | } else { 41 | count = testsCount 42 | } 43 | 44 | for tc = 0; tc < count; tc++ { 45 | s, err := runTest() 46 | if err == nil { 47 | good++ 48 | //log.Printf("[OK] %s", s) 49 | } else { 50 | log.Printf("[KO] %s", s) 51 | log.Printf(" ERR: %v", err) 52 | } 53 | } 54 | 55 | log.Println("") 56 | log.Printf("Results: %d/%d", good, count) 57 | } 58 | -------------------------------------------------------------------------------- /token.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // tokenType is a token type 8 | type tokenType int 9 | 10 | // token types are the types of the tokens used by the lexer 11 | const ( 12 | tokInvalid tokenType = iota 13 | 14 | // tokField is an alpha-numeric field 15 | tokField 16 | 17 | // keywords 18 | 19 | tokKeywordStart 20 | tokSelect // SELECT 21 | tokFrom // FROM 22 | tokWhere // WHERE 23 | tokStarting // STARTING 24 | tokAt // AT 25 | tokBetween // BETWEEN 26 | tokLimit // LIMIT 27 | tokKeywordEnd 28 | 29 | // operators 30 | 31 | tokLogicalOperatorStart 32 | tokAnd // && or AND 33 | tokOr // || or OR 34 | tokLogicalOperatorEnd 35 | 36 | tokComparisonOperatorStart 37 | tokEq // = 38 | tokNeq // != 39 | tokLt // < 40 | tokLte // <= 41 | tokGt // > 42 | tokGte // >= 43 | tokComparisonOperatorEnd 44 | 45 | tokInt 46 | tokFloat 47 | tokString 48 | 49 | // special values 50 | 51 | tokTrue // true 52 | tokFalse // false 53 | tokNull // null 54 | 55 | // misc 56 | 57 | tokLeftParenthesis // ( 58 | tokRightParenthesis // ) 59 | tokComma // , 60 | 61 | // tokEnd is the end token 62 | tokEnd = -1 63 | ) 64 | 65 | // token is the token found by the lexer 66 | type token struct { 67 | // the token type 68 | Type tokenType 69 | // the string value of this token 70 | Value string 71 | // the position into the parsed string 72 | Pos int 73 | } 74 | 75 | // Const returns the token's value as a Const 76 | func (tok token) Const() (*Const, error) { 77 | switch tok.Type { 78 | case tokTrue: 79 | return BoolConst(true), nil 80 | case tokFalse: 81 | return BoolConst(false), nil 82 | case tokNull: 83 | return NullConst(), nil 84 | case tokString: 85 | return StringConst(tok.Value), nil 86 | case tokInt, tokFloat: 87 | return ConstFromString(tok.Value), nil 88 | default: 89 | return nil, fmt.Errorf("Token type %v isn't a const", tok.Type) 90 | } 91 | } 92 | 93 | // isEnd checks if the token is a tokEnd 94 | func (tok token) isEnd() bool { return tok.Type == tokEnd } 95 | 96 | // isNumeric checks if the token is numeric 97 | func (tok token) isNumeric() bool { return tok.Type == tokInt || tok.Type == tokFloat } 98 | 99 | // isKeyword checks if the token is a keyword 100 | func (tok token) isKeyword() bool { return tok.Type > tokKeywordStart && tok.Type < tokKeywordEnd } 101 | 102 | // isOperator checks if the token is an operator 103 | func (tok token) isOperator() bool { return tok.isLogicalOperator() || tok.isComparisonOperator() } 104 | 105 | // isLogicalOperator checks if the token is a logical operator 106 | func (tok token) isLogicalOperator() bool { 107 | return tok.Type > tokLogicalOperatorStart && tok.Type < tokLogicalOperatorEnd 108 | } 109 | 110 | // isComparisonOperator checks if the token is a comparison operator 111 | func (tok token) isComparisonOperator() bool { 112 | return tok.Type > tokComparisonOperatorStart && tok.Type < tokComparisonOperatorEnd 113 | } 114 | 115 | // isConst tests if the token represents a constant value. If so, one can use 116 | // the Const() method to get the const value. 117 | func (tok token) isConst() bool { 118 | return tok.isNumeric() || tok.Type == tokString || tok.Type == tokTrue || 119 | tok.Type == tokFalse || tok.Type == tokNull 120 | } 121 | 122 | // isField tests if the token represents a field 123 | func (tok token) isField() bool { 124 | return tok.Type == tokField 125 | } 126 | 127 | func (tok token) String() string { 128 | return fmt.Sprintf("%d:%s(%s)", tok.Pos, tok.Type, tok.Value) 129 | } 130 | 131 | func (t tokenType) String() string { 132 | switch t { 133 | case tokEnd: 134 | return "End" 135 | case tokField: 136 | return "Field" 137 | case tokTrue: 138 | return "True" 139 | case tokFalse: 140 | return "False" 141 | case tokNull: 142 | return "Null" 143 | case tokInt: 144 | return "Int" 145 | case tokFloat: 146 | return "Float" 147 | case tokSelect: 148 | return "Select" 149 | case tokFrom: 150 | return "From" 151 | case tokWhere: 152 | return "Where" 153 | case tokStarting: 154 | return "Starting" 155 | case tokAt: 156 | return "At" 157 | case tokAnd: 158 | return "And" 159 | case tokOr: 160 | return "Or" 161 | case tokBetween: 162 | return "Between" 163 | case tokLimit: 164 | return "Limit" 165 | case tokEq: 166 | return "Eq" 167 | case tokNeq: 168 | return "Neq" 169 | case tokLt: 170 | return "Lt" 171 | case tokLte: 172 | return "Lte" 173 | case tokGt: 174 | return "Gt" 175 | case tokGte: 176 | return "Gte" 177 | case tokLeftParenthesis: 178 | return "tokLeftParenthesis" 179 | case tokRightParenthesis: 180 | return "tokRightParenthesis" 181 | case tokComma: 182 | return "Comma" 183 | } 184 | 185 | return "UNKNOWN" 186 | } 187 | -------------------------------------------------------------------------------- /token_test.go: -------------------------------------------------------------------------------- 1 | package charlatan 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestTokenIsEnd(t *testing.T) { 10 | assert.True(t, token{Type: tokEnd}.isEnd()) 11 | } 12 | 13 | func TestTokenIsNumeric(t *testing.T) { 14 | assert.True(t, token{Type: tokInt}.isNumeric()) 15 | assert.True(t, token{Type: tokFloat}.isNumeric()) 16 | } 17 | 18 | func TestTokenIsKeyword(t *testing.T) { 19 | assert.True(t, token{Type: tokSelect}.isKeyword()) 20 | assert.True(t, token{Type: tokFrom}.isKeyword()) 21 | assert.True(t, token{Type: tokWhere}.isKeyword()) 22 | assert.True(t, token{Type: tokStarting}.isKeyword()) 23 | 24 | for _, ty := range []tokenType{ 25 | tokSelect, tokFrom, tokWhere, tokStarting, 26 | } { 27 | assert.True(t, token{Type: ty}.isKeyword()) 28 | } 29 | } 30 | 31 | func TestTokenIsOperator(t *testing.T) { 32 | for _, ty := range []tokenType{ 33 | tokAnd, tokOr, tokEq, tokNeq, tokLt, tokLte, tokGt, tokGte, 34 | } { 35 | assert.True(t, token{Type: ty}.isOperator()) 36 | } 37 | } 38 | 39 | func TestTokenIsLogicalOperator(t *testing.T) { 40 | for _, ty := range []tokenType{tokAnd, tokOr} { 41 | assert.True(t, token{Type: ty}.isLogicalOperator()) 42 | } 43 | } 44 | 45 | func TestTokenIsComparisonOperator(t *testing.T) { 46 | for _, ty := range []tokenType{ 47 | tokEq, tokNeq, tokLt, tokLte, tokGt, tokGte, 48 | } { 49 | 50 | assert.True(t, token{Type: ty}.isComparisonOperator()) 51 | } 52 | } 53 | 54 | func TestTokenTypeString(t *testing.T) { 55 | for _, ty := range []tokenType{ 56 | tokField, tokInt, tokFloat, tokTrue, tokFalse, tokNull, tokSelect, 57 | tokFrom, tokWhere, tokStarting, tokAt, tokAnd, tokOr, tokEq, tokNeq, 58 | tokLt, tokLte, tokGt, tokGte, tokLeftParenthesis, tokRightParenthesis, 59 | tokComma, tokBetween, tokEnd, 60 | } { 61 | assert.NotEqual(t, "", ty.String()) 62 | assert.NotEqual(t, "UNKNOWN", ty.String()) 63 | } 64 | } 65 | --------------------------------------------------------------------------------