├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── FilterQL.md ├── LICENSE ├── README.md ├── codecov.yml ├── datasource ├── context.go ├── context_test.go ├── context_wrapper.go ├── context_wrapper_test.go ├── csv.go ├── csv_test.go ├── datatypes.go ├── datatypes_test.go ├── doc.go ├── files │ ├── README.md │ ├── file.go │ ├── file_test.go │ ├── filehandler.go │ ├── filehandler_test.go │ ├── filepager.go │ ├── filesource.go │ ├── filesource_test.go │ ├── filestore.go │ ├── json_test.go │ ├── scanner_csv.go │ ├── scanner_json.go │ ├── storesource.go │ └── tables │ │ ├── baseball │ │ └── appearances │ │ │ └── appearances.csv │ │ └── github │ │ └── issues.json ├── introspect.go ├── introspect_test.go ├── json.go ├── json_test.go ├── key.go ├── membtree │ ├── btree.go │ └── btree_test.go ├── memdb │ ├── db.go │ ├── db_test.go │ ├── index.go │ └── index_test.go ├── mockcsv │ └── mockcsv.go ├── mockcsvtestdata │ └── testdata.go ├── schemadb.go ├── schemadb_test.go ├── session.go └── sqlite │ ├── conn.go │ ├── schemawriter.go │ ├── source.go │ ├── sqlite_test.go │ └── sqlrewrite.go ├── dialects ├── _influxql │ ├── ast.go │ ├── dialect.go │ ├── parse_test.go │ └── parser.go └── example │ ├── README.md │ └── main.go ├── doc.go ├── examples ├── expressions │ └── main.go └── qlcsv │ ├── README.md │ ├── main.go │ └── users.csv ├── exec ├── command.go ├── ddl.go ├── errs.go ├── exec.go ├── exec_test.go ├── executor.go ├── groupby.go ├── join.go ├── mutations.go ├── order.go ├── projection.go ├── results.go ├── source.go ├── sqldriver.go ├── sqldriver_test.go ├── task.go ├── task_parallel.go ├── task_sequential.go ├── taskrunner.go └── where.go ├── expr ├── builtins │ ├── aggregations.go │ ├── builtins.go │ ├── builtins_test.go │ ├── cast.go │ ├── filter.go │ ├── hash_and_encode.go │ ├── json.go │ ├── list_map.go │ ├── logic.go │ ├── math.go │ ├── string.go │ ├── time.go │ └── url_email.go ├── dialect.go ├── dialect_test.go ├── funcs.go ├── funcs_test.go ├── include.go ├── include_test.go ├── node.go ├── node.pb.go ├── node.proto ├── node_test.go ├── parse.go ├── parse_test.go ├── stringutil.go └── stringutil_test.go ├── generators └── elasticsearch │ ├── es2gen │ ├── bridgeutil.go │ ├── esgenerator.go │ ├── esgenerator_test.go │ ├── estypes.go │ └── schema.go │ ├── esgen │ ├── bridgeutil.go │ ├── esgenerator.go │ ├── esgenerator_test.go │ ├── estypes.go │ ├── schema.go │ └── validate.go │ └── gentypes │ ├── errors.go │ └── gen.go ├── go.mod ├── go.sum ├── go.test.sh ├── lex ├── README.md ├── dialect.go ├── dialect_expr.go ├── dialect_expr_test.go ├── dialect_filterql.go ├── dialect_filterql_test.go ├── dialect_json.go ├── dialect_json_test.go ├── dialect_sql.go ├── dialect_sql_parse_test.go ├── dialect_sql_test.go ├── lexer.go ├── lexer_test.go ├── token.go └── token_test.go ├── plan ├── context.go ├── context_test.go ├── plan.go ├── plan.pb.go ├── plan.proto ├── plan_proto_test.go ├── plan_test.go ├── planner.go ├── planner_mutate.go ├── planner_select.go ├── planner_test.go ├── projection.go ├── sql_rewrite.go └── sql_rewrite_test.go ├── qlbdriver └── driver.go ├── rel ├── _bm │ └── bm_parse_test.go ├── filter.go ├── filter_test.go ├── parse_filterql.go ├── parse_filterql_test.go ├── parse_sql.go ├── parse_sql_test.go ├── sql.go ├── sql.pb.go ├── sql.proto ├── sql_proto_test.go ├── sql_rewrite.go ├── sql_rewrite_test.go ├── sql_test.go └── testsuite_test.go ├── schema ├── apply_schema.go ├── apply_schema_test.go ├── datasource.go ├── message.go ├── message_test.go ├── registry.go ├── registry_test.go ├── schema.go ├── schema.pb.go ├── schema.proto └── schema_test.go ├── testutil ├── harness.go ├── suite_test.go └── testsuite.go ├── updateglock.sh ├── value ├── coerce.go ├── coerce_test.go ├── value.go └── value_test.go └── vm ├── _archive ├── vm_bm_test.go ├── vm_sql.go └── vm_sql_test.go ├── _bm └── main.go ├── bm_test.go ├── datemath.go ├── datemath_test.go ├── filterqlvm.go ├── filterqlvm_test.go ├── sqlvm.go ├── sqlvm_test.go ├── vm.go ├── vm_bench_test.go └── vm_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | dialects/example/example 3 | examples/qlcsv/qlcsv 4 | examples/junk 5 | vm/_bm/_bm 6 | AARON 7 | cur.diff 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.13.x 5 | 6 | before_install: 7 | - go get -t -v ./... 8 | 9 | script: 10 | - bash go.test.sh 11 | 12 | after_success: 13 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014,2015,2016,2017,2018 Aaron Raddon 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | # - "**/*.pb.go" # ignore protobuf files 3 | - "expr/node.pb.go" 4 | - "plan/plan.pb.go" 5 | - "rel/sql.pb.go" 6 | - "schema/schema.pb.go" 7 | - "examples/**/*" 8 | - "datasource/mockcsvtestdata/*" 9 | - "dialects/**/*" 10 | - "generators/**/*" 11 | - "generators/*" 12 | - "testutil/*" -------------------------------------------------------------------------------- /datasource/context_wrapper_test.go: -------------------------------------------------------------------------------- 1 | package datasource_test 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | 8 | "github.com/araddon/dateparse" 9 | u "github.com/araddon/gou" 10 | 11 | "github.com/araddon/qlbridge/datasource" 12 | "github.com/araddon/qlbridge/expr" 13 | "github.com/araddon/qlbridge/value" 14 | ) 15 | 16 | // Our test struct, try as many different field types as possible 17 | type User struct { 18 | Name string 19 | Created time.Time 20 | Updated *time.Time 21 | Authenticated bool 22 | HasSession *bool 23 | Roles []string `json:"roles_list"` // See if we can do by alias 24 | BankAmount float64 25 | Address struct { 26 | City string 27 | Zip int 28 | } 29 | Data json.RawMessage 30 | Context u.JsonHelper 31 | Nope *time.Time 32 | } 33 | 34 | func (m *User) FullName() string { 35 | return m.Name + ", Jedi" 36 | } 37 | 38 | func TestStructWrapper(t *testing.T) { 39 | 40 | t1, _ := dateparse.ParseAny("12/18/2015") 41 | tr := true 42 | user := &User{ 43 | Name: "Yoda", 44 | Created: t1, 45 | Updated: &t1, 46 | Authenticated: true, 47 | HasSession: &tr, 48 | Roles: []string{"admin", "api"}, 49 | BankAmount: 55.5, 50 | } 51 | user.Address.City = "Phoenix" 52 | user.Address.Zip = 97811 53 | 54 | readers := []expr.ContextReader{ 55 | datasource.NewContextWrapper(user), 56 | datasource.NewContextSimpleNative(map[string]interface{}{ 57 | "str1": "str1", 58 | "int1": 1, 59 | "t1": t1, 60 | "Name": "notyoda", 61 | }), 62 | } 63 | 64 | nc := datasource.NewNestedContextReader(readers, time.Now()) 65 | expected := value.NewMapValue(map[string]interface{}{ 66 | "str1": "str1", 67 | "int1": 1, 68 | "Name": "Yoda", 69 | "Authenticated": true, 70 | "bankamount": 55.5, 71 | "FullName": "Yoda, Jedi", 72 | "Address.City": "Phoenix", 73 | "Roles": []string{"admin", "api"}, 74 | "roles_list": []string{"admin", "api"}, 75 | "Nope": nil, 76 | }) 77 | 78 | for k, v := range expected.Val() { 79 | //u.Infof("k:%v v:%#v", k, v) 80 | checkval(t, nc, k, v) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /datasource/csv_test.go: -------------------------------------------------------------------------------- 1 | package datasource_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | u "github.com/araddon/gou" 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/araddon/qlbridge/datasource" 12 | "github.com/araddon/qlbridge/schema" 13 | ) 14 | 15 | var ( 16 | _ = u.EMPTY 17 | testData = map[string]string{ 18 | "user.csv": `user_id,email,interests,reg_date,item_count 19 | 9Ip1aKbeZe2njCDM,"aaron@email.com","fishing","2012-10-17T17:29:39.738Z",82 20 | hT2impsOPUREcVPc,"bob@email.com","swimming","2009-12-11T19:53:31.547Z",12 21 | hT2impsabc345c,"not_an_email","swimming","2009-12-11T19:53:31.547Z",12`, 22 | } 23 | 24 | csvSource schema.Source = &datasource.CsvDataSource{} 25 | csvStringSource schema.Source = &csvStaticSource{testData: testData} 26 | ) 27 | 28 | type csvStaticSource struct { 29 | *datasource.CsvDataSource 30 | testData map[string]string 31 | } 32 | 33 | func (m *csvStaticSource) Open(connInfo string) (schema.Conn, error) { 34 | if data, ok := m.testData[connInfo]; ok { 35 | sr := strings.NewReader(data) 36 | return datasource.NewCsvSource(connInfo, 0, sr, make(<-chan bool, 1)) 37 | } 38 | return nil, fmt.Errorf("not found") 39 | } 40 | 41 | func TestCsvDataSource(t *testing.T) { 42 | csvIn, err := csvStringSource.Open("user.csv") 43 | assert.True(t, err == nil, "should not have error: %v", err) 44 | csvIter, ok := csvIn.(schema.ConnScanner) 45 | assert.True(t, ok) 46 | iterCt := 0 47 | for msg := csvIter.Next(); msg != nil; msg = csvIter.Next() { 48 | iterCt++ 49 | u.Infof("row: %v", msg.Body()) 50 | } 51 | assert.Equal(t, 3, iterCt) 52 | err = csvIn.Close() 53 | assert.Equal(t, nil, err) 54 | 55 | sr := strings.NewReader(testData["user.csv"]) 56 | csvIn, err = datasource.NewCsvSource("user.csv", 0, sr, make(<-chan bool, 1)) 57 | assert.Equal(t, nil, err) 58 | csvIn.Close() 59 | } 60 | -------------------------------------------------------------------------------- /datasource/datatypes_test.go: -------------------------------------------------------------------------------- 1 | package datasource_test 2 | 3 | import ( 4 | "database/sql/driver" 5 | "encoding/json" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/araddon/dateparse" 10 | 11 | u "github.com/araddon/gou" 12 | "github.com/stretchr/testify/assert" 13 | 14 | "github.com/araddon/qlbridge/datasource" 15 | "github.com/araddon/qlbridge/datasource/mockcsv" 16 | td "github.com/araddon/qlbridge/datasource/mockcsvtestdata" 17 | tu "github.com/araddon/qlbridge/testutil" 18 | ) 19 | 20 | var ( 21 | cats = []string{"sports", "politics", "worldnews"} 22 | catstr = strings.Join(cats, ",") 23 | jo = json.RawMessage([]byte(`{"name":"bob"}`)) 24 | t1 = dateparse.MustParse("2014-01-01") 25 | ) 26 | 27 | type dtData struct { 28 | Categories string `db:"categories"` 29 | Cat datasource.StringArray `db:"cat"` 30 | JsonCat datasource.StringArray `db:"json_cats"` 31 | Json1 *datasource.JsonWrapper `db:"j1"` 32 | Json2 *datasource.JsonHelperScannable `db:"j2"` 33 | Id string `db:"user_id"` 34 | T1 *datasource.TimeValue `db:"t1"` 35 | } 36 | 37 | func TestDataTypes(t *testing.T) { 38 | 39 | // Load in a "csv file" into our mock data store 40 | mockcsv.LoadTable(td.MockSchema.Name, "typestest", `user_id,categories,json_obj,json_cats,t1 41 | 9Ip1aKbeZe2njCDM,"sports,politics,worldnews","{""name"":""bob""}","[""sports"",""politics"",""worldnews""]","2014-01-01"`) 42 | 43 | data := dtData{} 44 | tu.ExecSqlSpec(t, &tu.QuerySpec{ 45 | Source: td.MockSchema.Name, 46 | Sql: `SELECT 47 | user_id, categories, categories AS cat, json_cats, t1, json_obj as j1, json_obj as j2 48 | FROM typestest;`, 49 | ExpectRowCt: 1, 50 | ValidateRowData: func() { 51 | u.Infof("%#v", data) 52 | assert.Equal(t, catstr, data.Categories) 53 | assert.Equal(t, cats, []string(data.Cat)) 54 | assert.Equal(t, cats, []string(data.JsonCat)) 55 | assert.Equal(t, t1, data.T1.Time()) 56 | assert.Equal(t, jo, json.RawMessage(*data.Json1)) 57 | j2, _ := json.Marshal(data.Json2) 58 | assert.Equal(t, string(jo), string(j2)) 59 | }, 60 | RowData: &data, 61 | }) 62 | by, err := json.Marshal(data) 63 | assert.Equal(t, nil, err) 64 | var data2 dtData 65 | err = json.Unmarshal(by, &data2) 66 | u.Infof("by %s", string(by)) 67 | assert.Equal(t, nil, err) 68 | assert.Equal(t, catstr, data2.Categories) 69 | assert.Equal(t, cats, []string(data2.Cat)) 70 | assert.Equal(t, cats, []string(data2.JsonCat)) 71 | assert.Equal(t, t1, data2.T1.Time()) 72 | assert.Equal(t, jo, json.RawMessage(*data2.Json1)) 73 | j2, _ := json.Marshal(data2.Json2) 74 | assert.Equal(t, string(jo), string(j2)) 75 | 76 | tu.ExecSqlSpec(t, &tu.QuerySpec{ 77 | Source: td.MockSchema.Name, 78 | Exec: "DROP TABLE typestest;", 79 | ExpectRowCt: -1, 80 | }) 81 | tu.TestSelect(t, `show tables;`, 82 | [][]driver.Value{{"orders"}, {"users"}}, 83 | ) 84 | } 85 | -------------------------------------------------------------------------------- /datasource/doc.go: -------------------------------------------------------------------------------- 1 | // Datasource package contains database/source type related. A few datasources 2 | // are implemented here (test, csv). This package also includes 3 | // schema base services (datasource registry). 4 | package datasource 5 | -------------------------------------------------------------------------------- /datasource/files/README.md: -------------------------------------------------------------------------------- 1 | 2 | File Data Source 3 | --------------------------------- 4 | 5 | Turn files into a SQL Queryable data source. 6 | 7 | Allows Cloud storage (Google Storage, S3, etc) files (csv, json, custom-protobuf) 8 | to be queried with traditional sql. Also allows these files to have custom 9 | serializations, compressions, encryptions. 10 | 11 | **Design Goal** 12 | * *Hackable Stores* easy to add to Google Storage, local files, s3, etc. 13 | * *Hackable File Formats* Protbuf files, WAL files, mysql-bin-log's etc. 14 | 15 | 16 | **Developing new stores or file formats** 17 | 18 | * *FileStore* defines file storage Factory (s3, google-storage, local files, sftp, etc) 19 | * *StoreReader* a configured, initilized instance of a specific *FileStore*. 20 | File storage reader(writer) for finding lists of files, and opening files. 21 | * *FileHandler* FileHandlers handle processing files from StoreReader, developers 22 | Register FileHandler implementations in Registry. FileHandlers will create 23 | `FileScanner` that iterates rows of this file. 24 | * *FileScanner* File Row Reading, how to transform contents of 25 | file into *qlbridge.Message* for use in query engine. 26 | Currently CSV, Json types. 27 | 28 | Example: Query CSV Files 29 | ---------------------------- 30 | We are going to create a CSV `database` of Baseball data from 31 | http://seanlahman.com/baseball-archive/statistics/ 32 | 33 | ```sh 34 | # download files to local /tmp 35 | mkdir -p /tmp/baseball 36 | cd /tmp/baseball 37 | curl -Ls http://seanlahman.com/files/database/baseballdatabank-2017.1.zip > bball.zip 38 | unzip bball.zip 39 | 40 | mv baseball*/core/*.csv . 41 | rm bball.zip 42 | rm -rf baseballdatabank-* 43 | 44 | # run a docker container locally 45 | docker run -e "LOGGING=debug" --rm -it -p 4000:4000 \ 46 | -v /tmp/baseball:/tmp/baseball \ 47 | gcr.io/dataux-io/dataux:latest 48 | 49 | 50 | ``` 51 | In another Console open Mysql: 52 | ```sql 53 | # connect to the docker container you just started 54 | mysql -h 127.0.0.1 -P4000 55 | 56 | 57 | -- Now create a new Source 58 | CREATE source baseball WITH { 59 | "type":"cloudstore", 60 | "schema":"baseball", 61 | "settings" : { 62 | "type": "localfs", 63 | "format": "csv", 64 | "path": "baseball/", 65 | "localpath": "/tmp" 66 | } 67 | }; 68 | 69 | show databases; 70 | 71 | use baseball; 72 | 73 | show tables; 74 | 75 | describe appearances 76 | 77 | select count(*) from appearances; 78 | 79 | select * from appearances limit 10; 80 | 81 | 82 | ``` 83 | 84 | TODO 85 | ---------------------------- 86 | 87 | * implement event-notification 88 | * SFTP 89 | * Change the store library? Currently https://github.com/lytics/cloudstorage but consider: 90 | * https://github.com/graymeta/stow 91 | * https://github.com/rook/rook 92 | * https://minio.io/ 93 | * http://rclone.org 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /datasource/files/file_test.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestFileTableNames(t *testing.T) { 10 | 11 | assert.Equal(t, "players", TableFromFileAndPath("", "tables/players.csv")) 12 | assert.Equal(t, "players", TableFromFileAndPath("", "tables/players/2017.csv")) 13 | 14 | assert.Equal(t, "players", TableFromFileAndPath("baseball/", "baseball/tables/players.csv")) 15 | assert.Equal(t, "players", TableFromFileAndPath("baseball/", "baseball/tables/players/2017.csv")) 16 | 17 | assert.Equal(t, "players", TableFromFileAndPath("baseball", "baseball/tables/players.csv")) 18 | assert.Equal(t, "players", TableFromFileAndPath("baseball", "baseball/tables/players/2017.csv")) 19 | 20 | // Cannot interpret this 21 | assert.Equal(t, "", TableFromFileAndPath("baseball", "baseball/tables/players/partition1/2017.csv")) 22 | } 23 | 24 | func TestFileInfo(t *testing.T) { 25 | 26 | fi := &FileInfo{} 27 | assert.NotEqual(t, "", fi.String()) 28 | 29 | } 30 | -------------------------------------------------------------------------------- /datasource/files/filehandler.go: -------------------------------------------------------------------------------- 1 | // Package files is a cloud (gcs, s3) and local file datasource that translates 2 | // json, csv, files into appropriate interface for qlbridge DataSource 3 | // so we can run queries. Provides FileHandler interface to allow 4 | // custom file type handling 5 | package files 6 | 7 | import ( 8 | "strings" 9 | "sync" 10 | 11 | "github.com/lytics/cloudstorage" 12 | 13 | "github.com/araddon/qlbridge/schema" 14 | ) 15 | 16 | var ( 17 | // the global file-scanners registry mutex 18 | registryMu sync.Mutex 19 | scanners = make(map[string]FileHandler) 20 | ) 21 | 22 | // FileHandler defines an interface for developers to build new File processing 23 | // to allow these custom file-types to be queried with SQL. 24 | // 25 | // Features of Filehandler 26 | // - File() Converts a cloudstorage object to a FileInfo that describes the File. 27 | // - Scanner() create a file-scanner, will allow the scanner to implement any 28 | // custom file-type reading (csv, protobuf, json, enrcyption). 29 | // 30 | // The File Reading, Opening, Listing is a separate layer, see FileSource 31 | // for the Cloudstorage layer. 32 | // 33 | // So it is a a factory to create Scanners for a speciffic format type such as csv, json 34 | type FileHandler interface { 35 | Init(store FileStore, ss *schema.Schema) error 36 | // File Each time the underlying FileStore finds a new file it hands it off 37 | // to filehandler to determine if it is File or not (directory?), and to to extract any 38 | // metadata such as partition, and parse out fields that may exist in File/Folder path 39 | File(path string, obj cloudstorage.Object) *FileInfo 40 | // Create a scanner for particiular file 41 | Scanner(store cloudstorage.StoreReader, fr *FileReader) (schema.ConnScanner, error) 42 | // FileAppendColumns provides a method that this file-handler is going to provide additional 43 | // columns to the files list table, ie we are going to extract column info from the 44 | // folder paths, file-names. 45 | // For example: `tables/appearances/2017/appearances.csv may extract the "2017" as "year" 46 | // and append that as column to all rows. 47 | // Optional: may be nil 48 | FileAppendColumns() []string 49 | } 50 | 51 | // FileHandlerTables - file handlers may optionally provide info about 52 | // tables contained in store 53 | type FileHandlerTables interface { 54 | FileHandler 55 | Tables() []*FileTable 56 | } 57 | 58 | // FileHandlerSchema - file handlers may optionally provide info about 59 | // tables contained 60 | type FileHandlerSchema interface { 61 | FileHandler 62 | schema.SourceTableSchema 63 | } 64 | 65 | // RegisterFileHandler Register a FileHandler available by the provided @scannerType 66 | func RegisterFileHandler(scannerType string, fh FileHandler) { 67 | if fh == nil { 68 | panic("File scanners must not be nil") 69 | } 70 | scannerType = strings.ToLower(scannerType) 71 | // u.Debugf("global FileHandler register: %v %T FileHandler:%p", scannerType, fh, fh) 72 | registryMu.Lock() 73 | defer registryMu.Unlock() 74 | if _, dupe := scanners[scannerType]; dupe { 75 | panic("Register called twice for FileHandler type " + scannerType) 76 | } 77 | scanners[scannerType] = fh 78 | } 79 | 80 | func scannerGet(scannerType string) (FileHandler, bool) { 81 | registryMu.Lock() 82 | defer registryMu.Unlock() 83 | scannerType = strings.ToLower(scannerType) 84 | scanner, ok := scanners[scannerType] 85 | return scanner, ok 86 | } 87 | -------------------------------------------------------------------------------- /datasource/files/filehandler_test.go: -------------------------------------------------------------------------------- 1 | package files_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/araddon/qlbridge/datasource/files" 9 | ) 10 | 11 | func TestFileHandlerRegister(t *testing.T) { 12 | assert.True(t, registryNilPanic(), "Should have paniced") 13 | assert.True(t, registryForPanic(), "Should have paniced") 14 | } 15 | func registryNilPanic() (paniced bool) { 16 | defer func() { 17 | if r := recover(); r != nil { 18 | paniced = true 19 | } 20 | }() 21 | files.RegisterFileHandler("testnil1", nil) 22 | return paniced 23 | } 24 | 25 | func registryForPanic() (paniced bool) { 26 | defer func() { 27 | if r := recover(); r != nil { 28 | paniced = true 29 | } 30 | }() 31 | files.RegisterFileHandler("test1", files.NewJsonHandlerTables(lineParser, []string{"issues"})) 32 | files.RegisterFileHandler("test1", files.NewJsonHandlerTables(lineParser, []string{"issues"})) 33 | return paniced 34 | } 35 | -------------------------------------------------------------------------------- /datasource/files/filestore.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "sync" 7 | 8 | u "github.com/araddon/gou" 9 | "github.com/lytics/cloudstorage" 10 | "github.com/lytics/cloudstorage/google" 11 | "github.com/lytics/cloudstorage/localfs" 12 | 13 | "github.com/araddon/qlbridge/schema" 14 | ) 15 | 16 | var ( 17 | // TODO: move to test files 18 | localFilesConfig = cloudstorage.Config{ 19 | Type: localfs.StoreType, 20 | AuthMethod: localfs.AuthFileSystem, 21 | LocalFS: "./tables", 22 | TmpDir: "/tmp/localcache", 23 | } 24 | 25 | // TODO: complete manufacture this from config 26 | gcsConfig = cloudstorage.Config{ 27 | Type: google.StoreType, 28 | AuthMethod: google.AuthGCEDefaultOAuthToken, 29 | Project: "lytics-dev", 30 | Bucket: "lytics-dataux-tests", 31 | TmpDir: "/tmp/localcache", 32 | } 33 | ) 34 | 35 | var ( 36 | // the global filestore registry mutex 37 | fileStoreMu sync.Mutex 38 | fileStores = make(map[string]FileStoreCreator) 39 | ) 40 | 41 | func init() { 42 | RegisterFileStore("gcs", createGCSFileStore) 43 | RegisterFileStore("localfs", createLocalFileStore) 44 | } 45 | 46 | // FileStoreLoader defines the interface for loading files 47 | func FileStoreLoader(ss *schema.Schema) (cloudstorage.StoreReader, error) { 48 | if ss == nil || ss.Conf == nil { 49 | return nil, fmt.Errorf("No config info for files source for %v", ss) 50 | } 51 | 52 | //u.Debugf("json conf:\n%s", ss.Conf.Settings.PrettyJson()) 53 | storeType := ss.Conf.Settings.String("type") 54 | if storeType == "" { 55 | return nil, fmt.Errorf("Expected 'type' in File Store definition conf") 56 | } 57 | 58 | fileStoreMu.Lock() 59 | storeType = strings.ToLower(storeType) 60 | fs, ok := fileStores[storeType] 61 | fileStoreMu.Unlock() 62 | 63 | if !ok { 64 | return nil, fmt.Errorf("Unrecognized filestore type %q expected [gcs,localfs]", storeType) 65 | } 66 | 67 | return fs(ss) 68 | } 69 | 70 | // FileStoreCreator defines a Factory type for creating FileStore 71 | type FileStoreCreator func(*schema.Schema) (FileStore, error) 72 | 73 | // FileStore Defines handler for reading Files, understanding 74 | // folders and how to create scanners/formatters for files. 75 | // Created by FileStoreCreator 76 | // 77 | // FileStoreCreator(schema) -> FileStore 78 | // FileStore.Objects() -> File 79 | // FileHandler(File) -> FileScanner 80 | // FileScanner.Next() -> Row 81 | type FileStore interface { 82 | cloudstorage.StoreReader 83 | } 84 | 85 | // RegisterFileStore global registry for Registering 86 | // implementations of FileStore factories of the provided @storeType 87 | func RegisterFileStore(storeType string, fs FileStoreCreator) { 88 | if fs == nil { 89 | panic("FileStore must not be nil") 90 | } 91 | storeType = strings.ToLower(storeType) 92 | u.Debugf("global FileStore register: %v %T FileStore:%p", storeType, fs, fs) 93 | fileStoreMu.Lock() 94 | defer fileStoreMu.Unlock() 95 | if _, dupe := fileStores[storeType]; dupe { 96 | panic("Register called twice for FileStore type " + storeType) 97 | } 98 | fileStores[storeType] = fs 99 | } 100 | 101 | func createGCSFileStore(ss *schema.Schema) (FileStore, error) { 102 | 103 | conf := ss.Conf.Settings 104 | 105 | c := gcsConfig 106 | if proj := conf.String("project"); proj != "" { 107 | c.Project = proj 108 | } 109 | if bkt := conf.String("bucket"); bkt != "" { 110 | bktl := strings.ToLower(bkt) 111 | // We don't actually need the gs:// because cloudstore does it 112 | if strings.HasPrefix(bktl, "gs://") && len(bkt) > 5 { 113 | bkt = bkt[5:] 114 | } 115 | c.Bucket = bkt 116 | } 117 | if jwt := conf.String("jwt"); jwt != "" { 118 | c.JwtFile = jwt 119 | } 120 | return cloudstorage.NewStore(&c) 121 | } 122 | 123 | func createLocalFileStore(ss *schema.Schema) (FileStore, error) { 124 | 125 | conf := ss.Conf.Settings 126 | 127 | localPath := conf.String("localpath") 128 | if localPath == "" { 129 | localPath = "./tables/" 130 | } 131 | c := cloudstorage.Config{ 132 | Type: localfs.StoreType, 133 | AuthMethod: localfs.AuthFileSystem, 134 | LocalFS: localPath, 135 | TmpDir: "/tmp/localcache", 136 | } 137 | if c.LocalFS == "" { 138 | return nil, fmt.Errorf(`"localfs" filestore requires a {"settings":{"localpath":"/path/to/files"}} to local files`) 139 | } 140 | return cloudstorage.NewStore(&c) 141 | } 142 | -------------------------------------------------------------------------------- /datasource/files/scanner_csv.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | u "github.com/araddon/gou" 5 | "github.com/lytics/cloudstorage" 6 | 7 | "github.com/araddon/qlbridge/datasource" 8 | "github.com/araddon/qlbridge/schema" 9 | ) 10 | 11 | var ( 12 | // ensuure our csv handler implements FileHandler interface 13 | _ FileHandler = (*csvFiles)(nil) 14 | ) 15 | 16 | func init() { 17 | RegisterFileHandler("csv", &csvFiles{}) 18 | } 19 | 20 | // the built in csv filehandler 21 | type csvFiles struct { 22 | appendcols []string 23 | } 24 | 25 | func (m *csvFiles) Init(store FileStore, ss *schema.Schema) error { return nil } 26 | func (m *csvFiles) FileAppendColumns() []string { return m.appendcols } 27 | func (m *csvFiles) File(path string, obj cloudstorage.Object) *FileInfo { 28 | return FileInfoFromCloudObject(path, obj) 29 | } 30 | func (m *csvFiles) Scanner(store cloudstorage.StoreReader, fr *FileReader) (schema.ConnScanner, error) { 31 | csv, err := datasource.NewCsvSource(fr.Table, 0, fr.F, fr.Exit) 32 | if err != nil { 33 | u.Errorf("Could not open file for csv reading %v", err) 34 | return nil, err 35 | } 36 | return csv, nil 37 | } 38 | -------------------------------------------------------------------------------- /datasource/files/scanner_json.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | u "github.com/araddon/gou" 5 | "github.com/lytics/cloudstorage" 6 | 7 | "github.com/araddon/qlbridge/datasource" 8 | "github.com/araddon/qlbridge/schema" 9 | ) 10 | 11 | var ( 12 | // ensuure our json handler implements FileHandler interface 13 | _ FileHandler = (*jsonHandler)(nil) 14 | ) 15 | 16 | func init() { 17 | RegisterFileHandler("json", &jsonHandler{}) 18 | } 19 | 20 | // the built in json filehandler 21 | type jsonHandler struct { 22 | parser datasource.FileLineHandler 23 | } 24 | 25 | // the built in json filehandler 26 | type jsonHandlerTables struct { 27 | tables []string 28 | FileHandler 29 | } 30 | 31 | // NewJsonHandler creates a json file handler for paging new-line 32 | // delimited rows of json file 33 | func NewJsonHandler(lh datasource.FileLineHandler) FileHandler { 34 | return &jsonHandler{lh} 35 | } 36 | 37 | // NewJsonHandler creates a json file handler for paging new-line 38 | // delimited rows of json file 39 | func NewJsonHandlerTables(lh datasource.FileLineHandler, tables []string) FileHandler { 40 | return &jsonHandlerTables{ 41 | FileHandler: &jsonHandler{lh}, 42 | tables: tables, 43 | } 44 | } 45 | 46 | func (m *jsonHandler) Init(store FileStore, ss *schema.Schema) error { return nil } 47 | func (m *jsonHandler) FileAppendColumns() []string { return nil } 48 | func (m *jsonHandler) File(path string, obj cloudstorage.Object) *FileInfo { 49 | return FileInfoFromCloudObject(path, obj) 50 | } 51 | func (m *jsonHandler) Scanner(store cloudstorage.StoreReader, fr *FileReader) (schema.ConnScanner, error) { 52 | js, err := datasource.NewJsonSource(fr.Table, fr.F, fr.Exit, m.parser) 53 | if err != nil { 54 | u.Errorf("Could not open file for json reading %v", err) 55 | return nil, err 56 | } 57 | return js, nil 58 | } 59 | func (m *jsonHandlerTables) Tables() []string { 60 | return m.tables 61 | } 62 | -------------------------------------------------------------------------------- /datasource/files/storesource.go: -------------------------------------------------------------------------------- 1 | package files 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | 7 | u "github.com/araddon/gou" 8 | "github.com/lytics/cloudstorage" 9 | "golang.org/x/net/context" 10 | "google.golang.org/api/iterator" 11 | 12 | "github.com/araddon/qlbridge/datasource" 13 | "github.com/araddon/qlbridge/schema" 14 | "github.com/araddon/qlbridge/value" 15 | ) 16 | 17 | var ( 18 | // Ensure we implement Source for our file source storage 19 | _ schema.Source = (*storeSource)(nil) 20 | 21 | // Connection Interfaces 22 | _ schema.Conn = (*storeSource)(nil) 23 | _ schema.ConnScanner = (*storeSource)(nil) 24 | ) 25 | 26 | // storeSource DataSource for reading lists of files/names/metadata of files 27 | // from the cloudstorage Store 28 | // 29 | // - readers: s3, gcs, local-fs 30 | type storeSource struct { 31 | f *FileSource 32 | table string 33 | tbl *schema.Table 34 | exit <-chan bool 35 | iter cloudstorage.ObjectIterator 36 | complete bool 37 | err error 38 | rowct uint64 39 | mu sync.Mutex 40 | } 41 | 42 | // newStoreSource reader 43 | func newStoreSource(table string, fs *FileSource) (*storeSource, error) { 44 | s := &storeSource{ 45 | f: fs, 46 | table: table, 47 | exit: make(<-chan bool, 1), 48 | } 49 | return s, nil 50 | } 51 | 52 | func (m *storeSource) Init() {} 53 | func (m *storeSource) Setup(*schema.Schema) error { return nil } 54 | func (m *storeSource) Tables() []string { return []string{m.table} } 55 | func (m *storeSource) Columns() []string { return m.f.fdbcols } 56 | func (m *storeSource) CreateIterator() schema.Iterator { return m } 57 | func (m *storeSource) Table(tableName string) (*schema.Table, error) { 58 | // u.Debugf("Table(%q), tbl nil?%v", tableName, m.tbl == nil) 59 | if m.tbl != nil { 60 | return m.tbl, nil 61 | } else { 62 | m.loadTable() 63 | } 64 | if m.tbl != nil { 65 | return m.tbl, nil 66 | } 67 | return nil, schema.ErrNotFound 68 | } 69 | func (m *storeSource) loadTable() error { 70 | m.mu.Lock() 71 | defer m.mu.Unlock() 72 | if m.tbl != nil { 73 | return nil 74 | } 75 | // u.Debugf("storeSource.loadTable(%q)", m.table) 76 | tbl := schema.NewTable(strings.ToLower(m.table)) 77 | columns := m.Columns() 78 | for i := range columns { 79 | columns[i] = strings.ToLower(columns[i]) 80 | tbl.AddField(schema.NewFieldBase(columns[i], value.StringType, 64, "string")) 81 | } 82 | tbl.SetColumns(columns) 83 | m.tbl = tbl 84 | return nil 85 | } 86 | func (m *storeSource) Open(connInfo string) (schema.Conn, error) { 87 | 88 | // u.Debugf("Open(%q)", connInfo) 89 | // Make a copy of itself 90 | s := &storeSource{ 91 | f: m.f, 92 | table: m.table, 93 | tbl: m.tbl, 94 | exit: make(<-chan bool, 1), 95 | } 96 | q := cloudstorage.Query{Delimiter: "", Prefix: m.f.path} 97 | q.Sorted() 98 | var err error 99 | s.iter, err = m.f.store.Objects(context.Background(), q) 100 | if err != nil { 101 | return nil, err 102 | } 103 | return s, nil 104 | } 105 | 106 | func (m *storeSource) Close() error { 107 | defer func() { 108 | if r := recover(); r != nil { 109 | u.Errorf("close error: %v", r) 110 | } 111 | }() 112 | return nil 113 | } 114 | 115 | func (m *storeSource) Next() schema.Message { 116 | 117 | select { 118 | case <-m.exit: 119 | return nil 120 | default: 121 | for { 122 | o, err := m.iter.Next() 123 | if err != nil { 124 | if err == iterator.Done { 125 | m.complete = true 126 | return nil 127 | } else { 128 | // Should we Retry? 129 | m.err = err 130 | return nil 131 | } 132 | } 133 | if o == nil { 134 | return nil 135 | } 136 | 137 | m.rowct++ 138 | 139 | fi := m.f.File(o) 140 | if fi == nil { 141 | u.Infof("ignoring path:%v %q is nil", m.f.path, o.Name()) 142 | continue 143 | } 144 | return datasource.NewSqlDriverMessageMap(m.rowct, fi.Values(), m.f.fdbcolidx) 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /datasource/introspect.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "database/sql/driver" 5 | "encoding/json" 6 | "time" 7 | 8 | u "github.com/araddon/gou" 9 | 10 | "github.com/araddon/qlbridge/schema" 11 | "github.com/araddon/qlbridge/value" 12 | ) 13 | 14 | var ( 15 | // IntrospectCount is default number of rows to evaluate for introspection 16 | // based schema discovery. 17 | IntrospectCount = 20 18 | ) 19 | 20 | // IntrospectSchema discover schema from contents of row introspection. 21 | func IntrospectSchema(s *schema.Schema, name string, iter schema.Iterator) error { 22 | tbl, err := s.Table(name) 23 | if err != nil { 24 | u.Errorf("Could not find table %q", name) 25 | return err 26 | } 27 | return IntrospectTable(tbl, iter) 28 | } 29 | 30 | // IntrospectTable accepts a table and schema Iterator and will 31 | // read a representative sample of rows, introspecting the results 32 | // to create a schema. Generally used for CSV, Json files to 33 | // create strongly typed schemas. 34 | func IntrospectTable(tbl *schema.Table, iter schema.Iterator) error { 35 | 36 | needsCols := len(tbl.Columns()) == 0 37 | nameIndex := make(map[int]string, len(tbl.Columns())) 38 | for i, colName := range tbl.Columns() { 39 | nameIndex[i] = colName 40 | } 41 | //u.Infof("s:%s INTROSPECT SCHEMA name %q", s, name) 42 | ct := 0 43 | for { 44 | msg := iter.Next() 45 | //u.Debugf("msg %#v", msg) 46 | if msg == nil || ct > IntrospectCount { 47 | break 48 | } 49 | switch mt := msg.Body().(type) { 50 | case []driver.Value: 51 | for i, v := range mt { 52 | 53 | k := nameIndex[i] 54 | _, exists := tbl.FieldMap[k] 55 | 56 | //u.Debugf("i:%v k:%s v: %T %v", i, k, v, v) 57 | switch val := v.(type) { 58 | case int, int64, int16, int32, uint16, uint64, uint32: 59 | tbl.AddFieldType(k, value.IntType) 60 | case time.Time, *time.Time: 61 | tbl.AddFieldType(k, value.TimeType) 62 | case bool: 63 | tbl.AddFieldType(k, value.BoolType) 64 | case float32, float64: 65 | tbl.AddFieldType(k, value.NumberType) 66 | case string: 67 | valType := value.ValueTypeFromStringAll(val) 68 | if !exists { 69 | tbl.AddFieldType(k, valType) 70 | //fld := tbl.FieldMap[k] 71 | //u.Debugf("add field? %+v", fld) 72 | //u.Debugf("%s = %v type: %T vt:%s new? %v", k, val, val, valType, !exists) 73 | } 74 | case map[string]interface{}: 75 | tbl.AddFieldType(k, value.JsonType) 76 | default: 77 | u.Debugf("not implemented: %T", val) 78 | } 79 | } 80 | case *SqlDriverMessageMap: 81 | if needsCols { 82 | nameIndex = make(map[int]string, len(mt.ColIndex)) 83 | for k2, ki := range mt.ColIndex { 84 | nameIndex[ki] = k2 85 | } 86 | } 87 | for i, v := range mt.Vals { 88 | 89 | k := nameIndex[i] 90 | // if k == "" { 91 | // for k2, ki := range mt.ColIndex { 92 | // if ki == i { 93 | // k = k2 94 | // break 95 | // } 96 | // } 97 | // } 98 | 99 | _, exists := tbl.FieldMap[k] 100 | 101 | //u.Debugf("i:%v k:%s v: %T %v", i, k, v, v) 102 | switch val := v.(type) { 103 | case int, int64, int16, int32, uint16, uint64, uint32: 104 | tbl.AddFieldType(k, value.IntType) 105 | case time.Time, *time.Time: 106 | tbl.AddFieldType(k, value.TimeType) 107 | case bool: 108 | tbl.AddFieldType(k, value.BoolType) 109 | case float32, float64, json.Number: 110 | tbl.AddFieldType(k, value.NumberType) 111 | case string: 112 | valType := value.ValueTypeFromStringAll(val) 113 | if !exists { 114 | tbl.AddFieldType(k, valType) 115 | //fld := tbl.FieldMap[k] 116 | //u.Debugf("add field? %+v", fld) 117 | //u.Debugf("%s = %v type: %T vt:%s new? %v", k, val, val, valType, !exists) 118 | } 119 | case map[string]interface{}: 120 | tbl.AddFieldType(k, value.JsonType) 121 | case []interface{}: 122 | tbl.AddFieldType(k, value.JsonType) 123 | case nil: 124 | // hm..... 125 | tbl.AddFieldType(k, value.JsonType) 126 | default: 127 | tbl.AddFieldType(k, value.JsonType) 128 | u.LogThrottle(u.WARN, 10, "not implemented: k:%v %T", k, val) 129 | } 130 | } 131 | default: 132 | u.Warnf("not implemented: %T", mt) 133 | } 134 | 135 | ct++ 136 | } 137 | if needsCols { 138 | cols := make([]string, len(tbl.Fields)) 139 | for i, f := range tbl.Fields { 140 | //u.Debugf("%+v", f) 141 | cols[i] = f.Name 142 | } 143 | tbl.SetColumns(cols) 144 | } 145 | 146 | //u.Debugf("%s: %v", tbl.Name, tbl.Columns()) 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /datasource/introspect_test.go: -------------------------------------------------------------------------------- 1 | package datasource_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/araddon/qlbridge/datasource" 10 | td "github.com/araddon/qlbridge/datasource/mockcsvtestdata" 11 | "github.com/araddon/qlbridge/schema" 12 | "github.com/araddon/qlbridge/testutil" 13 | "github.com/araddon/qlbridge/value" 14 | ) 15 | 16 | func TestMain(m *testing.M) { 17 | testutil.Setup() // will call flag.Parse() 18 | 19 | // load our mock data sources "users", "articles" 20 | td.LoadTestDataOnce() 21 | 22 | // Now run the actual Tests 23 | os.Exit(m.Run()) 24 | } 25 | 26 | func TestIntrospectedCsvSchema(t *testing.T) { 27 | sch := td.MockSchema 28 | 29 | tableName := "users" 30 | csvSrc, err := sch.OpenConn(tableName) 31 | assert.Equal(t, nil, err) 32 | scanner, ok := csvSrc.(schema.ConnScanner) 33 | assert.True(t, ok) 34 | 35 | err = datasource.IntrospectSchema(sch, tableName, scanner) 36 | assert.Equal(t, nil, err) 37 | tbl, err := sch.Table("users") 38 | assert.Equal(t, nil, err) 39 | assert.Equal(t, "users", tbl.Name) 40 | assert.Equal(t, 6, len(tbl.Fields)) 41 | 42 | refCt := tbl.FieldMap["referral_count"] 43 | assert.Equal(t, int(value.IntType), int(refCt.Type), "wanted int got %s", refCt.Type) 44 | 45 | userId := tbl.FieldMap["user_id"] 46 | assert.Equal(t, int(value.StringType), int(userId.Type), "wanted string got %s", userId.Type) 47 | 48 | jd := tbl.FieldMap["json_data"] 49 | assert.Equal(t, int(value.JsonType), int(jd.Type), "wanted json got %s", jd.Type) 50 | } 51 | -------------------------------------------------------------------------------- /datasource/json_test.go: -------------------------------------------------------------------------------- 1 | package datasource_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | "testing" 7 | 8 | u "github.com/araddon/gou" 9 | "github.com/stretchr/testify/assert" 10 | 11 | "github.com/araddon/qlbridge/datasource" 12 | "github.com/araddon/qlbridge/schema" 13 | ) 14 | 15 | var ( 16 | _ = u.EMPTY 17 | testJsonData = map[string]string{ 18 | "user.json": `{"user_id": "9Ip1aKbeZe2njCDM" ,"email":"aaron@email.com","interests":"fishing","reg_date":"2012-10-17T17:29:39.738Z","item_count":82} 19 | {"user_id": "hT2impsOPUREcVPc" ,"email":"bob@email.com","interests":"swimming","reg_date":"2009-12-11T19:53:31.547Z","item_count":12} 20 | {"user_id": "hT2impsabc345c" ,"email":"not_an_email","interests":"swimming","reg_date":"2009-12-11T19:53:31.547Z","item_count":12}`} 21 | 22 | jsonSource schema.Source = &datasource.JsonSource{} 23 | jsonStringSource schema.Source = &jsonStaticSource{files: testJsonData} 24 | ) 25 | 26 | type jsonStaticSource struct { 27 | *datasource.JsonSource 28 | files map[string]string 29 | } 30 | 31 | func (m *jsonStaticSource) Open(connInfo string) (schema.Conn, error) { 32 | if data, ok := m.files[connInfo]; ok { 33 | sr := strings.NewReader(data) 34 | return datasource.NewJsonSource(connInfo, ioutil.NopCloser(sr), make(<-chan bool, 1), nil) 35 | } 36 | return nil, schema.ErrNotFound 37 | } 38 | 39 | func TestJsonDataSource(t *testing.T) { 40 | jsonIn, err := jsonStringSource.Open("user.json") 41 | assert.Equal(t, nil, err, "should not have error: %v", err) 42 | iter, ok := jsonIn.(schema.ConnScanner) 43 | assert.True(t, ok) 44 | iterCt := 0 45 | for msg := iter.Next(); msg != nil; msg = iter.Next() { 46 | iterCt++ 47 | u.Infof("row: %v", msg.Body()) 48 | } 49 | assert.Equal(t, 3, iterCt, "should have 3 rows: %v", iterCt) 50 | } 51 | -------------------------------------------------------------------------------- /datasource/key.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "database/sql/driver" 5 | 6 | u "github.com/araddon/gou" 7 | 8 | "github.com/araddon/qlbridge/expr" 9 | "github.com/araddon/qlbridge/rel" 10 | "github.com/araddon/qlbridge/schema" 11 | ) 12 | 13 | var ( 14 | _ schema.Key = (*KeyInt)(nil) 15 | _ schema.Key = (*KeyInt64)(nil) 16 | _ schema.Key = (*KeyCol)(nil) 17 | ) 18 | 19 | // Variety of Key Types 20 | type ( 21 | KeyInt struct { 22 | Id int 23 | } 24 | KeyInt64 struct { 25 | Id int64 26 | } 27 | KeyCol struct { 28 | Name string 29 | Val driver.Value 30 | } 31 | ) 32 | 33 | func NewKeyInt(key int) KeyInt { return KeyInt{key} } 34 | func (m *KeyInt) Key() driver.Value { return driver.Value(m.Id) } 35 | 36 | //func (m KeyInt) Less(than Item) bool { return m.Id < than.(KeyInt).Id } 37 | 38 | func NewKeyInt64(key int64) KeyInt64 { return KeyInt64{key} } 39 | func (m *KeyInt64) Key() driver.Value { return driver.Value(m.Id) } 40 | 41 | func NewKeyCol(name string, val driver.Value) KeyCol { return KeyCol{name, val} } 42 | func (m KeyCol) Key() driver.Value { return m.Val } 43 | 44 | // Given a Where expression, lets try to create a key which 45 | // requires form `idenity = "value"` 46 | // 47 | func KeyFromWhere(wh interface{}) schema.Key { 48 | switch n := wh.(type) { 49 | case *rel.SqlWhere: 50 | return KeyFromWhere(n.Expr) 51 | case *expr.BinaryNode: 52 | if len(n.Args) != 2 { 53 | u.Warnf("need more args? %#v", n.Args) 54 | return nil 55 | } 56 | in, ok := n.Args[0].(*expr.IdentityNode) 57 | if !ok { 58 | u.Warnf("not identity? %T", n.Args[0]) 59 | return nil 60 | } 61 | // This only allows for identity = value 62 | // NOT: identity = expr(identity, arg) 63 | // 64 | switch valT := n.Args[1].(type) { 65 | case *expr.NumberNode: 66 | return NewKeyCol(in.Text, valT.Float64) 67 | case *expr.StringNode: 68 | return NewKeyCol(in.Text, valT.Text) 69 | //case *expr.FuncNode: 70 | default: 71 | u.Warnf("not supported arg? %#v", valT) 72 | } 73 | default: 74 | u.Warnf("not supported node type? %#v", n) 75 | } 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /datasource/memdb/db_test.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "database/sql/driver" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/araddon/dateparse" 10 | "github.com/stretchr/testify/assert" 11 | 12 | "github.com/araddon/qlbridge/datasource" 13 | "github.com/araddon/qlbridge/expr" 14 | "github.com/araddon/qlbridge/schema" 15 | "github.com/araddon/qlbridge/testutil" 16 | ) 17 | 18 | func TestMain(m *testing.M) { 19 | testutil.Setup() // will call flag.Parse() 20 | 21 | // Now run the actual Tests 22 | os.Exit(m.Run()) 23 | } 24 | 25 | func TestMemDb(t *testing.T) { 26 | 27 | created := dateparse.MustParse("2015/07/04") 28 | inrow := []driver.Value{122, "bob", "bob@email.com", created.In(time.UTC).Add(time.Hour * -24), []string{"not_admin"}} 29 | 30 | cols := []string{"user_id", "name", "email", "created", "roles"} 31 | db, err := NewMemDbData("users", [][]driver.Value{inrow}, cols) 32 | assert.Equal(t, nil, err) 33 | db.Init() 34 | db.Setup(nil) 35 | 36 | c, err := db.Open("users") 37 | assert.Equal(t, nil, err) 38 | dc, ok := c.(schema.ConnAll) 39 | assert.True(t, ok) 40 | 41 | dc.Put(nil, &datasource.KeyInt{Id: 123}, []driver.Value{123, "aaron", "email@email.com", created.In(time.UTC), []string{"admin"}}) 42 | row, err := dc.Get(123) 43 | assert.Equal(t, nil, err) 44 | assert.NotEqual(t, nil, row) 45 | di, ok := row.(*datasource.SqlDriverMessage) 46 | assert.True(t, ok) 47 | vals := di.Vals 48 | assert.Equal(t, 5, len(vals), "want 5 cols in user but got %v", len(vals)) 49 | assert.Equal(t, 123, vals[0].(int)) 50 | assert.Equal(t, "email@email.com", vals[2].(string)) 51 | 52 | _, err = dc.Put(nil, &datasource.KeyInt{Id: 225}, []driver.Value{}) 53 | assert.NotEqual(t, nil, err) 54 | 55 | dc.Put(nil, &datasource.KeyInt{Id: 123}, []driver.Value{123, "aaron", "aaron@email.com", created.In(time.UTC), []string{"root", "admin"}}) 56 | row, _ = dc.Get(123) 57 | assert.NotEqual(t, nil, row) 58 | vals2 := row.Body().([]driver.Value) 59 | 60 | assert.Equal(t, "aaron@email.com", vals2[2].(string)) 61 | assert.Equal(t, []string{"root", "admin"}, vals2[4], "Roles should match updated vals") 62 | assert.Equal(t, created, vals2[3], "created date should match updated vals") 63 | 64 | ct := 0 65 | for { 66 | msg := dc.Next() 67 | if msg == nil { 68 | break 69 | } 70 | ct++ 71 | } 72 | assert.Equal(t, 2, ct) 73 | err = dc.Close() 74 | assert.Equal(t, nil, err) 75 | 76 | // Schema 77 | tbl, err := db.Table("users") 78 | assert.Equal(t, nil, err) 79 | assert.Equal(t, cols, tbl.Columns()) 80 | assert.Equal(t, []string{"users"}, db.Tables()) 81 | 82 | // error testing 83 | _, err = NewMemDbData("users", [][]driver.Value{inrow}, nil) 84 | assert.NotEqual(t, nil, err) 85 | 86 | exprNode := expr.MustParse(`email == "bob@email.com"`) 87 | 88 | delCt, err := dc.DeleteExpression(nil, exprNode) 89 | assert.Equal(t, nil, err) 90 | assert.Equal(t, 1, delCt) 91 | 92 | delCt, err = dc.Delete(driver.Value(123)) 93 | assert.Equal(t, nil, err) 94 | assert.Equal(t, 1, delCt) 95 | 96 | key := datasource.KeyInt{Id: 123} 97 | _, err = dc.PutMulti(nil, []schema.Key{&key}, [][]driver.Value{{123, "aaron", "aaron@email.com", created.In(time.UTC), []string{"root", "admin"}}}) 98 | assert.Equal(t, nil, err) 99 | 100 | err = db.Close() 101 | assert.Equal(t, nil, err) 102 | 103 | // Make sure we can cancel/stop 104 | c2, err := db.Open("users") 105 | assert.Equal(t, nil, err) 106 | dc2, ok := c2.(schema.ConnAll) 107 | 108 | ct = 0 109 | for { 110 | msg := dc2.Next() 111 | if msg == nil { 112 | break 113 | } 114 | ct++ 115 | } 116 | assert.Equal(t, 0, ct) 117 | } 118 | -------------------------------------------------------------------------------- /datasource/memdb/index.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "database/sql/driver" 5 | "fmt" 6 | 7 | u "github.com/araddon/gou" 8 | "github.com/dchest/siphash" 9 | "github.com/hashicorp/go-memdb" 10 | 11 | "github.com/araddon/qlbridge/datasource" 12 | "github.com/araddon/qlbridge/schema" 13 | ) 14 | 15 | var ( 16 | _ = u.EMPTY 17 | // Indexes 18 | _ memdb.Indexer = (*indexWrapper)(nil) 19 | ) 20 | 21 | func makeId(dv driver.Value) uint64 { 22 | switch vt := dv.(type) { 23 | case int: 24 | return uint64(vt) 25 | case int64: 26 | return uint64(vt) 27 | case []byte: 28 | return siphash.Hash(456729, 1111581582, vt) 29 | case string: 30 | return siphash.Hash(456729, 1111581582, []byte(vt)) 31 | //by := append(make([]byte,0,8), byte(r), byte(r>>8), byte(r>>16), byte(r>>24), byte(r>>32), byte(r>>40), byte(r>>48), byte(r>>56)) 32 | case datasource.KeyCol: 33 | return makeId(vt.Val) 34 | } 35 | return 0 36 | } 37 | 38 | // Wrap the index so we can operate on rows 39 | type indexWrapper struct { 40 | t *schema.Table 41 | *schema.Index 42 | } 43 | 44 | func (s *indexWrapper) FromObject(obj interface{}) (bool, []byte, error) { 45 | switch row := obj.(type) { 46 | case *datasource.SqlDriverMessage: 47 | if len(row.Vals) < 0 { 48 | return false, nil, u.LogErrorf("No values in row?") 49 | } 50 | // Add the null character as a terminator 51 | val := fmt.Sprintf("%v", row.Vals[0]) 52 | val += "\x00" 53 | return true, []byte(val), nil 54 | case int, uint64, int64, string: 55 | // Add the null character as a terminator 56 | val := fmt.Sprintf("%v\x00", row) 57 | return true, []byte(val), nil 58 | default: 59 | return false, nil, u.LogErrorf("Unrecognized type %T", obj) 60 | } 61 | } 62 | 63 | func (s *indexWrapper) FromArgs(args ...interface{}) ([]byte, error) { 64 | if len(args) != 1 { 65 | return nil, fmt.Errorf("must provide only a single argument") 66 | } 67 | arg := fmt.Sprintf("%v", args[0]) 68 | // Add the null character as a terminator 69 | arg += "\x00" 70 | return []byte(arg), nil 71 | } 72 | 73 | func makeMemDbSchema(m *MemDb) *memdb.DBSchema { 74 | 75 | sindexes := make(map[string]*memdb.IndexSchema) 76 | 77 | for _, idx := range m.indexes { 78 | sidx := &memdb.IndexSchema{ 79 | Name: idx.Name, 80 | Indexer: &indexWrapper{Index: idx}, 81 | } 82 | if idx.PrimaryKey { 83 | sidx.Unique = true 84 | } 85 | sindexes[idx.Name] = sidx 86 | } 87 | /* 88 | { 89 | "id": &memdb.IndexSchema{ 90 | Name: "id", 91 | Unique: true, 92 | Indexer: &memdb.StringFieldIndex{Field: "ID"}, 93 | }, 94 | "foo": &memdb.IndexSchema{ 95 | Name: "foo", 96 | Indexer: &memdb.StringFieldIndex{Field: "Foo"}, 97 | }, 98 | }, 99 | */ 100 | s := memdb.DBSchema{ 101 | Tables: map[string]*memdb.TableSchema{ 102 | m.tbl.Name: { 103 | Name: m.tbl.Name, 104 | Indexes: sindexes, 105 | }, 106 | }, 107 | } 108 | return &s 109 | } 110 | -------------------------------------------------------------------------------- /datasource/memdb/index_test.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "database/sql/driver" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/araddon/qlbridge/datasource" 10 | ) 11 | 12 | func TestIndex(t *testing.T) { 13 | assert.Equal(t, uint64(264), makeId(driver.Value(int64(264)))) 14 | 15 | assert.Equal(t, uint64(0), makeId(driver.Value(nil))) 16 | 17 | assert.Equal(t, uint64(2404974478441873708), makeId([]byte("hello")), " %v", makeId([]byte("hello"))) 18 | 19 | assert.Equal(t, uint64(2404974478441873708), makeId("hello"), " %v", makeId("hello")) 20 | 21 | v := datasource.KeyCol{Val: 264} 22 | 23 | assert.Equal(t, uint64(264), makeId(v)) 24 | } 25 | -------------------------------------------------------------------------------- /datasource/mockcsvtestdata/testdata.go: -------------------------------------------------------------------------------- 1 | // Package mockcsvtestdata is csv test data only used for tests. 2 | package mockcsvtestdata 3 | 4 | import ( 5 | "sync" 6 | 7 | "github.com/araddon/qlbridge/datasource" 8 | "github.com/araddon/qlbridge/datasource/mockcsv" 9 | "github.com/araddon/qlbridge/expr/builtins" 10 | "github.com/araddon/qlbridge/plan" 11 | "github.com/araddon/qlbridge/schema" 12 | ) 13 | 14 | var ( 15 | loadData sync.Once 16 | MockSchema *schema.Schema 17 | 18 | // TestContext is a function to create plan.Context for a given test context. 19 | TestContext func(query string) *plan.Context 20 | ) 21 | 22 | func SetContextToMockCsv() { 23 | TestContext = func(query string) *plan.Context { 24 | ctx := plan.NewContext(query) 25 | ctx.DisableRecover = true 26 | ctx.Schema = MockSchema 27 | ctx.Session = datasource.NewMySqlSessionVars() 28 | return ctx 29 | } 30 | } 31 | 32 | func SchemaLoader(name string) (*schema.Schema, error) { 33 | return MockSchema, nil 34 | } 35 | 36 | func LoadTestDataOnce() { 37 | loadData.Do(func() { 38 | 39 | // Load in a "csv file" into our mock data store 40 | mockcsv.LoadTable(mockcsv.SchemaName, "users", `user_id,email,interests,reg_date,referral_count,json_data 41 | 9Ip1aKbeZe2njCDM,"aaron@email.com","fishing","2012-10-17T17:29:39.738Z",82,"{""name"":""aaron""}" 42 | hT2impsOPUREcVPc,"bob@email.com","swimming","2009-12-11T19:53:31.547Z",12,"{""name"":""bob""}" 43 | hT2impsabc345c,"not_an_email_2",,"2009-12-11T19:53:31.547Z",12,"{""name"":""notbob""}"`) 44 | 45 | mockcsv.LoadTable(mockcsv.SchemaName, "orders", `order_id,user_id,item_id,price,order_date,item_count 46 | 1,9Ip1aKbeZe2njCDM,1,22.50,"2012-12-24T17:29:39.738Z",82 47 | 2,9Ip1aKbeZe2njCDM,2,37.50,"2013-10-24T17:29:39.738Z",82 48 | 3,abcabcabc,1,22.50,"2013-10-24T17:29:39.738Z",82 49 | `) 50 | 51 | MockSchema = mockcsv.Schema() 52 | if MockSchema == nil { 53 | panic("MockSchema Must Exist") 54 | } 55 | 56 | SetContextToMockCsv() 57 | 58 | //reg.RefreshSchema(mockcsv.SchemaName) 59 | builtins.LoadAllBuiltins() 60 | }) 61 | } 62 | -------------------------------------------------------------------------------- /datasource/schemadb_test.go: -------------------------------------------------------------------------------- 1 | package datasource_test 2 | 3 | import ( 4 | "database/sql/driver" 5 | "testing" 6 | 7 | "github.com/araddon/qlbridge/datasource" 8 | "github.com/araddon/qlbridge/testutil" 9 | ) 10 | 11 | func TestSchemaShowStatements(t *testing.T) { 12 | 13 | // TODO: this test needs the "databases" ie system-schema not current-info-schema 14 | testutil.TestSelect(t, `show databases;`, 15 | [][]driver.Value{{"mockcsv"}}, 16 | ) 17 | // - rewrite show tables -> "use schema; select name from schema.tables;" 18 | testutil.TestSelect(t, `show tables;`, 19 | [][]driver.Value{{"orders"}, {"users"}}, 20 | ) 21 | testutil.TestSelect(t, `show tables from mockcsv;`, 22 | [][]driver.Value{{"orders"}, {"users"}}, 23 | ) 24 | testutil.TestSelect(t, `show tables in mockcsv;`, 25 | [][]driver.Value{{"orders"}, {"users"}}, 26 | ) 27 | 28 | // TODO: we need to detect other schemas? and error on non-existent schemas? 29 | //testutil.TestSelectErr(t, `show tables from non_existent;`, nil) 30 | 31 | // show table create 32 | createStmt := "CREATE TABLE `users` (\n" + 33 | " `user_id` varchar(255) DEFAULT NULL,\n" + 34 | " `email` varchar(255) DEFAULT NULL,\n" + 35 | " `interests` varchar(255) DEFAULT NULL,\n" + 36 | " `reg_date` datetime DEFAULT NULL,\n" + 37 | " `referral_count` bigint DEFAULT NULL,\n" + 38 | " `json_data` JSON\n" + 39 | ") ENGINE=InnoDB DEFAULT CHARSET=utf8;" 40 | testutil.TestSelect(t, `show create table users;`, 41 | [][]driver.Value{{"users", createStmt}}, 42 | ) 43 | 44 | // - rewrite show tables -> "use schema; select Table, Table_Type from schema.tables;" 45 | testutil.TestSelect(t, `show full tables;`, 46 | [][]driver.Value{{"orders", "BASE TABLE"}, {"users", "BASE TABLE"}}, 47 | ) 48 | testutil.TestSelect(t, `show tables like "us%";`, 49 | [][]driver.Value{{"users"}}, 50 | ) 51 | testutil.TestSelect(t, `show full tables like "us%";`, 52 | [][]driver.Value{{"users", "BASE TABLE"}}, 53 | ) 54 | testutil.TestSelect(t, `show full tables from mockcsv like "us%";`, 55 | [][]driver.Value{{"users", "BASE TABLE"}}, 56 | ) 57 | 58 | // SHOW [FULL] COLUMNS FROM tbl_name [FROM db_name] [like_or_where] 59 | testutil.TestSelect(t, `show columns from users;`, 60 | [][]driver.Value{ 61 | {"user_id", "string", "", "", "", ""}, 62 | {"email", "string", "", "", "", ""}, 63 | {"interests", "string", "", "", "", ""}, 64 | {"reg_date", "time", "", "", "", ""}, 65 | {"referral_count", "int", "", "", "", ""}, 66 | {"json_data", "json", "", "", "", ""}, 67 | }, 68 | ) 69 | testutil.TestSelect(t, `show columns FROM users FROM mockcsv;`, 70 | [][]driver.Value{ 71 | {"user_id", "string", "", "", "", ""}, 72 | {"email", "string", "", "", "", ""}, 73 | {"interests", "string", "", "", "", ""}, 74 | {"reg_date", "time", "", "", "", ""}, 75 | {"referral_count", "int", "", "", "", ""}, 76 | {"json_data", "json", "", "", "", ""}, 77 | }, 78 | ) 79 | testutil.TestSelect(t, `show columns from users WHERE Field Like "email";`, 80 | [][]driver.Value{ 81 | {"email", "string", "", "", "", ""}, 82 | }, 83 | ) 84 | testutil.TestSelect(t, `show full columns from users WHERE Field Like "email";`, 85 | [][]driver.Value{ 86 | {"email", "string", "", "", "", "", "", "", ""}, 87 | }, 88 | ) 89 | testutil.TestSelect(t, `show columns from users Like "user%";`, 90 | [][]driver.Value{ 91 | {"user_id", "string", "", "", "", ""}, 92 | }, 93 | ) 94 | // VARIABLES 95 | testutil.TestSelect(t, `show global variables like 'max_allowed*';`, 96 | [][]driver.Value{ 97 | {"max_allowed_packet", int64(datasource.MaxAllowedPacket)}, 98 | }, 99 | ) 100 | 101 | // DESCRIBE 102 | testutil.TestSelect(t, `describe users;`, 103 | [][]driver.Value{ 104 | {"user_id", "string", "", "", "", ""}, 105 | {"email", "string", "", "", "", ""}, 106 | {"interests", "string", "", "", "", ""}, 107 | {"reg_date", "time", "", "", "", ""}, 108 | {"referral_count", "int", "", "", "", ""}, 109 | {"json_data", "json", "", "", "", ""}, 110 | }, 111 | ) 112 | 113 | } 114 | -------------------------------------------------------------------------------- /datasource/session.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "database/sql/driver" 5 | "time" 6 | 7 | "github.com/araddon/qlbridge/expr" 8 | "github.com/araddon/qlbridge/plan" 9 | "github.com/araddon/qlbridge/value" 10 | ) 11 | 12 | const ( 13 | // Default Max Allowed packets for connections 14 | MaxAllowedPacket = 4194304 15 | ) 16 | 17 | // http://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html 18 | var mysqlGlobalVars *ContextSimple = NewMySqlGlobalVars() 19 | 20 | func RowsForSession(ctx *plan.Context) [][]driver.Value { 21 | 22 | ses := ctx.Session.Row() 23 | rows := make([][]driver.Value, 0, len(ses)) 24 | for k, v := range ses { 25 | rows = append(rows, []driver.Value{k, v.Value()}) 26 | } 27 | return rows 28 | } 29 | 30 | func NewMySqlSessionVars() expr.ContextReadWriter { 31 | ctx := NewContextSimple() 32 | ctx.Data["@@max_allowed_packet"] = value.NewIntValue(MaxAllowedPacket) 33 | ctx.Data["@@session.auto_increment_increment"] = value.NewIntValue(1) 34 | ctx.Data["@@session.tx_isolation"] = value.NewStringValue("REPEATABLE-READ") 35 | rdr := NewNestedContextReadWriter([]expr.ContextReader{ 36 | ctx, 37 | mysqlGlobalVars, 38 | }, ctx, time.Now()) 39 | return rdr 40 | } 41 | 42 | func NewMySqlGlobalVars() *ContextSimple { 43 | ctx := NewContextSimple() 44 | 45 | ctx.Data["@@session.auto_increment_increment"] = value.NewIntValue(1) 46 | ctx.Data["auto_increment_increment"] = value.NewIntValue(1) 47 | ctx.Data["@@session.tx_read_only"] = value.NewIntValue(1) 48 | //ctx.Data["@@session.auto_increment_increment"] = value.NewBoolValue(true) 49 | ctx.Data["@@character_set_client"] = value.NewStringValue("utf8") 50 | ctx.Data["@@character_set_connection"] = value.NewStringValue("utf8") 51 | ctx.Data["@@character_set_results"] = value.NewStringValue("utf8") 52 | ctx.Data["@@character_set_server"] = value.NewStringValue("utf8") 53 | ctx.Data["@@init_connect"] = value.NewStringValue("") 54 | ctx.Data["@@interactive_timeout"] = value.NewIntValue(28800) 55 | ctx.Data["@@license"] = value.NewStringValue("MIT") 56 | ctx.Data["@@lower_case_table_names"] = value.NewIntValue(0) 57 | ctx.Data["max_allowed_packet"] = value.NewIntValue(MaxAllowedPacket) 58 | ctx.Data["@@max_allowed_packet"] = value.NewIntValue(MaxAllowedPacket) 59 | ctx.Data["@@max_allowed_packets"] = value.NewIntValue(MaxAllowedPacket) 60 | ctx.Data["@@net_buffer_length"] = value.NewIntValue(16384) 61 | ctx.Data["@@net_write_timeout"] = value.NewIntValue(600) 62 | ctx.Data["@@query_cache_size"] = value.NewIntValue(1048576) 63 | ctx.Data["@@query_cache_type"] = value.NewStringValue("OFF") 64 | ctx.Data["@@sql_mode"] = value.NewStringValue("NO_ENGINE_SUBSTITUTION") 65 | ctx.Data["@@system_time_zone"] = value.NewStringValue("UTC") 66 | ctx.Data["@@time_zone"] = value.NewStringValue("SYSTEM") 67 | ctx.Data["@@tx_isolation"] = value.NewStringValue("REPEATABLE-READ") 68 | ctx.Data["@@version_comment"] = value.NewStringValue("DataUX (MIT), Release .0.9") 69 | ctx.Data["@@wait_timeout"] = value.NewIntValue(28800) 70 | return ctx 71 | } 72 | -------------------------------------------------------------------------------- /datasource/sqlite/schemawriter.go: -------------------------------------------------------------------------------- 1 | package sqlite 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/araddon/qlbridge/datasource" 9 | "github.com/araddon/qlbridge/schema" 10 | "github.com/araddon/qlbridge/value" 11 | ) 12 | 13 | var ( 14 | 15 | // normal tables 16 | //defaultSchemaTables = []string{"tables", "databases", "columns", "global_variables", "session_variables","functions", "procedures", "engines", "status", "indexes"} 17 | ) 18 | 19 | func init() { 20 | datasource.DialectWriterCols = append(datasource.DialectWriterCols, "sqlite") 21 | datasource.DialectWriters = append(datasource.DialectWriters, &sqliteWriter{}) 22 | } 23 | 24 | type sqliteWriter struct { 25 | } 26 | 27 | func (m *sqliteWriter) Dialect() string { 28 | return "sqlite" 29 | } 30 | func (m *sqliteWriter) FieldType(t value.ValueType) string { 31 | return ValueString(t) 32 | } 33 | 34 | // Table Implement Dialect Specific Writers 35 | // ie, mysql, postgres, cassandra all have different dialects 36 | // so the Create statements are quite different 37 | 38 | // Table output a CREATE TABLE statement using mysql dialect. 39 | func (m *sqliteWriter) Table(tbl *schema.Table) string { 40 | return TableToString(tbl) 41 | } 42 | 43 | // TableToString Table output a CREATE TABLE statement using mysql dialect. 44 | func TableToString(tbl *schema.Table) string { 45 | 46 | w := &bytes.Buffer{} 47 | //u.Infof("%s tbl=%p fields? %#v fields?%v", tbl.Name, tbl, tbl.FieldMap, len(tbl.Fields)) 48 | fmt.Fprintf(w, "CREATE TABLE `%s` (", tbl.Name) 49 | for i, fld := range tbl.Fields { 50 | if i != 0 { 51 | w.WriteByte(',') 52 | } 53 | fmt.Fprint(w, "\n ") 54 | WriteField(w, fld) 55 | } 56 | fmt.Fprint(w, "\n);") 57 | //tblStr := fmt.Sprintf("CREATE TABLE `%s` (\n\n);", tbl.Name, strings.Join(cols, ",")) 58 | //return tblStr, nil 59 | return w.String() 60 | } 61 | 62 | // WriteField write a schema.Field as string output for sqlite create statement 63 | // 64 | // https://www.sqlite.org/datatype3.html 65 | func WriteField(w *bytes.Buffer, fld *schema.Field) { 66 | fmt.Fprintf(w, "`%s` ", fld.Name) 67 | /* 68 | NULL. The value is a NULL value. 69 | INTEGER. The value is a signed integer, stored in 1, 2, 3, 4, 6, or 8 bytes depending on the magnitude of the value. 70 | REAL. The value is a floating point value, stored as an 8-byte IEEE floating point number. 71 | TEXT. The value is a text string, stored using the database encoding (UTF-8, UTF-16BE or UTF-16LE). 72 | BLOB. The value is a blob of data, stored exactly as it was input. 73 | */ 74 | //deflen := fld.Length 75 | switch fld.ValueType() { 76 | case value.BoolType: 77 | fmt.Fprint(w, "INTEGER") 78 | case value.IntType: 79 | fmt.Fprint(w, "INTEGER") 80 | case value.StringType: 81 | fmt.Fprintf(w, "text") 82 | case value.NumberType: 83 | fmt.Fprint(w, "REAL") 84 | case value.TimeType: 85 | fmt.Fprint(w, "text") 86 | case value.JsonType: 87 | fmt.Fprintf(w, "text") 88 | default: 89 | fmt.Fprint(w, "text") 90 | } 91 | if len(fld.Description) > 0 { 92 | fmt.Fprintf(w, " COMMENT %q", fld.Description) 93 | } 94 | } 95 | 96 | // TypeFromString given a string, return data type 97 | func TypeFromString(t string) value.ValueType { 98 | switch strings.ToLower(t) { 99 | case "integer": 100 | // This isn't necessarily true, as integer could be bool 101 | return value.IntType 102 | case "real": 103 | return value.NumberType 104 | default: 105 | return value.StringType 106 | } 107 | } 108 | 109 | // ValueString convert a value.ValueType into a sqlite type descriptor 110 | func ValueString(t value.ValueType) string { 111 | switch t { 112 | case value.NilType: 113 | return "text" 114 | case value.ErrorType: 115 | return "text" 116 | case value.UnknownType: 117 | return "text" 118 | case value.ValueInterfaceType: 119 | return "text" 120 | case value.NumberType: 121 | return "real" 122 | case value.IntType: 123 | return "integer" 124 | case value.BoolType: 125 | return "integer" 126 | case value.TimeType: 127 | return "text" 128 | case value.ByteSliceType: 129 | return "text" 130 | case value.StringType: 131 | return "text" 132 | case value.StringsType: 133 | return "text" 134 | case value.MapValueType: 135 | return "text" 136 | case value.MapIntType: 137 | return "text" 138 | case value.MapStringType: 139 | return "text" 140 | case value.MapNumberType: 141 | return "text" 142 | case value.MapBoolType: 143 | return "text" 144 | case value.SliceValueType: 145 | return "text" 146 | case value.StructType: 147 | return "text" 148 | case value.JsonType: 149 | return "text" 150 | default: 151 | return "text" 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /datasource/sqlite/sqlite_test.go: -------------------------------------------------------------------------------- 1 | package sqlite_test 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "encoding/json" 7 | "os" 8 | "sync" 9 | "testing" 10 | 11 | u "github.com/araddon/gou" 12 | td "github.com/araddon/qlbridge/datasource/mockcsvtestdata" 13 | "github.com/araddon/qlbridge/datasource/sqlite" 14 | "github.com/stretchr/testify/assert" 15 | 16 | // Ensure we import sqlite driver 17 | _ "github.com/mattn/go-sqlite3" 18 | 19 | "github.com/araddon/qlbridge/datasource" 20 | "github.com/araddon/qlbridge/plan" 21 | "github.com/araddon/qlbridge/schema" 22 | "github.com/araddon/qlbridge/testutil" 23 | ) 24 | 25 | /* 26 | TODO: 27 | - the schema doesn't exists/isn't getting loaded. 28 | 29 | */ 30 | var ( 31 | testFile = "./test.db" 32 | sch *schema.Schema 33 | loadData sync.Once 34 | ) 35 | 36 | func exitIfErr(err error) { 37 | if err != nil { 38 | panic(err.Error()) 39 | } 40 | } 41 | func LoadTestDataOnce(t *testing.T) { 42 | loadData.Do(func() { 43 | testutil.Setup() 44 | // load our mock data sources "users", "orders" 45 | td.LoadTestDataOnce() 46 | 47 | os.Remove(testFile) 48 | 49 | // It will be created if it doesn't exist. 50 | db, err := sql.Open("sqlite3", testFile) 51 | exitIfErr(err) 52 | err = db.Ping() 53 | exitIfErr(err) 54 | 55 | reg := schema.DefaultRegistry() 56 | by := []byte(`{ 57 | "name": "sqlite_test", 58 | "type": "sqlite", 59 | "settings" : { 60 | "file" : "test.db" 61 | } 62 | }`) 63 | 64 | conf := &schema.ConfigSource{} 65 | err = json.Unmarshal(by, conf) 66 | assert.Equal(t, nil, err) 67 | 68 | // Create Sqlite db schema 69 | for _, tablename := range td.MockSchema.Tables() { 70 | tbl, _ := td.MockSchema.Table(tablename) 71 | if tbl == nil { 72 | panic("missing table " + tablename) 73 | } 74 | //u.Infof("found schema for %s \n%s", tablename, tbl.String()) 75 | // for _, col := range tbl.Fields { 76 | // u.Debugf("%+v", col) 77 | // } 78 | u.Debugf("\n%s", sqlite.TableToString(tbl)) 79 | // Create table schema 80 | res, err := db.Exec(sqlite.TableToString(tbl)) 81 | assert.Equal(t, nil, err) 82 | assert.NotEqual(t, nil, res) 83 | //u.Infof("err=%v res=%+v", err, res) 84 | } 85 | db.Close() 86 | 87 | // From config, create schema 88 | err = reg.SchemaAddFromConfig(conf) 89 | assert.Equal(t, nil, err) 90 | 91 | // Get the Schema we just created 92 | s, ok := reg.Schema("sqlite_test") 93 | assert.Equal(t, true, ok) 94 | assert.NotEqual(t, nil, s) 95 | 96 | // set global to schema for text context 97 | sch = s 98 | 99 | // Copy, populate db 100 | for _, tablename := range td.MockSchema.Tables() { 101 | // Now go through the MockDB and copy data over 102 | baseConn, err := td.MockSchema.OpenConn(tablename) 103 | exitIfErr(err) 104 | 105 | sdbConn, err := s.OpenConn(tablename) 106 | assert.Equal(t, nil, err) 107 | assert.NotEqual(t, nil, sdbConn) 108 | userConn := sdbConn.(schema.ConnUpsert) 109 | cctx := context.Background() 110 | 111 | conn := baseConn.(schema.ConnScanner) 112 | for { 113 | msg := conn.Next() 114 | if msg == nil { 115 | break 116 | } 117 | sm := msg.(*datasource.SqlDriverMessageMap) 118 | k := sqlite.NewKey(sqlite.MakeId(sm.Vals[0])) 119 | if _, err := userConn.Put(cctx, k, sm.Vals); err != nil { 120 | u.Errorf("could not insert %v %#v", err, sm.Vals) 121 | } else { 122 | //u.Infof("inserted %v %#v", key, sm.Vals) 123 | } 124 | } 125 | 126 | err = sdbConn.Close() 127 | assert.Equal(t, nil, err) 128 | } 129 | 130 | td.TestContext = planContext 131 | }) 132 | } 133 | func TestMain(m *testing.M) { 134 | testutil.Setup() // will call flag.Parse() 135 | 136 | // Now run the actual Tests 137 | os.Exit(m.Run()) 138 | } 139 | 140 | func planContext(query string) *plan.Context { 141 | ctx := plan.NewContext(query) 142 | ctx.DisableRecover = true 143 | ctx.Schema = sch 144 | ctx.Session = datasource.NewMySqlSessionVars() 145 | return ctx 146 | } 147 | 148 | func TestSuite(t *testing.T) { 149 | defer func() { 150 | td.SetContextToMockCsv() 151 | }() 152 | LoadTestDataOnce(t) 153 | testutil.RunSimpleSuite(t) 154 | } 155 | -------------------------------------------------------------------------------- /dialects/_influxql/ast.go: -------------------------------------------------------------------------------- 1 | package influxql 2 | 3 | type Ast struct { 4 | Comments string `json:",omitempty"` // Any comments 5 | Select *SelectStmt 6 | } 7 | 8 | type SelectStmt struct { 9 | Columns []*Column // identify inputs and outputs, e.g. "SELECT fname, count(*) FROM ..." 10 | From *From // metric can be regex 11 | GroupBy []string // group by 12 | Alias string // Unique identifier of this query for start/stop/mgmt purposes 13 | //Where []*Expr `json:",omitempty"` // Filtering conditions, "SELECT ... WHERE x>y ... " 14 | } 15 | type From struct { 16 | Value string 17 | Regex bool 18 | } 19 | type Column struct { 20 | Name string 21 | } 22 | -------------------------------------------------------------------------------- /dialects/_influxql/dialect.go: -------------------------------------------------------------------------------- 1 | package influxql 2 | 3 | import ( 4 | u "github.com/araddon/gou" 5 | "github.com/araddon/qlbridge/lex" 6 | "strings" 7 | ) 8 | 9 | var ( 10 | // Tokens Specific to INFLUXDB 11 | TokenShortDesc lex.TokenType = 1000 12 | TokenLongDesc lex.TokenType = 1001 13 | TokenKind lex.TokenType = 1002 14 | ) 15 | 16 | var selectQl = []*lex.Clause{ 17 | {Token: lex.TokenSelect, Lexer: LexColumnsInflux}, 18 | {Token: lex.TokenFrom, Lexer: LexInfluxName}, 19 | {Token: lex.TokenGroupBy, Lexer: lex.LexColumns, Optional: true}, 20 | {Token: lex.TokenLimit, Lexer: lex.LexNumber, Optional: true}, 21 | {Token: lex.TokenInto, Lexer: lex.LexExpressionOrIdentity}, 22 | {Token: lex.TokenWhere, Lexer: lex.LexColumns, Optional: true}, 23 | } 24 | 25 | var InfluxQlDialect *lex.Dialect = &lex.Dialect{ 26 | Statements: []*lex.Clause{ 27 | {Token: lex.TokenSelect, Clauses: selectQl}, 28 | }, 29 | } 30 | 31 | func init() { 32 | lex.TokenNameMap[TokenShortDesc] = &lex.TokenInfo{Description: "SHORTDESC"} 33 | lex.TokenNameMap[TokenLongDesc] = &lex.TokenInfo{Description: "LONGDESC"} 34 | lex.TokenNameMap[TokenKind] = &lex.TokenInfo{Description: "kind"} 35 | // OverRide the Identity Characters in QLparse 36 | lex.IDENTITY_CHARS = "_./-" 37 | lex.LoadTokenInfo() 38 | InfluxQlDialect.Init() 39 | } 40 | 41 | // Handle influx columns 42 | // SELECT 43 | // valuect(item) AS stuff SHORTDESC "stuff" KIND INT 44 | // 45 | // Examples: 46 | // 47 | // (colx = y OR colb = b) 48 | // cola = 'a5'p 49 | // cola != "a5", colb = "a6" 50 | // REPLACE(cola,"stuff") != "hello" 51 | // FirstName = REPLACE(LOWER(name," ")) 52 | // cola IN (1,2,3) 53 | // cola LIKE "abc" 54 | // eq(name,"bob") AND age > 5 55 | // 56 | func LexColumnsInflux(l *lex.Lexer) lex.StateFn { 57 | 58 | l.SkipWhiteSpaces() 59 | 60 | keyWord := strings.ToLower(l.PeekWord()) 61 | 62 | u.Debugf("LexColumnsInflux r= '%v'", string(keyWord)) 63 | 64 | switch keyWord { 65 | case "if": 66 | l.ConsumeWord("if") 67 | l.Emit(lex.TokenIf) 68 | l.Push("LexColumnsInflux", LexColumnsInflux) 69 | return lex.LexColumns 70 | case "shortdesc": 71 | l.ConsumeWord("shortdesc") 72 | l.Emit(TokenShortDesc) 73 | l.Push("LexColumnsInflux", LexColumnsInflux) 74 | l.Push("lexIdentifier", lex.LexValue) 75 | return nil 76 | 77 | case "longdesc": 78 | l.ConsumeWord("longdesc") 79 | l.Emit(TokenLongDesc) 80 | l.Push("LexColumnsInflux", LexColumnsInflux) 81 | l.Push("lexIdentifier", lex.LexValue) 82 | return nil 83 | 84 | case "kind": 85 | l.ConsumeWord("kind") 86 | l.Emit(TokenKind) 87 | l.Push("LexColumnsInflux", lex.LexColumns) 88 | l.Push("lexIdentifier", lex.LexIdentifier) 89 | return nil 90 | 91 | } 92 | return lex.LexSelectClause 93 | } 94 | 95 | // lex value 96 | // 97 | // SIMPLE_NAME_VALUE | TABLE_NAME_VALUE | REGEX_VALUE 98 | func LexInfluxName(l *lex.Lexer) lex.StateFn { 99 | 100 | l.SkipWhiteSpaces() 101 | firstChar := l.Peek() 102 | u.Debugf("LexInfluxName: %v", string(firstChar)) 103 | 104 | switch firstChar { 105 | case '"': 106 | return lex.LexValue(l) 107 | case '/': 108 | // a regex 109 | return lex.LexRegex(l) 110 | } 111 | return lex.LexIdentifier 112 | } 113 | -------------------------------------------------------------------------------- /dialects/_influxql/parse_test.go: -------------------------------------------------------------------------------- 1 | package influxql 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "testing" 7 | 8 | u "github.com/araddon/gou" 9 | "github.com/araddon/qlbridge/lex" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var ( 14 | VerboseTests *bool = flag.Bool("vv", false, "Verbose Logging?") 15 | ) 16 | 17 | func TestMain(m *testing.M) { 18 | flag.Parse() 19 | if *VerboseTests { 20 | u.SetupLogging("debug") 21 | u.SetColorOutput() 22 | } 23 | // Now run the actual Tests 24 | os.Exit(m.Run()) 25 | } 26 | 27 | func tv(t lex.TokenType, v string) lex.Token { 28 | return lex.Token{T: t, V: v} 29 | } 30 | 31 | func verifyTokens(t *testing.T, sql string, tokens []lex.Token) { 32 | l := lex.NewLexer(sql, InfluxQlDialect) 33 | for _, goodToken := range tokens { 34 | tok := l.NextToken() 35 | u.Debugf("%#v %#v", tok, goodToken) 36 | assert.Equal(t, tok.V, goodToken.V, "has='%v' want='%v'", tok.V, goodToken.V) 37 | assert.Equal(t, tok.T, goodToken.T, "has='%v' want='%v'", tok.V, goodToken.V) 38 | } 39 | } 40 | 41 | func TestLexSimple(t *testing.T) { 42 | verifyTokens(t, `select * from "series with special characters!"`, 43 | []lex.Token{ 44 | tv(lex.TokenSelect, "select"), 45 | tv(lex.TokenStar, "*"), 46 | tv(lex.TokenFrom, "from"), 47 | tv(lex.TokenValue, "series with special characters!"), 48 | }) 49 | verifyTokens(t, `select * from /.*/ limit 1"`, 50 | []lex.Token{ 51 | tv(lex.TokenSelect, "select"), 52 | tv(lex.TokenStar, "*"), 53 | tv(lex.TokenFrom, "from"), 54 | tv(lex.TokenRegex, "/.*/"), 55 | tv(lex.TokenLimit, "limit"), 56 | tv(lex.TokenInteger, "1"), 57 | }) 58 | verifyTokens(t, `select * from /^stats\./i where time > now() - 1h;`, 59 | []lex.Token{ 60 | tv(lex.TokenSelect, "select"), 61 | tv(lex.TokenStar, "*"), 62 | tv(lex.TokenFrom, "from"), 63 | tv(lex.TokenRegex, "/^stats\\./i"), 64 | tv(lex.TokenWhere, "where"), 65 | tv(lex.TokenIdentity, "time"), 66 | tv(lex.TokenGT, ">"), 67 | tv(lex.TokenUdfExpr, "now"), 68 | tv(lex.TokenLeftParenthesis, "("), 69 | tv(lex.TokenRightParenthesis, ")"), 70 | tv(lex.TokenMinus, "-"), 71 | }) 72 | } 73 | 74 | func TestLexContinuous(t *testing.T) { 75 | verifyTokens(t, `select percentile(value,95) from response_times group by time(5m) 76 | into response_times.percentiles.5m.95`, 77 | []lex.Token{ 78 | tv(lex.TokenSelect, "select"), 79 | tv(lex.TokenUdfExpr, "percentile"), 80 | tv(lex.TokenLeftParenthesis, "("), 81 | tv(lex.TokenIdentity, "value"), 82 | tv(lex.TokenComma, ","), 83 | tv(lex.TokenInteger, "95"), 84 | tv(lex.TokenRightParenthesis, ")"), 85 | tv(lex.TokenFrom, "from"), 86 | tv(lex.TokenIdentity, "response_times"), 87 | tv(lex.TokenGroupBy, "group by"), 88 | tv(lex.TokenUdfExpr, "time"), 89 | tv(lex.TokenLeftParenthesis, "("), 90 | tv(lex.TokenDuration, "5m"), 91 | }) 92 | } 93 | -------------------------------------------------------------------------------- /dialects/_influxql/parser.go: -------------------------------------------------------------------------------- 1 | package influxql 2 | 3 | import ( 4 | "fmt" 5 | 6 | u "github.com/araddon/gou" 7 | "github.com/araddon/qlbridge/lex" 8 | ) 9 | 10 | /* 11 | 12 | Parser for InfluxDB ql 13 | 14 | */ 15 | 16 | // Parses string query 17 | func Parse(query string) (*Ast, error) { 18 | l := lex.NewLexer(query, InfluxQlDialect) 19 | p := Parser{l: l, qryText: query} 20 | return p.parse() 21 | } 22 | 23 | // parser evaluateslex.Tokens 24 | type Parser struct { 25 | l *lex.Lexer 26 | qryText string 27 | initialKeyword lex.Token 28 | curToken lex.Token 29 | } 30 | 31 | // parse the request 32 | func (m *Parser) parse() (*Ast, error) { 33 | 34 | comment := m.initialComment() 35 | u.Debug(comment) 36 | // Now, find First Keyword 37 | switch m.curToken.T { 38 | case lex.TokenSelect: 39 | m.initialKeyword = m.curToken 40 | return m.parseSelect(comment) 41 | default: 42 | return nil, fmt.Errorf("Unrecognized query, expected [SELECT] influx ql") 43 | } 44 | } 45 | 46 | func (m *Parser) initialComment() string { 47 | 48 | m.curToken = m.l.NextToken() 49 | comment := "" 50 | 51 | for { 52 | // We are going to loop until we find the first Non-Commentlex.Token 53 | switch m.curToken.T { 54 | case lex.TokenComment, lex.TokenCommentML: 55 | comment += m.curToken.V 56 | case lex.TokenCommentStart, lex.TokenCommentHash, lex.TokenCommentEnd, lex.TokenCommentSingleLine, lex.TokenCommentSlashes: 57 | // skip, currently ignore these 58 | default: 59 | // first non-commentlex.Token 60 | return comment 61 | } 62 | m.curToken = m.l.NextToken() 63 | } 64 | } 65 | 66 | // First keyword was SELECT, so use the SELECT parser rule-set 67 | func (m *Parser) parseSelect(comment string) (*Ast, error) { 68 | 69 | selStmt := SelectStmt{} 70 | ast := Ast{Comments: comment, Select: &selStmt} 71 | 72 | // we have already parsed SELECT lex.Token to get here, so this should be first col 73 | m.curToken = m.l.NextToken() 74 | if m.curToken.T != lex.TokenStar { 75 | if err := m.parseColumns(&selStmt); err != nil { 76 | return nil, err 77 | } 78 | } else { 79 | // * mark as star? 80 | return nil, fmt.Errorf("not implemented") 81 | } 82 | 83 | // FROM - required 84 | if m.curToken.T != lex.TokenFrom { 85 | return nil, fmt.Errorf("expected From") 86 | } 87 | 88 | // table/metric 89 | m.curToken = m.l.NextToken() 90 | if m.curToken.T != lex.TokenIdentity && m.curToken.T != lex.TokenValue { 91 | return nil, fmt.Errorf("expected from name fot %v", m.curToken) 92 | } else if m.curToken.T == lex.TokenRegex { 93 | selStmt.From = &From{Value: m.curToken.V, Regex: true} 94 | } 95 | 96 | // Where is optional 97 | if err := m.parseWhere(&selStmt); err != nil { 98 | return nil, err 99 | } 100 | // limit is optional 101 | // if err := m.parseLimit(&selStmt); err != nil { 102 | // return nil, err 103 | // } 104 | 105 | // we are finished, nice! 106 | return &ast, nil 107 | } 108 | 109 | func (m *Parser) parseColumns(stmt *SelectStmt) error { 110 | stmt.Columns = make([]*Column, 0) 111 | u.Infof("cols: %d", len(stmt.Columns)) 112 | return nil 113 | } 114 | 115 | func (m *Parser) parseWhere(stmt *SelectStmt) error { 116 | 117 | // Where is Optional, if we didn't use a where statement return 118 | if m.curToken.T == lex.TokenEOF || m.curToken.T == lex.TokenEOS { 119 | return nil 120 | } 121 | 122 | if m.curToken.T != lex.TokenWhere { 123 | return nil 124 | } 125 | 126 | u.Infof("wheres: %#v", stmt) 127 | return nil 128 | } 129 | -------------------------------------------------------------------------------- /dialects/example/README.md: -------------------------------------------------------------------------------- 1 | 2 | An example *PUBSUB* language we created for an example 3 | 4 | -------------------------------------------------------------------------------- /dialects/example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/araddon/qlbridge/lex" 6 | "strings" 7 | ) 8 | 9 | /* 10 | This example is meant to show how to create a new 11 | Dialect Language with a keyword"SUBSCRIBETO" 12 | then Lex an example of this syntax 13 | 14 | */ 15 | var ( 16 | // We need a token to recognize our "SUBSCRIBETO" keyword 17 | // in our PUBSUB dialect 18 | TokenSubscribeTo lex.TokenType = 1000 19 | 20 | // We are going to create our own Dialect now 21 | // that uses a "SUBSCRIBETO" keyword 22 | pubsub = &lex.Clause{Token: TokenSubscribeTo, Clauses: []*lex.Clause{ 23 | {Token: TokenSubscribeTo, Lexer: lex.LexColumns}, 24 | {Token: lex.TokenFrom, Lexer: LexMaybe}, 25 | {Token: lex.TokenWhere, Lexer: lex.LexColumns, Optional: true}, 26 | }} 27 | ourDialect = &lex.Dialect{ 28 | Name: "Subscribe To", Statements: []*lex.Clause{pubsub}, 29 | } 30 | ) 31 | 32 | func init() { 33 | // inject any new tokens into QLBridge.Lex describing the custom tokens we created 34 | lex.TokenNameMap[TokenSubscribeTo] = &lex.TokenInfo{Description: "subscribeto"} 35 | 36 | // OverRide the Identity Characters in lexer to allow a dash in identity 37 | lex.IDENTITY_CHARS = "_./-" 38 | lex.LoadTokenInfo() 39 | 40 | ourDialect.Init() 41 | } 42 | 43 | func verifyLexerTokens(l *lex.Lexer, tokens []lex.Token) { 44 | for _, goodToken := range tokens { 45 | tok := l.NextToken() 46 | if tok.T != goodToken.T || tok.V != goodToken.V { 47 | panic(fmt.Sprintf("bad token: %v but wanted %v\n", tok, goodToken)) 48 | } else { 49 | fmt.Printf("Got good token: %v\n", tok) 50 | } 51 | } 52 | } 53 | 54 | // Custom lexer for our maybe hash function 55 | // 56 | // SUBSCRIBE 57 | // valuect(item) AS stuff 58 | // FROM maybe(stuff) 59 | // WHERE x = y 60 | // 61 | func LexMaybe(l *lex.Lexer) lex.StateFn { 62 | 63 | l.SkipWhiteSpaces() 64 | 65 | keyWord := strings.ToLower(l.PeekWord()) 66 | 67 | switch keyWord { 68 | case "maybe": 69 | l.ConsumeWord("maybe") 70 | l.Emit(lex.TokenIdentity) 71 | return lex.LexExpressionOrIdentity 72 | } 73 | return lex.LexExpressionOrIdentity 74 | } 75 | 76 | func Tok(tok lex.TokenType, val string) lex.Token { return lex.Token{T: tok, V: val} } 77 | 78 | func main() { 79 | 80 | /* Many *ql languages support some type of columnar layout such as: 81 | name = value, name2 = value2 82 | */ 83 | l := lex.NewLexer(` 84 | SUBSCRIBETO 85 | count(x), Name 86 | FROM ourstream 87 | WHERE 88 | k = REPLACE(LOWER(Name),"cde","xxx");`, ourDialect) 89 | 90 | verifyLexerTokens(l, 91 | []lex.Token{ 92 | Tok(TokenSubscribeTo, "SUBSCRIBETO"), 93 | Tok(lex.TokenUdfExpr, "count"), 94 | Tok(lex.TokenLeftParenthesis, "("), 95 | Tok(lex.TokenIdentity, "x"), 96 | Tok(lex.TokenRightParenthesis, ")"), 97 | Tok(lex.TokenComma, ","), 98 | Tok(lex.TokenIdentity, "Name"), 99 | Tok(lex.TokenFrom, "FROM"), 100 | Tok(lex.TokenIdentity, "ourstream"), 101 | Tok(lex.TokenWhere, "WHERE"), 102 | Tok(lex.TokenIdentity, "k"), 103 | Tok(lex.TokenEqual, "="), 104 | Tok(lex.TokenUdfExpr, "REPLACE"), 105 | Tok(lex.TokenLeftParenthesis, "("), 106 | Tok(lex.TokenUdfExpr, "LOWER"), 107 | Tok(lex.TokenLeftParenthesis, "("), 108 | Tok(lex.TokenIdentity, "Name"), 109 | Tok(lex.TokenRightParenthesis, ")"), 110 | Tok(lex.TokenComma, ","), 111 | Tok(lex.TokenValue, "cde"), 112 | Tok(lex.TokenComma, ","), 113 | Tok(lex.TokenValue, "xxx"), 114 | Tok(lex.TokenRightParenthesis, ")"), 115 | Tok(lex.TokenEOS, ";"), 116 | }) 117 | } 118 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // QLBridge is a SQL Relational algebra and expression package 2 | // for embedding sql like functionality into your app. Includes 3 | // Lexer, Parsers, different SQL Dialects, as well as planners 4 | // and executors. 5 | package qlbridge 6 | 7 | import ( 8 | // Glock doesn't check test dependencies 9 | _ "github.com/go-sql-driver/mysql" 10 | ) 11 | -------------------------------------------------------------------------------- /examples/expressions/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/mail" 7 | "os" 8 | "time" 9 | 10 | "github.com/araddon/dateparse" 11 | u "github.com/araddon/gou" 12 | 13 | "github.com/araddon/qlbridge/datasource" 14 | "github.com/araddon/qlbridge/expr" 15 | "github.com/araddon/qlbridge/expr/builtins" 16 | "github.com/araddon/qlbridge/value" 17 | "github.com/araddon/qlbridge/vm" 18 | ) 19 | 20 | func init() { 21 | u.SetLogger(log.New(os.Stderr, "", 0), "debug") 22 | u.SetColorOutput() 23 | 24 | // load all of our built-in functions 25 | builtins.LoadAllBuiltins() 26 | } 27 | 28 | func main() { 29 | 30 | // Add a custom function to the VM to make available to expression language 31 | expr.FuncAdd("email_is_valid", &EmailIsValid{}) 32 | 33 | // This is the evaluation context which will be evaluated against the expressions 34 | evalContext := datasource.NewContextSimpleNative(map[string]interface{}{ 35 | "int5": 5, 36 | "str5": "5", 37 | "created": dateparse.MustParse("12/18/2015"), 38 | "bvalt": true, 39 | "bvalf": false, 40 | "user_id": "abc", 41 | "urls": []string{"http://google.com", "http://nytimes.com"}, 42 | "hits": map[string]int64{"google.com": 5, "bing.com": 1}, 43 | "email": "bob@bob.com", 44 | "emailbad": "bob", 45 | "mt": map[string]time.Time{ 46 | "event0": dateparse.MustParse("12/18/2015"), 47 | "event1": dateparse.MustParse("12/22/2015"), 48 | }, 49 | }) 50 | 51 | exprs := []string{ 52 | "int5 == 5", 53 | `6 > 5`, 54 | `6 > 5.5`, 55 | `(4 + 5) / 2`, 56 | `6 == (5 + 1)`, 57 | `2 * (3 + 5)`, 58 | `todate("12/12/2012")`, 59 | `created > "now-1M"`, // Date math 60 | `created > "now-10y"`, 61 | `user_id == "abc"`, 62 | `email_is_valid(email)`, 63 | `email_is_valid(emailbad)`, 64 | `email_is_valid("not_an_email")`, 65 | `EXISTS int5`, 66 | `!exists(user_id)`, 67 | `mt.event0 > now()`, // step into child of maps 68 | `mt.event0 < now()`, // step into child of maps 69 | `["portland"] LIKE "*land"`, 70 | `email contains "bob"`, 71 | `email NOT contains "bob"`, 72 | `[1,2,3] contains int5`, 73 | `[1,2,3,5] NOT contains int5`, 74 | `urls contains "http://google.com"`, 75 | `split("chicago,portland",",") LIKE "*land"`, 76 | `10 BETWEEN 1 AND 50`, 77 | `15.5 BETWEEN 1 AND "55.5"`, 78 | `created BETWEEN "now-50w" AND "12/18/2020"`, 79 | `toint(not_a_field) NOT IN ("a","b" 4.5)`, 80 | ` 81 | OR ( 82 | email != "bob@bob.com" 83 | AND ( 84 | NOT EXISTS not_a_field 85 | int5 == 5 86 | ) 87 | )`, 88 | } 89 | 90 | for _, expression := range exprs { 91 | // Same ast can be re-used safely concurrently 92 | exprAst := expr.MustParse(expression) 93 | // Evaluate AST in the vm 94 | val, _ := vm.Eval(evalContext, exprAst) 95 | v := val.Value() 96 | u.Debugf("%46s ==> %-35v T:%-15T ", expression, v, v) 97 | } 98 | } 99 | 100 | type EmailIsValid struct{} 101 | 102 | func (m *EmailIsValid) Validate(n *expr.FuncNode) (expr.EvaluatorFunc, error) { 103 | if len(n.Args) != 1 { 104 | return nil, fmt.Errorf("Expected 1 arg for EmailIsValid(arg) but got %s", n) 105 | } 106 | return func(ctx expr.EvalContext, args []value.Value) (value.Value, bool) { 107 | if args[0] == nil || args[0].Err() || args[0].Nil() { 108 | return value.BoolValueFalse, true 109 | } 110 | if _, err := mail.ParseAddress(args[0].ToString()); err == nil { 111 | return value.BoolValueTrue, true 112 | } 113 | 114 | return value.BoolValueFalse, true 115 | }, nil 116 | } 117 | func (m *EmailIsValid) Type() value.ValueType { return value.BoolType } 118 | -------------------------------------------------------------------------------- /examples/qlcsv/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Example App: Reading Csv via Stdio, and evaluating in QL VM 4 | ------------------------------------------------------------------ 5 | 6 | This is an example app to read a CSV file, and ouput query results after processing 7 | through a xQL expression evaluation VM which has a custom Func 8 | supplied into the VM eval engine `email_is_valid` 9 | 10 | ```sh 11 | 12 | go build 13 | 14 | # SQL: select cols including expression 15 | ./qlcsv -sql ' 16 | select 17 | user_id, email, item_count * 2, yy(reg_date) > 10 18 | FROM stdin' < users.csv 19 | 20 | # SQL: where guard 21 | ./qlcsv -sql 'select 22 | user_id AS theuserid, email, item_count * 2, reg_date FROM stdin 23 | WHERE yy(reg_date) > 10' < users.csv 24 | 25 | # SQL: add a custom function - email_is_valid 26 | ./qlcsv -sql 'select 27 | user_id AS theuserid, email, item_count * 2, reg_date 28 | FROM stdin 29 | WHERE email_is_valid(email)' < users.csv 30 | 31 | ./qlcsv -sql 'select count(*) as user_ct FROM stdin' < users.csv 32 | 33 | ```` 34 | 35 | 36 | ```go 37 | 38 | func main() { 39 | 40 | if sqlText == "" { 41 | u.Errorf("You must provide a valid select query in argument: --sql=\"select ...\"") 42 | return 43 | } 44 | 45 | // load all of our built-in functions 46 | builtins.LoadAllBuiltins() 47 | 48 | // Add a custom function to the VM to make available to SQL language 49 | expr.FuncAdd("email_is_valid", EmailIsValid) 50 | 51 | // Our file source of csv's is stdin 52 | stdIn, err := os.Open("/dev/stdin") 53 | if err != nil { 54 | u.Errorf("could not open stdin? %v", err) 55 | return 56 | } 57 | 58 | // We are registering the "csv" datasource, to show that 59 | // the backend/sources can be easily created/added. This csv 60 | // reader is an example datasource that is very, very simple. 61 | exit := make(chan bool) 62 | src, _ := datasource.NewCsvSource("stdin", 0, stdIn, exit) 63 | datasource.Register("csv", src) 64 | 65 | db, err := sql.Open("qlbridge", "csv:///dev/stdin") 66 | if err != nil { 67 | panic(err.Error()) 68 | } 69 | defer db.Close() 70 | 71 | rows, err := db.Query(sqlText) 72 | if err != nil { 73 | u.Errorf("could not execute query: %v", err) 74 | return 75 | } 76 | defer rows.Close() 77 | cols, _ := rows.Columns() 78 | 79 | // this is just stupid hijinx for getting pointers for unknown len columns 80 | readCols := make([]interface{}, len(cols)) 81 | writeCols := make([]string, len(cols)) 82 | for i, _ := range writeCols { 83 | readCols[i] = &writeCols[i] 84 | } 85 | fmt.Printf("\n\nScanning through CSV: (%v)\n\n", strings.Join(cols, ",")) 86 | for rows.Next() { 87 | rows.Scan(readCols...) 88 | fmt.Println(strings.Join(writeCols, ", ")) 89 | } 90 | fmt.Println("") 91 | } 92 | 93 | // Example of a custom Function, that we are adding into the Expression VM 94 | // 95 | // select 96 | // user_id AS theuserid, email, item_count * 2, reg_date 97 | // FROM stdio 98 | // WHERE email_is_valid(email) 99 | func EmailIsValid(ctx expr.EvalContext, email value.Value) (value.BoolValue, bool) { 100 | if _, err := mail.ParseAddress(email.ToString()); err == nil { 101 | return value.BoolValueTrue, true 102 | } 103 | 104 | return value.BoolValueFalse, true 105 | } 106 | 107 | 108 | ``` 109 | 110 | 111 | -------------------------------------------------------------------------------- /examples/qlcsv/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "flag" 7 | "fmt" 8 | "net/mail" 9 | "strings" 10 | 11 | // Side-Effect Import the qlbridge sql driver 12 | _ "github.com/araddon/qlbridge/qlbdriver" 13 | "github.com/araddon/qlbridge/schema" 14 | 15 | u "github.com/araddon/gou" 16 | "github.com/araddon/qlbridge/datasource" 17 | "github.com/araddon/qlbridge/expr" 18 | "github.com/araddon/qlbridge/expr/builtins" 19 | "github.com/araddon/qlbridge/value" 20 | ) 21 | 22 | var ( 23 | sqlText string 24 | flagCsvDelimiter = "," 25 | logging = "info" 26 | ) 27 | 28 | func init() { 29 | 30 | flag.StringVar(&logging, "logging", "info", "logging [ debug,info ]") 31 | flag.StringVar(&sqlText, "sql", "", "QL ish query multi-node such as [select user_id, yy(reg_date) from stdio];") 32 | flag.StringVar(&flagCsvDelimiter, "delimiter", ",", "delimiter: default = comma [t,|]") 33 | flag.Parse() 34 | 35 | u.SetupLogging(logging) 36 | u.SetColorOutput() 37 | } 38 | 39 | func main() { 40 | 41 | if sqlText == "" { 42 | u.Errorf("You must provide a valid select query in argument: --sql=\"select ...\"") 43 | return 44 | } 45 | 46 | // load all of our built-in functions 47 | builtins.LoadAllBuiltins() 48 | 49 | // Add a custom function to the VM to make available to SQL language 50 | expr.FuncAdd("email_is_valid", &EmailIsValid{}) 51 | 52 | // We are registering the "csv" datasource, to show that 53 | // the backend/sources can be easily created/added. This csv 54 | // reader is an example datasource that is very, very simple. 55 | exit := make(chan bool) 56 | src, _ := datasource.NewCsvSource("stdin", 0, bytes.NewReader([]byte("##")), exit) 57 | schema.RegisterSourceAsSchema("example_csv", src) 58 | 59 | db, err := sql.Open("qlbridge", "example_csv") 60 | if err != nil { 61 | panic(err.Error()) 62 | } 63 | defer db.Close() 64 | 65 | rows, err := db.Query(sqlText) 66 | if err != nil { 67 | u.Errorf("could not execute query: %v", err) 68 | return 69 | } 70 | defer rows.Close() 71 | cols, _ := rows.Columns() 72 | 73 | // this is just stupid hijinx for getting pointers for unknown len columns 74 | readCols := make([]interface{}, len(cols)) 75 | writeCols := make([]string, len(cols)) 76 | for i := range writeCols { 77 | readCols[i] = &writeCols[i] 78 | } 79 | fmt.Printf("\n\nScanning through CSV: (%v)\n\n", strings.Join(cols, ",")) 80 | for rows.Next() { 81 | rows.Scan(readCols...) 82 | fmt.Println(strings.Join(writeCols, ", ")) 83 | } 84 | fmt.Println("") 85 | } 86 | 87 | // Example of a custom Function, that we are adding into the Expression VM 88 | // 89 | // select 90 | // user_id AS theuserid, email, item_count * 2, reg_date 91 | // FROM stdio 92 | // WHERE email_is_valid(email) 93 | type EmailIsValid struct{} 94 | 95 | func (m *EmailIsValid) Validate(n *expr.FuncNode) (expr.EvaluatorFunc, error) { 96 | if len(n.Args) != 1 { 97 | return nil, fmt.Errorf("Expected 1 arg for EmailIsValid(arg) but got %s", n) 98 | } 99 | return func(ctx expr.EvalContext, args []value.Value) (value.Value, bool) { 100 | if args[0] == nil || args[0].Err() || args[0].Nil() { 101 | return value.BoolValueFalse, true 102 | } 103 | if _, err := mail.ParseAddress(args[0].ToString()); err == nil { 104 | return value.BoolValueTrue, true 105 | } 106 | 107 | return value.BoolValueFalse, true 108 | }, nil 109 | } 110 | func (m *EmailIsValid) Type() value.ValueType { return value.BoolType } 111 | -------------------------------------------------------------------------------- /examples/qlcsv/users.csv: -------------------------------------------------------------------------------- 1 | user_id,email,interests,reg_date,item_count,deleted 2 | 9Ip1aKbeZe2njCDM,"aaron@email.com","fishing","2012-10-17T17:29:39.738Z",82,false 3 | hT2impsOPUREcVPc,"bob@gmail.com","swimming","2009-12-11T19:53:31.547Z",12,true 4 | hT2impsabc345c,"not_an_email","swimming","2009-12-11T19:53:31.547Z",12,false -------------------------------------------------------------------------------- /exec/command.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | u "github.com/araddon/gou" 8 | 9 | "github.com/araddon/qlbridge/expr" 10 | "github.com/araddon/qlbridge/lex" 11 | "github.com/araddon/qlbridge/plan" 12 | "github.com/araddon/qlbridge/rel" 13 | "github.com/araddon/qlbridge/vm" 14 | ) 15 | 16 | var ( 17 | _ = u.EMPTY 18 | 19 | // Ensure that we implement the Task Runner interface 20 | _ TaskRunner = (*Command)(nil) 21 | ) 22 | 23 | // Command is executeable task for SET SQL commands 24 | type Command struct { 25 | *TaskBase 26 | p *plan.Command 27 | } 28 | 29 | // NewCommand creates new command exec task 30 | func NewCommand(ctx *plan.Context, p *plan.Command) *Command { 31 | m := &Command{ 32 | TaskBase: NewTaskBase(ctx), 33 | p: p, 34 | } 35 | return m 36 | } 37 | 38 | // Close Command 39 | func (m *Command) Close() error { 40 | if err := m.TaskBase.Close(); err != nil { 41 | return err 42 | } 43 | return nil 44 | } 45 | 46 | // Run Command 47 | func (m *Command) Run() error { 48 | //defer m.Ctx.Recover() 49 | defer close(m.msgOutCh) 50 | 51 | if m.Ctx.Session == nil { 52 | u.Warnf("no Context.Session?") 53 | return fmt.Errorf("no Context.Session?") 54 | } 55 | 56 | switch kw := m.p.Stmt.Keyword(); kw { 57 | case lex.TokenSet: 58 | return m.runSet() 59 | case lex.TokenRollback, lex.TokenCommit: 60 | u.Debugf("ignorning transaction, not implemented. %v", kw.String()) 61 | return nil 62 | default: 63 | u.Warnf("unrecognized command: kw=%v stmt:%s", kw, m.p.Stmt) 64 | } 65 | return ErrNotImplemented 66 | 67 | } 68 | func (m *Command) runSet() error { 69 | 70 | writeContext, ok := m.Ctx.Session.(expr.ContextWriter) 71 | if !ok || writeContext == nil { 72 | u.Warnf("expected context writer but no for %T", m.Ctx.Session) 73 | return fmt.Errorf("No write context?") 74 | } 75 | 76 | //u.Debugf("running set? %v", m.p.Stmt.String()) 77 | for _, col := range m.p.Stmt.Columns { 78 | err := evalSetExpression(col, m.Ctx.Session, col.Expr) 79 | if err != nil { 80 | u.Warnf("Could not evaluate [%s] err=%v", col.Expr, err) 81 | return err 82 | } 83 | } 84 | // for k, v := range m.Ctx.Session.Row() { 85 | // u.Infof("%p session? %s: %v", m.Ctx.Session, k, v.Value()) 86 | // } 87 | return nil 88 | } 89 | 90 | func evalSetExpression(col *rel.CommandColumn, ctx expr.ContextReadWriter, arg expr.Node) error { 91 | 92 | switch bn := arg.(type) { 93 | case *expr.BinaryNode: 94 | _, ok := bn.Args[0].(*expr.IdentityNode) 95 | if !ok { 96 | u.Warnf("expected identity but got %T in %s", bn.Args[0], arg.String()) 97 | return fmt.Errorf("Expected identity but got %T", bn.Args[0]) 98 | } 99 | rhv, ok := vm.Eval(ctx, bn.Args[1]) 100 | if !ok { 101 | u.Warnf("expected right side value but got %T in %s", bn.Args[1], arg.String()) 102 | return fmt.Errorf("Expected value but got %T", bn.Args[1]) 103 | } 104 | //u.Infof(`writeContext.Put("%v",%v)`, col.Key(), rhv.Value()) 105 | ctx.Put(col, ctx, rhv) 106 | case nil: 107 | // Special statements 108 | name := strings.ToLower(col.Name) 109 | switch { 110 | case strings.HasPrefix(name, "names ") || strings.HasPrefix(name, "character set"): 111 | // http://dev.mysql.com/doc/refman/5.7/en/charset-connection.html 112 | // hm, no idea what to do 113 | /* 114 | SET character_set_client = charset_name; 115 | SET character_set_results = charset_name; 116 | SET character_set_connection = charset_name; 117 | */ 118 | } 119 | default: 120 | u.Errorf("SET command only accepts binary nodes but got type: %#v", arg) 121 | return fmt.Errorf("Un recognized command %T", arg) 122 | } 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /exec/ddl.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | u "github.com/araddon/gou" 8 | 9 | "github.com/araddon/qlbridge/lex" 10 | "github.com/araddon/qlbridge/plan" 11 | "github.com/araddon/qlbridge/schema" 12 | ) 13 | 14 | var ( 15 | // Ensure that we implement the Task Runner interface 16 | _ TaskRunner = (*Create)(nil) 17 | _ TaskRunner = (*Drop)(nil) 18 | _ TaskRunner = (*Alter)(nil) 19 | ) 20 | 21 | type ( 22 | // Create is executeable task for SQL Create, Alter, Schema, Source etc. 23 | Create struct { 24 | *TaskBase 25 | p *plan.Create 26 | } 27 | // Drop is executeable task for SQL DROP. 28 | Drop struct { 29 | *TaskBase 30 | p *plan.Drop 31 | } 32 | // Alter is executeable task for SQL ALTER. 33 | Alter struct { 34 | *TaskBase 35 | p *plan.Alter 36 | } 37 | ) 38 | 39 | // NewCreate creates new create exec task 40 | func NewCreate(ctx *plan.Context, p *plan.Create) *Create { 41 | m := &Create{ 42 | TaskBase: NewTaskBase(ctx), 43 | p: p, 44 | } 45 | return m 46 | } 47 | 48 | // Close Create 49 | func (m *Create) Close() error { 50 | return m.TaskBase.Close() 51 | } 52 | 53 | // Run Create 54 | func (m *Create) Run() error { 55 | defer close(m.msgOutCh) 56 | 57 | cs := m.p.Stmt 58 | 59 | switch cs.Tok.T { 60 | case lex.TokenSource, lex.TokenSchema: 61 | 62 | /* 63 | // "sub_schema_name" will create a new child schema called "sub_schema_name" 64 | // that is added to "existing_schema_name" 65 | // of source type elasticsearch 66 | CREATE source sub_schema_name WITH { 67 | "type":"elasticsearch", 68 | "schema":"existing_schema_name", 69 | "settings" : { 70 | "apikey":"GET_YOUR_API_KEY" 71 | } 72 | }; 73 | */ 74 | // If we specify a parent schema to add this child schema to 75 | schemaName := cs.Identity 76 | by, err := json.MarshalIndent(cs.With, "", " ") 77 | if err != nil { 78 | u.Errorf("could not convert conf = %v ", cs.With) 79 | return fmt.Errorf("could not convert conf %v", cs.With) 80 | } 81 | 82 | sourceConf := &schema.ConfigSource{} 83 | err = json.Unmarshal(by, sourceConf) 84 | if err != nil { 85 | u.Errorf("could not convert conf = %v ", string(by)) 86 | return fmt.Errorf("could not convert conf %v", cs.With) 87 | } 88 | sourceConf.Name = schemaName 89 | 90 | reg := schema.DefaultRegistry() 91 | 92 | return reg.SchemaAddFromConfig(sourceConf) 93 | default: 94 | u.Warnf("unrecognized create/alter: kw=%v stmt:%s", cs.Tok, m.p.Stmt) 95 | } 96 | return ErrNotImplemented 97 | } 98 | 99 | // NewDrop creates new drop exec task. 100 | func NewDrop(ctx *plan.Context, p *plan.Drop) *Drop { 101 | m := &Drop{ 102 | TaskBase: NewTaskBase(ctx), 103 | p: p, 104 | } 105 | return m 106 | } 107 | 108 | // Close Drop 109 | func (m *Drop) Close() error { 110 | return m.TaskBase.Close() 111 | } 112 | 113 | // Run Drop 114 | func (m *Drop) Run() error { 115 | defer close(m.msgOutCh) 116 | 117 | cs := m.p.Stmt 118 | s := m.Ctx.Schema 119 | if s == nil { 120 | return fmt.Errorf("must have schema") 121 | } 122 | 123 | switch cs.Tok.T { 124 | case lex.TokenSource, lex.TokenSchema, lex.TokenTable: 125 | 126 | reg := schema.DefaultRegistry() 127 | return reg.SchemaDrop(s.Name, cs.Identity, cs.Tok.T) 128 | 129 | default: 130 | u.Warnf("unrecognized DROP: kw=%v stmt:%s", cs.Tok, m.p.Stmt) 131 | } 132 | return ErrNotImplemented 133 | } 134 | 135 | // NewAlter creates new ALTER exec task. 136 | func NewAlter(ctx *plan.Context, p *plan.Alter) *Alter { 137 | m := &Alter{ 138 | TaskBase: NewTaskBase(ctx), 139 | p: p, 140 | } 141 | return m 142 | } 143 | 144 | // Close Alter 145 | func (m *Alter) Close() error { 146 | return m.TaskBase.Close() 147 | } 148 | 149 | // Run Alter 150 | func (m *Alter) Run() error { 151 | defer close(m.msgOutCh) 152 | 153 | cs := m.p.Stmt 154 | 155 | switch cs.Tok.T { 156 | default: 157 | u.Warnf("unrecognized ALTER: kw=%v stmt:%s", cs.Tok, m.p.Stmt) 158 | } 159 | return ErrNotImplemented 160 | } 161 | -------------------------------------------------------------------------------- /exec/errs.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "database/sql/driver" 5 | "strings" 6 | ) 7 | 8 | // Create a multiple error type 9 | type errList []error 10 | 11 | func (e *errList) append(err error) { 12 | if err != nil { 13 | *e = append(*e, err) 14 | } 15 | } 16 | 17 | func (e errList) error() error { 18 | if len(e) == 0 { 19 | return nil 20 | } 21 | return e 22 | } 23 | 24 | func (e errList) Error() string { 25 | a := make([]string, len(e)) 26 | for i, v := range e { 27 | a[i] = v.Error() 28 | } 29 | return strings.Join(a, "\n") 30 | } 31 | 32 | func params(args []driver.Value) []interface{} { 33 | r := make([]interface{}, len(args)) 34 | for i, v := range args { 35 | r[i] = interface{}(v) 36 | } 37 | return r 38 | } 39 | -------------------------------------------------------------------------------- /exec/order.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "time" 7 | 8 | u "github.com/araddon/gou" 9 | 10 | "github.com/araddon/qlbridge/datasource" 11 | "github.com/araddon/qlbridge/expr" 12 | "github.com/araddon/qlbridge/plan" 13 | "github.com/araddon/qlbridge/vm" 14 | ) 15 | 16 | // Order 17 | type Order struct { 18 | *TaskBase 19 | p *plan.Order 20 | complete chan bool 21 | closed bool 22 | isComplete bool 23 | } 24 | 25 | // NewORder create new order by exec task 26 | func NewOrder(ctx *plan.Context, p *plan.Order) *Order { 27 | o := &Order{ 28 | TaskBase: NewTaskBase(ctx), 29 | p: p, 30 | complete: make(chan bool), 31 | } 32 | return o 33 | } 34 | 35 | func (m *Order) Close() error { 36 | m.Lock() 37 | if m.closed { 38 | m.Unlock() 39 | return nil 40 | } 41 | m.closed = true 42 | m.Unlock() 43 | 44 | // what should this be? 45 | ticker := time.NewTicker(30 * time.Second) 46 | defer ticker.Stop() 47 | 48 | //u.Infof("%p group by final Close() waiting for complete", m) 49 | select { 50 | case <-ticker.C: 51 | u.Warnf("order by timeout???? ") 52 | case <-m.complete: 53 | //u.Warnf("%p got groupbyfinal complete", m) 54 | } 55 | 56 | return m.TaskBase.Close() 57 | } 58 | 59 | func (m *Order) Run() error { 60 | defer m.Ctx.Recover() 61 | defer close(m.msgOutCh) 62 | 63 | outCh := m.MessageOut() 64 | inCh := m.MessageIn() 65 | 66 | colIndex := m.p.Stmt.ColIndexes() 67 | orderCt := len(m.p.Stmt.OrderBy) 68 | 69 | // are are going to hold entire row in memory while we are calculating 70 | // so obviously not scalable. 71 | sl := NewOrderMessages(m.p) 72 | 73 | msgReadLoop: 74 | for { 75 | 76 | select { 77 | case <-m.SigChan(): 78 | u.Warnf("got signal quit") 79 | return nil 80 | case msg, ok := <-inCh: 81 | if !ok { 82 | //u.Debugf("NICE, got closed channel shutdown") 83 | break msgReadLoop 84 | } else { 85 | var sdm *datasource.SqlDriverMessageMap 86 | 87 | switch mt := msg.(type) { 88 | case *datasource.SqlDriverMessageMap: 89 | sdm = mt 90 | default: 91 | 92 | msgReader, isContextReader := msg.(expr.ContextReader) 93 | if !isContextReader { 94 | err := fmt.Errorf("To use Join must use SqlDriverMessageMap but got %T", msg) 95 | u.Errorf("unrecognized msg %T", msg) 96 | close(m.TaskBase.sigCh) 97 | return err 98 | } 99 | 100 | sdm = datasource.NewSqlDriverMessageMapCtx(msg.Id(), msgReader, colIndex) 101 | } 102 | 103 | // We are going to use VM Engine to create a value for each statement in group by 104 | // then join each value together to create a unique key. 105 | keys := make([]string, orderCt) 106 | for i, col := range m.p.Stmt.OrderBy { 107 | if col.Expr != nil { 108 | if key, ok := vm.Eval(sdm, col.Expr); ok { 109 | //u.Debugf("msgtype:%T key:%q for-expr:%s", sdm, key, col.Expr) 110 | keys[i] = key.ToString() 111 | } else { 112 | // Is this an error? 113 | //u.Warnf("no key? %s for %+v", col.Expr, sdm) 114 | } 115 | } else { 116 | //u.Warnf("no col.expr? %#v", col) 117 | } 118 | } 119 | 120 | //u.Infof("found key:%s for %+v", key, sdm) 121 | sl.l = append(sl.l, &msgkey{keys, sdm}) 122 | } 123 | } 124 | } 125 | 126 | sort.Sort(sl) 127 | 128 | for _, m := range sl.l { 129 | //u.Debugf("got %s:%v msgs", key, vals) 130 | outCh <- m.msg 131 | } 132 | 133 | m.isComplete = true 134 | close(m.complete) 135 | 136 | return nil 137 | } 138 | 139 | type msgkey struct { 140 | keys []string 141 | msg *datasource.SqlDriverMessageMap 142 | } 143 | type OrderMessages struct { 144 | l []*msgkey 145 | invert []bool 146 | } 147 | 148 | func NewOrderMessages(p *plan.Order) *OrderMessages { 149 | invert := make([]bool, len(p.Stmt.OrderBy)) 150 | for i, col := range p.Stmt.OrderBy { 151 | //u.Debugf("invert? %s ORDER %v", col.Expr, col.Order) 152 | if col.Expr != nil { 153 | if !col.Asc() { 154 | invert[i] = true 155 | } 156 | } 157 | } 158 | return &OrderMessages{ 159 | l: make([]*msgkey, 0), 160 | invert: invert, 161 | } 162 | } 163 | func (m *OrderMessages) Len() int { 164 | return len(m.l) 165 | } 166 | func (m *OrderMessages) Less(i, j int) bool { 167 | for ki, key := range m.l[i].keys { 168 | if key < m.l[j].keys[ki] { 169 | if m.invert[ki] { 170 | return false 171 | } 172 | return true 173 | } else { 174 | if m.invert[ki] { 175 | return true 176 | } 177 | } 178 | } 179 | return false 180 | } 181 | func (m *OrderMessages) Swap(i, j int) { 182 | m.l[i], m.l[j] = m.l[j], m.l[i] 183 | } 184 | -------------------------------------------------------------------------------- /exec/source.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "fmt" 5 | 6 | u "github.com/araddon/gou" 7 | 8 | "github.com/araddon/qlbridge/plan" 9 | "github.com/araddon/qlbridge/schema" 10 | ) 11 | 12 | var ( 13 | _ = u.EMPTY 14 | 15 | // Ensure that we implement the Task Runner interface 16 | // to ensure this can run in exec engine 17 | _ TaskRunner = (*Source)(nil) 18 | ) 19 | 20 | // RequiresContext defines a Source which requires context. 21 | type RequiresContext interface { 22 | SetContext(ctx *plan.Context) 23 | } 24 | 25 | // Source defines a datasource execution task. It will Scan a data source for 26 | // rows to feed into exec dag of tasks. The source scanner uses iter.Next() 27 | // messages. The source may optionally allow Predicate PushDown, that is 28 | // use the SQL select/where to filter rows so its not a real table scan. This 29 | // interface is called ExecutorSource. 30 | // 31 | // Examples of Sources: 32 | // 1) table -- FROM table 33 | // 2) channels -- FROM stream 34 | // 3) join -- SELECT t1.name, t2.salary 35 | // FROM employee AS t1 36 | // INNER JOIN info AS t2 37 | // ON t1.name = t2.name; 38 | // 4) sub-select -- SELECT * FROM (SELECT 1, 2, 3) AS t1; 39 | type Source struct { 40 | *TaskBase 41 | p *plan.Source 42 | Scanner schema.ConnScanner 43 | ExecSource ExecutorSource 44 | JoinKey KeyEvaluator 45 | closed bool 46 | } 47 | 48 | // NewSource create a scanner to read from data source 49 | func NewSource(ctx *plan.Context, p *plan.Source) (*Source, error) { 50 | 51 | if p.Stmt == nil { 52 | return nil, fmt.Errorf("must have from for Source") 53 | } 54 | if p.Conn == nil { 55 | return nil, fmt.Errorf("Must have existing connection on Plan") 56 | } 57 | 58 | scanner, hasScanner := p.Conn.(schema.ConnScanner) 59 | 60 | // Some sources require context so we seed it here 61 | if sourceContext, needsContext := p.Conn.(RequiresContext); needsContext { 62 | sourceContext.SetContext(ctx) 63 | } 64 | 65 | if !hasScanner { 66 | e, hasSourceExec := p.Conn.(ExecutorSource) 67 | if hasSourceExec { 68 | s := &Source{ 69 | TaskBase: NewTaskBase(ctx), 70 | ExecSource: e, 71 | p: p, 72 | } 73 | return s, nil 74 | } 75 | u.Warnf("source %T does not implement datasource.Scanner", p.Conn) 76 | return nil, fmt.Errorf("%T Must Implement Scanner for %q", p.Conn, p.Stmt.String()) 77 | } 78 | s := &Source{ 79 | TaskBase: NewTaskBase(ctx), 80 | Scanner: scanner, 81 | p: p, 82 | } 83 | return s, nil 84 | } 85 | 86 | // NewSourceScanner A scanner to read from sub-query data source (join, sub-query, static) 87 | func NewSourceScanner(ctx *plan.Context, p *plan.Source, scanner schema.ConnScanner) *Source { 88 | s := &Source{ 89 | TaskBase: NewTaskBase(ctx), 90 | Scanner: scanner, 91 | p: p, 92 | } 93 | return s 94 | } 95 | 96 | func (m *Source) Copy() *Source { return &Source{} } 97 | 98 | func (m *Source) closeSource() error { 99 | m.Lock() 100 | defer m.Unlock() 101 | if m.closed { 102 | return nil 103 | } 104 | m.closed = true 105 | if m.Scanner != nil { 106 | if closer, ok := m.Scanner.(schema.Conn); ok { 107 | if err := closer.Close(); err != nil { 108 | return err 109 | } 110 | } 111 | } 112 | return nil 113 | } 114 | 115 | func (m *Source) Close() error { 116 | if err := m.closeSource(); err != nil { 117 | // Still need to close base right? 118 | return err 119 | } 120 | return m.TaskBase.Close() 121 | } 122 | 123 | func (m *Source) Run() error { 124 | defer m.Ctx.Recover() 125 | defer close(m.msgOutCh) 126 | 127 | if m.Scanner == nil { 128 | u.Warnf("no datasource configured?") 129 | return fmt.Errorf("No datasource found") 130 | } 131 | 132 | sigChan := m.SigChan() 133 | 134 | for item := m.Scanner.Next(); item != nil; item = m.Scanner.Next() { 135 | 136 | select { 137 | case <-sigChan: 138 | return nil 139 | case m.msgOutCh <- item: 140 | // continue 141 | } 142 | 143 | } 144 | return nil 145 | } 146 | -------------------------------------------------------------------------------- /exec/task_parallel.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | u "github.com/araddon/gou" 8 | 9 | "github.com/araddon/qlbridge/plan" 10 | ) 11 | 12 | var ( 13 | _ = u.EMPTY 14 | 15 | // Ensure that we implement the Tasks 16 | _ Task = (*TaskParallel)(nil) 17 | ) 18 | 19 | // A parallel set of tasks, this starts each child task and offers up 20 | // an output channel that is a merger of each child 21 | // 22 | // --> \ 23 | // --> - -> 24 | // --> / 25 | type TaskParallel struct { 26 | *TaskBase 27 | in TaskRunner 28 | runners []TaskRunner 29 | tasks []Task 30 | } 31 | 32 | func NewTaskParallel(ctx *plan.Context) *TaskParallel { 33 | return &TaskParallel{ 34 | TaskBase: NewTaskBase(ctx), 35 | runners: make([]TaskRunner, 0), 36 | tasks: make([]Task, 0), 37 | } 38 | } 39 | 40 | func (m *TaskParallel) PrintDag(depth int) { 41 | 42 | prefix := "" 43 | for i := 0; i < depth; i++ { 44 | prefix += "\t" 45 | } 46 | for i := 0; i < len(m.runners); i++ { 47 | t := m.runners[i] 48 | switch tt := t.(type) { 49 | case TaskPrinter: 50 | u.Warnf("%s%d %p task i:%v %T", prefix, depth, m, i, t) 51 | tt.PrintDag(depth + 1) 52 | default: 53 | u.Warnf("%s%d %p task i:%v %T", prefix, depth, m, i, t) 54 | } 55 | } 56 | } 57 | func (m *TaskParallel) Close() error { 58 | errs := make(errList, 0) 59 | for _, task := range m.tasks { 60 | if err := task.Close(); err != nil { 61 | errs.append(err) 62 | } 63 | } 64 | if len(errs) > 0 { 65 | return errs 66 | } 67 | return m.TaskBase.Close() 68 | } 69 | 70 | func (m *TaskParallel) Setup(depth int) error { 71 | m.setup = true 72 | if m.in != nil { 73 | for _, task := range m.runners { 74 | task.MessageInSet(m.in.MessageOut()) 75 | //u.Infof("parallel task in: #%d task p:%p %T %p", i, task, task, task.MessageIn()) 76 | } 77 | } 78 | for _, task := range m.runners { 79 | task.MessageOutSet(m.msgOutCh) 80 | } 81 | for i := 0; i < len(m.runners); i++ { 82 | //u.Debugf("%d Setup: %T", depth, m.runners[i]) 83 | if err := m.runners[i].Setup(depth + 1); err != nil { 84 | return err 85 | } 86 | } 87 | return nil 88 | } 89 | 90 | func (m *TaskParallel) Add(task Task) error { 91 | if m.setup { 92 | return fmt.Errorf("Cannot add task after Setup() called") 93 | } 94 | tr, ok := task.(TaskRunner) 95 | if !ok { 96 | panic(fmt.Sprintf("must be taskrunner %T", task)) 97 | } 98 | m.tasks = append(m.tasks, task) 99 | m.runners = append(m.runners, tr) 100 | return nil 101 | } 102 | 103 | func (m *TaskParallel) Children() []Task { return m.tasks } 104 | 105 | func (m *TaskParallel) Run() error { 106 | defer m.Ctx.Recover() // Our context can recover panics, save error msg 107 | defer func() { 108 | // TODO: find the culprit 109 | defer func() { 110 | if r := recover(); r != nil { 111 | //u.Errorf("panic on: %v", r) 112 | } 113 | }() 114 | //u.WarnT(8) 115 | close(m.msgOutCh) // closing output channels is the signal to stop 116 | }() 117 | 118 | // Either of the SigQuit, or error channel will 119 | // cause breaking out of message channels below 120 | select { 121 | case err := <-m.errCh: 122 | //m.errors = append(m.errors, err) 123 | u.Errorf("%v", err) 124 | case <-m.sigCh: 125 | 126 | default: 127 | } 128 | 129 | var wg sync.WaitGroup 130 | 131 | // start tasks in reverse order, so that by time 132 | // source starts up all downstreams have started 133 | for i := len(m.runners) - 1; i >= 0; i-- { 134 | wg.Add(1) 135 | go func(taskId int) { 136 | task := m.runners[taskId] 137 | //u.Infof("starting task %d-%d %T in:%p out:%p", m.depth, taskId, task, task.MessageIn(), task.MessageOut()) 138 | if err := task.Run(); err != nil { 139 | u.Errorf("%T.Run() errored %v", task, err) 140 | // TODO: what do we do with this error? send to error channel? 141 | } 142 | //u.Debugf("exiting taskId: %v %T", taskId, task) 143 | wg.Done() 144 | }(i) 145 | } 146 | 147 | wg.Wait() 148 | 149 | return nil 150 | } 151 | -------------------------------------------------------------------------------- /exec/taskrunner.go: -------------------------------------------------------------------------------- 1 | package exec 2 | -------------------------------------------------------------------------------- /exec/where.go: -------------------------------------------------------------------------------- 1 | package exec 2 | 3 | import ( 4 | u "github.com/araddon/gou" 5 | 6 | "github.com/araddon/qlbridge/datasource" 7 | "github.com/araddon/qlbridge/expr" 8 | "github.com/araddon/qlbridge/plan" 9 | "github.com/araddon/qlbridge/rel" 10 | "github.com/araddon/qlbridge/schema" 11 | "github.com/araddon/qlbridge/value" 12 | "github.com/araddon/qlbridge/vm" 13 | ) 14 | 15 | // Where execution of A filter to implement where clause 16 | type Where struct { 17 | *TaskBase 18 | filter expr.Node 19 | sel *rel.SqlSelect 20 | } 21 | 22 | // NewWhere create new Where Clause 23 | // filters vs final differ bc the Final does final column aliasing 24 | func NewWhere(ctx *plan.Context, p *plan.Where) *Where { 25 | if p.Final { 26 | return NewWhereFinal(ctx, p) 27 | } 28 | return NewWhereFilter(ctx, p.Stmt) 29 | } 30 | 31 | func NewWhereFinal(ctx *plan.Context, p *plan.Where) *Where { 32 | s := &Where{ 33 | TaskBase: NewTaskBase(ctx), 34 | sel: p.Stmt, 35 | filter: p.Stmt.Where.Expr, 36 | } 37 | cols := make(map[string]int) 38 | 39 | if len(p.Stmt.From) == 1 { 40 | cols = p.Stmt.ColIndexes() 41 | } else { 42 | // for _, col := range p.Stmt.Columns { 43 | // _, right, _ := col.LeftRight() 44 | // u.Debugf("p.Stmt col: %s %#v", right, col) 45 | // } 46 | 47 | for _, from := range p.Stmt.From { 48 | //u.Debugf("cols: %v", from.Columns) 49 | //u.Infof("source: %#v", from.Source) 50 | for _, col := range from.Source.Columns { 51 | _, right, _ := col.LeftRight() 52 | //u.Debugf("col: %s %#v", right, col) 53 | if _, ok := cols[right]; !ok { 54 | cols[right] = len(cols) 55 | } 56 | } 57 | } 58 | } 59 | 60 | //u.Debugf("found where columns: %d", len(cols)) 61 | 62 | s.Handler = whereFilter(s.filter, s, cols) 63 | return s 64 | } 65 | 66 | // NewWhereFilter filters vs final differ bc the Final does final column aliasing 67 | func NewWhereFilter(ctx *plan.Context, sql *rel.SqlSelect) *Where { 68 | s := &Where{ 69 | TaskBase: NewTaskBase(ctx), 70 | filter: sql.Where.Expr, 71 | } 72 | cols := sql.ColIndexes() 73 | s.Handler = whereFilter(s.filter, s, cols) 74 | return s 75 | } 76 | 77 | // NewHaving Filter 78 | func NewHaving(ctx *plan.Context, p *plan.Having) *Where { 79 | s := &Where{ 80 | TaskBase: NewTaskBase(ctx), 81 | filter: p.Stmt.Having, 82 | } 83 | s.Handler = whereFilter(p.Stmt.Having, s, p.Stmt.ColIndexes()) 84 | return s 85 | } 86 | 87 | func whereFilter(filter expr.Node, task TaskRunner, cols map[string]int) MessageHandler { 88 | out := task.MessageOut() 89 | 90 | //u.Debugf("prepare filter %s", filter) 91 | return func(ctx *plan.Context, msg schema.Message) bool { 92 | 93 | var filterValue value.Value 94 | var ok bool 95 | //u.Debugf("WHERE: T:%T body%#v", msg, msg.Body()) 96 | switch mt := msg.(type) { 97 | case *datasource.SqlDriverMessage: 98 | //u.Debugf("WHERE: T:%T vals:%#v", msg, mt.Vals) 99 | //u.Debugf("cols: %#v", cols) 100 | msgReader := mt.ToMsgMap(cols) 101 | filterValue, ok = vm.Eval(msgReader, filter) 102 | case *datasource.SqlDriverMessageMap: 103 | filterValue, ok = vm.Eval(mt, filter) 104 | if !ok { 105 | u.Warnf("wtf %s %#v", filter, mt) 106 | } 107 | //u.Debugf("WHERE: result:%v T:%T \n\trow:%#v \n\tvals:%#v", filterValue, msg, mt, mt.Values()) 108 | //u.Debugf("cols: %#v", cols) 109 | default: 110 | if msgReader, isContextReader := msg.(expr.ContextReader); isContextReader { 111 | filterValue, ok = vm.Eval(msgReader, filter) 112 | if !ok { 113 | u.Warnf("wat? %v filterval:%#v expr: %s", filter.String(), filterValue, filter) 114 | } 115 | } else { 116 | u.Errorf("could not convert to message reader: %T", msg) 117 | } 118 | } 119 | //u.Debugf("msg: %#v", msgReader) 120 | //u.Infof("evaluating: ok?%v result=%v filter expr: '%s'", ok, filterValue.ToString(), filter.String()) 121 | if !ok { 122 | u.Debugf("could not evaluate: %T %#v", msg, msg) 123 | return false 124 | } 125 | switch valTyped := filterValue.(type) { 126 | case value.BoolValue: 127 | if valTyped.Val() == false { 128 | //u.Debugf("Filtering out: T:%T v:%#v", valTyped, valTyped) 129 | return true 130 | } 131 | case nil: 132 | return false 133 | default: 134 | if valTyped.Nil() { 135 | return false 136 | } 137 | } 138 | 139 | //u.Debugf("about to send from where to forward: %#v", msg) 140 | select { 141 | case out <- msg: 142 | return true 143 | case <-task.SigChan(): 144 | return false 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /expr/builtins/json.go: -------------------------------------------------------------------------------- 1 | package builtins 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | u "github.com/araddon/gou" 8 | "github.com/jmespath/go-jmespath" 9 | 10 | "github.com/araddon/qlbridge/expr" 11 | "github.com/araddon/qlbridge/value" 12 | ) 13 | 14 | var _ = u.EMPTY 15 | 16 | // JsonPath jmespath json parser http://jmespath.org/ 17 | // 18 | // json_field = `[{"name":"n1","ct":8,"b":true, "tags":["a","b"]},{"name":"n2","ct":10,"b": false, "tags":["a","b"]}]` 19 | // 20 | // json.jmespath(json_field, "[?name == 'n1'].name | [0]") => "n1" 21 | // 22 | type JsonPath struct{} 23 | 24 | func (m *JsonPath) Type() value.ValueType { return value.UnknownType } 25 | func (m *JsonPath) Validate(n *expr.FuncNode) (expr.EvaluatorFunc, error) { 26 | if len(n.Args) != 2 { 27 | return nil, fmt.Errorf(`Expected 2 args for json.jmespath(field,json_val) but got %s`, n) 28 | } 29 | 30 | jsonPathExpr := "" 31 | switch jn := n.Args[1].(type) { 32 | case *expr.StringNode: 33 | jsonPathExpr = jn.Text 34 | default: 35 | return nil, fmt.Errorf("expected a string expression for jmespath got %T", jn) 36 | } 37 | 38 | parser := jmespath.NewParser() 39 | _, err := parser.Parse(jsonPathExpr) 40 | if err != nil { 41 | // if syntaxError, ok := err.(jmespath.SyntaxError); ok { 42 | // u.Warnf("%s\n%s\n", syntaxError, syntaxError.HighlightLocation()) 43 | // } 44 | return nil, err 45 | } 46 | return jsonPathEval(jsonPathExpr), nil 47 | } 48 | 49 | func jsonPathEval(expression string) expr.EvaluatorFunc { 50 | return func(ctx expr.EvalContext, args []value.Value) (value.Value, bool) { 51 | if args[0] == nil || args[0].Err() || args[0].Nil() { 52 | return nil, false 53 | } 54 | 55 | val := args[0].ToString() 56 | 57 | // Validate that this is valid json? 58 | var data interface{} 59 | if err := json.Unmarshal([]byte(val), &data); err != nil { 60 | return nil, false 61 | } 62 | 63 | result, err := jmespath.Search(expression, data) 64 | if err != nil { 65 | return nil, false 66 | } 67 | return value.NewValue(result), true 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /expr/builtins/math.go: -------------------------------------------------------------------------------- 1 | package builtins 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | 7 | "github.com/araddon/qlbridge/expr" 8 | "github.com/araddon/qlbridge/value" 9 | ) 10 | 11 | // Sqrt square root function. Must be able to coerce to number. 12 | // 13 | // sqrt(4) => 2, true 14 | // sqrt(9) => 3, true 15 | // sqrt(not_number) => 0, false 16 | // 17 | type Sqrt struct{} 18 | 19 | // Type is NumberType 20 | func (m *Sqrt) Type() value.ValueType { return value.NumberType } 21 | 22 | // Validate Must have 1 arg 23 | func (m *Sqrt) Validate(n *expr.FuncNode) (expr.EvaluatorFunc, error) { 24 | if len(n.Args) != 1 { 25 | return nil, fmt.Errorf("Expected exactly 1 args for sqrt(arg) but got %s", n) 26 | } 27 | return sqrtEval, nil 28 | } 29 | 30 | func sqrtEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool) { 31 | 32 | if args[0] == nil || args[0].Err() || args[0].Nil() { 33 | return value.NewNumberNil(), false 34 | } 35 | 36 | nv, ok := args[0].(value.NumericValue) 37 | if !ok { 38 | return value.NewNumberNil(), false 39 | } 40 | fv := nv.Float() 41 | fv = math.Sqrt(fv) 42 | return value.NewNumberValue(fv), true 43 | } 44 | 45 | // Pow exponents, raise x to the power of y 46 | // 47 | // pow(5,2) => 25, true 48 | // pow(3,2) => 9, true 49 | // pow(not_number,2) => NilNumber, false 50 | // 51 | type Pow struct{} 52 | 53 | // Type is Number 54 | func (m *Pow) Type() value.ValueType { return value.NumberType } 55 | 56 | // Must have 2 arguments, both must be able to be coerced to Number 57 | func (m *Pow) Validate(n *expr.FuncNode) (expr.EvaluatorFunc, error) { 58 | if len(n.Args) != 2 { 59 | return nil, fmt.Errorf("Expected 2 args for Pow(numer, power) but got %s", n) 60 | } 61 | return powerEval, nil 62 | } 63 | 64 | func powerEval(ctx expr.EvalContext, args []value.Value) (value.Value, bool) { 65 | 66 | if args[0] == nil || args[0].Err() || args[0].Nil() { 67 | return value.NewNumberNil(), false 68 | } 69 | if args[1] == nil || args[1].Err() || args[1].Nil() { 70 | return value.NewNumberNil(), false 71 | } 72 | fv, _ := value.ValueToFloat64(args[0]) 73 | pow, _ := value.ValueToFloat64(args[1]) 74 | if math.IsNaN(fv) || math.IsNaN(pow) { 75 | return value.NewNumberNil(), false 76 | } 77 | fv = math.Pow(fv, pow) 78 | return value.NewNumberValue(fv), true 79 | } 80 | -------------------------------------------------------------------------------- /expr/dialect_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/araddon/dateparse" 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/araddon/qlbridge/lex" 11 | "github.com/araddon/qlbridge/value" 12 | ) 13 | 14 | type testDialect struct { 15 | t lex.Token 16 | expect string 17 | } 18 | 19 | func TestDialectIdentityWriting(t *testing.T) { 20 | 21 | for _, td := range []testDialect{ 22 | {lex.Token{V: "name"}, "name"}, 23 | {lex.Token{Quote: '`', V: "has.period"}, "`has.period`"}, 24 | {lex.Token{Quote: '`', V: "has`.`period"}, "has.period"}, 25 | {lex.Token{V: "has space"}, "`has space`"}, 26 | } { 27 | dw := NewDefaultWriter() 28 | in := NewIdentityNode(&td.t) 29 | in.WriteDialect(dw) 30 | assert.Equal(t, td.expect, dw.String()) 31 | } 32 | 33 | for _, td := range []testDialect{ 34 | {lex.Token{V: "name"}, "name"}, 35 | {lex.Token{Quote: '`', V: "has.period"}, "'has.period'"}, 36 | {lex.Token{V: "has space"}, "'has space'"}, 37 | } { 38 | dw := NewDialectWriter('"', '\'') 39 | in := NewIdentityNode(&td.t) 40 | in.WriteDialect(dw) 41 | assert.Equal(t, td.expect, dw.String()) 42 | } 43 | 44 | for _, td := range []testDialect{ 45 | {lex.Token{V: "name"}, "name"}, 46 | {lex.Token{Quote: '`', V: "has.period"}, "[has.period]"}, 47 | {lex.Token{V: "has space"}, "[has space]"}, 48 | } { 49 | dw := NewDialectWriter('"', '[') 50 | in := NewIdentityNode(&td.t) 51 | in.WriteDialect(dw) 52 | assert.Equal(t, td.expect, dw.String()) 53 | } 54 | // strip Namespaces 55 | for _, td := range []testDialect{ 56 | {lex.Token{V: "name"}, "name"}, 57 | {lex.Token{Quote: '`', V: "table_name`.`fieldname"}, "fieldname"}, 58 | {lex.Token{V: "has space"}, "`has space`"}, 59 | } { 60 | dw := NewDefaultNoNamspaceWriter() 61 | in := NewIdentityNode(&td.t) 62 | in.WriteDialect(dw) 63 | assert.Equal(t, td.expect, dw.String()) 64 | } 65 | } 66 | func TestDialectValueWriting(t *testing.T) { 67 | // Test value writing 68 | for _, tc := range []struct { 69 | in value.Value 70 | out string 71 | }{ 72 | {value.NewValue(true), "true"}, 73 | {value.NewValue(22.2), "22.2"}, 74 | {value.NewValue(dateparse.MustParse("2017/08/08")), "\"2017-08-08 00:00:00 +0000 UTC\""}, 75 | {value.NewValue(22), "22"}, 76 | {value.NewValue("world"), `"world"`}, 77 | {value.NewValue(json.RawMessage(`{"name":"world"}`)), `{"name":"world"}`}, 78 | } { 79 | dw := NewDialectWriter('"', '[') 80 | dw.WriteValue(tc.in) 81 | assert.Equal(t, tc.out, dw.String()) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /expr/funcs.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | 7 | "github.com/araddon/qlbridge/value" 8 | ) 9 | 10 | var ( 11 | // The global function registry 12 | funcReg = NewFuncRegistry() 13 | ) 14 | 15 | type ( 16 | // EvaluatorFunc defines the evaluator func which may be stateful (or not) for 17 | // evaluating custom functions 18 | EvaluatorFunc func(ctx EvalContext, args []value.Value) (value.Value, bool) 19 | // CustomFunc allows custom functions to be added for run-time evaluation 20 | CustomFunc interface { 21 | // Type Define the Return Type of this function, or use value.Value for unknown. 22 | Type() value.ValueType 23 | // Validate is parse time syntax and type evaluation. Also returns the evaluation 24 | // function. 25 | Validate(n *FuncNode) (EvaluatorFunc, error) 26 | } 27 | // AggFunc allows custom functions to specify if they provide aggregation 28 | AggFunc interface { 29 | IsAgg() bool 30 | } 31 | // FuncResolver is a function resolution interface that allows 32 | // local/namespaced function resolution. 33 | FuncResolver interface { 34 | FuncGet(name string) (Func, bool) 35 | } 36 | 37 | // FuncRegistry contains lists of functions for different scope/run-time evaluation contexts. 38 | FuncRegistry struct { 39 | mu sync.RWMutex 40 | funcs map[string]Func 41 | aggs map[string]struct{} 42 | } 43 | ) 44 | 45 | // EmptyEvalFunc a no-op evaluation function for use in 46 | func EmptyEvalFunc(ctx EvalContext, args []value.Value) (value.Value, bool) { 47 | return value.NilValueVal, false 48 | } 49 | 50 | // NewFuncRegistry create a new function registry. By default their is a 51 | // global one, but you can have local function registries as well. 52 | func NewFuncRegistry() *FuncRegistry { 53 | return &FuncRegistry{ 54 | funcs: make(map[string]Func), 55 | aggs: make(map[string]struct{}), 56 | } 57 | } 58 | 59 | // Add a name/function to registry 60 | func (m *FuncRegistry) Add(name string, fn CustomFunc) { 61 | name = strings.ToLower(name) 62 | newFunc := Func{Name: name, CustomFunc: fn} 63 | m.mu.Lock() 64 | defer m.mu.Unlock() 65 | aggfn, hasAggFlag := fn.(AggFunc) 66 | if hasAggFlag { 67 | newFunc.Aggregate = aggfn.IsAgg() 68 | if newFunc.Aggregate { 69 | m.aggs[name] = struct{}{} 70 | } 71 | } 72 | m.funcs[name] = newFunc 73 | } 74 | 75 | // FuncGet gets a function from registry if it exists. 76 | func (m *FuncRegistry) FuncGet(name string) (Func, bool) { 77 | m.mu.RLock() 78 | fn, ok := m.funcs[name] 79 | m.mu.RUnlock() 80 | return fn, ok 81 | } 82 | 83 | // FuncAdd Global add Functions to the VM func registry occurs here. 84 | func FuncAdd(name string, fn CustomFunc) { 85 | funcReg.Add(name, fn) 86 | } 87 | -------------------------------------------------------------------------------- /expr/funcs_test.go: -------------------------------------------------------------------------------- 1 | package expr_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/araddon/qlbridge/expr" 9 | "github.com/araddon/qlbridge/expr/builtins" 10 | ) 11 | 12 | func TestFuncsRegistry(t *testing.T) { 13 | t.Parallel() 14 | 15 | builtins.LoadAllBuiltins() 16 | _, ok := expr.EmptyEvalFunc(nil, nil) 17 | assert.Equal(t, false, ok) 18 | 19 | } 20 | -------------------------------------------------------------------------------- /expr/include.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/araddon/qlbridge/lex" 7 | ) 8 | 9 | const ( 10 | maxIncludeDepth = 100 11 | ) 12 | 13 | var ( 14 | // If we hit max depth 15 | ErrMaxDepth = fmt.Errorf("Recursive Evaluation Error") 16 | ) 17 | 18 | // FindIncludes recursively descend down a node looking for all Include identities 19 | func FindIncludes(node Node) []string { 20 | return findAllIncludes(node, nil) 21 | } 22 | 23 | // InlineIncludes take an expression and resolve any includes so that 24 | // the included expression is "Inline" 25 | func InlineIncludes(ctx Includer, n Node) (Node, error) { 26 | return doInlineIncludes(ctx, n, 0) 27 | } 28 | func doInlineIncludes(ctx Includer, n Node, depth int) (Node, error) { 29 | // We need to make a copy, so we lazily use the To/From pb 30 | // We need the copy because we are going to mutate this node 31 | // but AST is assumed to be immuteable, and shared, since we are breaking 32 | // this contract we copy 33 | npb := n.NodePb() 34 | newNode := NodeFromNodePb(npb) 35 | return inlineIncludesDepth(ctx, newNode, depth) 36 | } 37 | func inlineIncludesDepth(ctx Includer, arg Node, depth int) (Node, error) { 38 | if depth > maxIncludeDepth { 39 | return nil, ErrMaxDepth 40 | } 41 | 42 | switch n := arg.(type) { 43 | // FuncNode, BinaryNode, BooleanNode, TriNode, UnaryNode, ArrayNode 44 | case NodeArgs: 45 | args := n.ChildrenArgs() 46 | for i, narg := range args { 47 | newNode, err := inlineIncludesDepth(ctx, narg, depth+1) 48 | if err != nil { 49 | return nil, err 50 | } 51 | if newNode != nil { 52 | args[i] = newNode 53 | } 54 | } 55 | return arg, nil 56 | case *IncludeNode: 57 | return resolveInclude(ctx, n, depth+1) 58 | default: 59 | //*NumberNode, *IdentityNode, *StringNode, nil, 60 | //*ValueNode, *NullNode: 61 | return arg, nil 62 | } 63 | } 64 | 65 | func resolveInclude(ctx Includer, inc *IncludeNode, depth int) (Node, error) { 66 | 67 | // if inc.inlineExpr != nil { 68 | // return inc.inlineExpr, nil 69 | // } 70 | 71 | n, err := ctx.Include(inc.Identity.Text) 72 | if err != nil { 73 | return nil, err 74 | } 75 | if n == nil { 76 | return nil, ErrIncludeNotFound 77 | } 78 | 79 | // Now inline, the inlines 80 | n, err = doInlineIncludes(ctx, n, depth) 81 | if err != nil { 82 | return nil, err 83 | } 84 | if inc.Negated() { 85 | inc.inlineExpr = NewUnary(lex.Token{T: lex.TokenNegate, V: "NOT"}, n) 86 | } else { 87 | inc.inlineExpr = n 88 | } 89 | 90 | inc.ExprNode = inc.inlineExpr 91 | 92 | return inc.inlineExpr, nil 93 | } 94 | 95 | func findAllIncludes(node Node, current []string) []string { 96 | switch n := node.(type) { 97 | case *IncludeNode: 98 | current = append(current, n.Identity.Text) 99 | case *BinaryNode: 100 | for _, arg := range n.Args { 101 | current = findAllIncludes(arg, current) 102 | } 103 | case *BooleanNode: 104 | for _, arg := range n.Args { 105 | current = findAllIncludes(arg, current) 106 | } 107 | case *UnaryNode: 108 | current = findAllIncludes(n.Arg, current) 109 | case *TriNode: 110 | for _, arg := range n.Args { 111 | current = findAllIncludes(arg, current) 112 | } 113 | case *ArrayNode: 114 | for _, arg := range n.Args { 115 | current = findAllIncludes(arg, current) 116 | } 117 | case *FuncNode: 118 | for _, arg := range n.Args { 119 | current = findAllIncludes(arg, current) 120 | } 121 | } 122 | return current 123 | } 124 | -------------------------------------------------------------------------------- /expr/node.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package expr; 3 | 4 | 5 | // protoc --proto_path=$GOPATH/src:$GOPATH/src/github.com/gogo/protobuf/protobuf:. --gofast_out=. node.proto 6 | 7 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 8 | 9 | option (gogoproto.marshaler_all) = true; 10 | option (gogoproto.sizer_all) = true; 11 | option (gogoproto.unmarshaler_all) = true; 12 | option (gogoproto.goproto_getters_all) = false; 13 | 14 | // The generic Expr 15 | message ExprPb { 16 | optional int32 op = 1 [(gogoproto.nullable) = true]; 17 | repeated ExprPb args = 2 [(gogoproto.nullable) = true]; 18 | 19 | optional string ident = 4 [(gogoproto.nullable) = true]; 20 | optional string val = 5 [(gogoproto.nullable) = true]; 21 | optional int64 ival = 6 [(gogoproto.nullable) = true]; 22 | optional bool bval = 7 [(gogoproto.nullable) = true]; 23 | optional double fval = 8 [(gogoproto.nullable) = true]; 24 | } 25 | 26 | // The generic Node, must be exactly one of these types 27 | message NodePb { 28 | optional BinaryNodePb bn = 1 [(gogoproto.nullable) = true]; 29 | optional BooleanNodePb booln = 2 [(gogoproto.nullable) = true]; 30 | optional UnaryNodePb un = 3 [(gogoproto.nullable) = true]; 31 | optional FuncNodePb fn = 4 [(gogoproto.nullable) = true]; 32 | optional TriNodePb tn = 5 [(gogoproto.nullable) = true]; 33 | optional ArrayNodePb an = 6 [(gogoproto.nullable) = true]; 34 | optional NumberNodePb nn = 10 [(gogoproto.nullable) = true]; 35 | optional ValueNodePb vn = 11 [(gogoproto.nullable) = true]; 36 | optional IdentityNodePb in = 12 [(gogoproto.nullable) = true]; 37 | optional StringNodePb sn = 13 [(gogoproto.nullable) = true]; 38 | optional IncludeNodePb incn = 14 [(gogoproto.nullable) = true]; 39 | optional NullNodePb niln = 15 [(gogoproto.nullable) = true]; 40 | } 41 | 42 | // Binary Node, two child args 43 | message BinaryNodePb { 44 | required int32 op = 1 [(gogoproto.nullable) = false]; 45 | optional bool paren = 2 [(gogoproto.nullable) = false]; 46 | repeated NodePb args = 3 [(gogoproto.nullable) = false]; 47 | } 48 | 49 | // Boolean Node, n child args 50 | message BooleanNodePb { 51 | required int32 op = 1 [(gogoproto.nullable) = false]; 52 | repeated NodePb args = 2 [(gogoproto.nullable) = false]; 53 | } 54 | 55 | // Include Node, two child args 56 | message IncludeNodePb { 57 | required int32 op = 1 [(gogoproto.nullable) = false]; 58 | required bool negated = 2 [(gogoproto.nullable) = false]; 59 | required IdentityNodePb identity = 3 [(gogoproto.nullable) = false]; 60 | } 61 | 62 | // Unary Node, one child 63 | message UnaryNodePb { 64 | required int32 op = 1 [(gogoproto.nullable) = false]; 65 | optional bool paren = 2 [(gogoproto.nullable) = false]; 66 | required NodePb arg = 3 [(gogoproto.nullable) = false]; 67 | } 68 | 69 | // Func Node, args are children 70 | message FuncNodePb { 71 | required string name = 1 [(gogoproto.nullable) = false]; 72 | repeated NodePb args = 2 [(gogoproto.nullable) = false]; 73 | } 74 | 75 | // Tri Node, may hve children 76 | message TriNodePb { 77 | required int32 op = 1 [(gogoproto.nullable) = false]; 78 | repeated NodePb args = 2 [(gogoproto.nullable) = false]; 79 | } 80 | 81 | // Array Node 82 | message ArrayNodePb { 83 | required int32 wrap = 1 [(gogoproto.nullable) = true]; 84 | repeated NodePb args = 3 [(gogoproto.nullable) = false]; 85 | } 86 | 87 | // String literal, no children 88 | message StringNodePb { 89 | optional bool noquote = 1 [(gogoproto.nullable) = true]; 90 | optional int32 quote = 2 [(gogoproto.nullable) = true]; 91 | optional string text = 3 [(gogoproto.nullable) = false]; 92 | } 93 | 94 | // Identity 95 | message IdentityNodePb { 96 | optional int32 quote = 1 [(gogoproto.nullable) = true]; 97 | optional string text = 3 [(gogoproto.nullable) = false]; 98 | } 99 | 100 | // Number Node 101 | message NumberNodePb { 102 | optional bool isint = 1 [(gogoproto.nullable) = false]; 103 | optional bool isfloat = 2 [(gogoproto.nullable) = false]; 104 | required int64 iv = 3 [(gogoproto.nullable) = false]; 105 | required double fv = 4 [(gogoproto.nullable) = false]; 106 | required string text = 5 [(gogoproto.nullable) = false]; 107 | } 108 | 109 | // Value Node 110 | message ValueNodePb { 111 | required int32 valuetype = 1 [(gogoproto.nullable) = false]; 112 | required bytes value = 2; 113 | } 114 | 115 | // NullNode 116 | message NullNodePb { 117 | optional int32 niltype = 1 [(gogoproto.nullable) = false]; 118 | } 119 | -------------------------------------------------------------------------------- /expr/stringutil_test.go: -------------------------------------------------------------------------------- 1 | package expr 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestIdentityQuoting(t *testing.T) { 10 | t.Parallel() 11 | 12 | assert.Equal(t, IdentityMaybeQuote('"', `na"me`), `"na""me"`) 13 | assert.Equal(t, IdentityMaybeQuote('"', "1name"), `"1name"`) 14 | assert.Equal(t, IdentityMaybeQuote('"', "name"), `name`) // don't escape 15 | assert.Equal(t, IdentityMaybeQuote('"', "na me"), `"na me"`) 16 | assert.Equal(t, IdentityMaybeQuote('"', "na#me"), `"na#me"`) 17 | // already escaped 18 | assert.Equal(t, IdentityMaybeQuote('"', `"name"`), `"name"`) // don't escape 19 | assert.Equal(t, IdentityMaybeQuote('\'', "'name'"), `'name'`) // don't escape 20 | assert.Equal(t, IdentityMaybeQuote('`', "`name`"), "`name`") // don't escape 21 | 22 | assert.Equal(t, IdentityMaybeQuote('`', "namex"), "namex") 23 | assert.Equal(t, IdentityMaybeQuote('`', "1name"), "`1name`") 24 | assert.Equal(t, IdentityMaybeQuote('`', "na`me"), "`na``me`") 25 | 26 | assert.Equal(t, IdentityMaybeQuote('`', "space name"), "`space name`") 27 | assert.Equal(t, IdentityMaybeQuote('`', "space name"), "`space name`") 28 | 29 | assert.Equal(t, IdentityMaybeQuoteStrict('`', "_uid"), "`_uid`") 30 | 31 | assert.Equal(t, IdentityMaybeQuote('`', "\xe2\x00"), "`\xe2\x00`") 32 | assert.Equal(t, IdentityMaybeQuote('`', "a\xe2\x00"), "`a\xe2\x00`") 33 | } 34 | 35 | func TestLiteralEscaping(t *testing.T) { 36 | t.Parallel() 37 | 38 | assert.Equal(t, LiteralQuoteEscape('"', `na"me`), `"na""me"`) 39 | assert.Equal(t, LiteralQuoteEscape('"', "1name"), `"1name"`) 40 | assert.Equal(t, LiteralQuoteEscape('"', "na me"), `"na me"`) 41 | assert.Equal(t, LiteralQuoteEscape('"', "na#me"), `"na#me"`) 42 | // already escaped 43 | assert.Equal(t, LiteralQuoteEscape('"', `"name"`), `"name"`) // don't escape 44 | assert.Equal(t, LiteralQuoteEscape('\'', "'name'"), `'name'`) // don't escape 45 | 46 | assert.Equal(t, LiteralQuoteEscape('\'', "namex"), "'namex'") 47 | assert.Equal(t, LiteralQuoteEscape('\'', "1name"), "'1name'") 48 | assert.Equal(t, LiteralQuoteEscape('\'', "na'me"), "'na''me'") 49 | 50 | assert.Equal(t, LiteralQuoteEscape('\'', "space name"), "'space name'") 51 | assert.Equal(t, LiteralQuoteEscape('\'', "space name"), "'space name'") 52 | 53 | newVal, wasUnEscaped := StringUnEscape('"', `Toys R\" Us`) 54 | assert.Equal(t, true, wasUnEscaped) 55 | assert.Equal(t, newVal, `Toys R" Us`) 56 | newVal, wasUnEscaped = StringUnEscape('"', `Toys R"" Us`) 57 | assert.Equal(t, true, wasUnEscaped) 58 | assert.Equal(t, newVal, `Toys R" Us`) 59 | } 60 | 61 | func TestLeftRight(t *testing.T) { 62 | t.Parallel() 63 | l, r, hasLeft := LeftRight("`table`.`column`") 64 | assert.True(t, l == "table" && hasLeft, "left, right, w quote: %s", l) 65 | assert.True(t, r == "column", "left, right, w quote & ns %s", l) 66 | 67 | l, r, hasLeft = LeftRight("`table.column`") 68 | assert.True(t, l == "" && !hasLeft, "no left bc escaped %s", l) 69 | assert.True(t, r == "table.column", "%s", l) 70 | 71 | // Un-escaped 72 | l, r, hasLeft = LeftRight("table.column") 73 | assert.True(t, l == "table" && hasLeft, "left, right no quote: %s", l) 74 | assert.True(t, r == "column", "left,right, no quote: %s", l) 75 | 76 | // Not sure i want to support this, legacy reasons we stupidly 77 | // allowed the left most part before the first period to be the 78 | // left, and the rest to be right. Now we should ??? no left? 79 | l, r, hasLeft = LeftRight("table.col.with.periods") 80 | assert.True(t, l == "table" && hasLeft, "no quote: %s", l) 81 | assert.True(t, r == "col.with.periods", "no quote: %s", l) 82 | 83 | l, r, hasLeft = LeftRight("`table.name`.`has.period`") 84 | assert.True(t, l == "table.name" && hasLeft, "recognize `left`.`right`: %s", l) 85 | assert.True(t, r == "has.period", "no quote: %s", l) 86 | } 87 | -------------------------------------------------------------------------------- /generators/elasticsearch/es2gen/esgenerator_test.go: -------------------------------------------------------------------------------- 1 | package es2gen_test 2 | -------------------------------------------------------------------------------- /generators/elasticsearch/es2gen/schema.go: -------------------------------------------------------------------------------- 1 | package es2gen 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/araddon/qlbridge/expr" 8 | "github.com/araddon/qlbridge/generators/elasticsearch/gentypes" 9 | ) 10 | 11 | // fieldType return the Elasticsearch field name for an identity node or an error. 12 | func fieldType(s gentypes.SchemaColumns, n expr.Node) (*gentypes.FieldType, error) { 13 | 14 | ident, ok := n.(*expr.IdentityNode) 15 | if !ok { 16 | return nil, fmt.Errorf("qlindex: expected an identity but found %T (%s)", n, n) 17 | } 18 | 19 | // This shotgun approach sucks, see https://github.com/lytics/lio/issues/7565 20 | ft, ok := s.ColumnInfo(ident.Text) 21 | if ok { 22 | return ft, nil 23 | } 24 | 25 | if ident.HasLeftRight() { 26 | ft, ok := s.ColumnInfo(ident.OriginalText()) 27 | if ok { 28 | return ft, nil 29 | } 30 | } 31 | 32 | // This is legacy crap, we stupidly used to allow this: 33 | // ticket to remove https://github.com/lytics/lio/issues/7565 34 | // 35 | // `key_name.field value` -> "key_name", "field value" 36 | // 37 | // check if key is left.right 38 | parts := strings.SplitN(ident.Text, ".", 2) 39 | if len(parts) == 2 { 40 | // Nested field lookup 41 | ft, ok = s.ColumnInfo(parts[0]) 42 | if ok { 43 | return ft, nil 44 | } 45 | } 46 | 47 | return nil, gentypes.MissingField(ident.OriginalText()) 48 | } 49 | -------------------------------------------------------------------------------- /generators/elasticsearch/esgen/esgenerator_test.go: -------------------------------------------------------------------------------- 1 | package esgen_test 2 | -------------------------------------------------------------------------------- /generators/elasticsearch/esgen/schema.go: -------------------------------------------------------------------------------- 1 | package esgen 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/araddon/qlbridge/expr" 9 | "github.com/araddon/qlbridge/generators/elasticsearch/gentypes" 10 | "github.com/araddon/qlbridge/value" 11 | ) 12 | 13 | func exprValueType(s gentypes.SchemaColumns, n expr.Node) value.ValueType { 14 | 15 | switch nt := n.(type) { 16 | case *expr.NumberNode: 17 | if !nt.IsInt { 18 | return value.NumberType 19 | } 20 | return value.IntType 21 | case *expr.StringNode: 22 | return value.StringType 23 | } 24 | return value.UnknownType 25 | } 26 | 27 | // scalar returns a JSONable representation of a scalar node type for use in ES 28 | // filters. 29 | // 30 | // Does not support Null. 31 | // 32 | func scalar(node expr.Node) (interface{}, bool) { 33 | switch n := node.(type) { 34 | 35 | case *expr.StringNode: 36 | return n.Text, true 37 | 38 | case *expr.NumberNode: 39 | if n.IsInt { 40 | // ES supports string encoded ints 41 | return n.Int64, true 42 | } 43 | return n.Float64, true 44 | 45 | case *expr.ValueNode: 46 | // Make sure this is a scalar value node 47 | switch n.Value.Type() { 48 | case value.BoolType, value.IntType, value.StringType, value.TimeType: 49 | return n.String(), true 50 | case value.NumberType: 51 | nn, ok := n.Value.(floatval) 52 | if !ok { 53 | return nil, false 54 | } 55 | return nn.Float(), true 56 | } 57 | case *expr.IdentityNode: 58 | if _, err := strconv.ParseBool(n.Text); err == nil { 59 | return n.Text, true 60 | } 61 | 62 | } 63 | return "", false 64 | } 65 | 66 | func fieldType(s gentypes.SchemaColumns, n expr.Node) (*gentypes.FieldType, error) { 67 | 68 | ident, ok := n.(*expr.IdentityNode) 69 | if !ok { 70 | return nil, fmt.Errorf("expected an identity but found %T (%s)", n, n) 71 | } 72 | 73 | // TODO: This shotgun approach sucks, see https://github.com/araddon/qlbridge/issues/159 74 | ft, ok := s.ColumnInfo(ident.Text) 75 | if ok { 76 | return ft, nil 77 | } 78 | 79 | //left, right, _ := expr.LeftRight(ident.Text) 80 | //u.Debugf("left:%q right:%q isNamespaced?%v key=%v", left, right, ident.HasLeftRight(), ident.OriginalText()) 81 | if ident.HasLeftRight() { 82 | ft, ok := s.ColumnInfo(ident.OriginalText()) 83 | if ok { 84 | return ft, nil 85 | } 86 | } 87 | 88 | // This is legacy, we stupidly used to allow this: 89 | // 90 | // `key_name.field value` -> "key_name", "field value" 91 | // 92 | // check if key is left.right 93 | parts := strings.SplitN(ident.Text, ".", 2) 94 | if len(parts) == 2 { 95 | // Nested field lookup 96 | ft, ok = s.ColumnInfo(parts[0]) 97 | if ok { 98 | return ft, nil 99 | } 100 | } 101 | 102 | return nil, gentypes.MissingField(ident.OriginalText()) 103 | } 104 | 105 | func fieldValueType(s gentypes.SchemaColumns, n expr.Node) (value.ValueType, error) { 106 | 107 | ident, ok := n.(*expr.IdentityNode) 108 | if !ok { 109 | return value.UnknownType, fmt.Errorf("expected an identity but found %T (%s)", n, n) 110 | } 111 | 112 | // TODO: This shotgun approach sucks, see https://github.com/araddon/qlbridge/issues/159 113 | vt, ok := s.Column(ident.Text) 114 | if ok { 115 | return vt, nil 116 | } 117 | 118 | //left, right, _ := expr.LeftRight(ident.Text) 119 | //u.Debugf("left:%q right:%q isNamespaced?%v key=%v", left, right, ident.HasLeftRight(), ident.OriginalText()) 120 | if ident.HasLeftRight() { 121 | vt, ok := s.Column(ident.OriginalText()) 122 | if ok { 123 | return vt, nil 124 | } 125 | } 126 | 127 | // This is legacy, we stupidly used to allow this: 128 | // 129 | // `key_name.field value` -> "key_name", "field value" 130 | // 131 | // check if key is left.right 132 | parts := strings.SplitN(ident.Text, ".", 2) 133 | if len(parts) == 2 { 134 | // Nested field lookup 135 | vt, ok = s.Column(parts[0]) 136 | if ok { 137 | return vt, nil 138 | } 139 | } 140 | 141 | return value.UnknownType, gentypes.MissingField(ident.OriginalText()) 142 | } 143 | -------------------------------------------------------------------------------- /generators/elasticsearch/esgen/validate.go: -------------------------------------------------------------------------------- 1 | package esgen 2 | 3 | import ( 4 | "fmt" 5 | 6 | u "github.com/araddon/gou" 7 | 8 | "github.com/araddon/qlbridge/expr" 9 | "github.com/araddon/qlbridge/generators/elasticsearch/gentypes" 10 | "github.com/araddon/qlbridge/lex" 11 | "github.com/araddon/qlbridge/rel" 12 | "github.com/araddon/qlbridge/value" 13 | ) 14 | 15 | var ( 16 | _ = u.EMPTY 17 | 18 | // Ensure our schema implments filter validation 19 | fakeValidator gentypes.FilterValidate 20 | ) 21 | 22 | func init() { 23 | tv := &TypeValidator{} 24 | fakeValidator = tv.FilterValidate 25 | } 26 | 27 | type TypeValidator struct { 28 | schema gentypes.SchemaColumns 29 | } 30 | 31 | func NewValidator(s gentypes.SchemaColumns) *TypeValidator { 32 | return &TypeValidator{schema: s} 33 | } 34 | 35 | func (m *TypeValidator) FilterValidate(stmt *rel.FilterStatement) error { 36 | return m.walkNode(stmt.Filter) 37 | } 38 | 39 | func (m *TypeValidator) walkNode(node expr.Node) error { 40 | 41 | //u.Debugf("%d m.expr T:%T %#v", depth, node, node) 42 | switch n := node.(type) { 43 | case *expr.UnaryNode: 44 | return m.urnaryNode(n) 45 | case *expr.BooleanNode: 46 | return m.booleanNode(n) 47 | case *expr.BinaryNode: 48 | return m.binaryNode(n) 49 | case *expr.TriNode: 50 | return m.triNode(n) 51 | case *expr.IdentityNode: 52 | return m.identityNode(n) 53 | case *expr.IncludeNode: 54 | // We assume included statement has don't its own validation 55 | return nil 56 | case *expr.FuncNode: 57 | return m.funcExpr(n) 58 | default: 59 | u.Warnf("not handled type validation %v %T", node, node) 60 | return fmt.Errorf("esgen: unsupported node in expression: %T (%s)", node, node) 61 | } 62 | } 63 | 64 | func (m *TypeValidator) identityNode(n *expr.IdentityNode) error { 65 | vt, ok := m.schema.Column(n.Text) 66 | if !ok { 67 | return gentypes.MissingField(n.OriginalText()) 68 | } 69 | if vt == value.UnknownType { 70 | return fmt.Errorf("Unknown Field Type %s", n) 71 | } 72 | return nil 73 | } 74 | 75 | func (m *TypeValidator) urnaryNode(n *expr.UnaryNode) error { 76 | switch n.Operator.T { 77 | case lex.TokenExists: 78 | in, ok := n.Arg.(*expr.IdentityNode) 79 | if !ok { 80 | return fmt.Errorf("Expected Identity field %s got %T", n, n.Arg) 81 | } 82 | return m.identityNode(in) 83 | 84 | case lex.TokenNegate: 85 | // TODO: validate that rhs = bool ? 86 | return m.walkNode(n.Arg) 87 | } 88 | return nil 89 | } 90 | 91 | func (m *TypeValidator) booleanNode(bn *expr.BooleanNode) error { 92 | for _, arg := range bn.Args { 93 | err := m.walkNode(arg) 94 | if err != nil { 95 | return err 96 | } 97 | } 98 | return nil 99 | } 100 | 101 | func (m *TypeValidator) binaryNode(node *expr.BinaryNode) error { 102 | 103 | // Type check binary expression arguments as they must be: 104 | // Identifier-Operator-Literal 105 | lhs, err := fieldValueType(m.schema, node.Args[0]) 106 | if err != nil { 107 | return err 108 | } 109 | //rhs := exprValueType(m.schema, node.Args[1]) 110 | 111 | switch op := node.Operator.T; op { 112 | case lex.TokenGE, lex.TokenLE, lex.TokenGT, lex.TokenLT: 113 | // es 5 now enforces that lhs, rhs must be same type no mixed 114 | switch lhs { 115 | case value.NumberType: 116 | // If left hand is number right hand needs to be number 117 | } 118 | case lex.TokenEqual, lex.TokenEqualEqual: 119 | // the VM supports both = and == 120 | case lex.TokenNE: 121 | // ident(0) != literal(1) 122 | case lex.TokenContains: 123 | // ident CONTAINS literal 124 | case lex.TokenLike: 125 | // ident LIKE literal 126 | case lex.TokenIN, lex.TokenIntersects: 127 | // Build up list of arguments 128 | } 129 | 130 | return nil 131 | } 132 | 133 | func (m *TypeValidator) triNode(node *expr.TriNode) error { 134 | return nil 135 | } 136 | 137 | func (m *TypeValidator) funcExpr(node *expr.FuncNode) error { 138 | return nil 139 | } 140 | -------------------------------------------------------------------------------- /generators/elasticsearch/gentypes/errors.go: -------------------------------------------------------------------------------- 1 | package gentypes 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // MissingFieldErrors are returned when a segment can't be evaluated due to a 8 | // referenced field missing from a schema. 9 | type MissingFieldError struct { 10 | Field string 11 | } 12 | 13 | // MissingField creates a new MissingFieldError for the given field. 14 | func MissingField(field string) *MissingFieldError { 15 | return &MissingFieldError{field} 16 | } 17 | 18 | func (m *MissingFieldError) Reason() string { return m.Error() } 19 | func (m *MissingFieldError) Status() int { return 400 } 20 | 21 | func (m *MissingFieldError) Error() string { 22 | return fmt.Sprintf("missing field %s", m.Field) 23 | } 24 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/araddon/qlbridge 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 7 | github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 8 | github.com/dchest/siphash v1.2.1 9 | github.com/go-sql-driver/mysql v1.4.1 10 | github.com/gogo/protobuf v1.3.1 11 | github.com/golang/protobuf v1.3.2 12 | github.com/google/btree v1.0.0 13 | github.com/hashicorp/go-memdb v1.0.4 14 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af 15 | github.com/jmoiron/sqlx v1.2.0 16 | github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d 17 | github.com/lytics/cloudstorage v0.2.1 18 | github.com/lytics/datemath v0.0.0-20180727225141-3ada1c10b5de 19 | github.com/mattn/go-sqlite3 v1.9.0 20 | github.com/mb0/glob v0.0.0-20160210091149-1eb79d2de6c4 21 | github.com/mssola/user_agent v0.5.0 22 | github.com/pborman/uuid v1.2.0 23 | github.com/stretchr/testify v1.4.0 24 | golang.org/x/net v0.0.0-20191021144547-ec77196f6094 25 | google.golang.org/api v0.11.0 26 | ) 27 | -------------------------------------------------------------------------------- /go.test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "" > coverage.txt 5 | 6 | for d in $(go list ./... | grep -v vendor); do 7 | go test -race -coverprofile=profile.out -covermode=atomic $d 8 | if [ -f profile.out ]; then 9 | cat profile.out >> coverage.txt 10 | rm profile.out 11 | fi 12 | done -------------------------------------------------------------------------------- /lex/README.md: -------------------------------------------------------------------------------- 1 | ### Lexing/Dialects 2 | 3 | QLBridge implements a few different Dialects: *Sql*, *FilterQL*, *Expressions*, *json* 4 | 5 | * *SQL* a subset, non-complete implementation of SQL 6 | * *FilterQL* A filtering language, think just the WHERE part of SQL but more DSL'ish with 7 | syntax `AND ( , , )` instead of ` AND AND ` 8 | * *Expression* Simple boolean logic expressions see https://github.com/araddon/qlbridge/blob/master/vm/vm_test.go#L57 for examples 9 | * *Json* Lexes json (instead of marshal) 10 | 11 | ### Creating a custom Lexer/Parser ie Dialect 12 | 13 | See example in `dialects/example` folder for a custom ql dialect, this 14 | example creates a mythical *SUBSCRIBETO* query language... 15 | ```go 16 | // Tokens Specific to our PUBSUB 17 | var TokenSubscribeTo lex.TokenType = 1000 18 | 19 | // Custom lexer for our maybe hash function 20 | func LexMaybe(l *ql.Lexer) ql.StateFn { 21 | 22 | l.SkipWhiteSpaces() 23 | 24 | keyWord := strings.ToLower(l.PeekWord()) 25 | 26 | switch keyWord { 27 | case "maybe": 28 | l.ConsumeWord("maybe") 29 | l.Emit(lex.TokenIdentity) 30 | return ql.LexExpressionOrIdentity 31 | } 32 | return ql.LexExpressionOrIdentity 33 | } 34 | 35 | func main() { 36 | 37 | // We are going to inject new tokens into qlbridge 38 | lex.TokenNameMap[TokenSubscribeTo] = &lex.TokenInfo{Description: "subscribeto"} 39 | 40 | // OverRide the Identity Characters in qlbridge to allow a dash in identity 41 | ql.IDENTITY_CHARS = "_./-" 42 | 43 | ql.LoadTokenInfo() 44 | ourDialect.Init() 45 | 46 | // We are going to create our own Dialect that uses a "SUBSCRIBETO" keyword 47 | pubsub = &ql.Statement{TokenSubscribeTo, []*ql.Clause{ 48 | {Token: TokenSubscribeTo, Lexer: ql.LexColumns}, 49 | {Token: lex.TokenFrom, Lexer: LexMaybe}, 50 | {Token: lex.TokenWhere, Lexer: ql.LexColumns, Optional: true}, 51 | }} 52 | ourDialect = &ql.Dialect{ 53 | "Subscribe To", []*ql.Statement{pubsub}, 54 | } 55 | 56 | l := ql.NewLexer(` 57 | SUBSCRIBETO 58 | count(x), Name 59 | FROM ourstream 60 | WHERE 61 | k = REPLACE(LOWER(Name),'cde','xxx'); 62 | `, ourDialect) 63 | 64 | } 65 | 66 | ``` 67 | 68 | 69 | -------------------------------------------------------------------------------- /lex/dialect.go: -------------------------------------------------------------------------------- 1 | package lex 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type ( 9 | // KeywordMatcher A Clause may supply a keyword matcher instead of keyword-token 10 | KeywordMatcher func(c *Clause, peekWord string, l *Lexer) bool 11 | 12 | // Dialect is a Language made up of multiple Statements. 13 | // Examples are {SQL, CQL, GRAPHQL} 14 | Dialect struct { 15 | Name string 16 | Statements []*Clause 17 | IdentityQuoting []byte 18 | inited bool 19 | } 20 | // Clause is a unique "Section" of a statement 21 | Clause struct { 22 | parent *Clause 23 | next *Clause 24 | prev *Clause 25 | keyword string // keyword is firstWord, not full word, "GROUP" portion of "GROUP BY" 26 | fullWord string // In event multi word such as "GROUP BY" 27 | multiWord bool // flag if is multi-word 28 | Optional bool // Is this Clause/Keyword optional? 29 | Repeat bool // Repeatable clause? 30 | Token TokenType // Token identifiyng start of clause, optional 31 | KeywordMatcher KeywordMatcher 32 | Lexer StateFn // Lex Function to lex clause, optional 33 | Clauses []*Clause // Children Clauses 34 | Name string 35 | } 36 | ) 37 | 38 | // Init Dialects have one time load-setup. 39 | func (m *Dialect) Init() { 40 | if m.inited { 41 | return 42 | } 43 | m.inited = true 44 | for _, s := range m.Statements { 45 | s.init() 46 | } 47 | } 48 | 49 | // MatchesKeyword 50 | func (c *Clause) MatchesKeyword(peekWord string, l *Lexer) bool { 51 | if c.KeywordMatcher != nil { 52 | return c.KeywordMatcher(c, peekWord, l) 53 | } else if c.keyword == peekWord && !c.multiWord { 54 | return true 55 | } else if c.multiWord { 56 | if strings.ToLower(l.PeekX(len(c.fullWord))) == c.fullWord { 57 | return true 58 | } 59 | } 60 | return false 61 | } 62 | func (c *Clause) init() { 63 | if c.KeywordMatcher == nil { 64 | // Find the Keyword, MultiWord options 65 | c.fullWord = c.Token.String() 66 | c.keyword = strings.ToLower(c.Token.MatchString()) 67 | c.multiWord = c.Token.MultiWord() 68 | } 69 | for i, clause := range c.Clauses { 70 | clause.init() 71 | clause.parent = c 72 | if i != 0 { // .prev is nil on first clause 73 | clause.prev = c.Clauses[i-1] 74 | } 75 | if i+1 < len(c.Clauses) { // .next is nil on last clause 76 | clause.next = c.Clauses[i+1] 77 | } 78 | } 79 | } 80 | func (c *Clause) String() string { 81 | if c.parent != nil { 82 | return fmt.Sprintf(``, c, c.Name, c.keyword, c.fullWord, c.multiWord, len(c.Clauses), c.parent.keyword) 83 | } 84 | return fmt.Sprintf(``, c, c.Name, c.keyword, c.fullWord, c.multiWord, len(c.Clauses)) 85 | } 86 | -------------------------------------------------------------------------------- /lex/dialect_expr.go: -------------------------------------------------------------------------------- 1 | package lex 2 | 3 | var ( 4 | expressionStatement = []*Clause{ 5 | {Token: TokenIdentity, Lexer: LexExpressionOrIdentity}, 6 | } 7 | 8 | // ExpressionDialect, is a Single Expression dialect, useful for parsing Single 9 | // function 10 | // 11 | // eq(tolower(item_name),"buy") 12 | ExpressionDialect *Dialect = &Dialect{ 13 | Statements: []*Clause{ 14 | {Token: TokenNil, Clauses: expressionStatement}, 15 | }, 16 | } 17 | 18 | logicalEpressions = []*Clause{ 19 | {Token: TokenNil, Lexer: LexLogical}, 20 | } 21 | 22 | // logical Expression Statement of the following functional format 23 | // 24 | // 5 > 4 => true 25 | // 4 + 5 => 9 26 | // tolower(item) + 12 > 4 27 | // 4 IN (4,5,6) 28 | // 29 | LogicalExpressionDialect *Dialect = &Dialect{ 30 | Statements: []*Clause{ 31 | {Token: TokenNil, Clauses: logicalEpressions}, 32 | }, 33 | } 34 | ) 35 | 36 | // NewExpressionLexer creates a new lexer for the input string using Expression Dialect. 37 | func NewExpressionLexer(input string) *Lexer { 38 | return NewLexer(input, ExpressionDialect) 39 | } 40 | -------------------------------------------------------------------------------- /lex/dialect_expr_test.go: -------------------------------------------------------------------------------- 1 | package lex 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestExprDialectInit(t *testing.T) { 10 | // Make sure we can init more than once, see if it panics 11 | ExpressionDialect.Init() 12 | for _, stmt := range ExpressionDialect.Statements { 13 | assert.NotEqual(t, "", stmt.String()) 14 | } 15 | } 16 | 17 | func tokenexpr(lexString string, runLex StateFn) Token { 18 | l := NewLexer(lexString, ExpressionDialect) 19 | runLex(l) 20 | return l.NextToken() 21 | } 22 | 23 | func verifyExprTokens(t *testing.T, expString string, tokens []Token) { 24 | l := NewLexer(expString, ExpressionDialect) 25 | for _, goodToken := range tokens { 26 | tok := l.NextToken() 27 | assert.Equal(t, tok.T, goodToken.T, "want='%v' has %v ", goodToken.T, tok.T) 28 | assert.Equal(t, tok.V, goodToken.V, "want='%v' has %v ", goodToken.V, tok.V) 29 | } 30 | } 31 | func verifyExpr2Tokens(t *testing.T, expString string, tokens []Token) { 32 | l := NewLexer(expString, LogicalExpressionDialect) 33 | for _, goodToken := range tokens { 34 | tok := l.NextToken() 35 | assert.Equal(t, tok.T, goodToken.T, "want='%v' has %v ", goodToken.T, tok.T) 36 | assert.Equal(t, tok.V, goodToken.V, "want='%v' has %v ", goodToken.V, tok.V) 37 | } 38 | } 39 | func TestLexExprDialect(t *testing.T) { 40 | verifyExprTokens(t, `eq(toint(item),5)`, 41 | []Token{ 42 | tv(TokenUdfExpr, "eq"), 43 | tv(TokenLeftParenthesis, "("), 44 | tv(TokenUdfExpr, "toint"), 45 | tv(TokenLeftParenthesis, "("), 46 | tv(TokenIdentity, "item"), 47 | tv(TokenRightParenthesis, ")"), 48 | tv(TokenComma, ","), 49 | tv(TokenInteger, "5"), 50 | tv(TokenRightParenthesis, ")"), 51 | }) 52 | 53 | verifyExprTokens(t, `eq(@@varfive,5)`, 54 | []Token{ 55 | tv(TokenUdfExpr, "eq"), 56 | tv(TokenLeftParenthesis, "("), 57 | tv(TokenIdentity, "@@varfive"), 58 | tv(TokenComma, ","), 59 | tv(TokenInteger, "5"), 60 | tv(TokenRightParenthesis, ")"), 61 | }) 62 | } 63 | 64 | func TestLexLogicalDialect(t *testing.T) { 65 | 66 | verifyExpr2Tokens(t, `4 > 5`, 67 | []Token{ 68 | tv(TokenInteger, "4"), 69 | tv(TokenGT, ">"), 70 | tv(TokenInteger, "5"), 71 | }) 72 | 73 | verifyExpr2Tokens(t, `item || 5`, 74 | []Token{ 75 | tv(TokenIdentity, "item"), 76 | tv(TokenOr, "||"), 77 | tv(TokenInteger, "5"), 78 | }) 79 | 80 | verifyExpr2Tokens(t, `10 > 5`, 81 | []Token{ 82 | tv(TokenInteger, "10"), 83 | tv(TokenGT, ">"), 84 | tv(TokenInteger, "5"), 85 | }) 86 | verifyExpr2Tokens(t, `toint(10 * 5)`, 87 | []Token{ 88 | tv(TokenUdfExpr, "toint"), 89 | tv(TokenLeftParenthesis, "("), 90 | tv(TokenInteger, "10"), 91 | tv(TokenMultiply, "*"), 92 | tv(TokenInteger, "5"), 93 | tv(TokenRightParenthesis, ")"), 94 | }) 95 | 96 | verifyExpr2Tokens(t, `6 == !eq(5,6)`, 97 | []Token{ 98 | tv(TokenInteger, "6"), 99 | tv(TokenEqualEqual, "=="), 100 | tv(TokenNegate, "!"), 101 | tv(TokenUdfExpr, "eq"), 102 | tv(TokenLeftParenthesis, "("), 103 | tv(TokenInteger, "5"), 104 | tv(TokenComma, ","), 105 | tv(TokenInteger, "6"), 106 | tv(TokenRightParenthesis, ")"), 107 | }) 108 | 109 | verifyExpr2Tokens(t, `(4 + 5)/2`, 110 | []Token{ 111 | tv(TokenLeftParenthesis, "("), 112 | tv(TokenInteger, "4"), 113 | tv(TokenPlus, "+"), 114 | tv(TokenInteger, "5"), 115 | tv(TokenRightParenthesis, ")"), 116 | tv(TokenDivide, "/"), 117 | tv(TokenInteger, "2"), 118 | }) 119 | 120 | verifyExpr2Tokens(t, `(4.5 + float(5))/2`, 121 | []Token{ 122 | tv(TokenLeftParenthesis, "("), 123 | tv(TokenFloat, "4.5"), 124 | tv(TokenPlus, "+"), 125 | tv(TokenUdfExpr, "float"), 126 | tv(TokenLeftParenthesis, "("), 127 | tv(TokenInteger, "5"), 128 | tv(TokenRightParenthesis, ")"), 129 | tv(TokenRightParenthesis, ")"), 130 | tv(TokenDivide, "/"), 131 | tv(TokenInteger, "2"), 132 | }) 133 | } 134 | -------------------------------------------------------------------------------- /lex/dialect_filterql.go: -------------------------------------------------------------------------------- 1 | package lex 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | var ( 8 | // FilterStatement a FilterQL statement. 9 | FilterStatement = []*Clause{ 10 | {Token: TokenFilter, Lexer: LexFilterClause, Optional: true}, 11 | {Token: TokenFrom, Lexer: LexIdentifier, Optional: true}, 12 | {Token: TokenLimit, Lexer: LexNumber, Optional: true}, 13 | {Token: TokenWith, Lexer: LexJsonOrKeyValue, Optional: true}, 14 | {Token: TokenAlias, Lexer: LexIdentifier, Optional: true}, 15 | {Token: TokenEOF, Lexer: LexEndOfStatement, Optional: false}, 16 | } 17 | // FilterSelectStatement Filter statement that also supports column projection. 18 | FilterSelectStatement = []*Clause{ 19 | {Token: TokenSelect, Lexer: LexSelectClause, Optional: false}, 20 | {Token: TokenFrom, Lexer: LexIdentifier, Optional: false}, 21 | {Token: TokenWhere, Lexer: LexConditionalClause, Optional: true}, 22 | {Token: TokenFilter, Lexer: LexFilterClause, Optional: true}, 23 | {Token: TokenLimit, Lexer: LexNumber, Optional: true}, 24 | {Token: TokenWith, Lexer: LexJsonOrKeyValue, Optional: true}, 25 | {Token: TokenAlias, Lexer: LexIdentifier, Optional: true}, 26 | {Token: TokenEOF, Lexer: LexEndOfStatement, Optional: false}, 27 | } 28 | // FilterQLDialect is a Where Clause filtering language slightly 29 | // more DSL'ish than SQL Where Clause. 30 | FilterQLDialect *Dialect = &Dialect{ 31 | Statements: []*Clause{ 32 | {Token: TokenFilter, Clauses: FilterStatement}, 33 | {Token: TokenSelect, Clauses: FilterSelectStatement}, 34 | }, 35 | IdentityQuoting: IdentityQuotingWSingleQuote, 36 | } 37 | ) 38 | 39 | // NewFilterQLLexer creates a new lexer for the input string using FilterQLDialect 40 | // which is dsl for where/filtering. 41 | func NewFilterQLLexer(input string) *Lexer { 42 | return NewLexer(input, FilterQLDialect) 43 | } 44 | 45 | // LexFilterClause Handle Filter QL Main Statement 46 | // 47 | // FILTER := ( | ) 48 | // 49 | // := ( AND | OR ) '(' ( | ) [, ( | ) ] ')' 50 | // 51 | // := 52 | // 53 | // Examples: 54 | // 55 | // FILTER 56 | /// AND ( 57 | // daysago(datefield) < 100 58 | // , domain(url) == "google.com" 59 | // , INCLUDE name_of_filter 60 | // , 61 | // , OR ( 62 | // momentum > 20 63 | // , propensity > 50 64 | // ) 65 | // ) 66 | // ALIAS myfilter 67 | // 68 | // FILTER x > 7 69 | // 70 | func LexFilterClause(l *Lexer) StateFn { 71 | 72 | if l.SkipWhiteSpacesNewLine() { 73 | l.Emit(TokenNewLine) 74 | debugf("%p LexFilterClause emit new line stack=%d", l, len(l.stack)) 75 | l.Push("LexFilterClause", LexFilterClause) 76 | return LexFilterClause 77 | } 78 | 79 | if l.IsComment() { 80 | l.Push("LexFilterClause", LexFilterClause) 81 | debugf("%p LexFilterClause comment stack=%d", l, len(l.stack)) 82 | return LexComment 83 | } 84 | 85 | keyWord := strings.ToLower(l.PeekWord()) 86 | 87 | debugf("%p LexFilterClause r=%-15q stack=%d", l, string(keyWord), len(l.stack)) 88 | 89 | switch keyWord { 90 | case "from", "with": 91 | return nil 92 | case "include": 93 | l.ConsumeWord(keyWord) 94 | l.Emit(TokenInclude) 95 | l.Push("LexFilterClause", LexFilterClause) 96 | return LexIdentifier 97 | case "and": 98 | l.ConsumeWord(keyWord) 99 | l.Emit(TokenLogicAnd) 100 | l.Push("LexFilterClause", LexFilterClause) 101 | return LexFilterClause 102 | case "or": 103 | l.ConsumeWord(keyWord) 104 | l.Emit(TokenLogicOr) 105 | l.Push("LexFilterClause", LexFilterClause) 106 | return LexFilterClause 107 | case "not": 108 | l.ConsumeWord(keyWord) 109 | l.Emit(TokenNegate) 110 | return LexFilterClause 111 | case "(": 112 | l.ConsumeWord(keyWord) 113 | l.Emit(TokenLeftParenthesis) 114 | l.Push("LexFilterClause", LexFilterClause) 115 | return LexFilterClause 116 | case ",": 117 | l.ConsumeWord(keyWord) 118 | l.Emit(TokenComma) 119 | l.Push("LexFilterClause", LexFilterClause) 120 | return LexFilterClause 121 | case ")": 122 | l.ConsumeWord(keyWord) 123 | l.Emit(TokenRightParenthesis) 124 | return nil 125 | } 126 | return LexExpression 127 | } 128 | -------------------------------------------------------------------------------- /lex/dialect_json.go: -------------------------------------------------------------------------------- 1 | package lex 2 | 3 | // NewJsonLexer Creates a new json dialect lexer for the input string. 4 | func NewJsonLexer(input string) *Lexer { 5 | return NewLexer(input, JsonDialect) 6 | } 7 | 8 | var ( 9 | jsonDialectStatement = []*Clause{ 10 | {Token: TokenNil, Lexer: LexJson}, 11 | } 12 | // JsonDialect, is a json lexer 13 | // 14 | // ["hello","world"] 15 | // {"name":"bob","apples":["honeycrisp","fuji"]} 16 | // 17 | JsonDialect *Dialect = &Dialect{ 18 | Statements: []*Clause{ 19 | {Token: TokenNil, Clauses: jsonDialectStatement}, 20 | }, 21 | } 22 | ) 23 | -------------------------------------------------------------------------------- /lex/dialect_sql_parse_test.go: -------------------------------------------------------------------------------- 1 | package lex_test 2 | 3 | import ( 4 | "testing" 5 | 6 | u "github.com/araddon/gou" 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/araddon/qlbridge/rel" 10 | ) 11 | 12 | func parseSqlTest(t *testing.T, sql string) { 13 | u.Debugf("parsing sql: %s", sql) 14 | sqlRequest, err := rel.ParseSql(sql) 15 | assert.Equal(t, nil, err) 16 | assert.NotEqual(t, nil, sqlRequest, "Must parse: %s \n\t%v", sql, err) 17 | if ss, ok := sqlRequest.(*rel.SqlSelect); ok { 18 | _, err2 := rel.ParseSqlSelect(sql) 19 | assert.Equal(t, nil, err2) 20 | pb := ss.ToPbStatement() 21 | pbb, err := pb.Marshal() 22 | assert.Equal(t, nil, err) 23 | ss2, err := rel.SqlFromPb(pbb) 24 | assert.Equal(t, nil, err) 25 | assert.True(t, ss.Equal(ss2)) 26 | } 27 | } 28 | func parseSqlError(t *testing.T, sql string) { 29 | u.Debugf("parse looking for error sql: %s", sql) 30 | _, err := rel.ParseSql(sql) 31 | assert.NotEqual(t, nil, err, "Must error on parse: %s", sql) 32 | } 33 | 34 | func TestSqlParser(t *testing.T) { 35 | 36 | parseSqlTest(t, `## this is a comment 37 | SELECT a FROM x;`) 38 | 39 | parseSqlError(t, `SELECT a FROM x LIMIT 1 NOTAWORD;`) 40 | 41 | parseSqlError(t, `SELECT a, tolower(b) AS b INTO newtable FROM FROM WHERE a != "hello";`) 42 | parseSqlTest(t, ` 43 | SELECT a.language, a.template, Count(*) AS count 44 | FROM 45 | (Select Distinct language, template FROM content WHERE language != "en" OFFSET 1) AS a 46 | Left Join users AS b 47 | On b.language = a.language AND b.template = b.template 48 | GROUP BY a.language, a.template`) 49 | parseSqlTest(t, ` 50 | SELECT a FROM x WHERE a IN (select ax FROM z); 51 | `) 52 | 53 | // CREATE 54 | parseSqlTest(t, `CREATE CONTINUOUSVIEW viewx AS SELECT a FROM tbl;`) 55 | parseSqlError(t, `CREATE FAKEITEM viewx;`) 56 | parseSqlTest(t, `CREATE OR REPLACE VIEW viewx 57 | AS SELECT a, b FROM mydb.tbl 58 | WITH stuff = "hello";`) 59 | parseSqlTest(t, `CREATE TABLE articles 60 | --comment-here 61 | ( 62 | ID int(11) NOT NULL AUTO_INCREMENT, 63 | Email char(150) NOT NULL DEFAULT '' UNIQUE COMMENT 'this-is-comment', 64 | stuff varchar(150), 65 | profile text, 66 | PRIMARY KEY (ID), 67 | visitct BIGINT, 68 | CONSTRAINT emails_fk FOREIGN KEY (Email) REFERENCES Emails (Email) 69 | ) ENGINE=InnoDB AUTO_INCREMENT=4080 DEFAULT CHARSET=utf8;`) 70 | 71 | // DROP 72 | parseSqlTest(t, `DROP CONTINUOUSVIEW viewx WITH stuff = "hello";`) 73 | } 74 | -------------------------------------------------------------------------------- /lex/token_test.go: -------------------------------------------------------------------------------- 1 | package lex 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestToken(t *testing.T) { 10 | l := NewSqlLexer("SELECT X FROM Y;") 11 | tok := l.NextToken() 12 | err := tok.Err(l) 13 | assert.NotEqual(t, "", err.Error()) 14 | err = tok.ErrMsg(l, "not now") 15 | assert.NotEqual(t, "", err.Error()) 16 | 17 | tok = TokenFromOp("select") 18 | assert.Equal(t, TokenSelect, tok.T) 19 | tok = TokenFromOp("noway") 20 | assert.Equal(t, TokenNil, tok.T) 21 | } 22 | -------------------------------------------------------------------------------- /plan/context.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | 7 | "golang.org/x/net/context" 8 | 9 | "github.com/araddon/qlbridge/expr" 10 | "github.com/araddon/qlbridge/rel" 11 | "github.com/araddon/qlbridge/schema" 12 | ) 13 | 14 | // NextIdFunc is the id generation function to give statements 15 | // their own id 16 | type NextIdFunc func() uint64 17 | 18 | // NextId is the global next id generation function 19 | var NextId NextIdFunc 20 | 21 | var rs = rand.New(rand.NewSource(time.Now().UnixNano())) 22 | 23 | func init() { 24 | NextId = mathRandId 25 | } 26 | 27 | func mathRandId() uint64 { 28 | return uint64(rs.Int63()) 29 | } 30 | 31 | // Context for plan of a Relational task has info about the query 32 | // projection, schema, function resolvers necessary to plan this statement. 33 | // - may be transported across network boundaries to particpate in dag of tasks 34 | // - holds references to in-mem data structures for schema 35 | // - holds references to original statement 36 | // - holds task specific state for errors, ids, etc (net.context) 37 | // - manages Recover() - to persist/transport state 38 | type Context struct { 39 | 40 | // Stateful Fields that are transported to participate across network/nodes 41 | context.Context // go context for cancelation in plan 42 | SchemaName string // schema name to load schema with 43 | id uint64 // unique id per request 44 | fingerprint uint64 // not unique per statement, used for getting prepared plans 45 | Raw string // Raw sql statement 46 | Stmt rel.SqlStatement // Original Statement 47 | Projection *Projection // Projection for this context optional 48 | 49 | // Local in-memory helpers not transported across network 50 | Session expr.ContextReadWriter // Session for this connection 51 | Schema *schema.Schema // this schema for this connection 52 | Funcs expr.FuncResolver // Local/Dialect specific functions 53 | 54 | // From configuration 55 | DisableRecover bool 56 | 57 | // Local State 58 | Errors []error 59 | errRecover interface{} 60 | } 61 | 62 | // NewContext plan context 63 | func NewContext(query string) *Context { 64 | return &Context{Raw: query} 65 | } 66 | func NewContextFromPb(pb *ContextPb) *Context { 67 | return &Context{id: pb.Id, fingerprint: pb.Fingerprint, SchemaName: pb.Schema} 68 | } 69 | 70 | // called by go routines/tasks to ensure any recovery panics are captured 71 | func (m *Context) Recover() { 72 | if m == nil { 73 | return 74 | } 75 | } 76 | func (m *Context) init() { 77 | if m.id == 0 { 78 | if m.Schema != nil { 79 | m.SchemaName = m.Schema.Name 80 | } 81 | if ss, ok := m.Stmt.(*rel.SqlSelect); ok { 82 | m.fingerprint = uint64(ss.FingerPrintID()) 83 | } 84 | m.id = NextId() 85 | } 86 | } 87 | 88 | // called by go routines/tasks to ensure any recovery panics are captured 89 | func (m *Context) ToPB() *ContextPb { 90 | m.init() 91 | pb := &ContextPb{} 92 | pb.Schema = m.SchemaName 93 | pb.Fingerprint = m.fingerprint 94 | pb.Id = m.id 95 | return pb 96 | } 97 | 98 | func (m *Context) Equal(c *Context) bool { 99 | if m == nil && c == nil { 100 | return true 101 | } 102 | if m == nil && c != nil { 103 | return false 104 | } 105 | if m != nil && c == nil { 106 | return false 107 | } 108 | m.init() 109 | c.init() 110 | if m.id != c.id { 111 | return false 112 | } 113 | if m.fingerprint != c.fingerprint { 114 | return false 115 | } 116 | if m.SchemaName != c.SchemaName { 117 | return false 118 | } 119 | return true 120 | } 121 | -------------------------------------------------------------------------------- /plan/context_test.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | //"github.com/araddon/qlbridge/plan" 8 | ) 9 | 10 | func TestContext(t *testing.T) { 11 | c := &Context{} 12 | var planNil *Context 13 | planNil.Recover() // make sure we don't panic 14 | assert.Equal(t, true, planNil.Equal(nil)) 15 | assert.Equal(t, false, planNil.Equal(c)) 16 | assert.Equal(t, false, c.Equal(nil)) 17 | 18 | selQuery := "Select 1;" 19 | c1 := NewContext(selQuery) 20 | c2 := NewContext(selQuery) 21 | // Should NOT be equal because the id is not the same 22 | assert.Equal(t, false, c1.Equal(c2)) 23 | 24 | c1pb := c1.ToPB() 25 | c1FromPb := NewContextFromPb(c1pb) 26 | // Should be equal 27 | assert.Equal(t, true, c1.Equal(c1FromPb)) 28 | c1FromPb.SchemaName = "what" 29 | assert.Equal(t, false, c1.Equal(c1FromPb)) 30 | c1FromPb.fingerprint = 88 // 31 | assert.Equal(t, false, c1.Equal(c1FromPb)) 32 | } 33 | -------------------------------------------------------------------------------- /plan/plan.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package plan; 3 | 4 | 5 | // protoc --proto_path=$GOPATH/src:$GOPATH/src/github.com/gogo/protobuf/protobuf:. --gofast_out=. plan.proto 6 | 7 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 8 | import "github.com/araddon/qlbridge/rel/sql.proto"; 9 | import "github.com/araddon/qlbridge/expr/node.proto"; 10 | //import "github.com/araddon/qlbridge/schema/schema.proto"; 11 | 12 | option (gogoproto.marshaler_all) = true; 13 | option (gogoproto.sizer_all) = true; 14 | option (gogoproto.unmarshaler_all) = true; 15 | option (gogoproto.goproto_getters_all) = false; 16 | 17 | // The generic Node, must be exactly one of these types 18 | message PlanPb { 19 | required bool parallel = 1 [(gogoproto.nullable) = false]; 20 | optional SelectPb select = 3 [(gogoproto.nullable) = true]; 21 | optional SourcePb source = 4 [(gogoproto.nullable) = true]; 22 | optional WherePb where = 5 [(gogoproto.nullable) = true]; 23 | optional HavingPb having = 6 [(gogoproto.nullable) = true]; 24 | optional GroupByPb groupBy = 7 [(gogoproto.nullable) = true]; 25 | optional OrderPb order = 8 [(gogoproto.nullable) = true]; 26 | optional JoinMergePb joinMerge = 9 [(gogoproto.nullable) = true]; 27 | optional JoinKeyPb joinKey = 10 [(gogoproto.nullable) = true]; 28 | optional rel.ProjectionPb projection = 11 [(gogoproto.nullable) = true]; 29 | repeated PlanPb children = 12 [(gogoproto.nullable) = true]; 30 | } 31 | 32 | // Select Plan 33 | message SelectPb { 34 | required rel.SqlSelectPb select = 1 [(gogoproto.nullable) = true]; 35 | optional ContextPb context = 2 [(gogoproto.nullable) = true]; 36 | } 37 | 38 | // Context 39 | message ContextPb { 40 | required string schema = 1 [(gogoproto.nullable) = false]; 41 | required uint64 id = 2 [(gogoproto.nullable) = false]; 42 | required uint64 fingerprint = 3 [(gogoproto.nullable) = false]; 43 | } 44 | 45 | // Source Plan is a plan for single source of select query, of which 46 | // many may exist (joins, sub-querys etc) 47 | message SourcePb { 48 | // do we need group-by, join, partition key for routing purposes? 49 | required bool needsHashableKey = 2 [(gogoproto.nullable) = false]; 50 | // Is this final projection or not? non finals are partial-sub-query types 51 | required bool final = 3 [(gogoproto.nullable) = false]; 52 | // Is this plan complete as is? skip remaining plan walk steps 53 | required bool complete = 4 [(gogoproto.nullable) = false]; 54 | required bool join = 5 [(gogoproto.nullable) = false]; 55 | required bool sourceExec = 6 [(gogoproto.nullable) = false]; 56 | optional bytes custom = 7 [(gogoproto.nullable) = true]; 57 | optional rel.SqlSourcePb sqlSource = 8; 58 | optional rel.ProjectionPb projection = 9; 59 | } 60 | 61 | // Where Plan 62 | message WherePb { 63 | optional rel.SqlSelectPb select = 1 [(gogoproto.nullable) = true]; 64 | required bool final = 2 [(gogoproto.nullable) = false]; 65 | } 66 | 67 | // Group By Plan 68 | message GroupByPb { 69 | optional rel.SqlSelectPb select = 1 [(gogoproto.nullable) = true]; 70 | } 71 | 72 | message HavingPb { 73 | optional rel.SqlSelectPb select = 1 [(gogoproto.nullable) = true]; 74 | } 75 | 76 | message OrderPb { 77 | optional rel.SqlSelectPb select = 1 [(gogoproto.nullable) = true]; 78 | } 79 | 80 | message JoinMergePb { 81 | optional expr.NodePb having = 1 [(gogoproto.nullable) = true]; 82 | } 83 | 84 | message JoinKeyPb { 85 | optional expr.NodePb having = 1 [(gogoproto.nullable) = true]; 86 | } -------------------------------------------------------------------------------- /plan/plan_test.go: -------------------------------------------------------------------------------- 1 | package plan_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | td "github.com/araddon/qlbridge/datasource/mockcsvtestdata" 8 | "github.com/araddon/qlbridge/testutil" 9 | ) 10 | 11 | func TestMain(m *testing.M) { 12 | testutil.Setup() // will call flag.Parse() 13 | 14 | // load our mock data sources "users", "articles" 15 | td.LoadTestDataOnce() 16 | 17 | // Now run the actual Tests 18 | os.Exit(m.Run()) 19 | } 20 | 21 | func TestRunTestSuite(t *testing.T) { 22 | testutil.RunDDLTests(t) 23 | testutil.RunTestSuite(t) 24 | } 25 | -------------------------------------------------------------------------------- /plan/planner.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "fmt" 5 | 6 | u "github.com/araddon/gou" 7 | ) 8 | 9 | var ( 10 | // Ensure our default planner meets Planner interface. 11 | _ Planner = (*PlannerDefault)(nil) 12 | ) 13 | 14 | // PlannerDefault is implementation of Planner that creates a dag of plan.Tasks 15 | // that will be turned into execution plan by executor. This is a simple 16 | // planner but can be over-ridden by providing a Planner that will 17 | // supercede any single or more visit methods. 18 | // - stateful, specific to a single request 19 | type PlannerDefault struct { 20 | Planner Planner 21 | Ctx *Context 22 | distinct bool 23 | children []Task 24 | } 25 | 26 | // NewPlanner creates a new default planner with context. 27 | func NewPlanner(ctx *Context) *PlannerDefault { 28 | p := &PlannerDefault{ 29 | Ctx: ctx, 30 | children: make([]Task, 0), 31 | } 32 | p.Planner = p 33 | return p 34 | } 35 | 36 | // WalkPreparedStatement not implemented 37 | func (m *PlannerDefault) WalkPreparedStatement(p *PreparedStatement) error { 38 | u.Debugf("WalkPreparedStatement %+v", p.Stmt) 39 | return ErrNotImplemented 40 | } 41 | 42 | // WalkCommand walks the command statement 43 | func (m *PlannerDefault) WalkCommand(p *Command) error { 44 | u.Debugf("WalkCommand %+v", p.Stmt) 45 | return nil 46 | } 47 | 48 | // WalkDrop walks the draop statement 49 | func (m *PlannerDefault) WalkDrop(p *Drop) error { 50 | u.Debugf("WalkDrop %+v", p.Stmt) 51 | return nil 52 | } 53 | 54 | // WalkCreate walk a Create Plan to create the dag of tasks for Create. 55 | func (m *PlannerDefault) WalkCreate(p *Create) error { 56 | u.Debugf("WalkCreate %#v", p) 57 | if len(p.Stmt.With) == 0 { 58 | return fmt.Errorf("CREATE {SCHEMA|SOURCE|DATABASE}") 59 | } 60 | return nil 61 | } 62 | 63 | // WalkAlter walk a ALTER Plan to create the dag of tasks forAlter. 64 | func (m *PlannerDefault) WalkAlter(p *Alter) error { 65 | u.Debugf("WalkAlter %#v", p) 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /plan/planner_mutate.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "fmt" 5 | 6 | u "github.com/araddon/gou" 7 | 8 | "github.com/araddon/qlbridge/schema" 9 | ) 10 | 11 | var ( 12 | _ = u.EMPTY 13 | ) 14 | 15 | func (m *PlannerDefault) WalkInto(p *Into) error { 16 | u.Debugf("VisitInto %+v", p.Stmt) 17 | return ErrNotImplemented 18 | } 19 | 20 | func upsertSource(ctx *Context, table string) (schema.ConnUpsert, error) { 21 | 22 | conn, err := ctx.Schema.OpenConn(table) 23 | if err != nil { 24 | u.Warnf("%p no schema for %q err=%v", ctx.Schema, table, err) 25 | return nil, err 26 | } 27 | 28 | mutatorSource, hasMutator := conn.(schema.ConnMutation) 29 | if hasMutator { 30 | mutator, err := mutatorSource.CreateMutator(ctx) 31 | if err != nil { 32 | u.Warnf("%p could not create mutator for %q err=%v", ctx.Schema, table, err) 33 | //return nil, err 34 | } else { 35 | return mutator, nil 36 | } 37 | } 38 | 39 | upsertDs, isUpsert := conn.(schema.ConnUpsert) 40 | if !isUpsert { 41 | return nil, fmt.Errorf("%T does not implement required schema.Upsert for upserts", conn) 42 | } 43 | return upsertDs, nil 44 | } 45 | 46 | func (m *PlannerDefault) WalkInsert(p *Insert) error { 47 | u.Debugf("VisitInsert %s", p.Stmt) 48 | src, err := upsertSource(m.Ctx, p.Stmt.Table) 49 | if err != nil { 50 | return err 51 | } 52 | p.Source = src 53 | return nil 54 | } 55 | 56 | func (m *PlannerDefault) WalkUpdate(p *Update) error { 57 | u.Debugf("VisitUpdate %+v", p.Stmt) 58 | src, err := upsertSource(m.Ctx, p.Stmt.Table) 59 | if err != nil { 60 | return err 61 | } 62 | p.Source = src 63 | return nil 64 | } 65 | 66 | func (m *PlannerDefault) WalkUpsert(p *Upsert) error { 67 | u.Debugf("VisitUpsert %+v", p.Stmt) 68 | src, err := upsertSource(m.Ctx, p.Stmt.Table) 69 | if err != nil { 70 | return err 71 | } 72 | p.Source = src 73 | return nil 74 | } 75 | 76 | func (m *PlannerDefault) WalkDelete(p *Delete) error { 77 | u.Debugf("VisitDelete %+v", p.Stmt) 78 | conn, err := m.Ctx.Schema.OpenConn(p.Stmt.Table) 79 | if err != nil { 80 | u.Warnf("%p no schema for %q err=%v", m.Ctx.Schema, p.Stmt.Table, err) 81 | return err 82 | } 83 | 84 | mutatorSource, hasMutator := conn.(schema.ConnMutation) 85 | if hasMutator { 86 | mutator, err := mutatorSource.CreateMutator(m.Ctx) 87 | if err != nil { 88 | u.Warnf("%p could not create mutator for %q err=%v", m.Ctx.Schema, p.Stmt.Table, err) 89 | //return nil, err 90 | } else { 91 | p.Source = mutator 92 | return nil 93 | } 94 | } 95 | 96 | deleteDs, isDelete := conn.(schema.ConnDeletion) 97 | if !isDelete { 98 | return fmt.Errorf("%T does not implement required schema.Deletion for deletions", conn) 99 | } 100 | p.Source = deleteDs 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /plan/planner_test.go: -------------------------------------------------------------------------------- 1 | package plan_test 2 | 3 | import ( 4 | "testing" 5 | 6 | u "github.com/araddon/gou" 7 | "github.com/stretchr/testify/assert" 8 | 9 | td "github.com/araddon/qlbridge/datasource/mockcsvtestdata" 10 | ) 11 | 12 | type plantest struct { 13 | q string 14 | cols int 15 | } 16 | 17 | var planTests = []plantest{ 18 | {"SELECT DATABASE()", 1}, 19 | } 20 | 21 | var _ = u.EMPTY 22 | 23 | func TestPlans(t *testing.T) { 24 | for _, pt := range planTests { 25 | ctx := td.TestContext(pt.q) 26 | u.Infof("running %s for plan check", pt.q) 27 | p := selectPlan(t, ctx) 28 | assert.True(t, p != nil) 29 | 30 | u.Infof("%#v", ctx.Projection) 31 | u.Infof("cols %#v", ctx.Projection) 32 | if pt.cols > 0 { 33 | // ensure our projection has these columns 34 | assert.True(t, len(ctx.Projection.Proj.Columns) == pt.cols, 35 | "expected %d cols got %v %#v", pt.cols, len(ctx.Projection.Proj.Columns), ctx.Projection.Proj) 36 | } 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /plan/sql_rewrite_test.go: -------------------------------------------------------------------------------- 1 | package plan_test 2 | -------------------------------------------------------------------------------- /qlbdriver/driver.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package driver registers a QL Bridge sql/driver named "qlbridge" 3 | 4 | Usage 5 | 6 | package main 7 | 8 | import ( 9 | "database/sql" 10 | _ "github.com/araddon/qlbridge/qlbdriver" 11 | ) 12 | 13 | func main() { 14 | 15 | db, err := sql.Open("qlbridge", "csv:///dev/stdin") 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | 20 | // Use db here 21 | 22 | } 23 | 24 | */ 25 | package qlbdriver 26 | 27 | import "github.com/araddon/qlbridge/exec" 28 | 29 | func init() { 30 | exec.RegisterSqlDriver() 31 | } 32 | -------------------------------------------------------------------------------- /rel/_bm/bm_parse_test.go: -------------------------------------------------------------------------------- 1 | package rel 2 | 3 | import ( 4 | "github.com/araddon/qlbridge/rel" 5 | "github.com/dataux/dataux/vendored/mixer/sqlparser" 6 | surgesql "github.com/surge/sqlparser" 7 | "testing" 8 | ) 9 | 10 | /* 11 | 12 | * surge/sqlparser is actually == vitess, branched 13 | * dataux/dataux/vendor/mixer/sqlparser is also derived from vitess -> mixer 14 | 15 | 16 | Benchmark testing, mostly used to try out different runtime strategies for speed 17 | 18 | 19 | BenchmarkVitessParser1 10000 139669 ns/op 20 | BenchmarkSurgeVitessParser1 10000 201577 ns/op 21 | BenchmarkQlbridgeParser1 50000 35545 ns/op 22 | 23 | 24 | # 5/25/2017 25 | BenchmarkVitessParser1-4 50000 37308 ns/op 26 | BenchmarkSurgeVitessParser1-4 30000 38899 ns/op 27 | BenchmarkQlbridgeParser1-4 50000 33327 ns/op 28 | 29 | 30 | 31 | go test -bench="Parser" 32 | 33 | go test -bench="QlbridgeParser" --cpuprofile cpu.out 34 | 35 | go tool pprof testutil.test cpu.out 36 | 37 | web 38 | 39 | 40 | */ 41 | 42 | func BenchmarkVitessParser1(b *testing.B) { 43 | b.StartTimer() 44 | for i := 0; i < b.N; i++ { 45 | _, err := sqlparser.Parse(` 46 | SELECT count(*), repository.name 47 | FROM github_watch 48 | GROUP BY repository.name, repository.language`) 49 | if err != nil { 50 | b.Fail() 51 | } 52 | } 53 | } 54 | 55 | func BenchmarkSurgeVitessParser1(b *testing.B) { 56 | b.StartTimer() 57 | for i := 0; i < b.N; i++ { 58 | _, err := surgesql.Parse(` 59 | SELECT count(*), repository.name 60 | FROM github_watch 61 | GROUP BY repository.name, repository.language`) 62 | if err != nil { 63 | b.Fail() 64 | } 65 | } 66 | } 67 | 68 | func BenchmarkQlbridgeParser1(b *testing.B) { 69 | b.StartTimer() 70 | for i := 0; i < b.N; i++ { 71 | _, err := rel.ParseSql(` 72 | SELECT count(*), repository.name 73 | FROM github_watch 74 | GROUP BY repository.name, repository.language`) 75 | if err != nil { 76 | b.Fail() 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /rel/filter_test.go: -------------------------------------------------------------------------------- 1 | package rel_test 2 | 3 | import ( 4 | "testing" 5 | 6 | u "github.com/araddon/gou" 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/araddon/qlbridge/rel" 10 | ) 11 | 12 | func filterEqual(t *testing.T, ql1, ql2 string) { 13 | u.Debugf("before: %s", ql1) 14 | f1, err := rel.ParseFilterQL(ql1) 15 | assert.Equal(t, nil, err) 16 | f2, err := rel.ParseFilterQL(ql2) 17 | assert.Equal(t, nil, err) 18 | assert.True(t, f1.Equal(f2), "Should Equal: \nf1:%s %s \nf2:%s %s", ql1, f1, ql2, f2.String()) 19 | assert.Equal(t, f1.String(), f2.String()) 20 | } 21 | 22 | func filterSelEqual(t *testing.T, ql1, ql2 string) { 23 | u.Debugf("before: %s", ql1) 24 | f1, err := rel.ParseFilterSelect(ql1) 25 | assert.Equal(t, nil, err) 26 | f2, err := rel.ParseFilterSelect(ql2) 27 | assert.Equal(t, nil, err) 28 | assert.True(t, f1.Equal(f2), "Should Equal: \nf1:%s %s \nf2:%s %s", ql1, f1, ql2, f2.String()) 29 | assert.Equal(t, f1.String(), f2.String()) 30 | } 31 | 32 | func TestFilterEquality(t *testing.T) { 33 | t.Parallel() 34 | 35 | filterEqual(t, `FILTER OR (x == "y")`, `FILTER x == "y"`) 36 | filterEqual(t, `FILTER NOT OR (x == "y")`, `FILTER NOT (x == "y")`) 37 | filterEqual(t, `FILTER NOT AND (x == "y")`, `FILTER NOT (x == "y")`) 38 | filterEqual(t, `FILTER AND (x == "y" , AND ( stuff == x ))`, `FILTER AND (x == "y" , stuff == x )`) 39 | filterSelEqual(t, `SELECT x, y FROM user FILTER NOT AND (x == "y")`, `SELECT x, y FROM user FILTER NOT (x == "y")`) 40 | filterSelEqual(t, `SELECT x, y FROM user FILTER NOT AND (x == "y") LIMIT 10 ALIAS bob`, `SELECT x, y FROM user FILTER NOT (x == "y") LIMIT 10 ALIAS bob`) 41 | 42 | rfs := rel.NewFilterSelect() 43 | assert.NotEqual(t, nil, rfs) 44 | 45 | // Some Manipulations to force un-equal compare 46 | fs1, _ := rel.ParseFilterSelect(`SELECT a FROM user FILTER NOT AND (x == "y")`) 47 | fs1b, _ := rel.ParseFilterSelect(`SELECT a FROM user FILTER NOT AND (x == "y")`) 48 | var fs2, fs3 *rel.FilterSelect 49 | 50 | assert.True(t, fs1.Equal(fs1b)) 51 | assert.True(t, fs3.Equal(fs2)) 52 | assert.True(t, !fs2.Equal(fs1)) 53 | assert.True(t, !fs1.Equal(fs2)) 54 | assert.Equal(t, "", fs2.String()) 55 | 56 | // Some Manipulations to force un-equal compare 57 | f1 := rel.MustParseFilter(`FILTER NOT AND (x == "y")`) 58 | f1b := rel.MustParseFilter(`FILTER NOT AND (x == "y")`) 59 | var f2, f3 *rel.FilterStatement 60 | 61 | assert.True(t, f1.Equal(f1b)) 62 | assert.True(t, f3.Equal(f2)) 63 | assert.True(t, !f2.Equal(f1)) 64 | assert.True(t, !f1.Equal(f2)) 65 | assert.Equal(t, "", f3.String()) 66 | 67 | tests := [][]string{ 68 | {`FILTER OR (x == "y")`, `--description 69 | FILTER OR (x == "y")`}, 70 | {`FILTER OR (x == "y")`, `FILTER OR (x == "8")`}, 71 | {`FILTER OR (x == "y") FROM user`, `FILTER OR (x == "y")`}, 72 | {`FILTER OR (x == "y") LIMIT 10`, `FILTER OR (x == "y")`}, 73 | {`FILTER NOT AND (x == "y") ALIAS xyz`, `FILTER NOT AND (x == "y");`}, 74 | {`FILTER NOT AND (x == "y") WITH conf_key = 22;`, `FILTER NOT AND (x == "y");`}, 75 | {`FILTER NOT AND (x == "y") WITH conf_key = 22;`, `FILTER NOT AND (x == "y") WITH conf_key = 25;`}, 76 | } 77 | 78 | for _, fl := range tests { 79 | ft1 := rel.MustParseFilter(fl[0]) 80 | ft2 := rel.MustParseFilter(fl[1]) 81 | assert.True(t, !ft1.Equal(ft2)) 82 | } 83 | 84 | tests = [][]string{ 85 | {`SELECT x, y FROM user FILTER NOT AND (x == "y") LIMIT 10 ALIAS bob`, `SELECT x, y, z FROM user FILTER NOT AND (x == "y") LIMIT 10 ALIAS bob`}, 86 | {`SELECT x, y FROM user FILTER NOT AND (x == "y") LIMIT 10 ALIAS bob`, `SELECT x, y FROM user FILTER NOT AND (x == "18") LIMIT 10 ALIAS bob`}, 87 | } 88 | 89 | for _, fl := range tests { 90 | fts1, _ := rel.ParseFilterSelect(fl[0]) 91 | fts2, _ := rel.ParseFilterSelect(fl[1]) 92 | assert.True(t, !fts1.Equal(fts2)) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /rel/sql_proto_test.go: -------------------------------------------------------------------------------- 1 | package rel_test 2 | 3 | import ( 4 | "testing" 5 | 6 | u "github.com/araddon/gou" 7 | "github.com/gogo/protobuf/proto" 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/araddon/qlbridge/rel" 11 | ) 12 | 13 | var pbTests = []string{ 14 | "SELECT hash(a) AS id, `z` FROM nothing;", 15 | `SELECT name FROM orders WHERE name = "bob";`, 16 | } 17 | 18 | func TestPb(t *testing.T) { 19 | t.Parallel() 20 | for _, sql := range pbTests { 21 | s, err := rel.ParseSql(sql) 22 | assert.True(t, err == nil, "Should not error on parse sql but got [%v] for %s", err, sql) 23 | ss := s.(*rel.SqlSelect) 24 | pb := ss.ToPbStatement() 25 | assert.True(t, pb != nil, "was nil PB: %#v", ss) 26 | pbBytes, err := proto.Marshal(pb) 27 | assert.True(t, err == nil, "Should not error on proto.Marshal but got [%v] for %s pb:%#v", err, sql, pb) 28 | ss2, err := rel.SqlFromPb(pbBytes) 29 | assert.True(t, err == nil, "Should not error from pb but got [%v] for %s ", err, sql) 30 | assert.True(t, ss.Equal(ss2), "Equal?") 31 | u.Infof("pre/post: \n\t%s\n\t%s", ss, ss2) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /rel/sql_rewrite_test.go: -------------------------------------------------------------------------------- 1 | package rel_test 2 | -------------------------------------------------------------------------------- /rel/testsuite_test.go: -------------------------------------------------------------------------------- 1 | package rel_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/araddon/qlbridge/testutil" 8 | ) 9 | 10 | func TestMain(m *testing.M) { 11 | testutil.Setup() // will call flag.Parse() 12 | 13 | // Now run the actual Tests 14 | os.Exit(m.Run()) 15 | } 16 | func TestSuite(t *testing.T) { 17 | testutil.RunTestSuite(t) 18 | } 19 | -------------------------------------------------------------------------------- /schema/apply_schema_test.go: -------------------------------------------------------------------------------- 1 | package schema_test 2 | 3 | import ( 4 | "database/sql/driver" 5 | "testing" 6 | 7 | "github.com/araddon/qlbridge/datasource" 8 | 9 | "github.com/araddon/qlbridge/datasource/memdb" 10 | "github.com/araddon/qlbridge/schema" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestApplySchema(t *testing.T) { 15 | a := schema.NewApplyer(func(s *schema.Schema) schema.Source { 16 | sdb := datasource.NewSchemaDb(s) 17 | s.InfoSchema.DS = sdb 18 | return sdb 19 | }) 20 | reg := schema.NewRegistry(a) 21 | a.Init(reg) 22 | 23 | inrow := []driver.Value{122, "bob", "bob@email.com"} 24 | db, err := memdb.NewMemDbData("users", [][]driver.Value{inrow}, []string{"user_id", "name", "email"}) 25 | assert.Equal(t, nil, err) 26 | 27 | s := schema.NewSchema("hello") 28 | s.DS = db 29 | err = a.AddOrUpdateOnSchema(s, s) 30 | assert.Equal(t, nil, err) 31 | 32 | err = a.AddOrUpdateOnSchema(s, "not_real") 33 | assert.NotEqual(t, nil, err) 34 | 35 | a.Drop(s, s) 36 | 37 | err = a.Drop(s, "fake") 38 | assert.NotEqual(t, nil, err) 39 | } 40 | -------------------------------------------------------------------------------- /schema/message.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import ( 4 | "database/sql/driver" 5 | "time" 6 | ) 7 | 8 | type ( 9 | // Message is an interface to describe a Row being processed by query engine/vm 10 | // or it is a message between distributed parts of the system. It provides a 11 | // Id() method which can be used by consistent-hash algorithms for routing a message 12 | // consistently to different processes/servers. 13 | // 14 | // Body() returns interface allowing this to be generic structure for routing 15 | // 16 | // see http://github.com/lytics/grid 17 | // 18 | Message interface { 19 | Id() uint64 20 | Body() interface{} 21 | } 22 | // MessageValues describes a message with array of driver.Value. 23 | MessageValues interface { 24 | Values() []driver.Value 25 | } 26 | // Key interface is the Unique Key identifying a row. 27 | Key interface { 28 | Key() driver.Value 29 | } 30 | // TimeMessage describes a message with a timestamp. 31 | TimeMessage interface { 32 | Ts() time.Time 33 | } 34 | ) 35 | 36 | // KeyUint implements Key interface and is simple uint64 key 37 | type KeyUint struct { 38 | ID uint64 39 | } 40 | 41 | // NewKeyUint simple new uint64 key 42 | func NewKeyUint(key uint64) *KeyUint { 43 | return &KeyUint{key} 44 | } 45 | 46 | // Key is key interface 47 | func (m *KeyUint) Key() driver.Value { 48 | return driver.Value(m.ID) 49 | } 50 | -------------------------------------------------------------------------------- /schema/message_test.go: -------------------------------------------------------------------------------- 1 | package schema_test 2 | 3 | import ( 4 | "database/sql/driver" 5 | "testing" 6 | 7 | "github.com/araddon/qlbridge/schema" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestMessage(t *testing.T) { 12 | k := schema.NewKeyUint(uint64(7)) 13 | assert.Equal(t, driver.Value(uint64(7)), k.Key()) 14 | } 15 | -------------------------------------------------------------------------------- /schema/registry_test.go: -------------------------------------------------------------------------------- 1 | package schema_test 2 | 3 | import ( 4 | "database/sql/driver" 5 | "sort" 6 | "testing" 7 | "time" 8 | 9 | "github.com/araddon/qlbridge/lex" 10 | 11 | "github.com/araddon/dateparse" 12 | "github.com/stretchr/testify/assert" 13 | 14 | "github.com/araddon/qlbridge/datasource" 15 | "github.com/araddon/qlbridge/datasource/memdb" 16 | "github.com/araddon/qlbridge/datasource/mockcsv" 17 | td "github.com/araddon/qlbridge/datasource/mockcsvtestdata" 18 | "github.com/araddon/qlbridge/schema" 19 | ) 20 | 21 | func TestRegistry(t *testing.T) { 22 | 23 | reg := schema.DefaultRegistry() 24 | reg.Init() 25 | 26 | created, _ := dateparse.ParseAny("2015/07/04") 27 | 28 | inrow := []driver.Value{122, "bob", "bob@email.com", created.In(time.UTC).Add(time.Hour * -24), []string{"not_admin"}} 29 | 30 | db, err := memdb.NewMemDbData("memdb_users", [][]driver.Value{inrow}, []string{"user_id", "name", "email", "created", "roles"}) 31 | assert.Equal(t, nil, err) 32 | 33 | c, err := db.Open("memdb_users") 34 | assert.Equal(t, nil, err) 35 | dc, ok := c.(schema.ConnAll) 36 | assert.True(t, ok) 37 | 38 | _, err = dc.Put(nil, &datasource.KeyInt{Id: 123}, []driver.Value{123, "aaron", "email@email.com", created.In(time.UTC), []string{"admin"}}) 39 | assert.Equal(t, nil, err) 40 | 41 | // We need to register our DataSource provider here 42 | err = schema.RegisterSourceAsSchema("memdb_reg_test", db) 43 | assert.Equal(t, nil, err) 44 | 45 | // Repeating this will cause error, dupe schema 46 | err = schema.RegisterSourceAsSchema("memdb_reg_test", db) 47 | assert.NotEqual(t, nil, err) 48 | 49 | reg.SchemaRefresh("memdb_reg_test") 50 | 51 | c2, err := schema.OpenConn("memdb_reg_test", "memdb_users") 52 | assert.Equal(t, nil, err) 53 | _, ok = c2.(schema.ConnAll) 54 | assert.True(t, ok) 55 | _, err = schema.OpenConn("invalid_schema", "memdb_users") 56 | assert.NotEqual(t, nil, err) 57 | 58 | sl := reg.Schemas() 59 | sort.Strings(sl) 60 | assert.Equal(t, []string{"memdb_reg_test", "mockcsv"}, sl) 61 | assert.NotEqual(t, "", reg.String()) 62 | 63 | schema.RegisterSourceType("alias_to_memdb", db) 64 | c, err = reg.GetSource("alias_to_memdb") 65 | assert.Equal(t, nil, err) 66 | assert.NotEqual(t, nil, c) 67 | 68 | c, err = reg.GetSource("fake_not_real") 69 | assert.NotEqual(t, nil, err) 70 | assert.Equal(t, nil, c) 71 | 72 | s := schema.NewSchema("hello-world") 73 | s.DS = db 74 | err = schema.RegisterSchema(s) 75 | assert.Equal(t, nil, err) 76 | err = schema.RegisterSchema(s) 77 | assert.NotEqual(t, nil, err) 78 | 79 | err = reg.SchemaAddChild("not.valid.schema", s) 80 | assert.NotEqual(t, nil, err) 81 | 82 | // test did panic bc nil source 83 | dp := didPanic(func() { 84 | schema.RegisterSourceType("nil_source", nil) 85 | }) 86 | assert.Equal(t, true, dp) 87 | // dupe causes panic 88 | dp = didPanic(func() { 89 | schema.RegisterSourceType("alias_to_memdb", db) 90 | }) 91 | assert.Equal(t, true, dp) 92 | 93 | if td.MockSchema == nil { 94 | t.Fatalf("mock schema not loaded?") 95 | } 96 | 97 | rs, _ := reg.Schema(td.MockSchema.Name) 98 | assert.NotEqual(t, nil, rs) 99 | assert.Equal(t, 2, len(rs.Tables())) 100 | 101 | // Load in a "csv file" into our mock data store 102 | mockcsv.LoadTable(td.MockSchema.Name, "droptable1", `user_id,t1 103 | 9Ip1aKbeZe2njCDM,"2014-01-01"`) 104 | 105 | rs, _ = reg.Schema(td.MockSchema.Name) 106 | assert.NotEqual(t, nil, rs) 107 | assert.Equal(t, 3, len(rs.Tables())) 108 | 109 | err = reg.SchemaDrop(td.MockSchema.Name, "droptable1", lex.TokenTable) 110 | assert.Equal(t, nil, err) 111 | 112 | err = reg.SchemaDrop("bad.schema", "droptable1", lex.TokenTable) 113 | assert.NotEqual(t, nil, err) 114 | 115 | err = reg.SchemaDrop("bad.schema", "bad.schema", lex.TokenSchema) 116 | assert.NotEqual(t, nil, err) 117 | 118 | err = reg.SchemaDrop(td.MockSchema.Name, "not.a.table", lex.TokenTable) 119 | assert.NotEqual(t, nil, err) 120 | 121 | err = reg.SchemaDrop(td.MockSchema.Name, "not.a.table", lex.TokenAnd) 122 | assert.NotEqual(t, nil, err) 123 | 124 | rs, _ = reg.Schema(td.MockSchema.Name) 125 | assert.NotEqual(t, nil, rs) 126 | assert.Equal(t, 2, len(rs.Tables())) 127 | 128 | reg.Init() 129 | } 130 | func didPanic(f func()) (dp bool) { 131 | defer func() { 132 | if r := recover(); r != nil { 133 | dp = true 134 | } 135 | }() 136 | f() 137 | return dp 138 | } 139 | -------------------------------------------------------------------------------- /schema/schema.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package schema; 4 | 5 | // protoc --go_out=. *.proto 6 | 7 | 8 | //import "github.com/araddon/qlbridge/rel/sql.proto"; 9 | //import "github.com/araddon/qlbridge/expr/node.proto"; 10 | 11 | //import "google/protobuf/any.proto"; 12 | 13 | // Partition describes a range of data (in a Table). 14 | // left-key is contained in this partition 15 | // right key is not contained in this partition, in the next partition. 16 | // So any value >= left-key, and < right-key is contained herein. 17 | message TablePartition { 18 | string table = 1; 19 | repeated string keys = 2; 20 | repeated Partition partitions = 3; 21 | } 22 | 23 | 24 | 25 | // Partition describes a range of data 26 | // the left-key is contained in this partition 27 | // the right key is not contained in this partition, in the next one 28 | message Partition { 29 | string id = 1; 30 | string left = 2; 31 | string right = 3; 32 | } 33 | 34 | message TablePb { 35 | // Name of table lowercased 36 | string name = 1; 37 | // Name of table (not lowercased) 38 | string nameOriginal = 2; 39 | // some dbs are more hiearchical (table-column-family) 40 | string parent = 3; 41 | // Character set, default = utf8 42 | uint32 Charset = 4; 43 | // Partitions in this table, optional may be empty 44 | TablePartition partition = 5; 45 | // Partition Count 46 | uint32 PartitionCt = 6; 47 | // List of indexes for this table 48 | repeated Index indexes = 7; 49 | // context json bytes 50 | bytes contextJson = 8; 51 | // List of Fields, in order 52 | repeated FieldPb fieldpbs = 9; 53 | } 54 | 55 | message FieldPb { 56 | string name = 1; 57 | string description = 2; 58 | string key = 3; 59 | string extra = 4; 60 | string data = 5; 61 | uint32 length = 6; 62 | uint32 type = 7; 63 | uint32 nativeType = 8; 64 | uint64 defLength = 9; 65 | bytes defVal = 11; 66 | bool indexed = 13; 67 | bool noNulls = 14; 68 | string collation = 15; 69 | repeated string roles = 16; 70 | repeated Index indexes = 17; 71 | bytes contextJson = 18; 72 | } 73 | 74 | // Index a description of how field(s) should be indexed for a table. 75 | message Index { 76 | string name = 1; 77 | repeated string fields = 2; 78 | bool primaryKey = 3; 79 | repeated string hashPartition = 4; 80 | int32 partitionSize = 5; 81 | } -------------------------------------------------------------------------------- /testutil/suite_test.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestSuite(t *testing.T) { 8 | RunTestSuite(t) 9 | } 10 | -------------------------------------------------------------------------------- /updateglock.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | 4 | cd $GOPATH/src/github.com/araddon/dateparse && git checkout master && git pull 5 | cd $GOPATH/src/github.com/araddon/gou && git checkout master && git pull 6 | cd $GOPATH/src/github.com/couchbaselabs/goforestdb && git checkout master && git pull 7 | # cd $GOPATH/src/github.com/dataux/dataux && git checkout master && git pull 8 | cd $GOPATH/src/github.com/davecgh/go-spew && git checkout master && git pull 9 | cd $GOPATH/src/github.com/dchest/siphash && git checkout master && git pull 10 | cd $GOPATH/src/github.com/go-sql-driver/mysql && git checkout master && git pull 11 | cd $GOPATH/src/github.com/gogo/protobuf && git checkout master && git pull 12 | cd $GOPATH/src/github.com/golang/protobuf && git checkout master && git pull 13 | cd $GOPATH/src/github.com/googleapis/gax-go && git checkout master && git pull 14 | cd $GOPATH/src/github.com/google/btree && git checkout master && git pull 15 | cd $GOPATH/src/github.com/hashicorp/go-immutable-radix && git checkout master && git pull 16 | cd $GOPATH/src/github.com/hashicorp/go-memdb && git checkout master && git pull 17 | cd $GOPATH/src/github.com/hashicorp/golang-lru && git checkout master && git pull 18 | cd $GOPATH/src/github.com/jmespath/go-jmespath && git checkout master && git pull 19 | cd $GOPATH/src/github.com/jmoiron/sqlx && git checkout master && git pull 20 | cd $GOPATH/src/github.com/kr/pretty && git checkout master && git pull 21 | cd $GOPATH/src/github.com/kr/pty && git checkout master && git pull 22 | cd $GOPATH/src/github.com/kr/text && git checkout master && git pull 23 | cd $GOPATH/src/github.com/leekchan/timeutil && git checkout master && git pull 24 | cd $GOPATH/src/github.com/lytics/cloudstorage && git checkout master && git pull 25 | cd $GOPATH/src/github.com/lytics/confl && git checkout master && git pull 26 | cd $GOPATH/src/github.com/lytics/datemath && git checkout master && git pull 27 | cd $GOPATH/src/github.com/mb0/glob && git checkout master && git pull 28 | cd $GOPATH/src/github.com/mssola/user_agent && git checkout master && git pull 29 | cd $GOPATH/src/github.com/pborman/uuid && git checkout master && git pull 30 | cd $GOPATH/src/github.com/rcrowley/go-metrics && git checkout master && git pull 31 | cd $GOPATH/src/github.com/stretchr/testify && git checkout master && git pull 32 | cd $GOPATH/src/github.com/go.opencensus.io && git checkout master && git pull 33 | 34 | cd $GOPATH/src/golang.org/x/crypto && git checkout master && git pull 35 | cd $GOPATH/src/golang.org/x/net && git checkout master && git pull 36 | cd $GOPATH/src/golang.org/x/text && git checkout master && git pull 37 | cd $GOPATH/src/golang.org/x/sys && git checkout master && git pull 38 | cd $GOPATH/src/golang.org/x/oauth2 && git checkout master && git pull 39 | 40 | 41 | cd $GOPATH/src/google.golang.org/api && git checkout master && git pull 42 | cd $GOPATH/src/google.golang.org/appengine && git checkout master && git pull 43 | cd $GOPATH/src/google.golang.org/genproto && git checkout master && git pull 44 | cd $GOPATH/src/google.golang.org/grpc && git checkout master && git pull 45 | cd $GOPATH/src/cloud.google.com/go/ && git checkout master && git pull 46 | 47 | 48 | #go get -u -v ./... 49 | 50 | #glock save github.com/araddon/qlbridge 51 | 52 | -------------------------------------------------------------------------------- /vm/_archive/vm_bm_test.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | "github.com/araddon/qlbridge/datasource" 5 | "github.com/araddon/qlbridge/expr" 6 | "github.com/araddon/qlbridge/value" 7 | "testing" 8 | ) 9 | 10 | /* 11 | 12 | go test -bench="Vm" 13 | 14 | 12/18/2014 15 | BenchmarkVmParse 50000 36411 ns/op 3578 B/op 118 allocs/op 16 | BenchmarkVmExecute 50000 45530 ns/op 4605 B/op 138 allocs/op 17 | 18 | 12/19/2014 (string contactenation in lex.PeekWord()) 19 | BenchmarkVmParse 100000 19346 ns/op 1775 B/op 33 allocs/op 20 | BenchmarkVmExecute 100000 27774 ns/op 2994 B/op 53 allocs/op 21 | 22 | 12/20/2014 (faster machine - d5) 23 | BenchmarkVmParse 200000 13374 ns/op 1775 B/op 33 allocs/op 24 | BenchmarkVmExecute 100000 17472 ns/op 2998 B/op 53 allocs/op 25 | BenchmarkVmExecuteNoParse 500000 3429 ns/op 737 B/op 17 allocs/op 26 | */ 27 | 28 | var bmSql = []string{ 29 | `select user_id, item_count * 2 as itemsx2, yy(reg_date) > 10 as regyy FROM stdio`, 30 | } 31 | 32 | func BenchmarkVmParse(b *testing.B) { 33 | b.ReportAllocs() 34 | b.StartTimer() 35 | for i := 0; i < b.N; i++ { 36 | for _, sqlText := range bmSql { 37 | _, err := expr.ParseSql(sqlText) 38 | if err != nil { 39 | panic(err.Error()) 40 | } 41 | } 42 | } 43 | } 44 | 45 | func verifyBenchmarkSql(t *testing.B, sql string, readContext datasource.ContextReader) *datasource.ContextSimple { 46 | 47 | sqlVm, err := NewSqlVm(sql) 48 | if err != nil { 49 | t.Fail() 50 | } 51 | 52 | writeContext := datasource.NewContextSimple() 53 | err = sqlVm.Execute(writeContext, readContext) 54 | if err != nil { 55 | t.Fail() 56 | } 57 | 58 | return writeContext 59 | } 60 | 61 | func BenchmarkVmExecute(b *testing.B) { 62 | msg := datasource.NewContextSimpleData( 63 | map[string]value.Value{ 64 | "int5": value.NewIntValue(5), 65 | "item_count": value.NewStringValue("5"), 66 | "reg_date": value.NewStringValue("2014/11/01"), 67 | "user_id": value.NewStringValue("abc")}, 68 | ) 69 | b.ReportAllocs() 70 | b.StartTimer() 71 | for i := 0; i < b.N; i++ { 72 | for _, sqlText := range bmSql { 73 | verifyBenchmarkSql(b, sqlText, msg) 74 | } 75 | } 76 | } 77 | 78 | func BenchmarkVmExecuteNoParse(b *testing.B) { 79 | readContext := datasource.NewContextSimpleData( 80 | map[string]value.Value{ 81 | "int5": value.NewIntValue(5), 82 | "item_count": value.NewStringValue("5"), 83 | "reg_date": value.NewStringValue("2014/11/01"), 84 | "user_id": value.NewStringValue("abc")}, 85 | ) 86 | sqlVm, err := NewSqlVm(bmSql[0]) 87 | if err != nil { 88 | b.Fail() 89 | } 90 | writeContext := datasource.NewContextSimple() 91 | b.ReportAllocs() 92 | b.StartTimer() 93 | for i := 0; i < b.N; i++ { 94 | err = sqlVm.Execute(writeContext, readContext) 95 | if err != nil { 96 | b.Fail() 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /vm/_bm/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "os" 7 | "runtime/pprof" 8 | "time" 9 | 10 | "github.com/araddon/qlbridge/datasource" 11 | "github.com/araddon/qlbridge/expr" 12 | "github.com/araddon/qlbridge/expr/builtins" 13 | "github.com/araddon/qlbridge/rel" 14 | "github.com/araddon/qlbridge/value" 15 | "github.com/araddon/qlbridge/vm" 16 | ) 17 | 18 | /* 19 | 20 | go build && time ./_bm --command=parse --cpuprofile=cpu.prof 21 | go tool pprof _bm cpu.prof 22 | 23 | 24 | go build && time ./_bm --command=vm --cpuprofile=cpu.prof 25 | go tool pprof _bm cpu.prof 26 | 27 | 28 | */ 29 | var ( 30 | cpuProfileFile string 31 | memProfileFile string 32 | logging = "info" 33 | command = "parse" 34 | 35 | msg = datasource.NewContextSimpleTs( 36 | map[string]value.Value{ 37 | "int5": value.NewIntValue(5), 38 | "item_count": value.NewStringValue("5"), 39 | "reg_date": value.NewStringValue("2014/11/01"), 40 | "user_id": value.NewStringValue("abc")}, 41 | time.Now(), 42 | ) 43 | ) 44 | 45 | func init() { 46 | 47 | flag.StringVar(&logging, "logging", "info", "logging [ debug,info ]") 48 | flag.StringVar(&cpuProfileFile, "cpuprofile", "", "cpuprofile") 49 | flag.StringVar(&memProfileFile, "memprofile", "", "memProfileFile") 50 | flag.StringVar(&command, "command", "parse", "command to run [parse,vm]") 51 | flag.Parse() 52 | 53 | builtins.LoadAllBuiltins() 54 | 55 | } 56 | 57 | func main() { 58 | 59 | if cpuProfileFile != "" { 60 | f, err := os.Create(cpuProfileFile) 61 | if err != nil { 62 | log.Fatal(err) 63 | } 64 | pprof.StartCPUProfile(f) 65 | defer pprof.StopCPUProfile() 66 | } 67 | 68 | switch command { 69 | case "parse": 70 | runParse(100000, `select user_id, item_count * 2 as itemsx2, yy(reg_date) > 10 as regyy FROM stdio`, msg) 71 | 72 | case "vm": 73 | runVm(100000, `select user_id, item_count * 2 as itemsx2, yy(reg_date) > 10 as regyy FROM stdio`, msg) 74 | } 75 | } 76 | 77 | func runParse(repeat int, sql string, readContext expr.ContextReader) { 78 | for i := 0; i < repeat; i++ { 79 | sel, err := rel.ParseSqlSelect(sql) 80 | if err != nil { 81 | panic(err.Error()) 82 | } 83 | writeContext := datasource.NewContextSimple() 84 | _, err = vm.EvalSql(sel, writeContext, readContext) 85 | if err != nil { 86 | panic(err.Error()) 87 | } 88 | } 89 | } 90 | 91 | func runVm(repeat int, sql string, readContext expr.ContextReader) { 92 | sel, err := rel.ParseSqlSelect(sql) 93 | if err != nil { 94 | panic(err.Error()) 95 | } 96 | 97 | for i := 0; i < repeat; i++ { 98 | 99 | writeContext := datasource.NewContextSimple() 100 | _, err = vm.EvalSql(sel, writeContext, readContext) 101 | //log.Println(writeContext.All()) 102 | if err != nil { 103 | panic(err.Error()) 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /vm/bm_test.go: -------------------------------------------------------------------------------- 1 | package vm_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | /* 9 | Benchmark testing, mostly used to try out different runtime strategies for speed 10 | 11 | 12 | BenchmarkReflectionKind 10000000 285 ns/op 13 | BenchmarkReflectionKind2 5000000 615 ns/op 14 | BenchmarkReflectionOurType 20000000 136 ns/op 15 | BenchmarkReflectionOurType2 10000000 192 ns/op 16 | BenchmarkReflectionOurType3 50000000 33.8 ns/op 17 | BenchmarkReflectionOurType4 20000000 90.5 ns/op 18 | BenchmarkReflectionOurType5 50000000 42.3 ns/op 19 | 20 | */ 21 | // go test -bench="Reflection" 22 | 23 | type OurType int 24 | 25 | const ( 26 | OurInt OurType = iota 27 | OurString 28 | OurBool 29 | ) 30 | 31 | func (m OurType) String() string { 32 | switch m { 33 | case OurInt: 34 | return "int" 35 | case OurString: 36 | return "string" 37 | default: 38 | return "unknown" 39 | } 40 | } 41 | 42 | type FakeNode interface { 43 | Kind() reflect.Kind 44 | OurType() OurType 45 | } 46 | 47 | type FakeNodeStuff struct { 48 | ot OurType 49 | rv reflect.Value 50 | val interface{} 51 | } 52 | 53 | func (m *FakeNodeStuff) Kind() reflect.Kind { 54 | switch m.ot { 55 | case OurInt: 56 | return m.rv.Kind() 57 | case OurString: 58 | return m.rv.Kind() 59 | default: 60 | panic("unknown") 61 | } 62 | } 63 | 64 | func (m *FakeNodeStuff) Kind2() reflect.Kind { 65 | return reflect.ValueOf(m.val).Kind() 66 | } 67 | 68 | func (m *FakeNodeStuff) OurType() OurType { 69 | return m.ot 70 | } 71 | 72 | func (m *FakeNodeStuff) OurType2() OurType { 73 | switch m.ot { 74 | case OurInt: 75 | return m.ot 76 | case OurString: 77 | return m.ot 78 | case OurBool: 79 | return m.ot 80 | default: 81 | panic("unknown") 82 | } 83 | } 84 | 85 | /* 86 | Notes: 87 | - type switching isn't that expensive, it is but not hugely so 88 | - you do want to memoize the reflect.Value though 89 | 90 | */ 91 | // We are going to test use of one time creation of Reflect Value, then 20 iterations 92 | func BenchmarkReflectionKind(b *testing.B) { 93 | b.StartTimer() 94 | for i := 0; i < b.N; i++ { 95 | n := FakeNodeStuff{ot: OurString, rv: reflect.ValueOf("hello")} 96 | for j := 0; j < 20; j++ { 97 | if k := n.Kind(); k != reflect.String { 98 | // 99 | } 100 | } 101 | } 102 | } 103 | func BenchmarkReflectionKind2(b *testing.B) { 104 | b.StartTimer() 105 | for i := 0; i < b.N; i++ { 106 | n := FakeNodeStuff{ot: OurString, val: "hello"} 107 | for j := 0; j < 20; j++ { 108 | if k := n.Kind2(); k != reflect.String { 109 | // 110 | } 111 | } 112 | } 113 | } 114 | func BenchmarkReflectionOurType(b *testing.B) { 115 | b.StartTimer() 116 | for i := 0; i < b.N; i++ { 117 | n := FakeNodeStuff{ot: OurString, rv: reflect.ValueOf("hello")} 118 | for j := 0; j < 20; j++ { 119 | if k := n.OurType(); k != OurString { 120 | // 121 | } 122 | } 123 | } 124 | } 125 | func BenchmarkReflectionOurType2(b *testing.B) { 126 | b.StartTimer() 127 | for i := 0; i < b.N; i++ { 128 | n := FakeNodeStuff{ot: OurString, rv: reflect.ValueOf("hello")} 129 | for j := 0; j < 20; j++ { 130 | if k := n.OurType2(); k != OurString { 131 | // 132 | } 133 | } 134 | } 135 | } 136 | func BenchmarkReflectionOurType3(b *testing.B) { 137 | b.StartTimer() 138 | for i := 0; i < b.N; i++ { 139 | n := FakeNodeStuff{ot: OurString} 140 | for j := 0; j < 20; j++ { 141 | if k := n.OurType(); k != OurString { 142 | // 143 | } 144 | } 145 | } 146 | } 147 | func BenchmarkReflectionOurType4(b *testing.B) { 148 | b.StartTimer() 149 | for i := 0; i < b.N; i++ { 150 | n := FakeNodeStuff{ot: OurString} 151 | for j := 0; j < 20; j++ { 152 | if k := n.OurType2(); k != OurString { 153 | // 154 | } 155 | } 156 | } 157 | } 158 | func BenchmarkReflectionOurType5(b *testing.B) { 159 | b.StartTimer() 160 | for i := 0; i < b.N; i++ { 161 | n := FakeNodeStuff{ot: OurString} 162 | for j := 0; j < 20; j++ { 163 | switch n.OurType() { 164 | case OurInt: 165 | // 166 | case OurString: 167 | // 168 | case OurBool: 169 | // 170 | default: 171 | panic("unknown") 172 | } 173 | } 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /vm/filterqlvm.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | u "github.com/araddon/gou" 5 | 6 | "github.com/araddon/qlbridge/expr" 7 | "github.com/araddon/qlbridge/rel" 8 | "github.com/araddon/qlbridge/value" 9 | ) 10 | 11 | var ( 12 | // a static nil includer whose job is to return errors 13 | // for vm's that don't have an includer 14 | noIncluder = &expr.IncludeContext{} 15 | ) 16 | 17 | type filterql struct { 18 | expr.EvalContext 19 | expr.Includer 20 | } 21 | 22 | // EvalFilerSelect evaluates a FilterSelect statement from read, into write context 23 | // 24 | // @writeContext = Write results of projection 25 | // @readContext = Message input, ie evaluate for Where/Filter clause 26 | func EvalFilterSelect(sel *rel.FilterSelect, writeContext expr.ContextWriter, readContext expr.EvalContext) (bool, bool) { 27 | 28 | ctx, ok := readContext.(expr.EvalIncludeContext) 29 | if !ok { 30 | ctx = &expr.IncludeContext{ContextReader: readContext} 31 | } 32 | // Check and see if we are where Guarded, which would discard the entire message 33 | if sel.FilterStatement != nil { 34 | 35 | matches, ok := Matches(ctx, sel.FilterStatement) 36 | if !ok { 37 | return false, ok 38 | } 39 | if !matches { 40 | return false, ok 41 | } 42 | } 43 | 44 | for _, col := range sel.Columns { 45 | 46 | if col.Guard != nil { 47 | ifColValue, ok := Eval(readContext, col.Guard) 48 | if !ok { 49 | u.Debugf("Could not evaluate if: T:%T v:%v", col.Guard, col.Guard.String()) 50 | continue 51 | } 52 | switch ifVal := ifColValue.(type) { 53 | case value.BoolValue: 54 | if ifVal.Val() == false { 55 | continue // filter out this col 56 | } 57 | default: 58 | continue 59 | } 60 | 61 | } 62 | 63 | v, ok := Eval(readContext, col.Expr) 64 | if ok { 65 | writeContext.Put(col, readContext, v) 66 | } 67 | 68 | } 69 | 70 | return true, true 71 | } 72 | 73 | // Matches executes a FilterQL statement against an evaluation context 74 | // returning true if the context matches. 75 | func MatchesInc(inc expr.Includer, cr expr.EvalContext, stmt *rel.FilterStatement) (bool, bool) { 76 | return matchesExpr(filterql{cr, inc}, stmt.Filter, 0) 77 | } 78 | 79 | // Matches executes a FilterQL statement against an evaluation context 80 | // returning true if the context matches. 81 | func Matches(cr expr.EvalContext, stmt *rel.FilterStatement) (bool, bool) { 82 | return matchesExpr(cr, stmt.Filter, 0) 83 | } 84 | 85 | // MatchesExpr executes a expr.Node expression against an evaluation context 86 | // returning true if the context matches. 87 | func MatchesExpr(cr expr.EvalContext, node expr.Node) (bool, bool) { 88 | return matchesExpr(cr, node, 0) 89 | } 90 | 91 | func matchesExpr(cr expr.EvalContext, n expr.Node, depth int) (bool, bool) { 92 | switch exp := n.(type) { 93 | case *expr.IdentityNode: 94 | if exp.Text == "*" || exp.Text == "match_all" { 95 | return true, true 96 | } 97 | } 98 | val, ok := Eval(cr, n) 99 | if !ok || val == nil { 100 | return false, ok 101 | } 102 | if bv, isBool := val.(value.BoolValue); isBool { 103 | return bv.Val(), ok 104 | } 105 | return false, true 106 | } 107 | -------------------------------------------------------------------------------- /vm/sqlvm.go: -------------------------------------------------------------------------------- 1 | package vm 2 | 3 | import ( 4 | u "github.com/araddon/gou" 5 | 6 | "github.com/araddon/qlbridge/expr" 7 | "github.com/araddon/qlbridge/rel" 8 | "github.com/araddon/qlbridge/value" 9 | ) 10 | 11 | // EvalSql Is a partial SQL statement evaluator (that doesn't get it all right). See 12 | // exec package for full sql evaluator. Can be used to evaluate a read context and write 13 | // results to write context. This does not project columns prior to running WHERE. 14 | // 15 | // @writeContext = Write out results of projection 16 | // @readContext = Message to evaluate does it match where clause? if so proceed to projection 17 | func EvalSql(sel *rel.SqlSelect, writeContext expr.ContextWriter, readContext expr.EvalContext) (bool, error) { 18 | 19 | // Check and see if we are where Guarded, which would discard the entire message 20 | if sel.Where != nil { 21 | 22 | whereValue, ok := Eval(readContext, sel.Where.Expr) 23 | if !ok { 24 | return false, nil 25 | } 26 | switch whereVal := whereValue.(type) { 27 | case value.BoolValue: 28 | if whereVal.Val() == false { 29 | return false, nil 30 | } 31 | // ok, continue 32 | default: 33 | return false, nil 34 | } 35 | } 36 | 37 | for _, col := range sel.Columns { 38 | 39 | if col.Guard != nil { 40 | ifColValue, ok := Eval(readContext, col.Guard) 41 | if !ok { 42 | continue 43 | } 44 | switch ifVal := ifColValue.(type) { 45 | case value.BoolValue: 46 | if ifVal.Val() == false { 47 | continue // filter out this col 48 | } 49 | default: 50 | continue // filter out 51 | } 52 | 53 | } 54 | 55 | v, ok := Eval(readContext, col.Expr) 56 | if !ok { 57 | u.Debugf("Could not evaluate: %s ctx: %#v", col.Expr, readContext) 58 | } else { 59 | // Write out the result of the evaluation 60 | writeContext.Put(col, readContext, v) 61 | } 62 | } 63 | 64 | return true, nil 65 | } 66 | -------------------------------------------------------------------------------- /vm/sqlvm_test.go: -------------------------------------------------------------------------------- 1 | package vm_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/araddon/dateparse" 7 | u "github.com/araddon/gou" 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/araddon/qlbridge/datasource" 11 | "github.com/araddon/qlbridge/expr" 12 | "github.com/araddon/qlbridge/rel" 13 | "github.com/araddon/qlbridge/value" 14 | "github.com/araddon/qlbridge/vm" 15 | ) 16 | 17 | var ( 18 | _ = u.EMPTY 19 | 20 | st1, _ = dateparse.ParseAny("12/18/2014") 21 | st2, _ = dateparse.ParseAny("12/18/2019") 22 | 23 | // This is the message context which will be added to all tests below 24 | // and be available to the VM runtime for evaluation by using 25 | // key's such as "int5" or "user_id" 26 | sqlData = datasource.NewContextSimpleData(map[string]value.Value{ 27 | "int5": value.NewIntValue(5), 28 | "str5": value.NewStringValue("5"), 29 | "created": value.NewTimeValue(st1), 30 | "updated": value.NewTimeValue(st2), 31 | "bvalt": value.NewBoolValue(true), 32 | "bvalf": value.NewBoolValue(false), 33 | "user_id": value.NewStringValue("abc"), 34 | "urls": value.NewStringsValue([]string{"abc", "123"}), 35 | "hits": value.NewMapIntValue(map[string]int64{"google.com": 5, "bing.com": 1}), 36 | "email": value.NewStringValue("bob@bob.com"), 37 | }) 38 | // list of tests 39 | sqlTests = []sqlTest{ 40 | st(`select int5 FROM mycontext`, map[string]interface{}{"int5": 5}), 41 | st(`select int5 FROM mycontext WHERE created < "now-1M"`, map[string]interface{}{"int5": 5}), 42 | st(`select int5 FROM mycontext WHERE not_a_field < "now-1M"`, map[string]interface{}{}), 43 | st(`select int5 IF EXISTS urls FROM mycontext WHERE created < "now-1M"`, map[string]interface{}{"int5": 5}), 44 | st(`select int5, str5 IF EXISTS not_a_field FROM mycontext WHERE created < "now-1M"`, map[string]interface{}{"int5": 5}), 45 | st(`select int5, str5 IF toint(str5) FROM mycontext WHERE created < "now-1M"`, map[string]interface{}{"int5": 5}), 46 | st(`select int5, "hello" AS hello IF user_id > true FROM mycontext WHERE created < "now-1M"`, map[string]interface{}{"int5": 5}), 47 | st(`select int5, todate("hello") AS hello FROM mycontext WHERE created < "now-1M"`, map[string]interface{}{"int5": 5}), 48 | // this should fail 49 | st(`select int5 FROM mycontext WHERE not_a_field > 10`, nil), 50 | st(`select int5 FROM mycontext WHERE user_id > true`, nil), 51 | st(`select int5 FROM mycontext WHERE int5 + 6`, nil), 52 | } 53 | ) 54 | 55 | func TestRunSqlTests(t *testing.T) { 56 | 57 | for _, test := range sqlTests { 58 | 59 | ss, err := rel.ParseSql(test.sql) 60 | assert.Equal(t, nil, err, "expected no error but got %v for %s", err, test.sql) 61 | 62 | sel, ok := ss.(*rel.SqlSelect) 63 | assert.True(t, ok, "expected rel.SqlSelect but got %T", ss) 64 | 65 | writeContext := datasource.NewContextSimple() 66 | _, err = vm.EvalSql(sel, writeContext, test.context) 67 | assert.Equal(t, nil, err, "expected no error but got %v for %s", err, test.sql) 68 | 69 | for key, v := range test.result.Data { 70 | v2, ok := writeContext.Get(key) 71 | assert.True(t, ok, "Expected ok for get %s output: %#v", key, writeContext.Data) 72 | assert.Equal(t, v2.Value(), v.Value(), "?? %s %v!=%v %T %T", key, v.Value(), v2.Value(), v.Value(), v2.Value()) 73 | } 74 | } 75 | } 76 | 77 | type sqlTest struct { 78 | sql string 79 | context expr.EvalContext 80 | result *datasource.ContextSimple // ?? what is this? 81 | rowct int // expected row count 82 | } 83 | 84 | func st(sql string, results map[string]interface{}) sqlTest { 85 | return sqlTest{sql: sql, result: datasource.NewContextSimpleNative(results), context: sqlData} 86 | } 87 | -------------------------------------------------------------------------------- /vm/vm_bench_test.go: -------------------------------------------------------------------------------- 1 | package vm_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/araddon/qlbridge/expr" 7 | "github.com/araddon/qlbridge/value" 8 | "github.com/araddon/qlbridge/vm" 9 | ) 10 | 11 | /* 12 | 13 | go test -bench="Vm" 14 | 15 | 16 | Benchmark testing for a few different aspects of vm 17 | 18 | BenchmarkVmFuncNew-4 2000000 789 ns/op 19 | BenchmarkVmFuncOld-4 300000 5741 ns/op 20 | 21 | 22 | */ 23 | 24 | // The new vm reflection-less func count 25 | func BenchmarkVmFuncNew(b *testing.B) { 26 | 27 | n, err := expr.ParseExpression("count(str5) + count(int5)") 28 | if err != nil { 29 | b.Fail() 30 | } 31 | 32 | b.StartTimer() 33 | for i := 0; i < b.N; i++ { 34 | val, ok := vm.Eval(msgContext, n) 35 | if !ok { 36 | b.Fail() 37 | } 38 | if iv, isInt := val.(value.IntValue); isInt { 39 | if iv.Val() != 2 { 40 | b.Fail() 41 | } 42 | } else { 43 | b.Fail() 44 | } 45 | } 46 | } 47 | --------------------------------------------------------------------------------