├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── NOTICE ├── README.md ├── api.go ├── batch.go ├── column.go ├── common.go ├── config.go ├── config_test.go ├── connection.go ├── connection_test.go ├── cover.out ├── dialect.go ├── dialect_registry.go ├── dialect_test.go ├── dml_builder.go ├── dml_provider.go ├── doc.go ├── docs └── README.md ├── examples ├── api.go ├── domain.go ├── server.go ├── service.go ├── service_client.go ├── service_test.go └── test │ ├── config │ ├── admin.json │ └── store.json │ ├── delete_expect_interests.json │ ├── delete_prepare_interests.json │ ├── init.json │ ├── persist_expect_interests.json │ ├── persist_prepare_interests.json │ ├── read_prepare_interests.json │ └── script │ └── database.sql ├── factory_registry.go ├── file_connection.go ├── file_dialect.go ├── file_dialect_test.go ├── file_manager.go ├── file_manager_factory.go ├── file_manager_test.go ├── file_scanner.go ├── go.mod ├── limiter.go ├── limiter_test.go ├── logger.go ├── manager.go ├── manager_factory.go ├── manager_factory_test.go ├── manager_registry.go ├── matcher.go ├── matcher └── id.go ├── matcher_test.go ├── query_builder.go ├── record_mapper.go ├── reserved.go ├── scanner.go ├── sql_connection.go ├── sql_dialect.go ├── sql_dialect_test.go ├── sql_manager.go ├── sql_manager_factory.go ├── sql_manager_test.go ├── sql_parser.go ├── sql_parser_test.go ├── sql_predicate.go ├── sql_predicate_test.go ├── sql_scanner.go ├── table.go ├── table_descriptor.go ├── table_descriptor_test.go ├── table_test.go └── test ├── file_config.json ├── store.json ├── test1_expect_users.json ├── traveler.csv ├── travelers1.json └── travelers2.json /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viant/dsc/1c53f76764a60a394921ff70b222ca5a85702f00/.gitignore -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## March 17 2020 0.16.1 2 | - Update sql dialect with session control 3 | 4 | ## Feb 13 2020 0.16.0 5 | - Added ColumnTypes to Scanner interface 6 | 7 | ## September 17 2019 0.14.1 8 | - Added vertica dialect 9 | - Added dialect IsKeyCheckSessionLevel 10 | 11 | ## August 5 2019 0.12.0 12 | - Added CopyLocalInsert batch support 13 | 14 | ## MAy 8 2019 0.10.0 15 | - Added config.Clone 16 | 17 | ## MAy 7 2019 0.9.0 18 | - Changed IsNullable as interface type 19 | - Added dialect.BulkInsertType 20 | - Added INSERT ALL support 21 | - Patched oracle show create table 22 | - Patched CreateDatastore ora dialect 23 | - Patched duplicate column DML builder 24 | 25 | ## April 28 2019 0.8.2 26 | - Patched nil pointer on keySetter 27 | 28 | ## April 15 2019 0.8.0 29 | - Secured raw description credentials 30 | * Added DsnDescription method 31 | * Removed SecuredDescriptor 32 | 33 | ## April 15 2019 0.7.1 34 | * Patched batch insert 35 | 36 | ## April 15 2019 0.7.0 37 | * Added dynamic sql driver 38 | * Added request limiter 39 | 40 | ## April 15 2019 0.6.5 41 | * Reduced connection time 42 | 43 | ## April 5 2019 0.6.4 44 | * Patched connection leackage 45 | 46 | ## March 12 2019 0.6.0 47 | * Added Dialect.Ping to check/wait if database if online 48 | 49 | ## Feb 23 2019 0.5.0 50 | * Added persist support with toolbox.Ranger, toolbox.Iterator data types 51 | 52 | ## Feb 6 2019 0.4.4 53 | * Patched getColumn with metadata 54 | 55 | ## Feb 6 2019 0.4.3 56 | * Patched show create table dialect embedding issue 57 | 58 | ## Jan 31 2018 0.4.2 59 | * Patched persisting row with nil primary key value 60 | 61 | ## Jan 19 2018 0.4.1 62 | * Change CreateTable specificiation data type 63 | * Update sql parser with mili column IN clause 64 | 65 | ## Dec 27 2018 0.3.0 66 | * Added casandra dialect 67 | * Change Persist to allow table without pk (for insert only operation) 68 | * Added CanHandleTransaction to dialect 69 | 70 | ## Nov 5 2018 0.2.0 71 | * Extended dialect with ShowCreateTable 72 | 73 | ## Jul 1 2016 (Alpha) 74 | 75 | * Initial Release. 76 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2012-2016 Viant. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Viant, inc. Dsc (Datastore connectivity) 2 | Copyright 2012-2016 Viant 3 | 4 | This product includes software developed at 5 | Viant (http://viantinc.com/) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Datastore Connectivity (dsc) 2 | 3 | [![Datastore Connectivity library for Go.](https://goreportcard.com/badge/github.com/viant/dsc)](https://goreportcard.com/report/github.com/viant/dsc) 4 | [![GoDoc](https://godoc.org/github.com/viant/dsc?status.svg)](https://godoc.org/github.com/viant/dsc) 5 | 6 | This library is compatible with Go 1.10+ 7 | 8 | Please refer to [`CHANGELOG.md`](CHANGELOG.md) if you encounter breaking changes. 9 | 10 | - [Motivation](#Motivation) 11 | - [Usage](#Usage) 12 | - [Prerequisites](#Prerequisites) 13 | - [Installation](#Installation) 14 | - [API Documentaion](#API-Documentation) 15 | - [Tests](#Tests) 16 | - [Examples](#Examples) 17 | - [License](#License) 18 | - [Credits and Acknowledgements](#Credits-and-Acknowledgements) 19 | 20 | 21 | 22 | ## Motivation 23 | 24 | This library was developed as part of dsunit (Datastore unit testibility library) to provide unified access to SQL, noSQL, 25 | or any other store that deals with structured data in SQL-ish way. 26 | 27 | 28 | ## Usage: 29 | 30 | 31 | The following is a very simple example of CRUD operations with dsc 32 | 33 | ```go 34 | package main 35 | 36 | import ( 37 | ) 38 | 39 | 40 | func main() { 41 | 42 | 43 | config := dsc.NewConfig("mysql", "[user]:[password]@[url]", "user:root,password:dev,url:tcp(127.0.0.1:3306)/mydb?parseTime=true") 44 | factory := NewManagerFactory() 45 | manager, err := factory.Create(config) 46 | if err != nil { 47 | panic(err.Error()) 48 | } 49 | 50 | // manager := factory.CreateFromURL("file:///etc/myapp/datastore.json") 51 | 52 | 53 | interest := Interest{} 54 | 55 | success, err:= manager.ReadSingle(&interest, "SELECT id, name, expiry, category FROM interests WHERE id = ?", []interface{}{id},nil) 56 | if err != nil { 57 | panic(err.Error()) 58 | } 59 | 60 | var interests = make([]Interest, 0) 61 | err:= manager.ReadAll(&interests, "SELECT id, name, expiry, category FROM interests", nil ,nil) 62 | if err != nil { 63 | panic(err.Error()) 64 | } 65 | 66 | fmt.Printf("all interests: %v\n", interests) 67 | 68 | interests = []Interest { 69 | Interest{Name:"Abc", ExpiryTimeInSecond:3600, Category:"xyz"}, 70 | Interest{Name:"Def", ExpiryTimeInSecond:3600, Category:"xyz"}, 71 | Interest{Id:20, Name:"Ghi", ExpiryTimeInSecond:3600, Category:"xyz"}, 72 | } 73 | 74 | 75 | inserted, updated, err:= manager.PersistAll(&interests, "interests", nil) 76 | if err != nil { 77 | panic(err.Error()) 78 | } 79 | deleted, err := manager.DeleteAll(&interests, "interests", nil) 80 | if err != nil { 81 | panic(err.Error()) 82 | } 83 | fmt.Printf("Inserted %v, updated: %v, deleted: %v\n", inserted, updated, deleted) 84 | 85 | } 86 | ``` 87 | 88 | More examples illustrating the use of the API are located in the 89 | [`examples`](examples) directory. 90 | 91 | Details about the API are available in the [`docs`](docs) directory. 92 | 93 | 94 | ## Prerequisites 95 | 96 | [Go](http://golang.org) version v1.5+ is required. 97 | 98 | To install the latest stable version of Go, visit 99 | [http://golang.org/dl/](http://golang.org/dl/) 100 | 101 | 102 | Target 103 | 104 | 105 | 106 | ## Installation: 107 | 108 | 1. Install Go 1.5+ and setup your environment as [Documented](http://golang.org/doc/code.html#GOPATH) here. 109 | 2. Get the client in your ```GOPATH``` : ```go get github.com/viant/dsc``` 110 | * To update the client library: ```go get -u github.com/viant/dsc``` 111 | 112 | 113 | ### Some Hints: 114 | 115 | * To run a go program directly: ```go run ``` 116 | * to build: ```go build -o ``` 117 | 118 | 119 | 120 | 121 | 122 | ## API Documentation 123 | 124 | API documentation is available in the [`docs`](docs/README.md) directory. 125 | 126 | 127 | ## Tests 128 | 129 | This library is packaged with a number of tests. Tests require Testify library. 130 | 131 | Before running the tests, you need to update the dependencies: 132 | 133 | $ go get . 134 | 135 | To run all the test cases with race detection: 136 | 137 | $ go test 138 | 139 | 140 | 141 | 142 | ## Examples 143 | 144 | A simple CRUD applications is provided in the [`examples`](examples) directory. 145 | 146 | 147 | ## GoCover 148 | 149 | [![GoCover](http://gocover.io/github.com/viant/dsc)](http://gocover.io/github.com/viant/dsc) 150 | 151 | 152 | 153 | ## License 154 | 155 | The source code is made available under the terms of the Apache License, Version 2, as stated in the file `LICENSE`. 156 | 157 | Individual files may be made available under their own specific license, 158 | all compatible with Apache License, Version 2. Please see individual files for details. 159 | 160 | 161 | 162 | 163 | ## Credits and Acknowledgements 164 | 165 | **Library Author:** Adrian Witas 166 | 167 | **Contributors:** Sudhakaran Dharmaraj 168 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "database/sql" 5 | "reflect" 6 | "time" 7 | ) 8 | 9 | //Scanner represents a datastore data scanner. This abstraction provides the ability to convert and assign datastore record of data to provided destination 10 | type Scanner interface { 11 | //Returns all columns specified in select statement. 12 | Columns() ([]string, error) 13 | 14 | //ColumnTypes return column types 15 | ColumnTypes() ([]ColumnType, error) 16 | 17 | //Scans datastore record data to convert and assign it to provided destinations, a destination needs to be pointer. 18 | Scan(dest ...interface{}) error 19 | } 20 | 21 | //ColumnValueProvider provides column values 22 | type ColumnValueProvider interface { 23 | ColumnValues() ([]interface{}, error) 24 | } 25 | 26 | //RecordMapper represents a datastore record mapper, it is responsible for mapping data record into application abstraction. 27 | type RecordMapper interface { 28 | //Maps data record by passing to the scanner references to the application abstraction 29 | Map(scanner Scanner) (interface{}, error) 30 | } 31 | 32 | //ParametrizedSQL represents a parmetrized SQL with its binding values, in order of occurrence. 33 | type ParametrizedSQL struct { 34 | SQL string //Sql 35 | Values []interface{} //binding parameter values 36 | Type int 37 | } 38 | 39 | //DmlProvider represents dml generator, which is responsible for providing parametrized sql, it takes operation type: 40 | //SqlTypeInsert = 0 41 | //SqlTypeUpdate = 1 42 | //SqlTypeDelete = 2 43 | // and instance to the application abstraction 44 | type DmlProvider interface { 45 | Get(operationType int, instance interface{}) *ParametrizedSQL 46 | 47 | KeySetter 48 | KeyGetter 49 | } 50 | 51 | //Manager represents datastore manager. 52 | type Manager interface { 53 | Config() *Config 54 | 55 | //ConnectionProvider returns connection provider 56 | ConnectionProvider() ConnectionProvider 57 | 58 | //Execute executes provided sql, with the arguments, '?' is used as placeholder for and arguments 59 | Execute(sql string, parameters ...interface{}) (sql.Result, error) 60 | 61 | //ExecuteAll executes all provided sql 62 | ExecuteAll(sqls []string) ([]sql.Result, error) 63 | 64 | //ExecuteOnConnection executes sql on passed in connection, this allowes to maintain transaction if supported 65 | ExecuteOnConnection(connection Connection, sql string, parameters []interface{}) (sql.Result, error) 66 | 67 | //ExecuteAllOnConnection executes all sql on passed in connection, this allowes to maintain transaction if supported 68 | ExecuteAllOnConnection(connection Connection, sqls []string) ([]sql.Result, error) 69 | 70 | //ReadSingle fetches a single record of data, it takes pointer to the result, sql query, binding parameters, record to application instance mapper 71 | ReadSingle(resultPointer interface{}, query string, parameters []interface{}, mapper RecordMapper) (success bool, err error) 72 | 73 | //ReadSingleOnConnection fetches a single record of data on connection, it takes connection, pointer to the result, sql query, binding parameters, record to application instance mapper 74 | ReadSingleOnConnection(connection Connection, resultPointer interface{}, query string, parameters []interface{}, mapper RecordMapper) (success bool, err error) 75 | 76 | //ReadAll reads all records, it takes pointer to the result slice , sql query, binding parameters, record to application instance mapper 77 | ReadAll(resultSlicePointer interface{}, query string, parameters []interface{}, mapper RecordMapper) error 78 | 79 | //ReadAllOnConnection reads all records, it takes connection, pointer to the result slice , sql query, binding parameters, record to application instance mapper 80 | ReadAllOnConnection(connection Connection, resultSlicePointer interface{}, query string, parameters []interface{}, mapper RecordMapper) error 81 | 82 | //ReadAllWithHandler reads data for passed in query and parameters, for each row reading handler will be called, to continue reading next row it needs to return true 83 | ReadAllWithHandler(query string, parameters []interface{}, readingHandler func(scanner Scanner) (toContinue bool, err error)) error 84 | 85 | //ReadAllOnWithHandlerOnConnection reads data for passed in query and parameters, on connection, for each row reading handler will be called, to continue reading next row, it needs to return true 86 | ReadAllOnWithHandlerOnConnection(connection Connection, query string, parameters []interface{}, readingHandler func(scanner Scanner) (toContinue bool, err error)) error 87 | 88 | //ReadAllOnWithHandlerOnConnection persists all passed in data to the table, it uses dml provider to generate DML for each row. 89 | PersistAll(slicePointer interface{}, table string, provider DmlProvider) (inserted int, updated int, err error) 90 | 91 | //PersistAllOnConnection persists all passed in data on connection to the table, it uses dml provider to generate DML for each row. 92 | PersistAllOnConnection(connection Connection, dataPointer interface{}, table string, provider DmlProvider) (inserted int, updated int, err error) 93 | 94 | //PersistSingle persists single row into table, it uses dml provider to generate DML to the row. 95 | PersistSingle(dataPointer interface{}, table string, provider DmlProvider) (inserted int, updated int, err error) 96 | 97 | //PersistSingleOnConnection persists single row on connection into table, it uses dml provider to generate DML to the row. 98 | PersistSingleOnConnection(connection Connection, dataPointer interface{}, table string, provider DmlProvider) (inserted int, updated int, err error) 99 | 100 | //connection persists all all row of data to passed in table, it uses key setter to optionally set back autoincrement value, and func to generate parametrized sql for the row. 101 | PersistData(connection Connection, data interface{}, table string, keySetter KeySetter, sqlProvider func(item interface{}) *ParametrizedSQL) (int, error) 102 | 103 | //DeleteAll deletes all record for passed in slice pointer from table, it uses key provider to take id/key for the record. 104 | DeleteAll(slicePointer interface{}, table string, keyProvider KeyGetter) (deleted int, err error) 105 | 106 | //DeleteAllOnConnection deletes all record on connection for passed in slice pointer from table, it uses key provider to take id/key for the record. 107 | DeleteAllOnConnection(connection Connection, resultPointer interface{}, table string, keyProvider KeyGetter) (deleted int, err error) 108 | 109 | //DeleteSingle deletes single row of data from table, it uses key provider to take id/key for the record. 110 | DeleteSingle(resultPointer interface{}, table string, keyProvider KeyGetter) (success bool, err error) 111 | 112 | //DeleteSingleOnConnection deletes single row of data on connection from table, it uses key provider to take id/key for the record. 113 | DeleteSingleOnConnection(connection Connection, resultPointer interface{}, table string, keyProvider KeyGetter) (success bool, err error) 114 | 115 | //ClassifyDataAsInsertableOrUpdatable classifies records are inserable and what are updatable. 116 | ClassifyDataAsInsertableOrUpdatable(connection Connection, slicePointer interface{}, table string, provider DmlProvider) (insertables, updatables []interface{}, err error) 117 | 118 | //TableDescriptorRegistry returns Table Descriptor Registry 119 | TableDescriptorRegistry() TableDescriptorRegistry 120 | } 121 | 122 | //DatastoreDialect represents datastore dialects. 123 | type DatastoreDialect interface { 124 | GetDatastores(manager Manager) ([]string, error) 125 | 126 | GetTables(manager Manager, datastore string) ([]string, error) 127 | 128 | DropTable(manager Manager, datastore string, table string) error 129 | 130 | CreateTable(manager Manager, datastore string, table string, specification interface{}) error 131 | 132 | CanCreateDatastore(manager Manager) bool 133 | 134 | CreateDatastore(manager Manager, datastore string) error 135 | 136 | CanDropDatastore(manager Manager) bool 137 | 138 | DropDatastore(manager Manager, datastore string) error 139 | 140 | GetCurrentDatastore(manager Manager) (string, error) 141 | 142 | //GetSequence returns a sequence number 143 | GetSequence(manager Manager, name string) (int64, error) 144 | 145 | //GetKeyName returns a name of TableColumn name that is a key, or coma separated list if complex key 146 | GetKeyName(manager Manager, datastore, table string) string 147 | 148 | //GetColumns returns TableColumn info 149 | GetColumns(manager Manager, datastore, table string) ([]Column, error) 150 | 151 | //IsAutoincrement returns true if autoicrement 152 | IsAutoincrement(manager Manager, datastore, table string) bool 153 | 154 | //Flag if data store can batch batch 155 | CanPersistBatch() bool 156 | 157 | //BulkInsert type 158 | BulkInsertType() string 159 | 160 | //DisableForeignKeyCheck disables fk check 161 | DisableForeignKeyCheck(manager Manager, connection Connection) error 162 | 163 | //EnableForeignKeyCheck disables fk check 164 | EnableForeignKeyCheck(manager Manager, connection Connection) error 165 | 166 | //IsKeyCheckSwitchSessionLevel returns true if key check is controlled on connection level (as opposed to globally on table level) 167 | IsKeyCheckSwitchSessionLevel() bool 168 | 169 | //Normalizes sql i.e for placeholders: dsc uses '?' for placeholder if some dialect use difference this method should take care of it 170 | NormalizeSQL(SQL string) string 171 | 172 | //EachTable iterate all current connection manager datastore tables 173 | EachTable(manager Manager, handler func(table string) error) error 174 | 175 | //ShowCreateTable returns DDL showing create table statement or error 176 | ShowCreateTable(manager Manager, table string) (string, error) 177 | 178 | //Init initializes connection 179 | Init(manager Manager, connection Connection) error 180 | 181 | //CanHandleTransaction returns true if driver can handle transaction 182 | CanHandleTransaction() bool 183 | 184 | //Checks if database is online 185 | Ping(manager Manager) error 186 | } 187 | 188 | type ColumnType interface { 189 | // Length returns the TableColumn type length for variable length TableColumn types such 190 | // as text and binary field types. If the type length is unbounded the value will 191 | // be math.MaxInt64 (any database limits will still apply). 192 | // If the TableColumn type is not variable length, such as an int, or if not supported 193 | // by the driver ok is false. 194 | Length() (length int64, ok bool) 195 | 196 | // DecimalSize returns the scale and precision of a decimal type. 197 | // If not applicable or if not supported ok is false. 198 | DecimalSize() (precision, scale int64, ok bool) 199 | 200 | // ScanType returns a Go type suitable for scanning into using Rows.Scan. 201 | // If a driver does not support this property ScanType will return 202 | // the type of an empty interface. 203 | ScanType() reflect.Type 204 | 205 | // Nullable returns whether the TableColumn may be null. 206 | // If a driver does not support this property ok will be false. 207 | Nullable() (nullable, ok bool) 208 | 209 | // DatabaseTypeName returns the database system name of the TableColumn type. If an empty 210 | // string is returned the driver type name is not supported. 211 | // Consult your driver documentation for a list of driver data types. Length specifiers 212 | // are not included. 213 | // Common type include "VARCHAR", "TEXT", "NVARCHAR", "DECIMAL", "BOOL", "INT", "BIGINT". 214 | DatabaseTypeName() string 215 | } 216 | 217 | //Column represents TableColumn type interface (compabible with *sql.ColumnType 218 | type Column interface { 219 | ColumnType 220 | // Name returns the name or alias of the TableColumn. 221 | Name() string 222 | } 223 | 224 | //TransactionManager represents a transaction manager. 225 | type TransactionManager interface { 226 | Begin() error 227 | 228 | Commit() error 229 | 230 | Rollback() error 231 | } 232 | 233 | //Connection represents a datastore connection 234 | type Connection interface { 235 | Config() *Config 236 | 237 | ConnectionPool() chan Connection 238 | 239 | //Close closes connection or it returns it back to the pool 240 | Close() error 241 | 242 | CloseNow() error //closes connecition, it does not return it back to the pool 243 | 244 | Unwrap(target interface{}) interface{} 245 | 246 | LastUsed() *time.Time 247 | 248 | SetLastUsed(ts *time.Time) 249 | 250 | TransactionManager 251 | } 252 | 253 | //ConnectionProvider represents a datastore connection provider. 254 | type ConnectionProvider interface { 255 | Get() (Connection, error) 256 | 257 | Config() *Config 258 | 259 | ConnectionPool() chan Connection 260 | 261 | SpawnConnectionIfNeeded() 262 | 263 | NewConnection() (Connection, error) 264 | 265 | Close() error 266 | } 267 | 268 | //ManagerFactory represents a manager factory. 269 | type ManagerFactory interface { 270 | //Creates manager, takes config pointer. 271 | Create(config *Config) (Manager, error) 272 | 273 | //Creates manager from url, can url needs to point to Config JSON. 274 | CreateFromURL(url string) (Manager, error) 275 | } 276 | 277 | //ManagerRegistry represents a manager registry. 278 | type ManagerRegistry interface { 279 | Get(name string) Manager 280 | 281 | Register(name string, manager Manager) 282 | } 283 | 284 | //KeySetter represents id/key mutator. 285 | type KeySetter interface { 286 | //SetKey sets autoincrement/sql value to the application domain instance. 287 | SetKey(instance interface{}, seq int64) 288 | } 289 | 290 | //KeyGetter represents id/key accessor. 291 | type KeyGetter interface { 292 | //Key returns key/id for the the application domain instance. 293 | Key(instance interface{}) []interface{} 294 | } 295 | -------------------------------------------------------------------------------- /batch.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "compress/gzip" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | type batch struct { 12 | processed int 13 | tempDir string 14 | tempFile string 15 | size int 16 | sql string 17 | writer *gzip.Writer 18 | values []interface{} 19 | placeholders string 20 | columns string 21 | dataIndexes []int 22 | firstSeq int64 23 | bulkInsertType string 24 | manager *AbstractManager 25 | sqlProvider func(item interface{}) *ParametrizedSQL 26 | updateId func(index int, seq int64) 27 | connection Connection 28 | table string 29 | } 30 | 31 | func (b *batch) flush() (int, error) { 32 | if b.sql == "" { 33 | return 0, nil 34 | } 35 | 36 | var dataIndexes = b.dataIndexes 37 | b.dataIndexes = []int{} 38 | switch b.bulkInsertType { 39 | case CopyLocalInsert: 40 | defer os.Remove(b.tempFile) 41 | err := b.writer.Flush() 42 | if err != nil { 43 | return 0, err 44 | } 45 | err = b.writer.Close() 46 | if err != nil { 47 | return 0, err 48 | } 49 | case BulkInsertAllType: 50 | b.sql += " SELECT 1 FROM DUAL" 51 | } 52 | result, err := b.manager.ExecuteOnConnection(b.connection, b.sql, b.values) 53 | b.dataIndexes = []int{} 54 | b.sql = "" 55 | b.values = []interface{}{} 56 | if err != nil { 57 | return 0, err 58 | } 59 | affected, err := result.RowsAffected() 60 | if err != nil { 61 | return 0, err 62 | } 63 | for _, i := range dataIndexes { 64 | b.firstSeq++ 65 | b.updateId(i, b.firstSeq) 66 | } 67 | b.firstSeq = 0 68 | return int(affected), nil 69 | } 70 | 71 | func (b *batch) expandedValues(parametrizedSQL *ParametrizedSQL) string { 72 | recordLine := b.manager.ExpandSQL(b.placeholders, parametrizedSQL.Values) 73 | if breakCount := strings.Count(recordLine, "\n"); breakCount > 0 { 74 | recordLine = strings.Replace(recordLine, "\n", "", breakCount) 75 | } 76 | return recordLine + "\n" 77 | } 78 | 79 | func (b *batch) transformFirst(parametrizedSQL *ParametrizedSQL) error { 80 | b.sql = parametrizedSQL.SQL 81 | b.values = parametrizedSQL.Values 82 | fragment := " VALUES" 83 | valuesIndex := strings.Index(parametrizedSQL.SQL, fragment) 84 | if beginIndex := strings.Index(parametrizedSQL.SQL, "("); beginIndex != -1 { 85 | names := string(parametrizedSQL.SQL[beginIndex+1:]) 86 | if endIndex := strings.Index(names, ")"); endIndex != -1 { 87 | b.columns = string(names[:endIndex]) 88 | } 89 | } 90 | b.placeholders = strings.Trim(strings.TrimSpace(string(parametrizedSQL.SQL[valuesIndex+7:])), "()") 91 | switch b.bulkInsertType { 92 | case CopyLocalInsert: 93 | b.tempDir = b.manager.config.GetString("tempDir", os.TempDir()) 94 | if b.columns == "" { 95 | return fmt.Errorf("columns were empty") 96 | } 97 | file, err := ioutil.TempFile(b.tempDir, "temp") 98 | if err != nil { 99 | return err 100 | } 101 | b.tempFile = file.Name() 102 | b.writer = gzip.NewWriter(file) 103 | if _, err := b.writer.Write([]byte(b.expandedValues(parametrizedSQL))); err != nil { 104 | return err 105 | } 106 | 107 | table := b.table 108 | b.sql = fmt.Sprintf(`COPY %v(%v) 109 | FROM LOCAL '%v' GZIP 110 | DELIMITER ',' 111 | NULL AS 'null' 112 | ENCLOSED BY '''' 113 | `, table, b.columns, file.Name()) 114 | b.values = make([]interface{}, 0) 115 | case UnionSelectInsert: 116 | valuesIndex := strings.Index(parametrizedSQL.SQL, " VALUES") 117 | selectAll := " SELECT " + b.expandedValues(parametrizedSQL) 118 | selectAll = b.manager.ExpandSQL(selectAll, parametrizedSQL.Values) 119 | parametrizedSQL.Values = []interface{}{} 120 | b.sql = b.sql[:valuesIndex] + " " + selectAll 121 | 122 | case BulkInsertAllType: 123 | b.sql = strings.Replace(b.sql, "INSERT ", "INSERT ALL ", 1) 124 | default: 125 | 126 | } 127 | return nil 128 | } 129 | 130 | func (b *batch) transformNext(parametrizedSQL *ParametrizedSQL) error { 131 | switch b.bulkInsertType { 132 | case CopyLocalInsert: 133 | _, err := b.writer.Write([]byte(b.expandedValues(parametrizedSQL))) 134 | return err 135 | case UnionSelectInsert: 136 | b.sql += "\nUNION ALL SELECT " + b.expandedValues(parametrizedSQL) 137 | case BulkInsertAllType: 138 | b.sql += fmt.Sprintf("\nINTO %v(%v) VALUES(%v)", b.table, b.columns, b.placeholders) 139 | b.values = append(b.values, parametrizedSQL.Values...) 140 | default: 141 | b.sql += fmt.Sprintf(",(%v)", b.placeholders) 142 | b.values = append(b.values, parametrizedSQL.Values...) 143 | } 144 | return nil 145 | } 146 | 147 | func (b *batch) persist(index int, item interface{}) error { 148 | parametrizedSQL := b.sqlProvider(item) 149 | if len(parametrizedSQL.Values) == 1 && parametrizedSQL.Type == SQLTypeUpdate { 150 | //nothing to udpate, one parameter is ID=? without values to update 151 | return nil 152 | } 153 | if parametrizedSQL.Type == SQLTypeInsert && b.size > 0 { 154 | if len(b.dataIndexes) > b.size { 155 | if _, err := b.flush(); err != nil { 156 | return err 157 | } 158 | } 159 | b.dataIndexes = append(b.dataIndexes, index) 160 | if isFirst := len(b.sql) == 0; isFirst { 161 | return b.transformFirst(parametrizedSQL) 162 | } 163 | return b.transformNext(parametrizedSQL) 164 | } 165 | result, err := b.manager.ExecuteOnConnection(b.connection, parametrizedSQL.SQL, parametrizedSQL.Values) 166 | if err != nil { 167 | return err 168 | } 169 | affected, err := result.RowsAffected() 170 | if err != nil { 171 | return err 172 | } 173 | b.processed += int(affected) 174 | seq, _ := result.LastInsertId() 175 | if b.size > 0 && b.firstSeq == 0 { 176 | b.firstSeq = seq 177 | } 178 | b.updateId(index, seq) 179 | return nil 180 | } 181 | 182 | func newBatch(table string, connection Connection, manager *AbstractManager, sqlProvider func(item interface{}) *ParametrizedSQL, updateId func(index int, seq int64)) *batch { 183 | dialect := GetDatastoreDialect(manager.Config().DriverName) 184 | var batchSize = manager.Config().GetInt(BatchSizeKey, defaultBatchSize) 185 | Logf("batch size: %v\n", batchSize) 186 | canUseBatch := dialect != nil && dialect.CanPersistBatch() && batchSize > 1 187 | if !canUseBatch { 188 | batchSize = 0 189 | } 190 | insertType := "" 191 | if dialect != nil { 192 | insertType = dialect.BulkInsertType() 193 | } 194 | return &batch{ 195 | connection: connection, 196 | updateId: updateId, 197 | sqlProvider: sqlProvider, 198 | size: batchSize, 199 | values: []interface{}{}, 200 | dataIndexes: []int{}, 201 | bulkInsertType: insertType, 202 | manager: manager, 203 | table: table, 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /column.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "github.com/viant/toolbox" 5 | "reflect" 6 | ) 7 | 8 | type TableColumn struct { 9 | ColumnName string `column:"column_name"` 10 | DataType string `column:"data_type"` 11 | DataTypeLength *int64 `column:"data_type_length"` 12 | NumericPrecision *int64 `column:"numeric_precision"` 13 | NumericScale *int64 `column:"numeric_scale"` 14 | IsNullable interface{} `column:"is_nullable"` 15 | Position int `column:"position"` 16 | scanType reflect.Type 17 | } 18 | 19 | func (c *TableColumn) Name() string { 20 | return c.ColumnName 21 | } 22 | 23 | func (c *TableColumn) Length() (length int64, ok bool) { 24 | if c.DataTypeLength == nil { 25 | return 0, false 26 | } 27 | return *c.DataTypeLength, true 28 | } 29 | 30 | func (c *TableColumn) DecimalSize() (precision, scale int64, ok bool) { 31 | if c.NumericPrecision == nil || c.NumericScale == nil { 32 | return 0, 0, false 33 | } 34 | return *c.NumericPrecision, *c.NumericScale, true 35 | } 36 | 37 | func (c *TableColumn) ScanType() reflect.Type { 38 | return c.scanType 39 | } 40 | 41 | func (c *TableColumn) Nullable() (nullable, ok bool) { 42 | if c.IsNullable == nil { 43 | return false, false 44 | } 45 | return toolbox.AsBoolean(c.IsNullable), true 46 | } 47 | 48 | // Common type include "VARCHAR", "TEXT", "NVARCHAR", "DECIMAL", "BOOL", "INT", "BIGINT". 49 | func (c *TableColumn) DatabaseTypeName() string { 50 | return c.DataType 51 | } 52 | 53 | //NewColumn create new TableColumn 54 | func NewColumn(name, typeName string, length, precision, scale *int64, scanType reflect.Type, nullable *bool) Column { 55 | var result = &TableColumn{ 56 | ColumnName: name, 57 | DataType: typeName, 58 | DataTypeLength: length, 59 | NumericPrecision: precision, 60 | NumericScale: scale, 61 | scanType: scanType, 62 | IsNullable: nullable, 63 | } 64 | return result 65 | } 66 | 67 | //NewSimpleColumn create simple TableColumn name 68 | func NewSimpleColumn(name, typeName string) Column { 69 | return &TableColumn{ 70 | ColumnName: name, 71 | DataType: typeName, 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /common.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "database/sql" 5 | ) 6 | 7 | const ( 8 | //SQLTypeInsert 0 constant for DML insert statement provider. 9 | SQLTypeInsert = 0 10 | //SQLTypeUpdate 1 constant for DML update statement provider. 11 | SQLTypeUpdate = 1 12 | //SQLTypeDelete 2 constant for DML delete statement provider. 13 | SQLTypeDelete = 2 14 | ) 15 | 16 | var sqlDbPointer = (*sql.DB)(nil) 17 | var sqlTxtPointer = (*sql.Tx)(nil) 18 | 19 | type sqlResult struct { 20 | lastInsertID int64 21 | rowsAffected int64 22 | } 23 | 24 | //LastInsertId returns the last inserted/autoincrement id. 25 | func (r *sqlResult) LastInsertId() (int64, error) { 26 | return r.lastInsertID, nil 27 | } 28 | 29 | //RowsAffected returns row affected by the last sql. 30 | func (r *sqlResult) RowsAffected() (int64, error) { 31 | return r.rowsAffected, nil 32 | } 33 | 34 | //NewSQLResult returns a new SqlResult 35 | func NewSQLResult(rowsAffected, lastInsertID int64) sql.Result { 36 | var result sql.Result = &sqlResult{lastInsertID: lastInsertID, rowsAffected: rowsAffected} 37 | return result 38 | } 39 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/scy/cred" 6 | "github.com/viant/scy/cred/secret" 7 | "github.com/viant/toolbox" 8 | "github.com/viant/toolbox/url" 9 | "strings" 10 | "sync" 11 | "sync/atomic" 12 | "time" 13 | ) 14 | 15 | // BatchSizeKey represents a config batch size parameter 16 | const BatchSizeKey = "batchSize" 17 | 18 | // Config represent datastore config. 19 | type Config struct { 20 | URL string 21 | //@deprecated use driver instead 22 | DriverName string `json:"-"` 23 | Driver string 24 | 25 | PoolSize int 26 | MaxPoolSize int 27 | //@deprecated use DSN instead 28 | Descriptor string `json:"-"` 29 | DSN string 30 | InitSQL []string 31 | Parameters map[string]interface{} 32 | Credentials string 33 | MaxRequestPerSecond int 34 | cred string 35 | username string 36 | password string 37 | dsnDescriptor string 38 | lock *sync.Mutex 39 | race uint32 40 | initRun bool 41 | CredConfig *cred.Generic `json:"-"` 42 | } 43 | 44 | // Get returns value for passed in parameter name or panic - please use Config.Has to check if value is present. 45 | func (c *Config) Get(name string) string { 46 | c.lock.Lock() 47 | defer c.lock.Unlock() 48 | if result, ok := c.Parameters[name]; ok { 49 | return toolbox.AsString(result) 50 | } 51 | return "" 52 | } 53 | 54 | // DsnDescriptor return dsn expanded descriptor or error 55 | func (c *Config) DsnDescriptor() (string, error) { 56 | if c.dsnDescriptor == "" { 57 | if err := c.Init(); err != nil { 58 | return "", err 59 | } 60 | } 61 | return c.dsnDescriptor, nil 62 | } 63 | 64 | // Get returns value for passed in parameter name or panic - please use Config.Has to check if value is present. 65 | func (c *Config) GetMap(name string) map[string]interface{} { 66 | c.lock.Lock() 67 | defer c.lock.Unlock() 68 | if result, ok := c.Parameters[name]; ok { 69 | if toolbox.IsMap(result) { 70 | return toolbox.AsMap(result) 71 | } 72 | } 73 | return nil 74 | } 75 | 76 | // GetInt returns value for passed in parameter name or defaultValue 77 | func (c *Config) GetInt(name string, defaultValue int) int { 78 | c.lock.Lock() 79 | defer c.lock.Unlock() 80 | if result, ok := c.Parameters[name]; ok { 81 | return toolbox.AsInt(result) 82 | } 83 | return defaultValue 84 | } 85 | 86 | // GetFloat returns value for passed in parameter name or defaultValue 87 | func (c *Config) GetFloat(name string, defaultValue float64) float64 { 88 | c.lock.Lock() 89 | defer c.lock.Unlock() 90 | if result, ok := c.Parameters[name]; ok { 91 | return toolbox.AsFloat(result) 92 | } 93 | return defaultValue 94 | } 95 | 96 | // GetDuration returns value for passed in parameter name or defaultValue 97 | func (c *Config) GetDuration(name string, multiplier time.Duration, defaultValue time.Duration) time.Duration { 98 | c.lock.Lock() 99 | defer c.lock.Unlock() 100 | if result, ok := c.Parameters[name]; ok { 101 | return time.Duration(toolbox.AsInt(result)) * multiplier 102 | } 103 | return defaultValue 104 | } 105 | 106 | // GetString returns value for passed in parameter name or defaultValue 107 | func (c *Config) GetString(name string, defaultValue string) string { 108 | c.lock.Lock() 109 | defer c.lock.Unlock() 110 | if result, ok := c.Parameters[name]; ok { 111 | return toolbox.AsString(result) 112 | } 113 | return defaultValue 114 | } 115 | 116 | // GetBoolean returns value for passed in parameter name or defaultValue 117 | func (c *Config) GetBoolean(name string, defaultValue bool) bool { 118 | c.lock.Lock() 119 | defer c.lock.Unlock() 120 | if result, ok := c.Parameters[name]; ok { 121 | return toolbox.AsBoolean(result) 122 | } 123 | return defaultValue 124 | } 125 | 126 | // HasDateLayout returns true if config has date layout, it checks dateLayout or dateFormat parameter names. 127 | func (c *Config) HasDateLayout() bool { 128 | c.lock.Lock() 129 | defer c.lock.Unlock() 130 | return toolbox.HasTimeLayout(c.Parameters) 131 | } 132 | 133 | // GetDateLayout returns date layout 134 | func (c *Config) GetDateLayout() string { 135 | c.lock.Lock() 136 | defer c.lock.Unlock() 137 | return toolbox.GetTimeLayout(c.Parameters) 138 | } 139 | 140 | // Has returns true if parameter with passed in name is present, otherwise it returns false. 141 | func (c *Config) Has(name string) bool { 142 | c.lock.Lock() 143 | defer c.lock.Unlock() 144 | if _, ok := c.Parameters[name]; ok { 145 | return true 146 | } 147 | return false 148 | } 149 | 150 | func (c *Config) initLock() { 151 | if c.lock == nil { 152 | if atomic.CompareAndSwapUint32(&c.race, 0, 1) { 153 | c.lock = &sync.Mutex{} 154 | } else { 155 | c.initLock() 156 | } 157 | } 158 | } 159 | 160 | func (c *Config) loadCredentials(ctx context.Context) error { 161 | if c.Credentials == "" { 162 | return nil 163 | } 164 | secrets := secret.New() 165 | config, err := secrets.GetCredentials(ctx, c.Credentials) 166 | if err != nil { 167 | return err 168 | } 169 | if len(c.Parameters) == 0 { 170 | c.Parameters = make(map[string]interface{}) 171 | } 172 | return c.ApplyCredentials(config) 173 | } 174 | 175 | func (c *Config) ApplyCredentials(config *cred.Generic) error { 176 | c.username = config.Username 177 | c.password = config.Password 178 | if len(c.Parameters) == 0 { 179 | c.Parameters = make(map[string]interface{}) 180 | } 181 | c.Parameters["username"] = c.username 182 | c.CredConfig = config 183 | return nil 184 | } 185 | 186 | // Init makes parameter map from encoded parameters if presents, expands descriptor with parameter value using [param_name] matching pattern. 187 | func (c *Config) Init() error { 188 | defer func() { c.initRun = true }() 189 | if c.cred == "" { 190 | c.cred = c.Credentials 191 | } 192 | c.initLock() 193 | if c.URL != "" && c.DriverName == "" { 194 | resource := url.NewResource(c.URL) 195 | if err := resource.Decode(c); err != nil { 196 | return err 197 | } 198 | } 199 | var lock = c.lock 200 | lock.Lock() 201 | defer lock.Unlock() 202 | if err := c.loadCredentials(context.Background()); err != nil { 203 | return err 204 | } 205 | if c.DriverName == "" { 206 | c.DriverName = c.Driver 207 | } 208 | if c.Descriptor == "" { 209 | c.Descriptor = c.DSN 210 | } 211 | c.dsnDescriptor = c.Descriptor 212 | 213 | c.dsnDescriptor = strings.Replace(c.dsnDescriptor, "[username]", c.username, 1) 214 | c.dsnDescriptor = strings.Replace(c.dsnDescriptor, "[password]", c.password, 1) 215 | for key, value := range c.Parameters { 216 | textValue, ok := value.(string) 217 | if !ok { 218 | continue 219 | } 220 | macro := "[" + key + "]" 221 | c.dsnDescriptor = strings.Replace(c.dsnDescriptor, macro, textValue, 1) 222 | } 223 | return nil 224 | } 225 | 226 | // Clone clones config 227 | func (c *Config) Clone() *Config { 228 | cred := c.cred 229 | if cred == "" { 230 | cred = c.Credentials 231 | } 232 | result := &Config{ 233 | DriverName: c.DriverName, 234 | URL: c.URL, 235 | InitSQL: c.InitSQL, 236 | Descriptor: c.Descriptor, 237 | Driver: c.Driver, 238 | DSN: c.DSN, 239 | PoolSize: c.PoolSize, 240 | MaxPoolSize: c.MaxPoolSize, 241 | MaxRequestPerSecond: c.MaxRequestPerSecond, 242 | Parameters: make(map[string]interface{}), 243 | username: c.username, 244 | password: c.password, 245 | dsnDescriptor: c.dsnDescriptor, 246 | lock: &sync.Mutex{}, 247 | Credentials: cred, 248 | cred: c.cred, 249 | } 250 | if len(c.Parameters) > 0 { 251 | for k, v := range c.Parameters { 252 | result.Parameters[k] = v 253 | } 254 | } 255 | return result 256 | } 257 | 258 | // NewConfig creates new Config, it takes the following parameters 259 | // descriptor - optional datastore connection string with macros that will be looked epxanded from for instance [user]:[password]@[url] 260 | // encodedParameters should be in the following format: :, ...,: 261 | func NewConfig(driverName string, descriptor string, encodedParameters string) *Config { 262 | var parameters = toolbox.MakeMap(encodedParameters, ":", ",") 263 | result := &Config{DriverName: driverName, PoolSize: 1, MaxPoolSize: 2, Descriptor: descriptor, DSN: descriptor, Driver: driverName, Parameters: parameters, 264 | lock: &sync.Mutex{}} 265 | result.Init() 266 | return result 267 | } 268 | 269 | // NewConfigWithParameters creates a new config with parameters 270 | func NewConfigWithParameters(driverName string, descriptor string, credential string, parameters map[string]interface{}) (*Config, error) { 271 | result := &Config{ 272 | DriverName: driverName, 273 | Descriptor: descriptor, 274 | DSN: descriptor, 275 | Driver: driverName, 276 | Credentials: credential, 277 | Parameters: parameters, 278 | lock: &sync.Mutex{}, 279 | } 280 | err := result.Init() 281 | return result, err 282 | } 283 | 284 | // NewConfigFromUrl returns new config from url 285 | func NewConfigFromURL(URL string) (*Config, error) { 286 | result := &Config{} 287 | var resource = url.NewResource(URL) 288 | err := resource.Decode(result) 289 | result.lock = &sync.Mutex{} 290 | if err == nil { 291 | err = result.Init() 292 | } 293 | return result, err 294 | } 295 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/dsc" 7 | "testing" 8 | ) 9 | 10 | func TestConfigHasDateLayout(t *testing.T) { 11 | { 12 | config := dsc.NewConfig("sqlite3", "[url]", "url:./test/foo.db") 13 | assert.False(t, config.HasDateLayout()) 14 | } 15 | 16 | { 17 | config := dsc.NewConfig("sqlite3", "[url]", "url:./test/foo.db,dateFormat:yyyy-MM-dd hh:mm:ss") 18 | assert.True(t, config.HasDateLayout()) 19 | } 20 | 21 | } 22 | 23 | func TestConfigHasParameterLayout(t *testing.T) { 24 | { 25 | config := dsc.NewConfig("sqlite3", "[url]", "url:./test/foo.db") 26 | assert.False(t, config.Has("url1")) 27 | } 28 | 29 | { 30 | config := dsc.NewConfig("sqlite3", "[url]", "url:./test/foo.db,dateFormat:yyyy-MM-dd hh:mm:ss") 31 | assert.True(t, config.Has("url")) 32 | } 33 | 34 | } 35 | 36 | func TestGetPanic(t *testing.T) { 37 | config := dsc.NewConfig("sqlite3", "[url]", "url:./test/foo.db") 38 | 39 | defer func() { 40 | if err := recover(); err != nil { 41 | expected := "Missing value in descriptor abc" 42 | actual := fmt.Sprintf("%v", err) 43 | assert.Equal(t, actual, expected, "Assert Kind") 44 | } 45 | }() 46 | 47 | config.Get("abc") 48 | 49 | } 50 | -------------------------------------------------------------------------------- /connection.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "log" 5 | "time" 6 | ) 7 | 8 | //AbstractConnection represents an abstract connection 9 | type AbstractConnection struct { 10 | Connection 11 | lastUsed *time.Time 12 | config *Config 13 | connectionPool chan Connection 14 | } 15 | 16 | //Config returns a datastore config 17 | func (ac *AbstractConnection) Config() *Config { 18 | return ac.config 19 | } 20 | 21 | //ConnectionPool returns a connection channel 22 | func (ac *AbstractConnection) ConnectionPool() chan Connection { 23 | return ac.connectionPool 24 | } 25 | 26 | //LastUsed returns a last used time 27 | func (ac *AbstractConnection) LastUsed() *time.Time { 28 | return ac.lastUsed 29 | } 30 | 31 | //SetLastUsed sets last used time 32 | func (ac *AbstractConnection) SetLastUsed(ts *time.Time) { 33 | ac.lastUsed = ts 34 | } 35 | 36 | //Close closes connection if pool is full or send it back to the pool 37 | func (ac *AbstractConnection) Close() error { 38 | channel := ac.Connection.ConnectionPool() 39 | config := ac.config 40 | if len(ac.Connection.ConnectionPool()) < config.MaxPoolSize { 41 | var connection = ac.Connection 42 | channel <- connection 43 | var ts = time.Now() 44 | connection.SetLastUsed(&ts) 45 | 46 | } else { 47 | return ac.Connection.CloseNow() 48 | } 49 | return nil 50 | } 51 | 52 | //Begin starts a transaction - this method is an abstract method 53 | func (ac *AbstractConnection) Begin() error { return nil } 54 | 55 | //Commit finishes current transaction - this method is an abstract method 56 | func (ac *AbstractConnection) Commit() error { return nil } 57 | 58 | //Rollback - discards transaction - this method is an abstract method 59 | func (ac *AbstractConnection) Rollback() error { return nil } 60 | 61 | //NewAbstractConnection create a new abstract connection 62 | func NewAbstractConnection(config *Config, connectionPool chan Connection, connection Connection) *AbstractConnection { 63 | return &AbstractConnection{config: config, connectionPool: connectionPool, Connection: connection} 64 | } 65 | 66 | //AbstractConnectionProvider represents an abstract/superclass ConnectionProvider 67 | type AbstractConnectionProvider struct { 68 | ConnectionProvider 69 | config *Config 70 | connectionPool chan Connection 71 | } 72 | 73 | //Config returns a datastore config, 74 | func (cp *AbstractConnectionProvider) Config() *Config { 75 | return cp.config 76 | } 77 | 78 | //ConnectionPool returns a ConnectionPool 79 | func (cp *AbstractConnectionProvider) ConnectionPool() chan Connection { 80 | return cp.connectionPool 81 | } 82 | 83 | //SpawnConnectionIfNeeded creates a new connection if connection pool has not reached size controlled by Config.PoolSize 84 | func (cp *AbstractConnectionProvider) SpawnConnectionIfNeeded() { 85 | config := cp.ConnectionProvider.Config() 86 | if config.PoolSize == 0 { 87 | config.PoolSize = 1 88 | } 89 | connectionPool := cp.ConnectionProvider.ConnectionPool() 90 | for i := len(connectionPool); i < config.PoolSize; i++ { 91 | connection, err := cp.ConnectionProvider.NewConnection() 92 | if err != nil { 93 | log.Printf("failed to create connection %v\n", err) 94 | break 95 | } 96 | 97 | select { 98 | case <-time.After(1 * time.Second): 99 | log.Fatalf("failed to add connection to queue (size: %v, cap:%v)", len(connectionPool), cap(connectionPool)) 100 | case connectionPool <- connection: 101 | } 102 | 103 | } 104 | } 105 | 106 | //Close closes a datastore connection or returns it to the pool (Config.PoolSize and Config.MaxPoolSize). 107 | func (cp *AbstractConnectionProvider) Close() error { 108 | 109 | poolsize := len(cp.connectionPool) 110 | for i := 0; i < poolsize; i++ { 111 | var connection Connection 112 | select { 113 | case <-time.After(1 * time.Second): 114 | case connection = <-cp.connectionPool: 115 | err := connection.CloseNow() 116 | if err != nil { 117 | return err 118 | } 119 | } 120 | } 121 | 122 | // 防止池子中再回收进新连接 123 | if len(cp.connectionPool) > 0 { 124 | cp.Close() 125 | } 126 | 127 | return nil 128 | } 129 | 130 | //Get returns a new datastore connection or error. 131 | func (cp *AbstractConnectionProvider) Get() (Connection, error) { 132 | cp.ConnectionProvider.SpawnConnectionIfNeeded() 133 | connectionPool := cp.ConnectionProvider.ConnectionPool() 134 | 135 | var result Connection 136 | select { 137 | case <-time.After(100 * time.Millisecond): 138 | { 139 | Logf("unable to acquire connection from pool, creating new connection ...") 140 | } 141 | case result = <-connectionPool: 142 | } 143 | if result == nil { 144 | var err error 145 | result, err = cp.ConnectionProvider.NewConnection() 146 | if err != nil { 147 | return nil, err 148 | } 149 | } 150 | return result, nil 151 | } 152 | 153 | //NewAbstractConnectionProvider create a new AbstractConnectionProvider 154 | func NewAbstractConnectionProvider(config *Config, connectionPool chan Connection, connectionProvider ConnectionProvider) *AbstractConnectionProvider { 155 | return &AbstractConnectionProvider{config: config, connectionPool: connectionPool, ConnectionProvider: connectionProvider} 156 | } 157 | -------------------------------------------------------------------------------- /connection_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "errors" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/dsc" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestConnectionConfig(t *testing.T) { 12 | config := &dsc.Config{} 13 | connection := newTestConnection(config) 14 | assert.Equal(t, config, connection.Config()) 15 | assert.Nil(t, connection.Begin()) 16 | assert.Nil(t, connection.Commit()) 17 | assert.Nil(t, connection.Rollback()) 18 | } 19 | 20 | func TestConnectionProvider(t *testing.T) { 21 | { 22 | config := &dsc.Config{} 23 | provider := newTestConnectionProvider(config) 24 | connection, err := provider.Get() 25 | assert.Nil(t, err) 26 | assert.NotNil(t, connection) 27 | provider.Close() 28 | } 29 | { 30 | config := &dsc.Config{MaxPoolSize: 3, PoolSize: 2} 31 | provider := newTestConnectionProvider(config) 32 | for i := 0; i < 3; i++ { 33 | connection, err := provider.Get() 34 | assert.Nil(t, err) 35 | assert.NotNil(t, connection) 36 | } 37 | provider.Close() 38 | } 39 | 40 | { 41 | config := &dsc.Config{MaxPoolSize: 3, PoolSize: 2} 42 | provider := newTestConnectionProvider(config) 43 | provider.error = errors.New("Test error") 44 | 45 | _, err := provider.Get() 46 | assert.NotNil(t, err) 47 | 48 | } 49 | 50 | { 51 | config := &dsc.Config{MaxPoolSize: 3, PoolSize: 2} 52 | provider := newTestConnectionProvider(config) 53 | sleep := int(2 * time.Second) 54 | provider.sleep = &sleep 55 | 56 | connection, err := provider.Get() 57 | assert.Nil(t, err) 58 | assert.NotNil(t, connection) 59 | } 60 | 61 | } 62 | 63 | type testConnection struct { 64 | *dsc.AbstractConnection 65 | } 66 | 67 | func (t *testConnection) CloseNow() error { 68 | return nil 69 | } 70 | 71 | func newTestConnection(config *dsc.Config) dsc.Connection { 72 | 73 | if config.MaxPoolSize == 0 { 74 | config.MaxPoolSize = 1 75 | } 76 | testConnection := &testConnection{} 77 | connection := dsc.NewAbstractConnection(config, make(chan dsc.Connection, config.MaxPoolSize), testConnection) 78 | testConnection.AbstractConnection = connection 79 | return testConnection 80 | } 81 | 82 | type testConnectionProvider struct { 83 | *dsc.AbstractConnectionProvider 84 | sleep *int 85 | error error 86 | } 87 | 88 | func (cp *testConnectionProvider) NewConnection() (dsc.Connection, error) { 89 | if cp.sleep != nil { 90 | time.Sleep(time.Duration(*cp.sleep)) 91 | } 92 | 93 | if cp.error != nil { 94 | return nil, cp.error 95 | } 96 | config := cp.Config() 97 | var testConnection = &testConnection{} 98 | var connection = testConnection 99 | var super = dsc.NewAbstractConnection(config, cp.ConnectionProvider.ConnectionPool(), connection) 100 | testConnection.AbstractConnection = super 101 | return connection, nil 102 | } 103 | 104 | func newTestConnectionProvider(config *dsc.Config) *testConnectionProvider { 105 | if config.MaxPoolSize == 0 { 106 | config.MaxPoolSize = 1 107 | } 108 | testConnectionProvider := &testConnectionProvider{} 109 | var connectionProvider = testConnectionProvider 110 | super := dsc.NewAbstractConnectionProvider(config, make(chan dsc.Connection, config.MaxPoolSize), connectionProvider) 111 | testConnectionProvider.AbstractConnectionProvider = super 112 | return connectionProvider 113 | } 114 | -------------------------------------------------------------------------------- /dialect.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var errUnsupportedOperation = errors.New("unsupported operation") 8 | 9 | type DefaultDialect struct{} 10 | 11 | func (d DefaultDialect) GetDatastores(manager Manager) ([]string, error) { 12 | return nil, nil 13 | } 14 | 15 | func (d DefaultDialect) GetTables(manager Manager, datastore string) ([]string, error) { 16 | return nil, nil 17 | } 18 | 19 | func (d DefaultDialect) DropTable(manager Manager, datastore string, table string) error { 20 | return nil 21 | } 22 | 23 | func (d DefaultDialect) IsKeyCheckSwitchSessionLevel() bool { 24 | return true 25 | } 26 | 27 | func (d DefaultDialect) CreateTable(manager Manager, datastore string, table string, options interface{}) error { 28 | return nil 29 | } 30 | 31 | func (d DefaultDialect) CanCreateDatastore(manager Manager) bool { 32 | return false 33 | } 34 | 35 | func (d DefaultDialect) GetColumns(manager Manager, datastore, table string) ([]Column, error) { 36 | return []Column{}, nil 37 | } 38 | 39 | func (d DefaultDialect) CreateDatastore(manager Manager, datastore string) error { 40 | return errUnsupportedOperation 41 | } 42 | 43 | func (d DefaultDialect) CanDropDatastore(manager Manager) bool { 44 | return false 45 | } 46 | 47 | func (d DefaultDialect) DropDatastore(manager Manager, datastore string) error { 48 | return errUnsupportedOperation 49 | } 50 | 51 | func (d DefaultDialect) GetCurrentDatastore(manager Manager) (string, error) { 52 | return "", nil 53 | } 54 | 55 | func (d DefaultDialect) BulkInsertType() string { 56 | return "" 57 | } 58 | 59 | func (d DefaultDialect) GetSequence(manager Manager, name string) (int64, error) { 60 | return 0, errUnsupportedOperation 61 | } 62 | 63 | func (d DefaultDialect) GetKeyName(manager Manager, datastore, table string) string { 64 | return "" 65 | } 66 | 67 | func (d DefaultDialect) IsAutoincrement(manager Manager, datastore, table string) bool { 68 | return false 69 | } 70 | 71 | func (d DefaultDialect) CanPersistBatch() bool { 72 | return false 73 | } 74 | 75 | func (d DefaultDialect) Init(manager Manager, connection Connection) error { 76 | return nil 77 | } 78 | 79 | //DisableForeignKeyCheck disables fk check 80 | func (d DefaultDialect) DisableForeignKeyCheck(manager Manager, connection Connection) error { 81 | return nil 82 | } 83 | 84 | //DisableForeignKeyCheck disables fk check 85 | func (d DefaultDialect) EnableForeignKeyCheck(manager Manager, connection Connection) error { 86 | return nil 87 | } 88 | 89 | func (d DefaultDialect) NormalizeSQL(SQL string) string { 90 | return SQL 91 | } 92 | 93 | func (d DefaultDialect) ShowCreateTable(manager Manager, table string) (string, error) { 94 | return "", errors.New("unsupported") 95 | } 96 | 97 | func (d DefaultDialect) CanHandleTransaction() bool { 98 | return false 99 | } 100 | 101 | //EachTable iterates each datastore table 102 | func (d DefaultDialect) EachTable(manager Manager, handler func(table string) error) error { 103 | dbname, err := d.GetCurrentDatastore(manager) 104 | if err != nil { 105 | return err 106 | } 107 | tables, err := d.GetTables(manager, dbname) 108 | if err != nil { 109 | return err 110 | } 111 | for _, table := range tables { 112 | if err := handler(table); err != nil { 113 | return err 114 | } 115 | } 116 | return err 117 | } 118 | 119 | func (d DefaultDialect) Ping(manager Manager) error { 120 | return nil 121 | } 122 | 123 | //NewDefaultDialect crates a defulat dialect. DefaultDialect can be used as a embeddable struct (super class). 124 | func NewDefaultDialect() DatastoreDialect { 125 | return &DefaultDialect{} 126 | } 127 | -------------------------------------------------------------------------------- /dialect_registry.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | var dialect = NewDefaultDialect() 4 | var datastoreDialectableRegistry = make(map[string]DatastoreDialect) 5 | 6 | //RegisterDatastoreDialect register DatastoreDialect for a driver. 7 | func RegisterDatastoreDialect(driver string, dialectable DatastoreDialect) { 8 | datastoreDialectableRegistry[driver] = dialectable 9 | } 10 | 11 | //GetDatastoreDialect returns DatastoreDialect for passed in driver. 12 | func GetDatastoreDialect(driver string) DatastoreDialect { 13 | if result, ok := datastoreDialectableRegistry[driver]; ok { 14 | return result 15 | } 16 | if isSQLDatabase(driver) { 17 | RegisterDatastoreDialect(driver, newAnsiSQLDialect()) 18 | return datastoreDialectableRegistry[driver] 19 | } 20 | panic("failed to lookup datastore dialect: " + driver) 21 | } 22 | 23 | func init() { 24 | RegisterDatastoreDialect("mysql", newMySQLDialect()) 25 | RegisterDatastoreDialect("pg", newPgDialect()) 26 | RegisterDatastoreDialect("postgres", newPgDialect()) 27 | RegisterDatastoreDialect("ora", newOraDialect()) 28 | RegisterDatastoreDialect("oci8", newOraDialect()) 29 | RegisterDatastoreDialect("sqlserver", newMsSQLDialect()) 30 | RegisterDatastoreDialect("sqlite3", newSQLLiteDialect()) 31 | RegisterDatastoreDialect("cql", newCasandraDialect()) 32 | RegisterDatastoreDialect("vertica", newVerticaDialect()) 33 | RegisterDatastoreDialect("odbc", newOdbcDialect()) 34 | RegisterDatastoreDialect("ndjson", &fileDialect{}) 35 | RegisterDatastoreDialect("tsv", &fileDialect{}) 36 | RegisterDatastoreDialect("csv", &fileDialect{}) 37 | 38 | } 39 | -------------------------------------------------------------------------------- /dialect_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/dsc" 7 | "testing" 8 | ) 9 | 10 | func TestDialect(t *testing.T) { 11 | dialect := dsc.NewDefaultDialect() 12 | var manager dsc.Manager 13 | 14 | { 15 | datastores, err := dialect.GetDatastores(manager) 16 | assert.Nil(t, datastores) 17 | assert.Nil(t, err) 18 | } 19 | 20 | { 21 | tables, err := dialect.GetTables(manager, "test") 22 | assert.Nil(t, tables) 23 | assert.Nil(t, err) 24 | } 25 | 26 | { 27 | err := dialect.DropTable(manager, "test", "test") 28 | assert.Nil(t, err) 29 | } 30 | 31 | { 32 | err := dialect.CreateTable(manager, "test", "test", "") 33 | assert.Nil(t, err) 34 | } 35 | 36 | { 37 | check := dialect.CanCreateDatastore(manager) 38 | assert.False(t, check) 39 | } 40 | 41 | { 42 | err := dialect.CreateDatastore(manager, "test") 43 | assert.NotNil(t, err) 44 | } 45 | 46 | { 47 | check := dialect.CanDropDatastore(manager) 48 | assert.False(t, check) 49 | } 50 | 51 | { 52 | err := dialect.DropDatastore(manager, "test") 53 | assert.NotNil(t, err) 54 | } 55 | 56 | { 57 | store, err := dialect.GetCurrentDatastore(manager) 58 | assert.Equal(t, "", store) 59 | assert.Nil(t, err) 60 | } 61 | { 62 | seq, err := dialect.GetSequence(manager, "test") 63 | assert.EqualValues(t, 0, seq) 64 | assert.NotNil(t, err) 65 | } 66 | 67 | { 68 | check := dialect.CanPersistBatch() 69 | assert.False(t, check) 70 | } 71 | 72 | } 73 | 74 | func TestDialectRegistry(t *testing.T) { 75 | dialect := dsc.GetDatastoreDialect("ndjson") 76 | assert.NotNil(t, dialect) 77 | 78 | defer func() { 79 | if err := recover(); err != nil { 80 | expected := "failed to lookup datastore dialect: test" 81 | actual := fmt.Sprintf("%v", err) 82 | assert.Equal(t, actual, expected, "Assert Kind") 83 | } 84 | }() 85 | dsc.GetDatastoreDialect("test") 86 | } 87 | -------------------------------------------------------------------------------- /dml_builder.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | var querySQLTemplate = "SELECT %v FROM %v WHERE %v" 9 | var insertSQLTemplate = "INSERT INTO %v(%v) VALUES(%v)" 10 | var updateSQLTemplate = "UPDATE %v SET %v WHERE %v" 11 | var deleteSQLTemplate = "DELETE FROM %v WHERE %v" 12 | 13 | //DmlBuilder represents a insert,update,delete statement builder. 14 | type DmlBuilder struct { 15 | TableDescriptor *TableDescriptor 16 | NonPkColumns *[]string 17 | Columns *[]string 18 | InsertSQL string 19 | UpdateSQL string 20 | DeleteSQL string 21 | } 22 | 23 | func (b *DmlBuilder) readValues(columns []string, valueProvider func(column string) interface{}) []interface{} { 24 | var result = make([]interface{}, len(columns)) 25 | for i, column := range columns { 26 | result[i] = valueProvider(column) 27 | } 28 | return result 29 | } 30 | 31 | func (b *DmlBuilder) readInsertValues(valueProvider func(column string) interface{}) []interface{} { 32 | var columns []string 33 | if b.TableDescriptor.Autoincrement { 34 | columns = *b.NonPkColumns 35 | } else { 36 | columns = *b.Columns 37 | } 38 | return b.readValues(columns, valueProvider) 39 | } 40 | 41 | //GetParametrizedSQL returns GetParametrizedSQL for passed in sqlType, and value provider. 42 | func (b *DmlBuilder) GetParametrizedSQL(sqlType int, valueProvider func(column string) interface{}) *ParametrizedSQL { 43 | switch sqlType { 44 | case SQLTypeInsert: 45 | return &ParametrizedSQL{ 46 | SQL: b.InsertSQL, 47 | Values: b.readInsertValues(valueProvider), 48 | Type: SQLTypeInsert, 49 | } 50 | 51 | case SQLTypeUpdate: 52 | return &ParametrizedSQL{ 53 | SQL: b.UpdateSQL, 54 | Values: b.readValues(*b.Columns, valueProvider), 55 | Type: SQLTypeUpdate, 56 | } 57 | case SQLTypeDelete: 58 | return &ParametrizedSQL{ 59 | SQL: b.DeleteSQL, 60 | Values: b.readValues(b.TableDescriptor.PkColumns, valueProvider), 61 | Type: SQLTypeDelete, 62 | } 63 | } 64 | panic(fmt.Sprintf("Unsupprted sqltype:%v", sqlType)) 65 | } 66 | 67 | func buildAssignValueSQL(columns []string, separator string) string { 68 | result := "" 69 | for _, column := range columns { 70 | if len(result) > 0 { 71 | result = result + separator 72 | } 73 | result = result + " " + column + " = ?" 74 | } 75 | return result 76 | } 77 | 78 | func buildInsertSQL(descriptor *TableDescriptor, columns []string, nonPkColumns []string) string { 79 | var insertColumns []string 80 | var insertValues []string = make([]string, 0) 81 | if descriptor.Autoincrement { 82 | insertColumns = append(insertColumns, nonPkColumns...) 83 | } else { 84 | insertColumns = append(insertColumns, columns...) 85 | } 86 | for range insertColumns { 87 | insertValues = append(insertValues, "?") 88 | } 89 | 90 | updateReserved(insertColumns) 91 | return fmt.Sprintf(insertSQLTemplate, descriptor.Table, strings.Join(insertColumns, ","), strings.Join(insertValues, ",")) 92 | } 93 | 94 | func buildUpdateSQL(descriptor *TableDescriptor, nonPkColumns []string) string { 95 | updateReserved(nonPkColumns) 96 | pk := append([]string{}, descriptor.PkColumns...) 97 | updateReserved(pk) 98 | return fmt.Sprintf(updateSQLTemplate, descriptor.Table, buildAssignValueSQL(nonPkColumns, ","), buildAssignValueSQL(pk, " AND ")) 99 | } 100 | 101 | func buildDeleteSQL(descriptor *TableDescriptor) string { 102 | pk := append([]string{}, descriptor.PkColumns...) 103 | updateReserved(pk) 104 | return fmt.Sprintf(deleteSQLTemplate, descriptor.Table, buildAssignValueSQL(pk, " AND ")) 105 | } 106 | 107 | //NewDmlBuilder returns a new DmlBuilder for passed in table descriptor. 108 | func NewDmlBuilder(descriptor *TableDescriptor) *DmlBuilder { 109 | pkMap := make(map[string]int) 110 | 111 | if len(descriptor.PkColumns) > 0 { 112 | for i, k := range descriptor.PkColumns { 113 | pkMap[strings.ToLower(k)] = i 114 | } 115 | } 116 | var nonPkColumns = make([]string, 0) 117 | for _, column := range descriptor.Columns { 118 | idx, ok := pkMap[strings.ToLower(column)] 119 | if ok { //update pk with right case 120 | descriptor.PkColumns[idx] = column 121 | } else { 122 | nonPkColumns = append(nonPkColumns, column) 123 | } 124 | } 125 | 126 | var columns = make([]string, 0) 127 | columns = append(columns, nonPkColumns...) 128 | columns = append(columns, descriptor.PkColumns...) 129 | return &DmlBuilder{ 130 | TableDescriptor: descriptor, 131 | NonPkColumns: &nonPkColumns, 132 | Columns: &columns, 133 | InsertSQL: buildInsertSQL(descriptor, columns, nonPkColumns), 134 | UpdateSQL: buildUpdateSQL(descriptor, nonPkColumns), 135 | DeleteSQL: buildDeleteSQL(descriptor), 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /dml_provider.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | 7 | "github.com/viant/toolbox" 8 | ) 9 | 10 | //metaDmlProvider represents tag mapping base dml provider. 11 | type metaDmlProvider struct { 12 | dmlBuilder *DmlBuilder 13 | columnToFieldNameMap map[string](map[string]string) 14 | } 15 | 16 | func (p *metaDmlProvider) pkColumns() []string { 17 | return p.dmlBuilder.TableDescriptor.PkColumns 18 | } 19 | 20 | //Key returns primary key values 21 | func (p *metaDmlProvider) Key(instance interface{}) []interface{} { 22 | result := p.readValues(instance, p.pkColumns()) 23 | return result 24 | } 25 | 26 | //SetKey sets a key on passed in instance pointer 27 | func (p *metaDmlProvider) SetKey(instancePointer interface{}, seq int64) { 28 | toolbox.AssertPointerKind(instancePointer, reflect.Struct, "instance") 29 | key := p.pkColumns()[0] 30 | columnSetting := p.columnToFieldNameMap[strings.ToLower(key)] 31 | if field, found := columnSetting["fieldName"]; found { 32 | var reflectable = reflect.ValueOf(instancePointer) 33 | if reflectable.Kind() == reflect.Ptr { 34 | field := reflectable.Elem().FieldByName(field) 35 | field.SetInt(seq) 36 | } 37 | 38 | } 39 | } 40 | 41 | func (p *metaDmlProvider) readValues(instance interface{}, columns []string) []interface{} { 42 | var result = make([]interface{}, len(columns)) 43 | var reflectable = reflect.ValueOf(instance) 44 | if reflectable.Kind() == reflect.Ptr { 45 | reflectable = reflectable.Elem() 46 | } 47 | for i, column := range columns { 48 | result[i] = p.readValue(reflectable, column) 49 | } 50 | return result 51 | } 52 | 53 | func (p *metaDmlProvider) mapValueIfNeeded(value interface{}, column string, columnSetting map[string]string) interface{} { 54 | if mapping, found := columnSetting["valueMap"]; found { 55 | stringValue := toolbox.AsString(value) 56 | reverseMapValue := toolbox.MakeReverseStringMap(mapping, ":", ",") 57 | if mappedValue, ok := reverseMapValue[stringValue]; ok { 58 | return mappedValue 59 | } 60 | } 61 | return value 62 | } 63 | 64 | func (p *metaDmlProvider) readValue(source reflect.Value, column string) interface{} { 65 | columnSetting := p.columnToFieldNameMap[strings.ToLower(column)] 66 | if fieldName, ok := columnSetting["fieldName"]; ok { 67 | field := source.FieldByName(fieldName) 68 | value := toolbox.UnwrapValue(&field) 69 | if toolbox.IsZero(field) && value != nil && toolbox.IsStruct(value) { 70 | value = nil 71 | } 72 | return p.mapValueIfNeeded(value, column, columnSetting) 73 | } 74 | return nil 75 | } 76 | 77 | //Get returns a ParametrizedSQL for specified sqlType and target instance. 78 | func (p *metaDmlProvider) Get(sqlType int, instance interface{}) *ParametrizedSQL { 79 | var reflectable = reflect.ValueOf(instance) 80 | if reflectable.Kind() == reflect.Ptr { 81 | reflectable = reflectable.Elem() 82 | } 83 | //toolbox.AssertKind(instance, reflect.Type, "instance") 84 | return p.dmlBuilder.GetParametrizedSQL(sqlType, func(column string) interface{} { 85 | return p.readValue(reflectable, column) 86 | }) 87 | } 88 | 89 | func newMetaDmlProvider(table string, targetType reflect.Type) (DmlProvider, error) { 90 | descriptor, err := NewTableDescriptor(table, targetType) 91 | if err != nil { 92 | return nil, err 93 | } 94 | dmlBuilder := NewDmlBuilder(descriptor) 95 | return &metaDmlProvider{dmlBuilder: dmlBuilder, 96 | columnToFieldNameMap: toolbox.NewFieldSettingByKey(targetType, "column")}, nil 97 | } 98 | 99 | //NewDmlProviderIfNeeded returns a new NewDmlProvider for a table and target type if passed provider was nil. 100 | func NewDmlProviderIfNeeded(provider DmlProvider, table string, targetType reflect.Type) (DmlProvider, error) { 101 | if provider != nil { 102 | return provider, nil 103 | } 104 | return newMetaDmlProvider(table, targetType) 105 | } 106 | 107 | //NewKeyGetterIfNeeded returns a new key getter if supplied keyGetter was nil for the target type 108 | func NewKeyGetterIfNeeded(keyGetter KeyGetter, table string, targetType reflect.Type) (KeyGetter, error) { 109 | if keyGetter != nil { 110 | return keyGetter, nil 111 | } 112 | return newMetaDmlProvider(table, targetType) 113 | } 114 | 115 | type mapDmlProvider struct { 116 | tableDescriptor *TableDescriptor 117 | dmlBuilder *DmlBuilder 118 | } 119 | 120 | func (p *mapDmlProvider) Key(instance interface{}) []interface{} { 121 | var record = toolbox.AsMap(instance) 122 | var result = make([]interface{}, len(p.tableDescriptor.PkColumns)) 123 | for i, column := range p.tableDescriptor.PkColumns { 124 | result[i] = record[column] 125 | } 126 | return result 127 | } 128 | 129 | func (p *mapDmlProvider) SetKey(instance interface{}, seq int64) { 130 | var record = toolbox.AsMap(instance) 131 | record[p.tableDescriptor.PkColumns[0]] = seq 132 | } 133 | 134 | func (p *mapDmlProvider) Get(sqlType int, instance interface{}) *ParametrizedSQL { 135 | var record = toolbox.AsMap(instance) 136 | return p.dmlBuilder.GetParametrizedSQL(sqlType, func(column string) interface{} { 137 | return record[column] 138 | }) 139 | } 140 | 141 | func NewMapDmlProvider(descriptor *TableDescriptor) DmlProvider { 142 | var result = &mapDmlProvider{ 143 | tableDescriptor: descriptor, 144 | dmlBuilder: NewDmlBuilder(descriptor), 145 | } 146 | return result 147 | } 148 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package dsc - datastore connectivity library 3 | This library provides connection capabilities to SQL, noSQL datastores or structured files providing sql layer on top of it. 4 | 5 | For native database/sql it is just a ("database/sql") proxy, and for noSQL it supports simple SQL that is being translated to 6 | put/get,scan,batch native NoSQL operations. 7 | 8 | 9 | Datastore Manager implements read, batch (no insert nor update), and delete operations. 10 | Read operation requires data record mapper, 11 | Persist operation requires dml provider. 12 | Delete operation requries key provider. 13 | 14 | Datastore Manager provides default record mapper and dml/key provider for a struct, if no actual implementation is passed in. 15 | 16 | The following tags can be used 17 | 18 | 1 column - name of datastore field/column 19 | 20 | 2 autoincrement - boolean flag to use autoincrement, in this case on insert the value can be automatically set back on application model class 21 | 22 | 3 primaryKey - boolean flag primary key 23 | 24 | 4 dateLayout - date layout check string to time.Time conversion 25 | 26 | 4 dateFormat - date format check java simple date format 27 | 28 | 5 sequence - name of sequence used to generate next id 29 | 30 | 6 transient - boolean flag to not map a field with record data 31 | 32 | 7 valueMap - value maping that will be applied after fetching a record and before writing it to datastore. 33 | For instance valueMap:"yes:true,no:false" would map yes to true, no to false 34 | 35 | 36 | 37 | Usage: 38 | 39 | type Interest struct { 40 | Id int `autoincrement:"true"` 41 | Name string 42 | ExpiryTimeInSecond int `column:"expiry"` 43 | Category string 44 | } 45 | 46 | manager := factory.CreateFromURL("file:///etc/mystore-config.json") 47 | interest := Interest{} 48 | 49 | intersts = make([]Interest, 0) 50 | err:= manager.ReadAll(&interests, SELECT id, name, expiry, category FROM interests", nil ,nil) 51 | if err != nil { 52 | panic(err.Error()) 53 | } 54 | ... 55 | 56 | inserted, updated, err:= manager.PersistAll(&intersts, "interests", nil) 57 | if err != nil { 58 | panic(err.Error()) 59 | } 60 | 61 | */ 62 | package dsc 63 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | This package describes the datastore connectivity (dsc) API in detail. 4 | 5 | 6 | 7 | - [Read operation](#Read-operation) 8 | - [Persist operation](#Persist-operation) 9 | - [Delete operation](#Delete-operation) 10 | - [Execute SQL](#Execute-SQL) 11 | - [API Reference](#API-Reference) 12 | 13 | 14 | ## Getting instance of Manager 15 | 16 | Each datastore may use different configuration parameters, so see for specific vendor/plugin details 17 | The best was of getting manager is by using ManagerFactory. 18 | 19 | 20 | ```go 21 | 22 | factory := dsc.NewManagerFactory() 23 | config := dsc.NewConfig("mysql", "[user]:[password]@[url]", "user:myuser,password:dev,url:tcp(127.0.0.1:3306)/mydbname?parseTime=true") 24 | manager, err := factory.Create(config) 25 | if err != nil { 26 | panic(err.Error()) 27 | } 28 | 29 | ... 30 | manager := factory.CreateFromURL("file:///etc/myapp/datastore.json") 31 | 32 | ``` 33 | 34 | 35 | /etc/myapp/datastore.json 36 | ```json 37 | { 38 | "DriverName": "bigquery", 39 | "Parameters": { 40 | "serviceAccountId": "****@developer.gserviceaccount.com", 41 | "privateKeyPath": "/etc/test_service.pem", 42 | "projectId": "spheric-arcadia-98015", 43 | "datasetId": "MyDataset", 44 | "dateFormat": "yyyy-MM-dd HH:mm:ss z" 45 | } 46 | } 47 | 48 | ``` 49 | 50 | 51 | Connection configuration examples: 52 | 53 | 54 | 55 | 56 | |Datastore | driver name | Descriptor | Example of encoded parameters | Native driver | 57 | |---|---|---|---|---| 58 | |MySql|mysql|[user]:[password]@[url]|**user**:myuser,**password**:dev,**url**:tcp(127.0.0.1:3306)/mydbname?parseTime=true|github.com/go-sql-driver/mysql| 59 | |Postgres|pg | postgres://[user]:[password]@[url] |**user**:myuser,**password**:dev,**url**:localhost/mydbname?sslmode=verify-full|github.com/lib/pq| 60 | |MS SQL Server|mssql|server=[server];user id=[user];password:[password];database:[database]|**user**:myuser,**password**:dev,**sevrver**:localhost,**database**:mydbname|github.com/denisenkom/go-mssqldb| 61 | |Oracle|ora|[user]/[password]@[url]|**user**:myuser,**password**:dev,**url**:localhost:1521/mysid|github.com/rana/ora| 62 | |BigQuery|bigquery|n/a|**serviceAccountId**:myseriveAccount,**privateKeyPath**:/somepath/key.pem,**projectId**:myproject,**datasetId**:MyDataset,**dateFormat**:yyyy-MM-dd hh:mm:ss z|github.com/viant/bgc| 63 | |Aerospike|aerospike|n/a|**host**:127.0.0.1,**port**:3000,**namespace**:test,**dateFormat**:yyyy-MM-dd hh:mm:ss|github.com/viant/asc| 64 | |Casandra|cql|127.0.0.1?keyspace=mydb||github.com/MichaelS11/go-cql-driver| 65 | |MongoDB|mgc|n/a|**host**:127.0.0.1,**dbname**:mydb|github.com/adrianwit/mgc| 66 | |Firebase|fbc|n/a|**DatabaseURL**:https://myproject.firebaseio.com,**ProjecrID**:myproject|github.com/adrianwit/fbc| 67 | |Firestore|fsc|n/a||github.com/adrianwit/fsc| 68 | |DynamoDB|dyndb|n/a| parameters: key, secret, region or use credentials with path to json file with these attributes |github.com/adrianwit/dyndbc| 69 | |New line delimiter JSON|ndjson|n/a|**url**:someUrl,**ext**:.json,**dateFormat**:yyyy-MM-dd hh:mm:ss|github.com/viant/dsc| 70 | 71 | 72 | 73 | Note that sql drivers use driver name and descriptor as sql.Open(driver, descriptor) 74 | 75 | 76 | 77 | ## Tags meta mapping 78 | 79 | This library allows custom reading and persisting records, but also automatic field mapping 80 | In this scenario if column tag is not defined the filed name will be used. 81 | 82 | 83 | |Tag name | Description | 84 | |---|---| 85 | |column | Column name of table in datastore | 86 | |primaryKey| Flag indicating if column is part of primary key| 87 | |autoincrement| Flag indicating if column uses autoincrement | 88 | |sequence| Sequence name (not implemented yey) | 89 | |dateFormat | SimpleDateFormat date format format style | 90 | |dateLayout | Golang date layout | 91 | |transient | Ignores filed in datastore operations | 92 | |valueMap | value mapping after fetching record, and before persisting data| 93 | 94 | 95 | ```go 96 | 97 | 98 | type User struct { 99 | Id int `autoincrement:"true"` 100 | Name string `column:"name"` 101 | DateOfBirth time.Time `column:"dob" dateFormat:"yyyyy-MM-dd HH:mm:ss"` 102 | SessionId string `transient:"true"` 103 | } 104 | 105 | 106 | 107 | 108 | ``` 109 | 110 | 111 | ## Read operation 112 | 113 | High-level datastore read operation involves the following: 114 | 1 Opening connection 115 | 2 Opening cursor/statement with SQL specyfing data source 116 | 3 Fetching data 117 | * Mapping each fetched data record into application domain class 118 | 4 Closing cursor/statement 119 | 5 Closing connection 120 | 121 | This api has been design to hide the most of these operation. 122 | Read operation just requires SELECT statement and optionally record mapper. 123 | 124 | Behind the scene, for NoSQL datastore this library comes with basic Query parser to easily map structured Query into NoSQL fields. 125 | 126 | 127 | ### Reading with with default record mapper 128 | 129 | It is possible to use default MetaRecordMapper that uses tags defined in application model class. The following tags are supported 130 | 1 column - name of datastore field/column 131 | 2 autoincrement - boolean flag to use autoincrement, in this case on insert the value can be automatically set back on application model class 132 | 3 primaryKey - boolean flag primary key 133 | 4 dateLayout - date layout check string to time.Time conversion 134 | 4 dateFormat - date format check java simple date format 135 | 5 sequence - name of sequence used to generate next id 136 | 6 transient - boolean flag to not map a field with record data 137 | 7 valueMap - value maping that will be applied after fetching a record and before writing it to datastore. 138 | For instance valueMap:"yes:true,no:false" would map yes to true, no to false 139 | 140 | 141 | 142 | ```go 143 | 144 | type User struct { 145 | Id int `autoincrement:"true"` 146 | Username string //if column and field are same - no mapping needed 147 | LastAccessTime *time.Time `column:"last_access_time" dateLayout:"2006-01-02 15:04:05"` 148 | CachedField bool `transient:"true"` 149 | } 150 | 151 | //reading single record 152 | user := User{} 153 | success, err:= manager.ReadSingle(&user, "SELECT id, username FROM users WHERE id = ?", []interface{}{1}, nil) 154 | 155 | 156 | //reading all records 157 | var users=make([]User, 0) 158 | err:= manager.ReadAll(&users, "SELECT id, username FROM users", nil, nil) 159 | 160 | 161 | ``` 162 | 163 | 164 | ### Reading with custom record mapper 165 | 166 | In this scenario RecordMapper is responsible for mapping datatstore data record into application model class. 167 | 168 | ```go 169 | 170 | type UserRecordMapper struct {} 171 | 172 | func (this *UserRecordMapper) Map(scanner dsc.Scanner) (interface{}, error) { 173 | user := User{} 174 | var name = "" 175 | scanner.Scan( 176 | &user.Id, 177 | &name 178 | ) 179 | user.Username := name 180 | 181 | return &user, nil 182 | } 183 | 184 | 185 | //reading single record 186 | user := User{} 187 | var recordMapper dsc.RecordMapper = &UserRecordMapper{} 188 | success, err:= manager.ReadSingle(&user, "SELECT id, username FROM users WHERE id = ?", []interface{}{1}, recordMapper) 189 | ... 190 | 191 | //reading all records 192 | var users=make([]User, 0) 193 | var recordMapper dsc.RecordMapper = &UserRecordMapper{} 194 | err:= manager.ReadAll(&users, "SELECT id, username FROM users", nil, recordMapper) 195 | 196 | 197 | ``` 198 | 199 | ### Reading with custom reading handler 200 | 201 | In this scenario custom reading handler is responsible for mapping data record in the application domain class. 202 | The reading handler returns bool flag to instruct reader to continue reading more records. 203 | 204 | ```go 205 | 206 | // single record reading 207 | user := &User{} 208 | err:= manager.ReadAllWithHandler("SELECT id, username FROM users WHERE id = ?", []interface{}{1}, func(scanner dsc.Scanner) (toContinue bool, err error) { 209 | err =scanner.Scan(&user.Id, &user.Username) 210 | if err != nil { 211 | return fale, err 212 | } 213 | return false, nil //since handler only needs one record it returns false (toContinue) 214 | }) 215 | //... 216 | 217 | // all records reading 218 | var users = make([]User, 0) 219 | err:= manager.ReadAllWithHandler("SELECT id, username FROM users WHERE username LIKE ?", []interface{}{"Adam"}, func(scanner dsc.Scanner) (toContinue bool, err error) { 220 | user := &User{} 221 | err =scanner.Scan(&user.Id, &user.Username) 222 | if err != nil { 223 | return fale, err 224 | } 225 | users := append(users, user) 226 | return true, nil // since handlers needs all recors it returns true 227 | }) 228 | //... 229 | 230 | ``` 231 | 232 | 233 | 234 | ## Persist operation 235 | 236 | High-level persisting operation involves the following: 237 | 1 Opening connection and transaction if possible 238 | 2 Identifying which items needs to be inserted or updated if possible 239 | 3 Inserting all insertable items, retrieve autoincrement/sequence value if available 240 | 4 Updating all updatable items 241 | 5 Issuing commit or rollback transaction if possible 242 | 6 Closing connection 243 | 244 | 245 | This api has been design to hide the most of these operation. 246 | Persist operation just requires data, target table and optionally DML provider, which needs to provide parametrized sql for all INSERT, UPDATE and DELETE operations. 247 | 248 | Behind the scene, for NoSQL datastore this library comes with basic DML parser to easily map structured DML statement into NoSQL similar operation. 249 | 250 | ## Persisting with default DmlProvider 251 | 252 | Similarly like with default MetaRecordMapper, it is possible to use tags definition on application model class to automate all operations required by DmlProvider. 253 | MetaDmlProvider is an implementation that uses meta tags on application model class. 254 | 255 | ```go 256 | 257 | type User struct { 258 | Id int 259 | Username string 260 | } 261 | 262 | 263 | user := User{Id:1, Username:"Sir Edi"} 264 | inserted, updated, err:= manager.PersistSingle(&user, "users", nil) 265 | 266 | 267 | 268 | users := []User { 269 | User{ 270 | Id:1, 271 | Username:"Sir Edi", 272 | }, 273 | User{ 274 | Username:"Bogi", 275 | }, 276 | } 277 | inserted, updated, err:= manager.PersistAll(&users, "users", nil) 278 | ``` 279 | 280 | 281 | ## Persisting with custom DmlProvider 282 | 283 | In this scenario DmlProvider implementation is responsible for providing parametrized sql, primary key columns, and values, updating autoincrement fields if needed. 284 | 285 | ```go 286 | 287 | type User struct { 288 | Id int 289 | Username string 290 | } 291 | 292 | type UserDmlProvider struct {} 293 | 294 | func (this UserDmlProvider) Get(operationType int, instance interface{}) *dsc.ParametrizedSQL { 295 | user:=instance.(User) 296 | switch operationType { 297 | case dsc.SqlTypeInsert: 298 | return &dsc.ParametrizedSQL{ 299 | Sql :"INSERT INTO users(id, username) VALUES(?, ?)", 300 | Values: []interface{}{user.Id, user.Username}, 301 | 302 | } 303 | 304 | case dsc.SqlTypeUpdate: 305 | return &dsc.ParametrizedSQL{ 306 | Sql :"UPDATE users SET username = ? WHERE id = ?", 307 | Values: []interface{}{user.Id, user.Username}, 308 | 309 | } 310 | 311 | } 312 | panic(fmt.Sprintf("unsupported sql type:%v", operationType)) 313 | } 314 | 315 | 316 | func (this UserDmlProvider) SetKey(instance interface{}, seq int64) { 317 | user:=instance.(*User) 318 | user.Id = int(seq) 319 | } 320 | 321 | 322 | func (this UserDmlProvider) Key(instance interface{}) [] interface{} { 323 | user:=instance.(User) 324 | return []interface{}{user.Id} 325 | } 326 | 327 | 328 | func (this UserDmlProvider) PkColumns() []string { 329 | return []string{"id"} 330 | } 331 | 332 | func NewUserDmlProvider() dsc.DmlProvider { 333 | var dmlProvider dsc.DmlProvider = &UserDmlProvider{} 334 | return dmlProvider 335 | } 336 | 337 | 338 | user := User{Id:1, Username:"Sir Edi"} 339 | inserted, updated, err:= manager.PersistSingle(&user, "users", NewUserDmlProvider()) 340 | 341 | users := []User { 342 | User{ 343 | Id:1, 344 | Username:"Sir Edi", 345 | }, 346 | User{ 347 | Username:"Bogi", 348 | }, 349 | } 350 | inserted, updated, err:= manager.PersistAll(&users, "users", NewUserDmlProvider()) 351 | 352 | 353 | ``` 354 | 355 | 356 | 357 | ## Delete operation 358 | 359 | 360 | ### Deleting with default KeyProvider 361 | 362 | 363 | ```go 364 | 365 | type User struct { 366 | Id int 367 | Username string 368 | } 369 | 370 | 371 | user := User{Id:1, Username:"Sir Edi"} 372 | success, err:= manager.DeleteSingle(&user, "users", nil) 373 | 374 | 375 | users := []User { 376 | User{ 377 | Id:1, 378 | }, 379 | User{ 380 | Id:2, 381 | }, 382 | } 383 | deleted, err:= manager.DeleteAll(&users, "users", nil) 384 | ``` 385 | 386 | 387 | ## Execut SQL commands 388 | 389 | To execute any command supported by given datastore Execute method has been provided. 390 | 391 | 392 | ## API Reference 393 | 394 | - [API Interfaces](./../api.go) 395 | 396 | -------------------------------------------------------------------------------- /examples/api.go: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | //InterestService a test service 4 | type InterestService interface { 5 | 6 | //GetById returns interest by id 7 | GetByID(id int) *GetByIDResponse 8 | 9 | //GetByIds returns interests by passed in ids 10 | GetByIDs(id ...int) *GetByIDsResponse 11 | 12 | //PersistTable persists passed in interests 13 | Persist(interests []*Interest) *PersistResponse 14 | 15 | //DeleteById deletes interestes by id. 16 | DeleteByID(id int) *Response 17 | } 18 | 19 | //Response represents a response. 20 | type Response struct { 21 | Status string 22 | Message string 23 | } 24 | 25 | //GetByIDResponse represents get by id response. 26 | type GetByIDResponse struct { 27 | Response 28 | Result *Interest 29 | } 30 | 31 | //GetByIDsResponse represents a get by ids response. 32 | type GetByIDsResponse struct { 33 | Response 34 | Result []*Interest 35 | } 36 | 37 | //PersistResponse represents a persist response. 38 | type PersistResponse struct { 39 | Response 40 | Result []*Interest 41 | } 42 | -------------------------------------------------------------------------------- /examples/domain.go: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | //Interest represents an generic interest. 4 | type Interest struct { 5 | ID int `autoincrement:"true"` 6 | Name string 7 | Category string 8 | Status *bool `valueMap:"yes:true,no:false"` 9 | GroupName string `transient:"true"` 10 | } 11 | -------------------------------------------------------------------------------- /examples/server.go: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | import ( 4 | "fmt" 5 | "github.com/viant/toolbox" 6 | "log" 7 | "net/http" 8 | ) 9 | 10 | var version = "/v1/" 11 | var interestURI = version + "interest/" 12 | 13 | //StartServer starts interests web service 14 | func StartServer(configFile string, port string) { 15 | 16 | service, err := NewInterestService(configFile) 17 | if err != nil { 18 | panic(fmt.Sprintf("failed to create service due to %v", err)) 19 | } 20 | 21 | interestRouter := toolbox.NewServiceRouter( 22 | toolbox.ServiceRouting{ 23 | HTTPMethod: "GET", 24 | URI: interestURI + "{id}", 25 | Handler: service.GetByID, 26 | Parameters: []string{"id"}, 27 | }, 28 | toolbox.ServiceRouting{ 29 | HTTPMethod: "GET", 30 | URI: interestURI + "{ids}", 31 | Handler: service.GetByIDs, 32 | Parameters: []string{"ids"}, 33 | }, 34 | toolbox.ServiceRouting{ 35 | HTTPMethod: "POST", 36 | URI: interestURI, 37 | Handler: service.Persist, 38 | Parameters: []string{"interests"}, 39 | }, 40 | toolbox.ServiceRouting{ 41 | HTTPMethod: "DELETE", 42 | URI: interestURI + "{id}", 43 | Handler: service.DeleteByID, 44 | Parameters: []string{"id"}, 45 | }, 46 | ) 47 | 48 | http.HandleFunc(interestURI, func(response http.ResponseWriter, request *http.Request) { 49 | 50 | errorHandler := func(message string) { 51 | response.WriteHeader(http.StatusInternalServerError) 52 | err := interestRouter.WriteResponse(toolbox.NewJSONEncoderFactory(), &Response{Status: "error", Message: message}, request, response) 53 | if err != nil { 54 | fmt.Printf("failed to write response :%v", err) 55 | } 56 | } 57 | defer func() { 58 | if err := recover(); err != nil { 59 | errorHandler(fmt.Sprintf("%v", err)) 60 | } 61 | }() 62 | 63 | err := interestRouter.Route(response, request) 64 | if err != nil { 65 | errorHandler(fmt.Sprintf("%v", err)) 66 | } 67 | }) 68 | 69 | fmt.Printf("Started interest server on port %v\n", port) 70 | log.Fatal(http.ListenAndServe(":"+port, nil)) 71 | } 72 | -------------------------------------------------------------------------------- /examples/service.go: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/viant/dsc" 7 | "github.com/viant/toolbox" 8 | ) 9 | 10 | type interestServiceImpl struct { 11 | manager dsc.Manager 12 | } 13 | 14 | func setErrorStatus(response *Response, err error) { 15 | response.Message = err.Error() 16 | response.Status = "error" 17 | } 18 | 19 | func (s *interestServiceImpl) GetByID(id int) *GetByIDResponse { 20 | response := &GetByIDResponse{Response: Response{Status: "ok"}} 21 | interest := &Interest{} 22 | success, err := s.manager.ReadSingle(interest, "SELECT id, name, category, status FROM interests WHERE id = ?", []interface{}{id}, nil) 23 | if err != nil { 24 | setErrorStatus(&response.Response, err) 25 | return response 26 | } 27 | 28 | if success { 29 | response.Result = interest 30 | } 31 | return response 32 | } 33 | 34 | func (s *interestServiceImpl) GetByIDs(ids ...int) *GetByIDsResponse { 35 | response := &GetByIDsResponse{Response: Response{Status: "ok"}} 36 | var result = make([]*Interest, 0) 37 | err := s.manager.ReadAll(&result, fmt.Sprintf("SELECT id, name, category, status FROM interests WHERE id IN(%v)", toolbox.JoinAsString(ids, ",")), nil, nil) 38 | if err != nil { 39 | setErrorStatus(&response.Response, err) 40 | return response 41 | } 42 | response.Result = result 43 | return response 44 | } 45 | 46 | func (s *interestServiceImpl) Persist(interests []*Interest) *PersistResponse { 47 | response := &PersistResponse{Response: Response{Status: "ok"}} 48 | inserted, updated, err := s.manager.PersistAll(&interests, "interests", nil) 49 | if err != nil { 50 | setErrorStatus(&response.Response, err) 51 | return response 52 | } 53 | response.Result = interests 54 | response.Message = fmt.Sprintf("inserted %v, updated %v", inserted, updated) 55 | return response 56 | 57 | } 58 | 59 | func (s *interestServiceImpl) DeleteByID(id int) *Response { 60 | response := &Response{Status: "ok"} 61 | _, err := s.manager.DeleteSingle(&Interest{ID: id}, "interests", nil) 62 | if err != nil { 63 | setErrorStatus(response, err) 64 | return response 65 | } 66 | return response 67 | } 68 | 69 | //NewInterestService creates a new interests service 70 | func NewInterestService(configURL string) (InterestService, error) { 71 | 72 | manager, err := dsc.NewManagerFactory().CreateFromURL(configURL) 73 | 74 | if err != nil { 75 | return nil, err 76 | } 77 | var result InterestService = &interestServiceImpl{manager: manager} 78 | return result, nil 79 | } 80 | -------------------------------------------------------------------------------- /examples/service_client.go: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/viant/toolbox" 7 | ) 8 | 9 | type interestServiceClient struct { 10 | url string 11 | } 12 | 13 | func setError(err error, response *Response) { 14 | response.Message = err.Error() 15 | response.Status = "error" 16 | } 17 | 18 | func (c *interestServiceClient) GetByID(id int) *GetByIDResponse { 19 | response := &GetByIDResponse{} 20 | err := toolbox.RouteToService("get", fmt.Sprintf("%v%v", c.url, id), nil, response) 21 | if err != nil { 22 | setError(err, &response.Response) 23 | } 24 | return response 25 | } 26 | 27 | func (c *interestServiceClient) GetByIDs(ids ...int) *GetByIDsResponse { 28 | response := &GetByIDsResponse{} 29 | err := toolbox.RouteToService("get", fmt.Sprintf("%v%v", c.url, toolbox.JoinAsString(ids, ",")), nil, response) 30 | if err != nil { 31 | setError(err, &response.Response) 32 | } 33 | return response 34 | } 35 | 36 | func (c *interestServiceClient) Persist(interests []*Interest) *PersistResponse { 37 | response := &PersistResponse{} 38 | err := toolbox.RouteToService("post", fmt.Sprintf("%v", c.url), &interests, response) 39 | if err != nil { 40 | setError(err, &response.Response) 41 | } 42 | 43 | return response 44 | } 45 | 46 | func (c *interestServiceClient) DeleteByID(id int) *Response { 47 | response := &Response{} 48 | err := toolbox.RouteToService("delete", fmt.Sprintf("%v%v", c.url, id), nil, response) 49 | if err != nil { 50 | setError(err, response) 51 | } 52 | return response 53 | } 54 | 55 | //NewInterestServiceClient creates a new InterestService client 56 | func NewInterestServiceClient(server string) InterestService { 57 | var result InterestService = &interestServiceClient{url: fmt.Sprintf("http://%v%v", server, interestURI)} 58 | return result 59 | } 60 | -------------------------------------------------------------------------------- /examples/service_test.go: -------------------------------------------------------------------------------- 1 | package examples_test 2 | 3 | import ( 4 | _ "github.com/go-sql-driver/mysql" 5 | _ "github.com/viant/dsc" 6 | "testing" 7 | "time" 8 | 9 | "fmt" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/viant/dsc/examples" 12 | "github.com/viant/dsunit" 13 | "github.com/viant/toolbox/url" 14 | ) 15 | 16 | // 17 | //func init() { 18 | // go func() { 19 | // resource := url.NewResource("test/config/store.json") 20 | // examples.StartServer(resource.URL, "8084") 21 | // }() 22 | // time.Sleep(2 * time.Second) 23 | //} 24 | 25 | func getServices() ([]examples.InterestService, error) { 26 | 27 | resource := url.NewResource("test/config/store.json") 28 | local, err := examples.NewInterestService(resource.URL) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | return []examples.InterestService{local}, nil 34 | 35 | //client := examples.NewInterestServiceClient("127.0.0.1:8084") 36 | //return []examples.InterestService{local, client}, nil 37 | 38 | } 39 | 40 | func TestRead(t *testing.T) { 41 | dsunit.InitFromURL(t, "test/init.json") 42 | 43 | dsunit.PrepareFor(t, "mytestdb", "test/", "Read") 44 | services, err := getServices() 45 | if err != nil { 46 | t.Errorf("failed to get services %v", err) 47 | } 48 | 49 | for _, service := range services { 50 | { 51 | response := service.GetByID(1) 52 | assert.Equal(t, "ok", response.Status, response.Message) 53 | assert.NotNil(t, response) 54 | assert.NotNil(t, response.Result) 55 | assert.Equal(t, "Abc", response.Result.Name) 56 | assert.Equal(t, true, *response.Result.Status) 57 | } 58 | 59 | { 60 | response := service.GetByIDs(1, 3) 61 | assert.NotNil(t, response) 62 | assert.Equal(t, "ok", response.Status, response.Message) 63 | assert.Equal(t, 2, len(response.Result)) 64 | assert.Equal(t, "Abc", response.Result[0].Name) 65 | assert.Equal(t, "Cde", response.Result[1].Name) 66 | } 67 | 68 | } 69 | } 70 | 71 | func TestPersist(t *testing.T) { 72 | 73 | services, err := getServices() 74 | if err != nil { 75 | t.Errorf("failed to get services %v", err) 76 | } 77 | 78 | for _, service := range services { 79 | { 80 | falseValue := false 81 | dsunit.InitFromURL(t, "test/init.json") 82 | dsunit.PrepareFor(t, "mytestdb", "test/", "Persist") 83 | response := service.GetByID(1) 84 | assert.Equal(t, "ok", response.Status, response.Message) 85 | 86 | interest := response.Result 87 | interest.Category = "Alphabet" 88 | 89 | var interests = make([]*examples.Interest, 0) 90 | interests = append(interests, interest) 91 | interests = append(interests, &examples.Interest{Name: "Klm", Category: "Ubf", Status: &falseValue, GroupName: "AAAA"}) 92 | persistResponse := service.Persist(interests) 93 | assert.NotNil(t, persistResponse) 94 | assert.Equal(t, "ok", persistResponse.Status, persistResponse.Message) 95 | 96 | assert.NotNil(t, persistResponse.Result) 97 | assert.Equal(t, 2, len(persistResponse.Result)) 98 | dsunit.ExpectFor(t, "mytestdb", dsunit.FullTableDatasetCheckPolicy, "test/", "Persist") 99 | } 100 | } 101 | 102 | } 103 | 104 | func TestPersistAll(t *testing.T) { 105 | dsunit.InitFromURL(t, "test/init.json") 106 | 107 | services, err := getServices() 108 | if err != nil { 109 | t.Errorf("failed to get services %v", err) 110 | } 111 | 112 | service := services[0] 113 | var interests = make([]*examples.Interest, 0) 114 | for i := 1; i <= 1000; i++ { 115 | var status = true 116 | interests = append(interests, &examples.Interest{ 117 | Name: fmt.Sprintf("Name %v", i), 118 | Category: "cat", 119 | Status: &status, 120 | GroupName: "abc", 121 | }) 122 | } 123 | startTime := time.Now().Unix() 124 | persistResponse := service.Persist(interests) 125 | assert.Equal(t, "ok", persistResponse.Status) 126 | endTime := time.Now().Unix() 127 | var elapsed = endTime - startTime 128 | assert.True(t, elapsed < 60) //elapsed should 100k should be under 30 sec 129 | } 130 | 131 | func TestDelete(t *testing.T) { 132 | services, err := getServices() 133 | if err != nil { 134 | t.Errorf("failed to get services %v", err) 135 | } 136 | for _, service := range services { 137 | { 138 | dsunit.InitFromURL(t, "test/init.json") 139 | dsunit.PrepareFor(t, "mytestdb", "test/", "Delete") 140 | 141 | deleteResponse := service.DeleteByID(1) 142 | assert.NotNil(t, deleteResponse) 143 | assert.Equal(t, "ok", deleteResponse.Status, deleteResponse.Message) 144 | 145 | dsunit.ExpectFor(t, "mytestdb", dsunit.FullTableDatasetCheckPolicy, "test/", "Delete") 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /examples/test/config/admin.json: -------------------------------------------------------------------------------- 1 | { 2 | "DriverName": "mysql", 3 | "Descriptor": "[user]:[password]@[url]", 4 | "Parameters": { 5 | "user": "root", 6 | "password": "dev", 7 | "url": "tcp(127.0.0.1:3306)/mysql?parseTime=true" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /examples/test/config/store.json: -------------------------------------------------------------------------------- 1 | { 2 | "DriverName": "mysql", 3 | "Descriptor": "[user]:[password]@[url]", 4 | "Parameters": { 5 | "user": "root", 6 | "password": "dev", 7 | "url": "tcp(127.0.0.1:3306)/mytestdb?parseTime=true" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /examples/test/delete_expect_interests.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "id":"", "name":"Bcd", "category":"EFG"}, 3 | { "id":"", "name":"Cde", "category":"FGH"} 4 | ] -------------------------------------------------------------------------------- /examples/test/delete_prepare_interests.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "name":"Abc", "category":"DEF"}, 3 | { "name":"Bcd", "category":"EFG"}, 4 | { "name":"Cde", "category":"FGH"} 5 | ] -------------------------------------------------------------------------------- /examples/test/init.json: -------------------------------------------------------------------------------- 1 | { 2 | "Datastore": "mytestdb", 3 | "ConfigURL": "test/config/store.json", 4 | "Admin":{ 5 | "Datastore":"mysql", 6 | "ConfigURL":"test/config/admin.json" 7 | }, 8 | "Scripts":[{"URL":"test/script/database.sql"}] 9 | } 10 | 11 | -------------------------------------------------------------------------------- /examples/test/persist_expect_interests.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "id":"", "name":"Abc", "category":"Alphabet"}, 3 | { "id":"", "name":"Bcd", "category":"EFG"}, 4 | { "id":"", "name":"Cde", "category":"FGH"}, 5 | { "id":"", "name":"Klm", "category":"Ubf", "groupname":""} 6 | ] -------------------------------------------------------------------------------- /examples/test/persist_prepare_interests.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "name":"Abc", "category":"DEF","status":"yes"}, 3 | { "name":"Bcd", "category":"EFG"}, 4 | { "name":"Cde", "category":"FGH"} 5 | ] -------------------------------------------------------------------------------- /examples/test/read_prepare_interests.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "name":"Abc", "category":"DEF","status":"yes"}, 3 | { "name":"Bcd", "category":"EFG"}, 4 | { "name":"Cde", "category":"FGH"} 5 | ] -------------------------------------------------------------------------------- /examples/test/script/database.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS interests; 2 | 3 | CREATE TABLE `interests` ( 4 | `id` INT(11) NOT NULL AUTO_INCREMENT, 5 | `name` VARCHAR(255) DEFAULT NULL, 6 | `category` VARCHAR(255) DEFAULT NULL, 7 | `status` VARCHAR(10) DEFAULT NULL, 8 | `groupname` VARCHAR(10) DEFAULT NULL, 9 | PRIMARY KEY (`id`) 10 | ) ENGINE = InnoDB; 11 | 12 | -------------------------------------------------------------------------------- /factory_registry.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | var managerFactories = make(map[string]ManagerFactory) 10 | 11 | func init() { 12 | var managerFactory ManagerFactory = &sqlManagerFactory{} 13 | sqlDrivers := []string{"mysql", "ora", "pg", "postgres", "mssql", "sqlite3", "odbc", "cql", "oci8"} 14 | for _, driver := range sqlDrivers { 15 | RegisterManagerFactory(driver, managerFactory) 16 | } 17 | RegisterManagerFactory("ndjson", &jsonFileManagerFactory{}) 18 | RegisterManagerFactory("csv", &delimiteredFileManagerFactory{","}) 19 | RegisterManagerFactory("tsv", &delimiteredFileManagerFactory{"\t"}) 20 | } 21 | 22 | //RegisterManagerFactory registers manager factory for passed in driver. 23 | func RegisterManagerFactory(driver string, factory ManagerFactory) { 24 | managerFactories[driver] = factory 25 | } 26 | 27 | func isSQLDatabase(driver string) bool { 28 | _, err := sql.Open(driver, "") 29 | if err == nil { 30 | return true 31 | } 32 | return !strings.Contains(err.Error(), "unknown driver") 33 | } 34 | 35 | //GetManagerFactory returns a manager factory for passed in driver, or error. 36 | func GetManagerFactory(driver string) (ManagerFactory, error) { 37 | if result, ok := managerFactories[driver]; ok { 38 | return result, nil 39 | } 40 | 41 | if isSQLDatabase(driver) { 42 | var managerFactory ManagerFactory = &sqlManagerFactory{} 43 | RegisterManagerFactory(driver, managerFactory) 44 | return managerFactory, nil 45 | } 46 | return nil, fmt.Errorf("failed to lookup manager factory for '%v' ", driver) 47 | } 48 | -------------------------------------------------------------------------------- /file_connection.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/viant/toolbox" 7 | "os" 8 | ) 9 | 10 | type fileConnection struct { 11 | *AbstractConnection 12 | URL string 13 | ext string 14 | files map[string]*os.File 15 | } 16 | 17 | func (fc *fileConnection) Close() error { 18 | for _, file := range fc.files { 19 | file.Close() 20 | } 21 | return nil 22 | } 23 | 24 | func getFile(filename string, connection Connection) (*os.File, error) { 25 | fileConn, ok := connection.(*fileConnection) 26 | if !ok { 27 | return nil, fmt.Errorf("invalid connection type") 28 | } 29 | var err error 30 | if _, ok := fileConn.files[filename]; !ok { 31 | if len(fileConn.files) == 0 { 32 | fileConn.files = make(map[string]*os.File) 33 | } 34 | if !toolbox.FileExists(filename) { 35 | fileConn.files[filename], err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644) 36 | } else { 37 | fileConn.files[filename], err = os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0644) 38 | } 39 | if err != nil { 40 | return nil, err 41 | } 42 | } 43 | return fileConn.files[filename], nil 44 | 45 | } 46 | 47 | func (fc *fileConnection) Unwrap(target interface{}) interface{} { 48 | return errors.New("unsupported") 49 | } 50 | 51 | type fileConnectionProvider struct { 52 | *AbstractConnectionProvider 53 | } 54 | 55 | func (cp *fileConnectionProvider) NewConnection() (Connection, error) { 56 | config := cp.Config() 57 | url := config.Get("url") 58 | ext := config.Get("ext") 59 | var fileConnection = &fileConnection{URL: url, ext: ext} 60 | var connection = fileConnection 61 | var super = NewAbstractConnection(config, cp.ConnectionProvider.ConnectionPool(), connection) 62 | fileConnection.AbstractConnection = super 63 | return connection, nil 64 | } 65 | 66 | func newFileConnectionProvider(config *Config) ConnectionProvider { 67 | if config.MaxPoolSize == 0 { 68 | config.MaxPoolSize = 1 69 | } 70 | fileConnectionProvider := &fileConnectionProvider{} 71 | var connectionProvider ConnectionProvider = fileConnectionProvider 72 | super := NewAbstractConnectionProvider(config, make(chan Connection, config.MaxPoolSize), connectionProvider) 73 | fileConnectionProvider.AbstractConnectionProvider = super 74 | return connectionProvider 75 | } 76 | -------------------------------------------------------------------------------- /file_dialect.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "path" 7 | "strings" 8 | ) 9 | 10 | type fileDialect struct { 11 | DefaultDialect 12 | } 13 | 14 | //DropTable drops a table in datastore managed by passed in manager. 15 | func (d fileDialect) DropTable(manager Manager, datastore string, table string) error { 16 | fileManager, ok := manager.(*FileManager) 17 | if !ok { 18 | return fmt.Errorf("invalid store manager: %T, expected %T", &FileManager{}, manager) 19 | } 20 | 21 | tableURL := fileManager.getTableURL(manager, table) 22 | exists, err := fileManager.service.Exists(tableURL) 23 | if err != nil { 24 | return err 25 | } 26 | if !exists { 27 | return nil 28 | } 29 | 30 | object, err := fileManager.service.StorageObject(tableURL) 31 | if err != nil { 32 | return err 33 | } 34 | if object == nil { 35 | return nil 36 | } 37 | return fileManager.service.Delete(object) 38 | } 39 | 40 | //GetTables return tables names for passed in datastore managed by passed in manager. 41 | func (d fileDialect) GetTables(manager Manager, datastore string) ([]string, error) { 42 | fileManager, ok := manager.(*FileManager) 43 | if !ok { 44 | return nil, fmt.Errorf("Invalid store manager: %T, expected %T", &FileManager{}, manager) 45 | } 46 | 47 | exists, err := fileManager.service.Exists(fileManager.baseURL.URL) 48 | if err != nil { 49 | return nil, err 50 | } 51 | if !exists { 52 | return []string{}, nil 53 | } 54 | 55 | objects, err := fileManager.service.List(fileManager.baseURL.URL) 56 | ext := "." + manager.Config().Get("ext") 57 | var result = make([]string, 0) 58 | for _, object := range objects { 59 | if object.IsFolder() { 60 | continue 61 | } 62 | parsedURL, err := url.Parse(object.URL()) 63 | if err != nil { 64 | return nil, err 65 | } 66 | _, name := path.Split(parsedURL.Path) 67 | if strings.HasSuffix(name, ext) { 68 | result = append(result, name) 69 | } 70 | } 71 | return result, nil 72 | } 73 | 74 | //GetCurrentDatastore returns url, base path 75 | func (d fileDialect) GetCurrentDatastore(manager Manager) (string, error) { 76 | return manager.Config().Get("url"), nil 77 | } 78 | -------------------------------------------------------------------------------- /file_dialect_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/viant/dsc" 6 | "testing" 7 | ) 8 | 9 | func TestFileDialect(t *testing.T) { 10 | 11 | config := dsc.NewConfig("ndjson", "[url]", "dateFormat:yyyy-MM-dd hh:mm:ss,ext:json,url:test/") 12 | manager, err := dsc.NewManagerFactory().Create(config) 13 | assert.Nil(t, err) 14 | dialect := dsc.GetDatastoreDialect("ndjson") 15 | 16 | tables, err := dialect.GetTables(manager, "") 17 | assert.Nil(t, err) 18 | assert.True(t, len(tables) > 0) 19 | 20 | assert.False(t, dialect.CanCreateDatastore(manager)) 21 | assert.False(t, dialect.CanDropDatastore(manager)) 22 | assert.False(t, dialect.CanPersistBatch()) 23 | _, err = dialect.GetDatastores(manager) 24 | assert.Nil(t, err) 25 | err = dialect.DropDatastore(manager, "abc") 26 | assert.NotNil(t, err, "Unsupported") 27 | err = dialect.CreateDatastore(manager, "abc") 28 | assert.NotNil(t, err, "Unsupported") 29 | 30 | _, err = dialect.GetSequence(manager, "abc") 31 | assert.NotNil(t, err, "Unsupported") 32 | 33 | } 34 | -------------------------------------------------------------------------------- /file_manager_factory.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "github.com/viant/toolbox" 5 | ) 6 | 7 | type jsonFileManagerFactory struct{} 8 | 9 | func (f *jsonFileManagerFactory) Create(config *Config) (Manager, error) { 10 | var connectionProvider = newFileConnectionProvider(config) 11 | fileManager := NewFileManager(toolbox.NewJSONEncoderFactory(), toolbox.NewJSONDecoderFactory(), "", config) 12 | super := NewAbstractManager(config, connectionProvider, fileManager) 13 | fileManager.AbstractManager = super 14 | err := fileManager.Init() 15 | if err != nil { 16 | return nil, err 17 | } 18 | return fileManager, nil 19 | } 20 | 21 | func (f jsonFileManagerFactory) CreateFromURL(URL string) (Manager, error) { 22 | config, err := NewConfigFromURL(URL) 23 | if err != nil { 24 | return nil, err 25 | } 26 | return f.Create(config) 27 | } 28 | 29 | type delimiteredFileManagerFactory struct { 30 | delimiter string 31 | } 32 | 33 | func (f *delimiteredFileManagerFactory) Create(config *Config) (Manager, error) { 34 | var connectionProvider = newFileConnectionProvider(config) 35 | fileManager := NewFileManager(&delimiterEncoderFactory{delimiter: f.delimiter}, &delimiterDecoderFactory{}, f.delimiter, config) 36 | super := NewAbstractManager(config, connectionProvider, fileManager) 37 | fileManager.AbstractManager = super 38 | return fileManager, nil 39 | } 40 | 41 | func (f delimiteredFileManagerFactory) CreateFromURL(URL string) (Manager, error) { 42 | config, err := NewConfigFromURL(URL) 43 | if err != nil { 44 | return nil, err 45 | } 46 | return f.Create(config) 47 | } 48 | -------------------------------------------------------------------------------- /file_manager_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/viant/dsc" 6 | "github.com/viant/toolbox/url" 7 | "os" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | type MostLikedCity struct { 13 | City string 14 | Visits int 15 | Souvenirs []string 16 | } 17 | 18 | type Traveler struct { 19 | Id int 20 | Name string 21 | LastVisitTime time.Time 22 | Achievements []string 23 | MostLikedCity MostLikedCity 24 | VisitedCities []struct { 25 | City string 26 | Visits int 27 | } 28 | } 29 | 30 | func TestInsert(t *testing.T) { 31 | config := dsc.NewConfig("ndjson", "[url]", "dateFormat:yyyy-MM-dd hh:mm:ss,ext:json,url:/test/") 32 | manager, err := dsc.NewManagerFactory().Create(config) 33 | assert.Nil(t, err) 34 | for i := 0; i < 10; i++ { 35 | connection, err := manager.ConnectionProvider().Get() 36 | assert.Nil(t, err) 37 | defer connection.Close() 38 | assert.NotNil(t, connection) 39 | } 40 | 41 | } 42 | 43 | func TestPersist(t *testing.T) { 44 | config := dsc.NewConfig("ndjson", "[url]", "dateFormat:yyyy-MM-dd hh:mm:ss,ext:json,url:test/") 45 | manager, err := dsc.NewManagerFactory().Create(config) 46 | assert.Nil(t, err) 47 | dialect := dsc.GetDatastoreDialect("ndjson") 48 | datastore, err := dialect.GetCurrentDatastore(manager) 49 | assert.Nil(t, err) 50 | 51 | filePath := url.NewResource("test/travelers2.json") 52 | os.Create(filePath.ParsedURL.Path) 53 | 54 | err = dialect.DropTable(manager, datastore, "travelers2") 55 | assert.Nil(t, err) 56 | 57 | travelers := make([]*Traveler, 2) 58 | travelers[0] = &Traveler{ 59 | Id: 10, 60 | Name: "Cook", 61 | LastVisitTime: time.Now(), 62 | Achievements: []string{"abc", "jhi"}, 63 | MostLikedCity: MostLikedCity{City: "Cracow", Visits: 4}, 64 | } 65 | 66 | travelers[1] = &Traveler{ 67 | Id: 20, 68 | Name: "Robin", 69 | LastVisitTime: time.Now(), 70 | Achievements: []string{"w", "a"}, 71 | MostLikedCity: MostLikedCity{"Moscow", 3, []string{"s3", "sN"}}, 72 | } 73 | 74 | inserted, updated, err := manager.PersistAll(&travelers, "travelers2", nil) 75 | assert.Nil(t, err) 76 | assert.Equal(t, 2, inserted) 77 | assert.Equal(t, 0, updated) 78 | 79 | travelers[1].Achievements = []string{"z", "g"} 80 | inserted, updated, err = manager.PersistSingle(travelers[1], "travelers2", nil) 81 | assert.Nil(t, err) 82 | 83 | assert.Equal(t, 0, inserted) 84 | assert.Equal(t, 1, updated) 85 | 86 | success, err := manager.DeleteSingle(travelers[0], "travelers2", nil) 87 | assert.Nil(t, err) 88 | assert.True(t, success) 89 | 90 | } 91 | 92 | func TestRead(t *testing.T) { 93 | config := dsc.NewConfig("ndjson", "[url]", "dateFormat:yyyy-MM-dd hh:mm:ss,ext:json,url:test/") 94 | manager, err := dsc.NewManagerFactory().Create(config) 95 | assert.Nil(t, err) 96 | 97 | { 98 | travelers := make([][]interface{}, 0) 99 | err := manager.ReadAll(&travelers, " SELECT id, name, lastVisitTime, visitedCities, achievements, mostLikedCity, LastVisitTime FROM travelers1 WHERE id IN(?)", []interface{}{1}, nil) 100 | assert.Nil(t, err) 101 | assert.Equal(t, 1, len(travelers)) 102 | assert.EqualValues(t, 1, travelers[0][0]) 103 | assert.EqualValues(t, "Rob", travelers[0][1]) 104 | } 105 | 106 | { 107 | var travelers = make([]Traveler, 0) 108 | err = manager.ReadAll(&travelers, " SELECT id, name, lastVisitTime, visitedCities, achievements, mostLikedCity, LastVisitTime FROM travelers1 WHERE id IN(?, ?)", []interface{}{1, 4}, nil) 109 | assert.Nil(t, err) 110 | if assert.Equal(t, 2, len(travelers)) { 111 | traveler := travelers[0] 112 | assert.Equal(t, 1, traveler.Id) 113 | assert.Equal(t, "Rob", traveler.Name) 114 | assert.Equal(t, 2, len(traveler.VisitedCities)) 115 | assert.Equal(t, 3, traveler.VisitedCities[0].Visits) 116 | assert.Equal(t, "Warsaw", traveler.VisitedCities[0].City) 117 | assert.Equal(t, 2, len(traveler.Achievements)) 118 | assert.Equal(t, int64(1456801800), traveler.LastVisitTime.Unix()) 119 | } 120 | } 121 | 122 | { 123 | var travelers = make([]Traveler, 0) 124 | err = manager.ReadAll(&travelers, " SELECT id, name, lastVisitTime, visitedCities, achievements, mostLikedCity, LastVisitTime FROM travelers1", nil, nil) 125 | assert.Nil(t, err) 126 | assert.Equal(t, 4, len(travelers)) 127 | 128 | { 129 | traveler := travelers[0] 130 | assert.Equal(t, 1, traveler.Id) 131 | assert.Equal(t, "Rob", traveler.Name) 132 | assert.Equal(t, 2, len(traveler.VisitedCities)) 133 | assert.Equal(t, 3, traveler.VisitedCities[0].Visits) 134 | assert.Equal(t, "Warsaw", traveler.VisitedCities[0].City) 135 | assert.Equal(t, 2, len(traveler.Achievements)) 136 | assert.Equal(t, int64(1456801800), traveler.LastVisitTime.Unix()) 137 | } 138 | } 139 | 140 | { 141 | traveler := Traveler{} 142 | success, err := manager.ReadSingle(&traveler, " SELECT id, name, lastVisitTime, visitedCities, achievements, mostLikedCity, LastVisitTime FROM travelers1 WHERE id = ?", []interface{}{1}, nil) 143 | assert.Nil(t, err) 144 | assert.True(t, success) 145 | } 146 | 147 | { 148 | traveler := Traveler{} 149 | success, err := manager.ReadSingle(&traveler, " SELECT id, name, lastVisitTime, visitedCities, achievements, mostLikedCity, LastVisitTime FROM travelers1 WHERE id IN(?)", []interface{}{1}, nil) 150 | assert.Nil(t, err) 151 | assert.True(t, success) 152 | } 153 | 154 | { 155 | traveler := make([]interface{}, 0) 156 | success, err := manager.ReadSingle(&traveler, " SELECT id, name, lastVisitTime, visitedCities, achievements, mostLikedCity, LastVisitTime FROM travelers1 WHERE id IN(?)", []interface{}{1}, nil) 157 | assert.Nil(t, err) 158 | assert.True(t, success) 159 | } 160 | 161 | { 162 | traveler := make([]interface{}, 0) 163 | success, err := manager.ReadSingle(&traveler, " SELECT id, name, lastVisitTime, visitedCities, achievements, mostLikedCity, LastVisitTime FROM travelers1 WHERE id IN(?)", []interface{}{1}, nil) 164 | assert.Nil(t, err) 165 | assert.True(t, success) 166 | } 167 | 168 | } 169 | 170 | func TestReadCsv(t *testing.T) { 171 | config := dsc.NewConfig("csv", "[url]", "dateFormat:yyyy-MM-dd hh:mm:ss,ext:csv,url:test/") 172 | manager, err := dsc.NewManagerFactory().Create(config) 173 | assert.Nil(t, err) 174 | 175 | { 176 | travelers := make([][]interface{}, 0) 177 | err := manager.ReadAll(&travelers, " SELECT id, name FROM traveler", nil, nil) 178 | assert.Nil(t, err) 179 | assert.Equal(t, 4, len(travelers)) 180 | assert.EqualValues(t, "1", travelers[0][0]) 181 | assert.EqualValues(t, "Bob", travelers[0][1]) 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /file_scanner.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/viant/toolbox" 6 | "strings" 7 | ) 8 | 9 | //FileScanner represents a file scanner to transfer record to destinations. 10 | type FileScanner struct { 11 | columns []string 12 | columnTypes []ColumnType 13 | converter toolbox.Converter 14 | Values map[string]interface{} 15 | } 16 | 17 | //Columns returns columns of the processed record. 18 | func (s *FileScanner) Columns() ([]string, error) { 19 | return s.columns, nil 20 | } 21 | 22 | //Columns returns columns of the processed record. 23 | func (s *FileScanner) ColumnTypes() ([]ColumnType, error) { 24 | return s.columnTypes, nil 25 | } 26 | 27 | //Scan reads file record values to assign it to passed in destinations. 28 | func (s *FileScanner) Scan(destinations ...interface{}) (err error) { 29 | if len(destinations) == 1 { 30 | if toolbox.IsMap(destinations[0]) { 31 | var record = toolbox.AsMap(destinations[0]) 32 | for k, v := range s.Values { 33 | record[k] = v 34 | } 35 | return nil 36 | } 37 | } 38 | var columns, _ = s.Columns() 39 | for i, dest := range destinations { 40 | if value, found := s.Values[columns[i]]; found { 41 | switch val := value.(type) { 42 | case json.Number: 43 | var number interface{} 44 | if strings.Contains(val.String(), ".") { 45 | 46 | number, err = val.Float64() 47 | if err == nil { 48 | err = s.converter.AssignConverted(dest, number) 49 | } 50 | } else { 51 | number, err = val.Int64() 52 | if err == nil { 53 | err = s.converter.AssignConverted(dest, number) 54 | } 55 | } 56 | break 57 | 58 | default: 59 | err = s.converter.AssignConverted(dest, value) 60 | 61 | } 62 | if err != nil { 63 | return err 64 | } 65 | } 66 | } 67 | return nil 68 | } 69 | 70 | //NewFileScanner create a new file scanner, it takes config, and columns as parameters. 71 | func NewFileScanner(config *Config, columns []string, columnTypes []ColumnType) *FileScanner { 72 | converter := toolbox.NewColumnConverter(config.GetDateLayout()) 73 | return &FileScanner{ 74 | converter: *converter, 75 | columnTypes: columnTypes, 76 | columns: columns, 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/viant/dsc 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/denisenkom/go-mssqldb v0.12.3 7 | github.com/go-sql-driver/mysql v1.7.0 8 | github.com/mattn/go-sqlite3 v1.14.16 9 | github.com/pkg/errors v0.9.1 10 | github.com/stretchr/testify v1.8.1 11 | github.com/viant/dsunit v0.10.10 12 | github.com/viant/toolbox v0.34.5 13 | ) 14 | 15 | require ( 16 | cloud.google.com/go/compute/metadata v0.2.0 // indirect 17 | github.com/davecgh/go-spew v1.1.1 // indirect 18 | github.com/go-errors/errors v1.4.2 // indirect 19 | github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect 20 | github.com/golang-sql/sqlexp v0.1.0 // indirect 21 | github.com/golang/protobuf v1.5.2 // indirect 22 | github.com/lunixbochs/vtclean v1.0.0 // indirect 23 | github.com/pmezard/go-difflib v1.0.0 // indirect 24 | github.com/viant/assertly v0.9.0 // indirect 25 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect 26 | golang.org/x/net v0.6.0 // indirect 27 | golang.org/x/oauth2 v0.5.0 // indirect 28 | golang.org/x/sys v0.5.0 // indirect 29 | golang.org/x/term v0.5.0 // indirect 30 | google.golang.org/appengine v1.6.7 // indirect 31 | google.golang.org/protobuf v1.28.0 // indirect 32 | gopkg.in/yaml.v2 v2.4.0 // indirect 33 | gopkg.in/yaml.v3 v3.0.1 // indirect 34 | ) 35 | -------------------------------------------------------------------------------- /limiter.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | "time" 7 | ) 8 | 9 | type timeWindow struct { 10 | Start time.Time 11 | End time.Time 12 | } 13 | 14 | //Limiter represents resource limter 15 | type Limiter struct { 16 | count int64 17 | max int 18 | duration time.Duration 19 | win *timeWindow 20 | mux *sync.Mutex 21 | } 22 | 23 | //Acquire checks if limit for current time window was not exhausted or sleep 24 | func (l *Limiter) Acquire() { 25 | 26 | for { 27 | window := l.window() 28 | if int(atomic.AddInt64(&l.count, 1)) <= l.max { 29 | return 30 | } 31 | duration := window.End.Sub(time.Now()) 32 | if duration > 0 { 33 | time.Sleep(duration) 34 | } 35 | } 36 | } 37 | 38 | func (l *Limiter) window() *timeWindow { 39 | l.mux.Lock() 40 | defer l.mux.Unlock() 41 | if time.Now().After(l.win.End) { 42 | l.win.Start = time.Now() 43 | l.win.End = time.Now().Add(l.duration) 44 | atomic.StoreInt64(&l.count, 0) 45 | } 46 | return l.win 47 | } 48 | 49 | //NewLimiter creates a new limiter 50 | func NewLimiter(duration time.Duration, max int) *Limiter { 51 | if max == 0 { 52 | max = 1 53 | } 54 | return &Limiter{ 55 | mux: &sync.Mutex{}, 56 | duration: duration, 57 | win: &timeWindow{ 58 | Start: time.Now(), 59 | End: time.Now().Add(duration), 60 | }, 61 | max: max, 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /limiter_test.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "sync" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestLimiter_Acquire(t *testing.T) { 11 | 12 | limiter := NewLimiter(10*time.Millisecond, 10) 13 | 14 | startTime := time.Now() 15 | waitGroup := &sync.WaitGroup{} 16 | waitGroup.Add(1000) 17 | for i := 0; i < 1000; i++ { 18 | go func() { 19 | defer waitGroup.Done() 20 | limiter.Acquire() 21 | }() 22 | 23 | } 24 | waitGroup.Wait() 25 | elapsed := time.Now().Sub(startTime) 26 | assert.True(t, elapsed >= time.Second) 27 | } 28 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import "fmt" 4 | 5 | //Log represent log function 6 | type Log func(format string, args ...interface{}) 7 | 8 | //Logf - function to log debug info 9 | var Logf Log = VoidLogger 10 | 11 | //VoidLogger represent logger that do not log 12 | func VoidLogger(format string, args ...interface{}) { 13 | 14 | } 15 | 16 | //StdoutLogger represents stdout logger 17 | func StdoutLogger(format string, args ...interface{}) { 18 | fmt.Print(fmt.Sprintf(format, args...) + "\n") 19 | } 20 | -------------------------------------------------------------------------------- /manager_factory.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type managerFactoryProxy struct{} 8 | 9 | //Create creates a new manager for the passed in config. 10 | func (f managerFactoryProxy) Create(config *Config) (Manager, error) { 11 | if config.DriverName == "" { 12 | return nil, fmt.Errorf("DriverName was empty %v", config) 13 | } 14 | factory, err := GetManagerFactory(config.DriverName) 15 | if err != nil { 16 | return nil, fmt.Errorf("failed to lookup manager factory for `%v`, make sure you have imported required implmentation", config.DriverName) 17 | } 18 | config.Init() 19 | return factory.Create(config) 20 | } 21 | 22 | //CreateFromURL create a new manager from URL, url resource should be a JSON Config 23 | func (f managerFactoryProxy) CreateFromURL(URL string) (Manager, error) { 24 | config, err := NewConfigFromURL(URL) 25 | if err != nil { 26 | return nil, err 27 | } 28 | factory, err := GetManagerFactory(config.DriverName) 29 | if err != nil { 30 | return nil, fmt.Errorf("failed to lookup manager factory for `%v`, make sure you have imported required implmentation", config.DriverName) 31 | } 32 | return factory.Create(config) 33 | } 34 | 35 | //NewManagerFactory create a new manager factory. 36 | func NewManagerFactory() ManagerFactory { 37 | var manager ManagerFactory = &managerFactoryProxy{} 38 | return manager 39 | } 40 | -------------------------------------------------------------------------------- /manager_factory_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/viant/dsc" 6 | "testing" 7 | ) 8 | 9 | func TestCreateFromURL(t *testing.T) { 10 | factory := dsc.NewManagerFactory() 11 | { 12 | _, err := factory.CreateFromURL("test/file_config3.json") 13 | assert.NotNil(t, err) 14 | } 15 | 16 | { 17 | _, err := factory.CreateFromURL("test/file_config.json") 18 | assert.NotNil(t, err) 19 | } 20 | } 21 | 22 | func TestMissingDricer(t *testing.T) { 23 | factory := dsc.NewManagerFactory() 24 | { 25 | _, err := factory.CreateFromURL("test/file_config3.json") 26 | assert.NotNil(t, err) 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /manager_registry.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "github.com/viant/toolbox" 5 | "sync" 6 | ) 7 | 8 | type commonManagerRegistry struct { 9 | mux *sync.RWMutex 10 | registry map[string](Manager) 11 | } 12 | 13 | func (r commonManagerRegistry) Register(name string, manager Manager) { 14 | r.mux.Lock() 15 | defer r.mux.Unlock() 16 | if previousManager, found := r.registry[name]; found { 17 | _ = previousManager.ConnectionProvider().Close() 18 | } 19 | r.registry[name] = manager 20 | } 21 | 22 | func (r commonManagerRegistry) Get(name string) Manager { 23 | r.mux.RLock() 24 | result, ok := r.registry[name] 25 | r.mux.RUnlock() 26 | if ok { 27 | return result 28 | } 29 | return nil 30 | } 31 | 32 | func (r commonManagerRegistry) Names() []string { 33 | r.mux.RLock() 34 | defer r.mux.RUnlock() 35 | return toolbox.MapKeysToStringSlice(r.registry) 36 | } 37 | 38 | //NewManagerRegistry create a new ManagerRegistry 39 | func NewManagerRegistry() ManagerRegistry { 40 | var result = &commonManagerRegistry{ 41 | registry: make(map[string](Manager)), 42 | mux: &sync.RWMutex{}, 43 | } 44 | return result 45 | } 46 | -------------------------------------------------------------------------------- /matcher.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import "github.com/viant/toolbox" 4 | 5 | type valueMatcher struct { 6 | optionallyEnclosingChar string 7 | terminatorChars string 8 | } 9 | 10 | func (m valueMatcher) Match(input string, offset int) (matched int) { 11 | var i = 0 12 | isValueEnclosed := false 13 | if input[offset:offset+1] == m.optionallyEnclosingChar { 14 | isValueEnclosed = true 15 | i++ 16 | } 17 | for ; i < len(input)-offset; i++ { 18 | aChar := input[offset+i : offset+i+1] 19 | if isValueEnclosed { 20 | if aChar == m.optionallyEnclosingChar && input[offset+i-1:offset+i] != "\\" { 21 | i++ 22 | break 23 | } 24 | 25 | } else { 26 | for j := 0; j < len(m.terminatorChars); j++ { 27 | if aChar == m.terminatorChars[j:j+1] { 28 | return i 29 | } 30 | } 31 | } 32 | } 33 | if isValueEnclosed { 34 | if input[offset+i-1:offset+i] == m.optionallyEnclosingChar { 35 | return i 36 | } 37 | return 0 38 | } 39 | 40 | return i 41 | } 42 | 43 | type valuesMatcher struct { 44 | valuesGroupingBeginChar string 45 | valuesGroupingEndChar string 46 | valueSeparator string 47 | valueOptionallyEnclosedWithChar string 48 | valueTerminatorCharacters string 49 | } 50 | 51 | func (m valuesMatcher) Match(input string, offset int) (matched int) { 52 | if input[offset:offset+len(m.valuesGroupingBeginChar)] != m.valuesGroupingBeginChar { 53 | return 0 54 | } 55 | valueMatcher := valueMatcher{optionallyEnclosingChar: m.valueOptionallyEnclosedWithChar, terminatorChars: m.valueTerminatorCharacters} 56 | whitespaceMatcher := toolbox.CharactersMatcher{Chars: " \n\t"} 57 | 58 | i := len(m.valuesGroupingBeginChar) 59 | var firstIteration = true 60 | //"a(1, 2, 3)a" 61 | var maxLoopCount = len(input) - (offset + 1) 62 | for ; i < maxLoopCount; firstIteration = false { 63 | aChar := input[offset+i : offset+i+1] 64 | if aChar == m.valueSeparator { 65 | if firstIteration { 66 | return 0 67 | } 68 | i++ 69 | continue 70 | } 71 | whitespaceMatched := whitespaceMatcher.Match(input, offset+i) 72 | if whitespaceMatched > 0 { 73 | i += whitespaceMatched 74 | continue 75 | } 76 | 77 | valueMatched := valueMatcher.Match(input, offset+i) 78 | if valueMatched == 0 { 79 | if firstIteration { 80 | return 0 81 | } 82 | break 83 | } 84 | i += valueMatched 85 | 86 | } 87 | if offset+i < len(input) && input[offset+i:offset+i+1] != m.valuesGroupingEndChar { 88 | return 0 89 | } 90 | return i + 1 91 | } 92 | -------------------------------------------------------------------------------- /matcher/id.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import "unicode" 4 | 5 | var dotRune = rune('.') 6 | var underscoreRune = rune('_') 7 | 8 | // LiteralMatcher represents a matcher that finds any literals in the input 9 | type IdMatcher struct{} 10 | 11 | // Match matches a literal in the input, it returns number of character matched. 12 | func (m IdMatcher) Match(input string, offset int) int { 13 | var matched = 0 14 | if offset >= len(input) { 15 | return matched 16 | } 17 | for i, r := range input[offset:] { 18 | if i == 0 { 19 | if !(unicode.IsLetter(r) || unicode.IsDigit(r)) { 20 | break 21 | } 22 | } else if !(unicode.IsLetter(r) || r == '-' || unicode.IsDigit(r) || r == dotRune || r == underscoreRune) { 23 | break 24 | } 25 | matched++ 26 | } 27 | return matched 28 | } 29 | -------------------------------------------------------------------------------- /matcher_test.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestMatchValue(t *testing.T) { 9 | matcher := valueMatcher{optionallyEnclosingChar: "'", terminatorChars: ", \n\t)"} 10 | assert.Equal(t, 4, matcher.Match(" ?23? ", 2)) 11 | assert.Equal(t, 2, matcher.Match(" ?2)3? ", 2)) 12 | assert.Equal(t, 7, matcher.Match(" 'a2\\'4'a", 2)) 13 | } 14 | 15 | func TestMatchValues(t *testing.T) { 16 | matcher := valuesMatcher{ 17 | valuesGroupingBeginChar: "(", 18 | valuesGroupingEndChar: ")", 19 | valueSeparator: ",", 20 | valueOptionallyEnclosedWithChar: "'", 21 | valueTerminatorCharacters: ", \n\t)"} 22 | assert.Equal(t, 9, matcher.Match("a(1, 2, 3)a", 1)) 23 | assert.Equal(t, 17, matcher.Match("a('a', '(b)', 'd')", 1)) 24 | } 25 | -------------------------------------------------------------------------------- /query_builder.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/viant/toolbox" 8 | ) 9 | 10 | var queryAllSQLTemplate = "SELECT %v FROM %v" 11 | 12 | //QueryBuilder represetns a query builder. It builds simple select sql. 13 | type QueryBuilder struct { 14 | QueryHint string 15 | TableDescriptor *TableDescriptor 16 | } 17 | 18 | //BuildQueryAll builds query all data without where clause 19 | func (qb *QueryBuilder) BuildQueryAll(columns []string) *ParametrizedSQL { 20 | var columnsLiteral = qb.QueryHint + " " + strings.Join(columns, ",") 21 | table := qb.TableDescriptor.From() 22 | return &ParametrizedSQL{ 23 | SQL: fmt.Sprintf(queryAllSQLTemplate, columnsLiteral, table), 24 | Values: make([]interface{}, 0), 25 | } 26 | 27 | } 28 | 29 | //BuildQueryOnPk builds ParametrizedSQL for passed in query columns and pk values. 30 | func (qb *QueryBuilder) BuildQueryOnPk(columns []string, pkRowValues [][]interface{}) *ParametrizedSQL { 31 | return qb.BuildQueryWithInColumns(columns, append([]string{}, qb.TableDescriptor.PkColumns...), pkRowValues) 32 | } 33 | 34 | //BuildQueryOnPk builds ParametrizedSQL for passed in query columns and pk values. 35 | func (qb *QueryBuilder) BuildQueryWithInColumns(columns []string, inCriteriaColumns []string, pkRowValues [][]interface{}) *ParametrizedSQL { 36 | columns = append([]string{}, columns...) 37 | updateReserved(columns) 38 | var columnsLiteral = qb.QueryHint + " " + strings.Join(columns, ",") 39 | updateReserved(inCriteriaColumns) 40 | var inColumns = strings.Join(inCriteriaColumns, ",") 41 | var sqlArguments = make([]interface{}, 0) 42 | var criteria = "" 43 | var multiValuePk = false 44 | for _, pkValues := range pkRowValues { 45 | if len(pkValues) > 1 { 46 | multiValuePk = true 47 | } 48 | var rowCriteria = strings.Repeat("?,", len(pkValues)) 49 | rowCriteria = rowCriteria[0 : len(rowCriteria)-1] 50 | 51 | sqlArguments = append(sqlArguments, pkValues...) 52 | if len(criteria) > 0 { 53 | criteria = criteria + "," 54 | } 55 | if multiValuePk { 56 | criteria = criteria + "(" + rowCriteria + ")" 57 | } else { 58 | criteria = criteria + rowCriteria 59 | } 60 | } 61 | 62 | var whereCriteria = inColumns + " IN (" + criteria + ")" 63 | if multiValuePk { 64 | whereCriteria = "(" + inColumns + ") IN (" + criteria + ")" 65 | } 66 | table := qb.TableDescriptor.From() 67 | return &ParametrizedSQL{ 68 | SQL: fmt.Sprintf(querySQLTemplate, columnsLiteral, table, whereCriteria), 69 | Values: sqlArguments, 70 | } 71 | 72 | } 73 | 74 | //BuildBatchedQueryOnPk builds batches of ParametrizedSQL for passed in query columns and pk values. Batch size specifies number of rows in one parametrized sql. 75 | func (qb *QueryBuilder) BuildBatchedInQuery(columns []string, pkRowValues [][]interface{}, inColumns []string, batchSize int) []*ParametrizedSQL { 76 | var result = make([]*ParametrizedSQL, 0) 77 | toolbox.Process2DSliceInBatches(pkRowValues, batchSize, func(batch [][]interface{}) { 78 | sqlWithArguments := qb.BuildQueryWithInColumns(columns, inColumns, batch) 79 | result = append(result, sqlWithArguments) 80 | }) 81 | return result 82 | } 83 | 84 | //BuildBatchedQueryOnPk builds batches of ParametrizedSQL for passed in query columns and pk values. Batch size specifies number of rows in one parametrized sql. 85 | func (qb *QueryBuilder) BuildBatchedQueryOnPk(columns []string, pkRowValues [][]interface{}, batchSize int) []*ParametrizedSQL { 86 | var result = make([]*ParametrizedSQL, 0) 87 | toolbox.Process2DSliceInBatches(pkRowValues, batchSize, func(batch [][]interface{}) { 88 | sqlWithArguments := qb.BuildQueryOnPk(columns, batch) 89 | result = append(result, sqlWithArguments) 90 | }) 91 | return result 92 | } 93 | 94 | //NewQueryBuilder create anew QueryBuilder, it takes table descriptor and optional query hit to include it in the queries 95 | func NewQueryBuilder(descriptor *TableDescriptor, queryHint string) QueryBuilder { 96 | queryBuilder := QueryBuilder{TableDescriptor: descriptor, QueryHint: queryHint} 97 | return queryBuilder 98 | } 99 | -------------------------------------------------------------------------------- /record_mapper.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | 8 | "github.com/viant/toolbox" 9 | ) 10 | 11 | type metaRecordMapper struct { 12 | converter toolbox.Converter 13 | structType interface{} 14 | columnToFieldMap map[string](map[string]string) 15 | usePointer bool 16 | } 17 | 18 | //NewMetaRecordMapped creates a new MetaRecordMapped to map a data record to a struct, it takes target struct and flag if it is a pointer as parameters. 19 | func NewMetaRecordMapped(targetType interface{}, usePointer bool) RecordMapper { 20 | structType := targetType 21 | if usePointer { 22 | var originalType = targetType.(reflect.Type).Elem() 23 | structType = originalType 24 | } 25 | var result = &metaRecordMapper{ 26 | converter: *toolbox.NewColumnConverter(""), 27 | structType: structType, 28 | usePointer: usePointer, 29 | columnToFieldMap: toolbox.NewFieldSettingByKey(targetType, "column")} 30 | return result 31 | } 32 | 33 | func normalizeColumnKey(column string) string { 34 | var result = strings.ToLower(column) 35 | result = strings.Replace(result, "_", "", len(result)) 36 | return result 37 | } 38 | 39 | func (rm *metaRecordMapper) getValueMappingCount(columns []string) int { 40 | result := 0 41 | for _, key := range columns { 42 | mapping, ok := rm.columnToFieldMap[key] 43 | if !ok { 44 | mapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)] 45 | } 46 | if ok { 47 | if _, found := mapping["valueMap"]; found { 48 | result++ 49 | } 50 | } 51 | } 52 | return result 53 | } 54 | 55 | func (rm *metaRecordMapper) allocateValueMapByKey(columns []string) map[string]interface{} { 56 | var valuesPointers = make([]interface{}, rm.getValueMappingCount(columns)) 57 | index := 0 58 | var result = make(map[string]interface{}) 59 | for _, key := range columns { 60 | mapping, ok := rm.columnToFieldMap[key] 61 | if !ok { 62 | mapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)] 63 | } 64 | if ok { 65 | if _, found := mapping["valueMap"]; found { 66 | result[key] = &valuesPointers[index] 67 | index++ 68 | } 69 | } 70 | } 71 | return result 72 | } 73 | 74 | func (rm *metaRecordMapper) applyFieldMapValuesIfNeeded(fieldsValueMap map[string]interface{}, structPointer reflect.Value) error { 75 | for key, rawValue := range fieldsValueMap { 76 | valueMapping, ok := rm.columnToFieldMap[key] 77 | if !ok { 78 | valueMapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)] 79 | } 80 | fieldName := valueMapping["fieldName"] 81 | field := structPointer.Elem().FieldByName(fieldName) 82 | unwrappedValue := reflect.ValueOf(rawValue).Elem() 83 | if unwrappedValue.IsNil() { 84 | if field.Kind() != reflect.Ptr { 85 | return fmt.Errorf("failed to apply value map on %v, unable to set nil", fieldName) 86 | } 87 | continue 88 | } 89 | 90 | rawValue = unwrappedValue.Interface() 91 | var value string 92 | if valueAsBytes, ok := rawValue.([]byte); ok { 93 | value = string(valueAsBytes) 94 | } else { 95 | value = toolbox.AsString(rawValue) 96 | } 97 | 98 | valueMap := toolbox.MakeStringMap(valueMapping["valueMap"], ":", ",") 99 | stringValue := toolbox.AsString(value) 100 | if mappedValue, found := valueMap[stringValue]; found { 101 | fieldValuePointer := field.Addr().Interface() 102 | err := rm.converter.AssignConverted(fieldValuePointer, mappedValue) 103 | if err != nil { 104 | return fmt.Errorf("failed to map record, unable convert,dur to %v", err) 105 | 106 | } 107 | } else { 108 | return fmt.Errorf("failed to map record, unable to find valid mapping, want one of %s, but had %v", valueMap, stringValue) 109 | } 110 | 111 | } 112 | return nil 113 | } 114 | 115 | func (rm *metaRecordMapper) scanData(scanner Scanner) (result interface{}, err error) { 116 | structType := toolbox.DiscoverTypeByKind(rm.structType, reflect.Struct) 117 | structPointer := reflect.New(structType) 118 | resultStruct := structPointer.Elem() 119 | columns, _ := scanner.Columns() 120 | var fieldValuePointers = make([]interface{}, len(columns)) 121 | var fieldsValueMap map[string]interface{} 122 | 123 | hasFieldValueMap := rm.getValueMappingCount(columns) > 0 124 | if hasFieldValueMap { 125 | fieldsValueMap = rm.allocateValueMapByKey(columns) 126 | } 127 | for i, key := range columns { 128 | 129 | fieldMapping, ok := rm.columnToFieldMap[key] 130 | if !ok { 131 | fieldMapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)] 132 | } 133 | if ok { 134 | fieldName := fieldMapping["fieldName"] 135 | field := resultStruct.FieldByName(fieldName) 136 | 137 | if _, found := fieldMapping["valueMap"]; found { 138 | fieldValuePointers[i] = fieldsValueMap[key] 139 | continue 140 | } 141 | fieldValuePointers[i] = field.Addr().Interface() 142 | 143 | } else { 144 | return nil, fmt.Errorf("unable to map column %v to %v, avaialble: %v", key, rm.columnToFieldMap[key], rm.columnToFieldMap) 145 | } 146 | } 147 | err = scanner.Scan(fieldValuePointers...) 148 | if err != nil { 149 | return nil, fmt.Errorf("failed to scan data: %v\n", err) 150 | } 151 | 152 | if hasFieldValueMap { 153 | err := rm.applyFieldMapValuesIfNeeded(fieldsValueMap, structPointer) 154 | if err != nil { 155 | return nil, err 156 | } 157 | } 158 | 159 | if !rm.usePointer { 160 | result = structPointer.Elem().Interface() 161 | return result, err 162 | } 163 | return structPointer.Interface(), err 164 | } 165 | 166 | func (rm *metaRecordMapper) mapFromValues(vaues []interface{}) (result interface{}, err error) { 167 | return nil, nil 168 | } 169 | 170 | func (rm *metaRecordMapper) Map(scanner Scanner) (result interface{}, err error) { 171 | return rm.scanData(scanner) 172 | } 173 | 174 | type columnarRecordMapper struct { 175 | usePointer bool 176 | targetSlice reflect.Type 177 | } 178 | 179 | //NewColumnarRecordMapper creates a new ColumnarRecordMapper, to map a data record to slice. 180 | func NewColumnarRecordMapper(usePointer bool, targetSlice reflect.Type) RecordMapper { 181 | return &columnarRecordMapper{usePointer, targetSlice} 182 | } 183 | 184 | func (rm *columnarRecordMapper) Map(scanner Scanner) (interface{}, error) { 185 | result, _, err := ScanRow(scanner) 186 | if err != nil { 187 | return nil, err 188 | } 189 | if rm.usePointer { 190 | return result, nil 191 | } 192 | return result, nil 193 | } 194 | 195 | type mapRecordMapper struct { 196 | usePointer bool 197 | targetSlice reflect.Type 198 | } 199 | 200 | func (rm *mapRecordMapper) Map(scanner Scanner) (interface{}, error) { 201 | result, _, err := ScanRow(scanner) 202 | if err != nil { 203 | return nil, err 204 | } 205 | columns, _ := scanner.Columns() 206 | 207 | aMap := make(map[string]interface{}) 208 | for i, column := range columns { 209 | aMap[column] = result[i] 210 | } 211 | if rm.usePointer { 212 | return &aMap, nil 213 | } 214 | return aMap, nil 215 | } 216 | 217 | //NewMapRecordMapper creates a new ColumnarRecordMapper, to map a data record to slice. 218 | func NewMapRecordMapper(usePointer bool, targetSlice reflect.Type) RecordMapper { 219 | return &mapRecordMapper{usePointer, targetSlice} 220 | } 221 | 222 | //NewRecordMapper create a new record mapper, if struct is passed it would be MetaRecordMapper, or for slice ColumnRecordMapper 223 | func NewRecordMapper(targetType reflect.Type) RecordMapper { 224 | switch targetType.Kind() { 225 | case reflect.Struct: 226 | var mapper = NewMetaRecordMapped(targetType, false) 227 | return mapper 228 | 229 | case reflect.Map: 230 | var mapper = NewMapRecordMapper(false, targetType) 231 | return mapper 232 | case reflect.Slice: 233 | var mapper = NewColumnarRecordMapper(false, targetType) 234 | return mapper 235 | case reflect.Ptr: 236 | if targetType.Elem().Kind() == reflect.Slice { 237 | var mapper = NewColumnarRecordMapper(true, targetType.Elem()) 238 | return mapper 239 | } else if targetType.Elem().Kind() == reflect.Struct { 240 | var mapper = NewMetaRecordMapped(targetType, true) 241 | return mapper 242 | } 243 | default: 244 | panic(fmt.Sprintf("unsupported type: %v ", targetType.Name())) 245 | } 246 | return nil 247 | } 248 | 249 | //NewRecordMapperIfNeeded create a new mapper if passed in mapper is nil. It takes target type for the record mapper. 250 | func NewRecordMapperIfNeeded(mapper RecordMapper, targetType reflect.Type) RecordMapper { 251 | if mapper != nil { 252 | return mapper 253 | } 254 | return NewRecordMapper(targetType) 255 | } 256 | 257 | //ScanRow takes scanner to scans row. 258 | func ScanRow(scanner Scanner) ([]interface{}, []string, error) { 259 | columns, _ := scanner.Columns() 260 | count := len(columns) 261 | 262 | var err error 263 | var rowValues = make([]interface{}, count) 264 | 265 | provider, ok := scanner.(ColumnValueProvider) 266 | if ok { 267 | if values, err := provider.ColumnValues(); err == nil { 268 | if err = scanner.Scan(values...); err != nil { 269 | return nil, nil, fmt.Errorf("failed to scan row due to %v", err) 270 | } 271 | for i, v := range values { 272 | if v == nil { 273 | continue 274 | } 275 | 276 | for i := 0; i < 2; i++ { 277 | valuePtr := reflect.ValueOf(v) 278 | if valuePtr.Kind() != reflect.Ptr { 279 | break 280 | } 281 | v = valuePtr.Interface() 282 | if v == nil || valuePtr.IsNil() { 283 | break 284 | } 285 | v = valuePtr.Elem().Interface() 286 | } 287 | 288 | if v == nil { 289 | continue 290 | } 291 | rowValues[i] = v 292 | } 293 | return rowValues, columns, nil 294 | } 295 | } 296 | 297 | var valuePointers = make([]interface{}, count) 298 | for i := range rowValues { 299 | valuePointers[i] = &rowValues[i] 300 | } 301 | 302 | err = scanner.Scan(valuePointers...) 303 | if err != nil { 304 | return nil, nil, fmt.Errorf("failed to scan row due to %v", err) 305 | } 306 | 307 | for i := range rowValues { 308 | var value interface{} 309 | rawValue := rowValues[i] 310 | b, ok := rawValue.([]byte) 311 | if ok { 312 | value = string(b) 313 | } else { 314 | value = rawValue 315 | } 316 | rowValues[i] = value 317 | } 318 | 319 | return rowValues, columns, nil 320 | } 321 | -------------------------------------------------------------------------------- /reserved.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | var reservedKeyword = map[string]bool{ 9 | "key": true, 10 | "primary": true, 11 | "select": true, 12 | "from": true, 13 | "in": true, 14 | "table": true, 15 | "column": true, 16 | "constraint": true, 17 | "foreign": true, 18 | "index": true, 19 | "all": true, 20 | "and": true, 21 | "or": true, 22 | "as": true, 23 | "asc": true, 24 | "desc": true, 25 | "begin": true, 26 | "break": true, 27 | "between": true, 28 | "by": true, 29 | "order": true, 30 | "is": true, 31 | "database": true, 32 | } 33 | 34 | func updateReserved(pk []string) { 35 | if os.Getenv("SQLQuoteReserved") == "" { 36 | return 37 | } 38 | for i := range pk { 39 | if reservedKeyword[strings.ToLower(pk[i])] { 40 | if strings.Count(pk[i], "`") == 0 { 41 | pk[i] = "`" + pk[i] + "`" 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /scanner.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import "github.com/viant/toolbox" 4 | 5 | type scanner struct { 6 | scanner Scanner 7 | } 8 | 9 | func (s *scanner) Columns() ([]string, error) { 10 | return s.scanner.Columns() 11 | } 12 | 13 | func (s *scanner) ColumnTypes() ([]ColumnType, error) { 14 | return s.scanner.ColumnTypes() 15 | } 16 | 17 | func (s *scanner) Scan(destinations ...interface{}) error { 18 | if len(destinations) == 1 { 19 | if toolbox.IsMap(destinations[0]) { 20 | aMap := toolbox.AsMap(destinations[0]) 21 | values, columns, err := ScanRow(s) 22 | if err != nil { 23 | return err 24 | } 25 | for i, column := range columns { 26 | aMap[column] = values[i] 27 | } 28 | return nil 29 | } 30 | } 31 | err := s.scanner.Scan(destinations...) 32 | return err 33 | } 34 | 35 | func NewScanner(s Scanner) Scanner { 36 | return &scanner{s} 37 | } 38 | -------------------------------------------------------------------------------- /sql_connection.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | const ( 10 | connMaxLifetimeMsKey = "connMaxLifetimeMs" 11 | defaultConnMaxLifetimeMs = 1000 12 | maxIdleConnsKey = "maxIdleConns" 13 | ) 14 | 15 | type sqlConnection struct { 16 | canHandleTransaction bool 17 | *AbstractConnection 18 | db *sql.DB 19 | tx *sql.Tx 20 | init bool 21 | } 22 | 23 | func (c *sqlConnection) CloseNow() error { 24 | db, err := asSQLDb(c.db) 25 | if err != nil { 26 | return err 27 | } 28 | db.SetConnMaxLifetime(1000 * time.Millisecond) 29 | return db.Close() 30 | } 31 | 32 | func (c *sqlConnection) Begin() error { 33 | if !c.canHandleTransaction { 34 | return nil 35 | } 36 | db, err := asSQLDb(c.db) 37 | if err != nil { 38 | return err 39 | } 40 | tx, err := db.Begin() 41 | if err != nil { 42 | return err 43 | } 44 | c.tx = tx 45 | return nil 46 | } 47 | 48 | func (c *sqlConnection) Unwrap(target interface{}) interface{} { 49 | if target == sqlDbPointer { 50 | return c.db 51 | } else if target == sqlTxtPointer { 52 | return c.tx 53 | } 54 | panic(fmt.Sprintf("unsupported target type %v", target)) 55 | } 56 | 57 | func (c *sqlConnection) Commit() error { 58 | if !c.canHandleTransaction { 59 | return nil 60 | } 61 | if c.tx == nil { 62 | return fmt.Errorf("no active transaction") 63 | } 64 | err := c.tx.Commit() 65 | c.tx = nil 66 | return err 67 | } 68 | 69 | func (c *sqlConnection) Rollback() error { 70 | if !c.canHandleTransaction { 71 | return nil 72 | } 73 | if c.tx == nil { 74 | return fmt.Errorf("no active transaction") 75 | } 76 | err := c.tx.Rollback() 77 | c.tx = nil 78 | return err 79 | } 80 | 81 | type sqlConnectionProvider struct { 82 | *AbstractConnectionProvider 83 | } 84 | 85 | func (c *sqlConnectionProvider) NewConnection() (Connection, error) { 86 | config := c.ConnectionProvider.Config() 87 | dsn, err := config.DsnDescriptor() 88 | if err != nil { 89 | return nil, err 90 | } 91 | db, err := sql.Open(config.DriverName, dsn) 92 | if err != nil { 93 | return nil, fmt.Errorf("failed to open connection to %v on %v due to %v", config.DriverName, config.Descriptor, err) 94 | } 95 | if len(config.InitSQL) > 0 { 96 | for _, SQL := range config.InitSQL { 97 | if _, err = db.Exec(SQL); err != nil { 98 | return nil, fmt.Errorf("failed to execute init SQL %v on %v due to %v", SQL, config.Descriptor, err) 99 | } 100 | } 101 | } 102 | dialect := GetDatastoreDialect(config.DriverName) 103 | var sqlConnection = &sqlConnection{db: db, canHandleTransaction: dialect.CanHandleTransaction()} 104 | var connection Connection = sqlConnection 105 | var super = NewAbstractConnection(config, c.ConnectionProvider.ConnectionPool(), connection) 106 | sqlConnection.AbstractConnection = super 107 | 108 | return connection, nil 109 | } 110 | 111 | func (c *sqlConnectionProvider) Get() (Connection, error) { 112 | result, err := c.AbstractConnectionProvider.Get() 113 | if err != nil { 114 | return nil, err 115 | } 116 | db, err := asSQLDb(result.Unwrap(sqlDbPointer)) 117 | if err != nil { 118 | return nil, err 119 | } 120 | 121 | if result.LastUsed() != nil && (time.Now().Sub(*result.LastUsed()) > 60*time.Second) { 122 | err = db.Ping() 123 | } 124 | 125 | if err == nil { 126 | return result, nil 127 | } 128 | 129 | if c.config.Has(connMaxLifetimeMsKey) { 130 | connMaxLifetime := c.config.GetDuration(connMaxLifetimeMsKey, time.Millisecond, defaultConnMaxLifetimeMs) 131 | if connMaxLifetime != 0 { 132 | db.SetConnMaxLifetime(connMaxLifetime) 133 | } 134 | } 135 | if c.config.Has(maxIdleConnsKey) { 136 | db.SetMaxIdleConns(c.config.GetInt(maxIdleConnsKey, 1)) 137 | } 138 | 139 | result, err = c.NewConnection() 140 | if err != nil { 141 | return nil, err 142 | } 143 | return result, nil 144 | } 145 | 146 | func newSQLConnectionProvider(config *Config) ConnectionProvider { 147 | if config.MaxPoolSize == 0 { 148 | config.MaxPoolSize = 1 149 | } 150 | sqlConnectionProvider := &sqlConnectionProvider{} 151 | var connectionProvider ConnectionProvider = sqlConnectionProvider 152 | super := NewAbstractConnectionProvider(config, make(chan Connection, config.MaxPoolSize), connectionProvider) 153 | sqlConnectionProvider.AbstractConnectionProvider = super 154 | return connectionProvider 155 | } 156 | -------------------------------------------------------------------------------- /sql_dialect_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | _ "github.com/denisenkom/go-mssqldb" 5 | _ "github.com/mattn/go-sqlite3" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/dsc" 8 | "testing" 9 | ) 10 | 11 | func TestSqlAllSqlDialect(t *testing.T) { 12 | var dialect dsc.DatastoreDialect 13 | for _, driver := range []string{"sqlite3", "mysql", "pg", "ora", "mssql"} { 14 | dialect = dsc.GetDatastoreDialect(driver) 15 | assert.NotNil(t, dialect) 16 | } 17 | } 18 | func TestSqlDialect(t *testing.T) { 19 | 20 | dialect := dsc.GetDatastoreDialect("sqlite3") 21 | config := dsc.NewConfig("sqlite3", "[url]", "url:./test/bar.db") 22 | factory := dsc.NewManagerFactory() 23 | manager, err := factory.Create(config) 24 | assert.Nil(t, err) 25 | 26 | assert.Nil(t, dialect.CreateDatastore(manager, "bar")) 27 | assert.Nil(t, dialect.DropDatastore(manager, "bar")) 28 | 29 | assert.Nil(t, dialect.CreateTable(manager, "bar", "table1", "`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,`username` varchar(255) DEFAULT NULL,`active` tinyint(1) DEFAULT '1',`salary` decimal(7,2) DEFAULT NULL,`comments` text,`last_access_time` timestamp DEFAULT CURRENT_TIMESTAMP")) 30 | assert.Nil(t, dialect.CreateTable(manager, "bar", "table2", "`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,`username` varchar(255) DEFAULT NULL,`active` tinyint(1) DEFAULT '1',`salary` decimal(7,2) DEFAULT NULL,`comments` text,`last_access_time` timestamp DEFAULT CURRENT_TIMESTAMP")) 31 | tables, err := dialect.GetTables(manager, "bar") 32 | assert.Nil(t, err) 33 | assert.Equal(t, []string{"table1", "table2"}, tables) 34 | 35 | seq, err := dialect.GetSequence(manager, "table1") 36 | assert.Nil(t, err) 37 | assert.EqualValues(t, 1, seq) 38 | 39 | columns, err := dialect.GetColumns(manager, "bar.db", "table1") 40 | assert.Nil(t, err) 41 | assert.EqualValues(t, 6, len(columns)) 42 | 43 | pk := dialect.GetKeyName(manager, "bar.db", "table1") 44 | assert.Nil(t, err) 45 | assert.EqualValues(t, "id", pk) 46 | 47 | ddl, err := dialect.ShowCreateTable(manager, "table1") 48 | assert.Nil(t, err) 49 | 50 | assert.EqualValues(t, `CREATE TABLE table1( 51 | id INTEGER PRIMARY KEY , 52 | username varchar(255), 53 | active tinyint(1), 54 | salary decimal(7,2), 55 | comments text, 56 | last_access_time timestamp);`, ddl) 57 | 58 | assert.Nil(t, dialect.DropTable(manager, "bar", "table1")) 59 | assert.False(t, dialect.CanPersistBatch()) 60 | datastore, err := dialect.GetCurrentDatastore(manager) 61 | assert.Equal(t, "bar.db", datastore) 62 | datastores, err := dialect.GetDatastores(manager) 63 | assert.Equal(t, []string{"bar.db"}, datastores) 64 | 65 | assert.Nil(t, dialect.DropDatastore(manager, "bar")) 66 | 67 | { 68 | mySQLDialect := dsc.GetDatastoreDialect("mysql") 69 | assert.True(t, mySQLDialect.CanPersistBatch()) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /sql_manager.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/pkg/errors" 7 | "reflect" 8 | "time" 9 | ) 10 | 11 | func asSQLDb(wrapped interface{}) (*sql.DB, error) { 12 | if result, ok := wrapped.(*sql.DB); ok { 13 | return result, nil 14 | } 15 | wrappedType := reflect.ValueOf(wrapped) 16 | return nil, fmt.Errorf(fmt.Sprintf("failed cast as sql.DB: was %v !", wrappedType.Type())) 17 | } 18 | 19 | func asSQLTx(wrapped interface{}) (*sql.Tx, error) { 20 | if wrapped == nil { 21 | return nil, nil 22 | } 23 | if result, ok := wrapped.(*sql.Tx); ok { 24 | return result, nil 25 | } 26 | wrappedType := reflect.ValueOf(wrapped) 27 | return nil, fmt.Errorf(fmt.Sprintf("failed cast as sql.Tx: was %v !", wrappedType.Type())) 28 | } 29 | 30 | func asScanner(wrapped interface{}) (Scanner, error) { 31 | sqlRows, ok := wrapped.(*sql.Rows) 32 | if !ok { 33 | return nil, errors.Errorf("expected :%T, but had: %T", sqlRows, wrapped) 34 | } 35 | return &sqlScanner{sqlRows}, nil 36 | } 37 | 38 | type sqlScanner struct { 39 | *sql.Rows 40 | } 41 | 42 | func (s *sqlScanner) ColumnTypes() ([]ColumnType, error) { 43 | types, err := s.Rows.ColumnTypes() 44 | if err != nil || len(types) == 0 { 45 | return nil, err 46 | } 47 | var columnTypes = make([]ColumnType, len(types)) 48 | for i := range types { 49 | columnTypes[i] = types[i] 50 | } 51 | return columnTypes, nil 52 | } 53 | 54 | type sqlExecutor interface { 55 | Exec(sql string, parameters ...interface{}) (sql.Result, error) 56 | } 57 | 58 | type sqlManager struct { 59 | *AbstractManager 60 | } 61 | 62 | func (m *sqlManager) initConnectionIfNeeded(connection Connection) error { 63 | if sqlConnection, ok := connection.(*sqlConnection); ok { 64 | if sqlConnection.init { 65 | return nil 66 | } 67 | sqlConnection.init = true 68 | dialect := GetDatastoreDialect(m.config.DriverName) 69 | return dialect.Init(m, connection) 70 | } 71 | return nil 72 | } 73 | 74 | func (m *sqlManager) ExecuteOnConnection(connection Connection, sql string, args []interface{}) (sql.Result, error) { 75 | m.Acquire() 76 | db, err := asSQLDb(connection.Unwrap(sqlDbPointer)) 77 | if err == nil { 78 | err = m.initConnectionIfNeeded(connection) 79 | } 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | var executable sqlExecutor = db 85 | tx, err := asSQLTx(connection.Unwrap(sqlTxtPointer)) 86 | if err != nil { 87 | return nil, err 88 | } 89 | if tx != nil { 90 | executable = tx 91 | } 92 | if args == nil { 93 | args = make([]interface{}, 0) 94 | } 95 | 96 | dialect := GetDatastoreDialect(m.config.DriverName) 97 | sql = dialect.NormalizeSQL(sql) 98 | result, err := executable.Exec(sql, args...) 99 | if !dialect.CanHandleTransaction() { 100 | result = NewSQLResult(1, 0) 101 | } 102 | Logf("[%v]:%v %v", m.config.username, sql, args) 103 | if err != nil { 104 | return nil, fmt.Errorf("failed to execute %w: %v %v on %v", err.Error(), sql, args, m.Manager.Config().Parameters) 105 | } 106 | return result, err 107 | } 108 | 109 | func (m *sqlManager) ReadAllOnWithHandlerOnConnection(connection Connection, query string, args []interface{}, readingHandler func(scanner Scanner) (toContinue bool, err error)) error { 110 | m.Acquire() 111 | startTime := time.Now() 112 | db, err := asSQLDb(connection.Unwrap((*sql.DB)(nil))) 113 | if err == nil { 114 | err = m.initConnectionIfNeeded(connection) 115 | } 116 | if err != nil { 117 | return err 118 | } 119 | 120 | dialect := GetDatastoreDialect(m.config.DriverName) 121 | query = dialect.NormalizeSQL(query) 122 | Logf("[%v]:%v", m.config.username, query) 123 | 124 | sqlStatement, sqlError := db.Prepare(query) 125 | if sqlError != nil { 126 | return fmt.Errorf("failed to prepare sql: %v with %v due to:%v\n\t", query, args, sqlError.Error()) 127 | } 128 | 129 | Logf("[%v]:prepare time: %v\n", m.config.username, time.Now().Sub(startTime)) 130 | 131 | defer sqlStatement.Close() 132 | rows, queryError := m.executeQuery(sqlStatement, query, args) 133 | if queryError != nil { 134 | return fmt.Errorf(fmt.Sprintf("failed to execute sql: %v with %v due to:%v\n\t", query, args, queryError.Error())) 135 | } 136 | Logf("[%v]:execute time: %v\n", m.config.username, time.Now().Sub(startTime)) 137 | 138 | defer rows.Close() 139 | 140 | for rows.Next() { 141 | scanner, _ := asScanner(rows) 142 | 143 | toContinue, err := readingHandler(NewScanner(scanner)) 144 | if err != nil { 145 | return err 146 | } 147 | if !toContinue { 148 | break 149 | } 150 | } 151 | Logf("[%v]:fetched time: %v\n", m.config.username, time.Now().Sub(startTime)) 152 | return rows.Err() 153 | } 154 | 155 | func (m *sqlManager) executeQuery(sqlStatement *sql.Stmt, query string, args []interface{}) (rows *sql.Rows, err error) { 156 | if args == nil { 157 | args = make([]interface{}, 0) 158 | } 159 | rows, err = sqlStatement.Query(args...) 160 | if err != nil { 161 | return nil, err 162 | } 163 | return rows, nil 164 | } 165 | -------------------------------------------------------------------------------- /sql_manager_factory.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | type sqlManagerFactory struct{} 4 | 5 | func (mf *sqlManagerFactory) Create(config *Config) (Manager, error) { 6 | if err := config.Init(); err != nil { 7 | return nil, err 8 | } 9 | var connectionProvider = newSQLConnectionProvider(config) 10 | sqlManager := &sqlManager{} 11 | var self Manager = sqlManager 12 | super := NewAbstractManager(config, connectionProvider, self) 13 | sqlManager.AbstractManager = super 14 | return self, nil 15 | } 16 | 17 | func (mf sqlManagerFactory) CreateFromURL(URL string) (Manager, error) { 18 | config, err := NewConfigFromURL(URL) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return mf.Create(config) 23 | } 24 | -------------------------------------------------------------------------------- /sql_manager_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "fmt" 5 | _ "github.com/go-sql-driver/mysql" 6 | _ "github.com/mattn/go-sqlite3" 7 | "github.com/stretchr/testify/assert" 8 | "github.com/viant/dsc" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func GetManager(t *testing.T) dsc.Manager { 14 | config := dsc.NewConfig("sqlite3", "[url]", "url:./test/foo.db") 15 | factory := dsc.NewManagerFactory() 16 | manager, err := factory.Create(config) 17 | sqls := []string{ 18 | "DROP TABLE IF EXISTS users", 19 | "CREATE TABLE `users` (`id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,`username` varchar(255) DEFAULT NULL,`active` tinyint(1) DEFAULT '1',`salary` decimal(7,2) DEFAULT NULL,`comments` text,`last_access_time` timestamp DEFAULT CURRENT_TIMESTAMP)", 20 | "INSERT INTO users(username, active, salary, comments, last_access_time) VALUES('Edi', 1, 43000, 'no comments', '2010-05-28T15:36:56.200')", 21 | } 22 | assert.Nil(t, err) 23 | 24 | for _, sql := range sqls { 25 | _, err := manager.Execute(sql) 26 | if err != nil { 27 | t.Fatalf("failed to init database %v", err) 28 | } 29 | } 30 | return manager 31 | } 32 | 33 | func GetSqlManager(t *testing.T) dsc.Manager { 34 | config := dsc.NewConfig("mysql", 35 | "[user]:[password]@[url]", 36 | "user:root,password:dev,url:tcp(localhost:3306)/mydbname?parseTime=true") 37 | factory := dsc.NewManagerFactory() 38 | config.MaxPoolSize = 10 39 | manager, err := factory.Create(config) 40 | sqls := []string{ 41 | "DROP TABLE IF EXISTS users", 42 | "CREATE TABLE `users` (" + 43 | "`id` INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT," + 44 | "`username` varchar(255) DEFAULT NULL," + 45 | "`active` tinyint(1) DEFAULT '1'," + 46 | "`salary` decimal(7,2) DEFAULT NULL," + 47 | "`comments` text," + 48 | "`last_access_time` timestamp DEFAULT CURRENT_TIMESTAMP" + 49 | ") ENGINE = InnoDB", 50 | "INSERT INTO users(username, active, salary, comments, last_access_time) VALUES('Edi', 1, 43000, 'no comments', '2010-05-28T15:36:56.200'),('Sam', 0, 43000, 'test comments', '2010-05-28T15:36:56.200')", 51 | } 52 | assert.Nil(t, err) 53 | for _, sql := range sqls { 54 | _, err := manager.Execute(sql) 55 | if err != nil { 56 | t.Fatalf("failed to init database %v", err) 57 | } 58 | } 59 | return manager 60 | } 61 | 62 | type User struct { 63 | Id int `autoincrement:"true"` 64 | Username string 65 | Active bool 66 | LastAccessTime *time.Time `column:"last_access_time" dateFormat:"2006-01-02 15:04:05"` 67 | Salary float64 `column:"salary"` 68 | Comments string 69 | } 70 | 71 | func (this User) String() string { 72 | return fmt.Sprintf("Id: %v, Name: %v, Active:%v, Salary: %v Comments: %v, Last Access Time %v\n", this.Id, this.Username, this.Active, this.Salary, this.Comments, this.LastAccessTime) 73 | } 74 | 75 | type UserRecordMapper struct{} 76 | 77 | func (this *UserRecordMapper) Map(scanner dsc.Scanner) (interface{}, error) { 78 | user := User{} 79 | err := scanner.Scan( 80 | &user.Id, 81 | &user.Username, 82 | &user.Active, 83 | &user.Salary, 84 | &user.Comments, 85 | &user.LastAccessTime, 86 | ) 87 | if err != nil { 88 | return nil, err 89 | } 90 | return &user, nil 91 | } 92 | 93 | func TestConnection(t *testing.T) { 94 | manager := GetManager(t) 95 | for i := 0; i < 10; i++ { 96 | connection, err := manager.ConnectionProvider().Get() 97 | assert.Nil(t, err) 98 | defer connection.Close() 99 | 100 | } 101 | manager.ConnectionProvider().Close() 102 | } 103 | 104 | func TestExecuteWithError(t *testing.T) { 105 | manager := GetManager(t) 106 | _, err := manager.Execute("SEL ", 1) 107 | assert.NotNil(t, err) 108 | 109 | _, err = manager.ExecuteAll([]string{"SEL "}) 110 | assert.NotNil(t, err) 111 | user := &User{Id: 1} 112 | 113 | _, err = manager.ReadSingle(&user, "SELECT id, username FROM a id = ?", []interface{}{1}, nil) 114 | assert.NotNil(t, err) 115 | 116 | var users = make([]User, 1) 117 | err = manager.ReadAll(&users, "SELECT id, username FROM a id = ?", []interface{}{1}, nil) 118 | assert.NotNil(t, err) 119 | 120 | users[0] = User{} 121 | _, _, err = manager.PersistAll(&users, "asd", nil) 122 | assert.NotNil(t, err) 123 | 124 | _, _, err = manager.PersistSingle(&users[0], "asd", nil) 125 | assert.NotNil(t, err) 126 | 127 | _, err = manager.DeleteSingle(&users[0], "asd", nil) 128 | assert.NotNil(t, err) 129 | 130 | _, err = manager.DeleteAll(&users, "asd", nil) 131 | assert.NotNil(t, err) 132 | } 133 | 134 | func TestReadSingleWithCustomHandler(t *testing.T) { 135 | manager := GetManager(t) 136 | user := &User{} 137 | 138 | err := manager.ReadAllWithHandler("SELECT id, username FROM users WHERE id = ?", []interface{}{1}, func(scanner dsc.Scanner) (bool, error) { 139 | err := scanner.Scan(&user.Id, &user.Username) 140 | if err != nil { 141 | t.Errorf("Error %v", err) 142 | } 143 | return false, nil 144 | }) 145 | 146 | if err != nil { 147 | t.Error("failed test: " + err.Error()) 148 | } 149 | assert.Equal(t, "Edi", user.Username) 150 | 151 | } 152 | func TestReadSingleWithCustomMapperDataset(t *testing.T) { 153 | manager := GetManager(t) 154 | singleUser := User{} 155 | var recordMapper dsc.RecordMapper = &UserRecordMapper{} 156 | success, err := manager.ReadSingle(&singleUser, "SELECT id, username, active, salary, comments,last_access_time FROM users WHERE id = ?", []interface{}{1}, recordMapper) 157 | if err != nil { 158 | t.Error("failed test: " + err.Error()) 159 | } 160 | assert.Equal(t, true, success, "Should fetch a user") 161 | assert.Equal(t, "Edi", singleUser.Username) 162 | 163 | } 164 | 165 | func TestReadAllWithCustomMapperDataset(t *testing.T) { 166 | manager := GetManager(t) 167 | var users = make([]User, 0) 168 | var recordMapper dsc.RecordMapper = &UserRecordMapper{} 169 | err := manager.ReadAll(&users, "SELECT id, username, active, salary, comments,last_access_time FROM users WHERE id = ?", []interface{}{1}, recordMapper) 170 | if err != nil { 171 | t.Error("failed test: " + err.Error()) 172 | } 173 | assert.Equal(t, 1, len(users)) 174 | user := users[0] 175 | assert.Equal(t, "Edi", user.Username) 176 | 177 | } 178 | 179 | func TestReadSingleWithDefaultMetaMapper(t *testing.T) { 180 | manager := GetManager(t) 181 | singleUser := User{} 182 | 183 | success, err := manager.ReadSingle(&singleUser, "SELECT id, username, active, salary, comments,last_access_time FROM users WHERE id = ?", []interface{}{1}, nil) 184 | if err != nil { 185 | t.Error("failed test: " + err.Error()) 186 | } 187 | assert.Equal(t, true, success, "Should fetch a user") 188 | assert.Equal(t, true, singleUser.Active) 189 | } 190 | 191 | //func TestExecessiveReadAll(t *testing.T) { 192 | // manager := GetSqlManager(t) 193 | // for i := 0; i < 10000; i++ { 194 | // users := make([]*User, 0) 195 | // err := manager.ReadAll(&users, "SELECT * FROM users", nil, nil) 196 | // if err != nil { 197 | // t.Fatal("failed test:", err.Error()) 198 | // } 199 | // } 200 | //} 201 | 202 | func TestReadAllWithDefaultMetaMapper(t *testing.T) { 203 | manager := GetManager(t) 204 | var users = make([]User, 0) 205 | 206 | err := manager.ReadAll(&users, "SELECT id, username, active, salary, comments,last_access_time FROM users WHERE id = ?", []interface{}{1}, nil) 207 | if err != nil { 208 | t.Error("failed test: " + err.Error()) 209 | } 210 | assert.Equal(t, 1, len(users)) 211 | user := users[0] 212 | assert.Equal(t, "Edi", user.Username) 213 | } 214 | 215 | func TestReadSingleRowAsSlice(t *testing.T) { 216 | manager := GetManager(t) 217 | var aUser = make([]interface{}, 0) 218 | 219 | success, err := manager.ReadSingle(&aUser, "SELECT id, username, active, salary, comments,last_access_time FROM users WHERE id = ?", []interface{}{1}, nil) 220 | if err != nil { 221 | t.Error("failed test: " + err.Error()) 222 | } 223 | assert.Equal(t, true, success, "Should fetch a user") 224 | assert.EqualValues(t, 1, aUser[0].(int64)) 225 | } 226 | 227 | func TestReadSingleRowAsMap(t *testing.T) { 228 | manager := GetManager(t) 229 | var aUser = make(map[string]interface{}) 230 | 231 | success, err := manager.ReadSingle(&aUser, "SELECT id, username, active, salary, comments,last_access_time FROM users WHERE id = ?", []interface{}{1}, nil) 232 | if err != nil { 233 | t.Error("failed test: " + err.Error()) 234 | } 235 | assert.Equal(t, true, success, "Should fetch a user") 236 | assert.EqualValues(t, 1, aUser["id"].(int64)) 237 | 238 | } 239 | 240 | func TestReadSingleAllRowAsSlice(t *testing.T) { 241 | manager := GetManager(t) 242 | var users = make([][]interface{}, 0) 243 | 244 | err := manager.ReadAll(&users, "SELECT id, username, active, salary, comments,last_access_time FROM users WHERE id = ?", []interface{}{1}, nil) 245 | if err != nil { 246 | t.Error("failed test: " + err.Error()) 247 | } 248 | 249 | assert.Equal(t, 1, len(users)) 250 | user := users[0] 251 | 252 | assert.Equal(t, "Edi", user[1].(string)) 253 | } 254 | 255 | func TestReadSingleAllRowAsMap(t *testing.T) { 256 | manager := GetManager(t) 257 | var users = make([]map[string]interface{}, 0) 258 | 259 | err := manager.ReadAll(&users, "SELECT id, username, active, salary, comments,last_access_time FROM users WHERE id = ?", []interface{}{1}, nil) 260 | if err != nil { 261 | t.Error("failed test: " + err.Error()) 262 | } 263 | 264 | assert.Equal(t, 1, len(users)) 265 | user := users[0] 266 | 267 | assert.Equal(t, "Edi", user["username"].(string)) 268 | } 269 | 270 | type UserDmlProvider struct{} 271 | 272 | func (this UserDmlProvider) Get(operationType int, instance interface{}) *dsc.ParametrizedSQL { 273 | user := instance.(User) 274 | switch operationType { 275 | case dsc.SQLTypeInsert: 276 | return &dsc.ParametrizedSQL{ 277 | SQL: "INSERT INTO users(id, username) VALUES(?, ?)", 278 | Values: []interface{}{user.Id, user.Username}, 279 | } 280 | 281 | case dsc.SQLTypeUpdate: 282 | return &dsc.ParametrizedSQL{ 283 | SQL: "UPDATE users SET username = ? WHERE id = ?", 284 | Values: []interface{}{user.Id, user.Username}, 285 | } 286 | 287 | } 288 | panic(fmt.Sprintf("unsupported sql type:%v", operationType)) 289 | } 290 | 291 | func (this UserDmlProvider) SetKey(instance interface{}, seq int64) { 292 | user := instance.(*User) 293 | user.Id = int(seq) 294 | } 295 | 296 | func (this UserDmlProvider) Key(instance interface{}) []interface{} { 297 | user := instance.(User) 298 | return []interface{}{user.Id} 299 | } 300 | 301 | func (this UserDmlProvider) PkColumns() []string { 302 | return []string{"id"} 303 | } 304 | 305 | func NewUserDmlProvider() dsc.DmlProvider { 306 | var dmlProvider dsc.DmlProvider = &UserDmlProvider{} 307 | return dmlProvider 308 | } 309 | 310 | func TestPersistAllWithCustomDmlProvider(t *testing.T) { 311 | manager := GetManager(t) 312 | users := []User{ 313 | { 314 | Id: 1, 315 | Username: "Sir Edi", 316 | Active: false, 317 | Salary: 32432.3, 318 | }, 319 | { 320 | Username: "Bogi", 321 | Active: true, 322 | Salary: 32432.3, 323 | }, 324 | } 325 | inserted, updated, err := manager.PersistAll(&users, "users", NewUserDmlProvider()) 326 | if err != nil { 327 | t.Error("failed test: " + err.Error()) 328 | } 329 | assert.Equal(t, 1, inserted) 330 | assert.Equal(t, 0, updated) 331 | } 332 | 333 | func TestPersistAllWithDefaultDmlProvider(t *testing.T) { 334 | manager := GetManager(t) 335 | users := []User{ 336 | { 337 | Id: 1, 338 | Username: "Sir Edi", 339 | Active: false, 340 | Salary: 32432.3, 341 | }, 342 | { 343 | Username: "Bogi", 344 | Active: true, 345 | Salary: 32432.3, 346 | }, 347 | } 348 | inserted, updated, err := manager.PersistAll(&users, "users", nil) 349 | if err != nil { 350 | t.Error("failed test: " + err.Error()) 351 | } 352 | assert.Equal(t, 2, users[1].Id, "autoincrement value should be set") 353 | assert.Equal(t, 1, inserted) 354 | assert.Equal(t, 1, updated) 355 | user := User{Username: "KLK", Active: false} 356 | inserted, updated, err = manager.PersistSingle(&user, "users", nil) 357 | if err != nil { 358 | t.Error("failed test: " + err.Error()) 359 | } 360 | assert.Equal(t, 1, inserted) 361 | assert.Equal(t, 3, user.Id, "autoincrement value should be set") 362 | 363 | } 364 | 365 | func TestPersistSingleWithDefaultDmlProvider(t *testing.T) { 366 | manager := GetManager(t) 367 | users := []User{ 368 | { 369 | Id: 1, 370 | Username: "Sir Edi", 371 | Active: false, 372 | Salary: 32432.3, 373 | }, 374 | { 375 | Username: "Bogi", 376 | Active: true, 377 | Salary: 32432.3, 378 | }, 379 | } 380 | inserted, updated, err := manager.PersistSingle(&users[0], "users", nil) 381 | if err != nil { 382 | t.Error("failed test: " + err.Error()) 383 | } 384 | assert.Equal(t, 1, updated) 385 | assert.Equal(t, 0, inserted) 386 | 387 | inserted, updated, err = manager.PersistSingle(&users[1], "users", nil) 388 | if err != nil { 389 | t.Error("failed test: " + err.Error()) 390 | } 391 | 392 | assert.Equal(t, 2, users[1].Id, "autoicrement value should be set") 393 | assert.Equal(t, 1, inserted) 394 | assert.Equal(t, 0, updated) 395 | 396 | } 397 | 398 | func TestDeleteAll(t *testing.T) { 399 | manager := GetManager(t) 400 | users := []User{ 401 | { 402 | Id: 1, 403 | Username: "Sir Edi", 404 | Active: false, 405 | Salary: 32432.3, 406 | }, 407 | { 408 | Username: "Bogi", 409 | Active: true, 410 | Salary: 32432.3, 411 | }, 412 | } 413 | _, _, err := manager.PersistAll(&users, "users", nil) 414 | assert.Nil(t, err) 415 | deleted, err := manager.DeleteAll(users, "users", nil) 416 | assert.Nil(t, err) 417 | assert.Equal(t, 2, deleted) 418 | } 419 | 420 | func TestDeleteSingle(t *testing.T) { 421 | manager := GetManager(t) 422 | users := []User{ 423 | { 424 | Id: 1, 425 | Username: "Sir Edi", 426 | Active: false, 427 | Salary: 32432.3, 428 | }, 429 | { 430 | Username: "Bogi", 431 | Active: true, 432 | Salary: 32432.3, 433 | }, 434 | } 435 | _, _, err := manager.PersistAll(&users, "users", nil) 436 | assert.Nil(t, err) 437 | deleted, err := manager.DeleteSingle(&users[0], "users", nil) 438 | assert.Nil(t, err) 439 | assert.True(t, deleted) 440 | } 441 | -------------------------------------------------------------------------------- /sql_parser_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/dsc" 8 | "github.com/viant/toolbox" 9 | ) 10 | 11 | func TestAggregationQueryParser(t *testing.T) { 12 | 13 | parser := dsc.NewQueryParser() 14 | { 15 | query, err := parser.Parse("SELECT col1, SUM(col2) FROM bar GROUP BY 1") 16 | if err != nil { 17 | t.Fatalf(err.Error()) 18 | } 19 | assert.NotNil(t, query, "should have query") 20 | assert.Equal(t, 2, len(query.Columns)) 21 | 22 | assert.Equal(t, "col1", query.Columns[0].Name) 23 | 24 | assert.Equal(t, "SUM(col2)", query.Columns[1].Expression) 25 | 26 | assert.Equal(t, "SUM", query.Columns[1].Function) 27 | assert.Equal(t, "col2", query.Columns[1].FunctionArguments) 28 | 29 | assert.Equal(t, "f1", query.Columns[1].Alias) 30 | 31 | assert.Equal(t, 1, len(query.GroupBy)) 32 | assert.Equal(t, "col1", query.Columns[0].Name) 33 | 34 | assert.Equal(t, "bar", query.Table) 35 | } 36 | 37 | { 38 | query, err := parser.Parse("SELECT col1, SUM(col2) FROM bar WHERE col3 > 7 GROUP BY 1") 39 | if err != nil { 40 | t.Fatalf(err.Error()) 41 | } 42 | assert.NotNil(t, query, "should have query") 43 | assert.Equal(t, 2, len(query.Columns)) 44 | 45 | assert.Equal(t, "col1", query.Columns[0].Name) 46 | assert.Equal(t, "SUM(col2)", query.Columns[1].Expression) 47 | assert.Equal(t, "SUM", query.Columns[1].Function) 48 | assert.Equal(t, "col2", query.Columns[1].FunctionArguments) 49 | assert.Equal(t, "f1", query.Columns[1].Alias) 50 | assert.Equal(t, 1, len(query.GroupBy)) 51 | assert.Equal(t, "col1", query.Columns[0].Name) 52 | assert.Equal(t, "bar", query.Table) 53 | } 54 | 55 | } 56 | 57 | func TestQueryParser(t *testing.T) { 58 | parser := dsc.NewQueryParser() 59 | { 60 | query, err := parser.Parse("SELECT abc FROM bar") 61 | if err != nil { 62 | t.Fatalf(err.Error()) 63 | } 64 | assert.NotNil(t, query, "should have query") 65 | assert.Equal(t, 1, len(query.Columns)) 66 | assert.Equal(t, "abc", query.Columns[0].Name) 67 | assert.Equal(t, "bar", query.Table) 68 | 69 | } 70 | 71 | { 72 | query, err := parser.Parse("SELECT id,\nevent_type,\nquantity,\ntimestamp,\nquery_string\nFROM events t") 73 | assert.Nil(t, err) 74 | assert.Equal(t, 5, len(query.Columns)) 75 | assert.Equal(t, "id", query.Columns[0].Name) 76 | assert.Equal(t, "events", query.Table) 77 | assert.Equal(t, "t", query.Alias) 78 | } 79 | 80 | { 81 | query, err := parser.Parse("SELECT c1, c2 FROM bar") 82 | assert.Nil(t, err) 83 | assert.Equal(t, []string{"c1", "c2"}, query.ColumnNames()) 84 | } 85 | 86 | { 87 | query, err := parser.Parse("SELECT * FROM foo") 88 | if err != nil { 89 | t.Fatalf(err.Error()) 90 | } 91 | assert.NotNil(t, query, "should have query") 92 | assert.Equal(t, true, query.AllField) 93 | assert.Equal(t, "foo", query.Table) 94 | 95 | } 96 | 97 | { 98 | query, err := parser.Parse("SELECT * FROM foo WHERE column1 = 2") 99 | if err != nil { 100 | t.Fatalf(err.Error()) 101 | } 102 | assert.NotNil(t, query, "should have query") 103 | assert.Equal(t, true, query.AllField) 104 | assert.Equal(t, "foo", query.Table) 105 | assert.Equal(t, 1, len(query.Criteria)) 106 | assert.Equal(t, "column1", query.Criteria[0].LeftOperand) 107 | assert.Equal(t, "=", query.Criteria[0].Operator) 108 | assert.Equal(t, "2", query.Criteria[0].RightOperand) 109 | 110 | } 111 | 112 | { 113 | query, err := parser.Parse("SELECT * FROM foo WHERE column1 = 2 AND column2 != ?") 114 | if err != nil { 115 | t.Fatalf(err.Error()) 116 | } 117 | assert.NotNil(t, query, "should have query") 118 | assert.Equal(t, true, query.AllField) 119 | assert.Equal(t, "foo", query.Table) 120 | assert.Equal(t, 2, len(query.Criteria)) 121 | { 122 | assert.Equal(t, "column1", query.Criteria[0].LeftOperand) 123 | assert.Equal(t, "=", query.Criteria[0].Operator) 124 | assert.Equal(t, "2", query.Criteria[0].RightOperand) 125 | assert.Equal(t, "AND", query.LogicalOperator) 126 | 127 | } 128 | { 129 | assert.Equal(t, "column2", query.Criteria[1].LeftOperand) 130 | assert.Equal(t, "!=", query.Criteria[1].Operator) 131 | assert.Equal(t, "?", query.Criteria[1].RightOperand) 132 | assert.Equal(t, "AND", query.LogicalOperator) 133 | 134 | } 135 | 136 | } 137 | 138 | { 139 | query, err := parser.Parse("SELECT abc FROM bar WHERE id IN (1, 2, ?)") 140 | if err != nil { 141 | t.Fatalf(err.Error()) 142 | } 143 | assert.NotNil(t, query, "should have query") 144 | assert.Equal(t, 1, len(query.Columns)) 145 | assert.Equal(t, "abc", query.Columns[0].Name) 146 | assert.Equal(t, "bar", query.Table) 147 | assert.Equal(t, 1, len(query.Criteria)) 148 | assert.Equal(t, 3, len(query.Criteria[0].RightOperands)) 149 | 150 | assert.EqualValues(t, "1", query.Criteria[0].RightOperands[0]) 151 | assert.Equal(t, "?", query.Criteria[0].RightOperands[2]) 152 | } 153 | 154 | { 155 | query, err := parser.Parse("SELECT abc FROM bar WHERE id NOT IN (1, 2, ?)") 156 | if err != nil { 157 | t.Fatalf(err.Error()) 158 | } 159 | assert.NotNil(t, query, "should have query") 160 | assert.Equal(t, 1, len(query.Columns)) 161 | assert.Equal(t, "abc", query.Columns[0].Name) 162 | assert.Equal(t, "bar", query.Table) 163 | assert.Equal(t, 1, len(query.Criteria)) 164 | assert.True(t, query.Criteria[0].Inverse) 165 | assert.Equal(t, 3, len(query.Criteria[0].RightOperands)) 166 | 167 | assert.EqualValues(t, "1", query.Criteria[0].RightOperands[0]) 168 | assert.Equal(t, "?", query.Criteria[0].RightOperands[2]) 169 | } 170 | 171 | { 172 | query, err := parser.Parse("SELECT key, username, active, salary, comments,last_access_time FROM users WHERE key = ?") 173 | if err != nil { 174 | t.Fatalf(err.Error()) 175 | } 176 | assert.NotNil(t, query, "should have query") 177 | } 178 | 179 | //SELECT id, username, active, salary, comments,last_access_time FROM users WHERE key = ? 180 | 181 | { 182 | query, err := parser.Parse("SELECT * FROM foo WHERE id IS NULL") 183 | if err != nil { 184 | t.Fatalf(err.Error()) 185 | } 186 | assert.NotNil(t, query, "should have query") 187 | assert.Equal(t, true, query.AllField) 188 | assert.Equal(t, "foo", query.Table) 189 | assert.Equal(t, "id", query.Criteria[0].LeftOperand) 190 | assert.Equal(t, "IS", query.Criteria[0].Operator) 191 | assert.Nil(t, query.Criteria[0].RightOperand) 192 | 193 | } 194 | 195 | { 196 | query, err := parser.Parse("SELECT * FROM foo WHERE id BETWEEN 1 AND 2") 197 | if err != nil { 198 | t.Fatalf(err.Error()) 199 | } 200 | assert.NotNil(t, query, "should have query") 201 | assert.Equal(t, true, query.AllField) 202 | assert.Equal(t, "foo", query.Table) 203 | assert.Equal(t, "id", query.Criteria[0].LeftOperand) 204 | assert.Equal(t, "BETWEEN", query.Criteria[0].Operator) 205 | assert.Equal(t, 2, len(query.Criteria[0].RightOperands)) 206 | assert.Equal(t, "1", query.Criteria[0].RightOperands[0]) 207 | assert.Equal(t, "2", query.Criteria[0].RightOperands[1]) 208 | 209 | } 210 | 211 | { 212 | query, err := parser.Parse("SELECT * FROM foo WHERE id LIKE '%1%'") 213 | if err != nil { 214 | t.Fatalf(err.Error()) 215 | } 216 | assert.NotNil(t, query, "should have query") 217 | assert.Equal(t, true, query.AllField) 218 | assert.Equal(t, "foo", query.Table) 219 | assert.Equal(t, "id", query.Criteria[0].LeftOperand) 220 | assert.Equal(t, "LIKE", query.Criteria[0].Operator) 221 | assert.Equal(t, "'%1%'", query.Criteria[0].RightOperand) 222 | } 223 | 224 | { 225 | query, err := parser.Parse("SELECT foo, bar FROM table WHERE (foo, bar) IN ((1,2),(3,4)) ") 226 | assert.Nil(t, err) 227 | assert.Equal(t, false, query.AllField) 228 | assert.Equal(t, "table", query.Table) 229 | assert.Equal(t, "(foo, bar)", query.Criteria[0].LeftOperand) 230 | assert.Equal(t, "IN", query.Criteria[0].Operator) 231 | assert.Equal(t, []interface{}{ 232 | "(1,2)", 233 | "(3,4)", 234 | }, query.Criteria[0].RightOperands) 235 | 236 | } 237 | 238 | { 239 | _, err := parser.Parse("SELECT* FROM foo") 240 | assert.NotNil(t, err) 241 | } 242 | 243 | { 244 | _, err := parser.Parse("SELECT * FROM foo WHERE id IS a") 245 | assert.NotNil(t, err) 246 | } 247 | 248 | { 249 | _, err := parser.Parse("SELECT * FROM foo WHERE id IS NOT a") 250 | assert.NotNil(t, err) 251 | } 252 | 253 | { 254 | _, err := parser.Parse("SELECT * FROM foo WHERE id BETWEEN a b") 255 | assert.NotNil(t, err) 256 | } 257 | 258 | { 259 | _, err := parser.Parse("SELECT * FROM foo WHERE id IN a") 260 | assert.NotNil(t, err) 261 | } 262 | 263 | { 264 | _, err := parser.Parse("SELECT * FROM foo WHERE id NOT a") 265 | assert.NotNil(t, err) 266 | } 267 | 268 | { 269 | _, err := parser.Parse("SELECT a v s FROM foo WHERE id NOT a") 270 | assert.NotNil(t, err) 271 | } 272 | 273 | { 274 | _, err := parser.Parse(",SELECT a v s FROM foo WHERE id NOT a") 275 | assert.NotNil(t, err) 276 | } 277 | 278 | { 279 | _, err := parser.Parse(",SELECT a ,FROM foo WHERE id NOT a") 280 | assert.NotNil(t, err) 281 | } 282 | 283 | { 284 | _, err := parser.Parse(",SELECT a FROM foo ,WHERE id NOT a") 285 | assert.NotNil(t, err) 286 | } 287 | 288 | { 289 | _, err := parser.Parse("SELECT * a FROM foo ,WHERE id NOT a") 290 | assert.NotNil(t, err) 291 | } 292 | 293 | { 294 | _, err := parser.Parse("SELECT a AS 1 FROM foo WHERE id NOT a") 295 | assert.NotNil(t, err) 296 | } 297 | 298 | { 299 | _, err := parser.Parse("SELECT a FROM foo WHERE id BETWEEN") 300 | assert.NotNil(t, err) 301 | } 302 | 303 | { 304 | _, err := parser.Parse("SELECT a FROM foo WHERE id BETWEEN 1") 305 | assert.NotNil(t, err) 306 | } 307 | 308 | { 309 | _, err := parser.Parse("SELECT a FROM foo WHERE id BETWEEN 1 AND") 310 | assert.NotNil(t, err) 311 | } 312 | 313 | { 314 | _, err := parser.Parse("SELECT a FROM foo WHERE id LIKE") 315 | assert.NotNil(t, err) 316 | } 317 | 318 | { 319 | _, err := parser.Parse("SELECT a FROM foo WHERE id = 1 AVC") 320 | assert.NotNil(t, err) 321 | } 322 | 323 | { 324 | _, err := parser.Parse("SELECT a FROM foo WHERE") 325 | assert.NotNil(t, err) 326 | } 327 | 328 | { 329 | _, err := parser.Parse("SELECT a FROM foo WHERE ,") 330 | assert.NotNil(t, err) 331 | } 332 | 333 | { 334 | _, err := parser.Parse("SELECT a FROM foo WHERE in IN(1,)") 335 | assert.NotNil(t, err) 336 | } 337 | 338 | { 339 | _, err := parser.Parse("SELECT a FROM foo WHERE in IN(1") 340 | assert.NotNil(t, err) 341 | } 342 | 343 | { 344 | _, err := parser.Parse("SELECT a") 345 | assert.NotNil(t, err) 346 | } 347 | 348 | } 349 | 350 | func TestInvalidDml(t *testing.T) { 351 | 352 | parser := dsc.NewDmlParser() 353 | { 354 | _, err := parser.Parse(".INSERT INTO users(id, name, last_access_time) VALUES(?, ?, 2 )") 355 | assert.NotNil(t, err) 356 | } 357 | 358 | { 359 | _, err := parser.Parse(".INSERT users(id, name, last_access_time) VALUES(?, ?, 2 )") 360 | assert.NotNil(t, err) 361 | } 362 | { 363 | _, err := parser.Parse("INSERT INTO ") 364 | assert.NotNil(t, err) 365 | } 366 | { 367 | _, err := parser.Parse("INSERT INTO users ") 368 | assert.NotNil(t, err) 369 | } 370 | { 371 | _, err := parser.Parse(".INSERT users(id,) VALUES(?, ?, 2 )") 372 | assert.NotNil(t, err) 373 | } 374 | { 375 | _, err := parser.Parse(".INSERT users(id VALUES(?, ?, 2 )") 376 | assert.NotNil(t, err) 377 | } 378 | { 379 | _, err := parser.Parse(".INSERT users(id)") 380 | assert.NotNil(t, err) 381 | } 382 | { 383 | _, err := parser.Parse("INSERT INTO users(id, name, last_access_time) VALUES") 384 | assert.NotNil(t, err) 385 | } 386 | 387 | { 388 | _, err := parser.Parse("INSERT INTO ,") 389 | assert.NotNil(t, err) 390 | } 391 | 392 | { 393 | _, err := parser.Parse("INSERT ,") 394 | assert.NotNil(t, err) 395 | } 396 | { 397 | _, err := parser.Parse("INSERT INTO users(id, name, last_access_time) VALUES(1,") 398 | assert.NotNil(t, err) 399 | } 400 | 401 | { 402 | _, err := parser.Parse("INSERT INTO users(id,1) VALUES(1,") 403 | assert.NotNil(t, err) 404 | } 405 | 406 | { 407 | _, err := parser.Parse("INSERT INTO users(id -) VALUES(1,") 408 | assert.NotNil(t, err) 409 | } 410 | 411 | { 412 | _, err := parser.Parse("UPDATEusers SET name = 'Smith', last_access_time = ? WHERE id = 2") 413 | assert.NotNil(t, err) 414 | } 415 | { 416 | _, err := parser.Parse("UPDATE users") 417 | assert.NotNil(t, err) 418 | } 419 | 420 | { 421 | _, err := parser.Parse("UPDATE users SE") 422 | assert.NotNil(t, err.Error()) 423 | } 424 | 425 | { 426 | _, err := parser.Parse("UPDATE users SET") 427 | assert.NotNil(t, err) 428 | } 429 | { 430 | _, err := parser.Parse("UPDATEusers SET name WHERE id = 2") 431 | assert.NotNil(t, err) 432 | } 433 | 434 | { 435 | _, err := parser.Parse("UPDATE users SET name = 1 WHERE") 436 | assert.NotNil(t, err) 437 | } 438 | 439 | { 440 | _, err := parser.Parse("UPDATE users SET name = 1 WHERE ,") 441 | assert.NotNil(t, err) 442 | } 443 | 444 | { 445 | _, err := parser.Parse("UPDATE users SET , = 1") 446 | assert.NotNil(t, err) 447 | } 448 | { 449 | _, err := parser.Parse("UPDATE users SET a ,") 450 | assert.NotNil(t, err) 451 | } 452 | { 453 | _, err := parser.Parse("UPDATE users SET a = ,") 454 | assert.NotNil(t, err) 455 | } 456 | 457 | { 458 | _, err := parser.Parse("UPDATE users SET name =") 459 | assert.NotNil(t, err) 460 | } 461 | 462 | { 463 | _, err := parser.Parse("DELETE") 464 | assert.NotNil(t, err) 465 | } 466 | 467 | { 468 | _, err := parser.Parse("DELETE ,FROM") 469 | assert.NotNil(t, err) 470 | } 471 | 472 | { 473 | _, err := parser.Parse("DELETE FROM ,") 474 | assert.NotNil(t, err) 475 | } 476 | 477 | { 478 | _, err := parser.Parse("DELETE FROM users WHERE") 479 | assert.NotNil(t, err) 480 | } 481 | 482 | { 483 | _, err := parser.Parse("DELETE FROM users ,") 484 | assert.NotNil(t, err) 485 | } 486 | 487 | } 488 | 489 | func TestInsertStatement(t *testing.T) { 490 | parser := dsc.NewDmlParser() 491 | statement, err := parser.Parse("INSERT INTO users(id, name, last_access_time) VALUES(?, ?, 2 )") 492 | assert.Nil(t, err, "should not have errors") 493 | assert.Equal(t, "INSERT", statement.Type) 494 | assert.Equal(t, "users", statement.Table) 495 | assert.Equal(t, 3, len(statement.Columns)) 496 | assert.Equal(t, 3, len(statement.Values)) 497 | assert.Equal(t, "2", statement.Values[2]) 498 | } 499 | 500 | func TestUpdateStatement(t *testing.T) { 501 | parser := dsc.NewDmlParser() 502 | 503 | { 504 | statement, err := parser.Parse("UPDATE users SET name = 'Smith', last_access_time = ?") 505 | assert.Nil(t, err, "should not have errors") 506 | assert.Equal(t, "UPDATE", statement.Type) 507 | assert.Equal(t, "users", statement.Table) 508 | assert.Equal(t, 2, len(statement.Columns)) 509 | assert.Equal(t, 2, len(statement.Values)) 510 | assert.Equal(t, "'Smith'", statement.Values[0]) 511 | assert.Equal(t, 0, len(statement.Criteria)) 512 | } 513 | { 514 | statement, err := parser.Parse("UPDATE users SET name = 'Smith', last_access_time = ? WHERE id = 2") 515 | assert.Nil(t, err, "should not have errors") 516 | assert.Equal(t, "UPDATE", statement.Type) 517 | assert.Equal(t, "users", statement.Table) 518 | assert.Equal(t, 2, len(statement.Columns)) 519 | assert.Equal(t, 2, len(statement.Values)) 520 | assert.Equal(t, "'Smith'", statement.Values[0]) 521 | assert.Equal(t, 1, len(statement.Criteria)) 522 | assert.Equal(t, "id", statement.Criteria[0].LeftOperand) 523 | assert.Equal(t, "=", statement.Criteria[0].Operator) 524 | assert.Equal(t, "2", statement.Criteria[0].RightOperand) 525 | } 526 | 527 | } 528 | 529 | func TestDeleteStatement(t *testing.T) { 530 | parse := dsc.NewDmlParser() 531 | { 532 | statement, err := parse.Parse("DELETE FROM users ") 533 | assert.Nil(t, err, "should not have errors") 534 | assert.Equal(t, "DELETE", statement.Type) 535 | assert.Equal(t, "users", statement.Table) 536 | assert.Equal(t, 0, len(statement.Criteria)) 537 | 538 | } 539 | 540 | { 541 | statement, err := parse.Parse("DELETE FROM users WHERE id = 2") 542 | assert.Nil(t, err, "should not have errors") 543 | assert.Equal(t, "DELETE", statement.Type) 544 | assert.Equal(t, "users", statement.Table) 545 | assert.Equal(t, 1, len(statement.Criteria)) 546 | assert.Equal(t, "id", statement.Criteria[0].LeftOperand) 547 | assert.Equal(t, "=", statement.Criteria[0].Operator) 548 | assert.Equal(t, "2", statement.Criteria[0].RightOperand) 549 | 550 | } 551 | 552 | } 553 | 554 | func TestCriteriaValues(t *testing.T) { 555 | parser := dsc.NewQueryParser() 556 | 557 | query, err := parser.Parse("SELECT * FROM foo WHERE column1 = 2 AND column2 = ? AND column3 = 'abc' AND column4 = ? AND column5 = true") 558 | assert.Nil(t, err) 559 | { 560 | iterator := toolbox.NewSliceIterator([]string{"3", "a"}) 561 | values, err := query.CriteriaValues(iterator) 562 | assert.Nil(t, err) 563 | assert.Equal(t, 5, len(values)) 564 | assert.Equal(t, []interface{}{"2", "3", "abc", "a", "true"}, values) 565 | } 566 | { 567 | iterator := toolbox.NewSliceIterator([]string{"3"}) 568 | _, err = query.CriteriaValues(iterator) 569 | assert.NotNil(t, err) 570 | } 571 | { 572 | parse := dsc.NewDmlParser() 573 | statement, err := parse.Parse("UPDATE users SET a = ? WHERE id = ?") 574 | iterator := toolbox.NewSliceIterator([]string{}) 575 | _, err = statement.ColumnValueMap(iterator) 576 | assert.NotNil(t, err) 577 | } 578 | 579 | { 580 | parse := dsc.NewDmlParser() 581 | statement, err := parse.Parse("UPDATE users SET a = ? WHERE id = ?") 582 | iterator := toolbox.NewSliceIterator([]string{}) 583 | _, err = statement.ColumnValues(iterator) 584 | assert.NotNil(t, err) 585 | } 586 | } 587 | -------------------------------------------------------------------------------- /sql_predicate.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/viant/toolbox" 7 | "strings" 8 | ) 9 | 10 | func getOperandValue(operand interface{}, parameters toolbox.Iterator) (interface{}, error) { 11 | if operand != "?" { 12 | return operand, nil 13 | } 14 | var values = make([]interface{}, 1) 15 | if !parameters.HasNext() { 16 | return "", fmt.Errorf("missing binding parameters ?, %v", parameters) 17 | } 18 | err := parameters.Next(&values[0]) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return values[0], nil 23 | } 24 | 25 | func getOperandValues(operands []interface{}, parameters toolbox.Iterator) ([]interface{}, error) { 26 | var result = make([]interface{}, 0) 27 | for _, operand := range operands { 28 | operand, err := getOperandValue(operand, parameters) 29 | if err != nil { 30 | return nil, err 31 | } 32 | result = append(result, operand) 33 | } 34 | return result, nil 35 | } 36 | 37 | //NewSQLCriterionPredicate create a new predicate for passed in SQLCriterion 38 | func NewSQLCriterionPredicate(criterion *SQLCriterion, parameters toolbox.Iterator) (toolbox.Predicate, error) { 39 | if criterion.Operator == "" { 40 | return nil, errors.New("criterion.Operator was empty") 41 | } 42 | switch strings.ToLower(criterion.Operator) { 43 | case "in": 44 | operands, err := getOperandValues(criterion.RightOperands, parameters) 45 | if err != nil { 46 | return nil, fmt.Errorf("missing binding parameters for %v", criterion) 47 | } 48 | return toolbox.NewInPredicate(operands...), nil 49 | case "like": 50 | operand, err := getOperandValue(criterion.RightOperand, parameters) 51 | if err != nil { 52 | return nil, fmt.Errorf("missing binding parameters for %v", criterion) 53 | } 54 | return toolbox.NewLikePredicate(toolbox.AsString(operand)), nil 55 | case "between": 56 | operands, err := getOperandValues(criterion.RightOperands, parameters) 57 | if err != nil || len(operands) != 2 { 58 | return nil, fmt.Errorf("missing binding parameters for %v", criterion) 59 | } 60 | return toolbox.NewBetweenPredicate(operands[0], operands[1]), nil 61 | case "is": 62 | return toolbox.NewNilPredicate(), nil 63 | default: 64 | operand, err := getOperandValue(criterion.RightOperand, parameters) 65 | if err != nil { 66 | return nil, fmt.Errorf("missing binding parameters for %v", criterion) 67 | } 68 | return toolbox.NewComparablePredicate(criterion.Operator, operand), nil 69 | } 70 | } 71 | 72 | type booleanPredicate struct { 73 | operator string 74 | leftOperand bool 75 | } 76 | 77 | func (p *booleanPredicate) Apply(value interface{}) bool { 78 | rightOperand := toolbox.AsBoolean(value) 79 | switch strings.ToLower(p.operator) { 80 | case "or": 81 | return p.leftOperand || rightOperand 82 | case "and": 83 | return p.leftOperand && rightOperand 84 | } 85 | return false 86 | } 87 | 88 | //NewBooleanPredicate returns a new boolean predicate. It takes left operand and logical operator: 'OR' or 'AND' 89 | func NewBooleanPredicate(leftOperand bool, operator string) toolbox.Predicate { 90 | return &booleanPredicate{operator, leftOperand} 91 | } 92 | 93 | type sqlCriteriaPredicate struct { 94 | *SQLCriteria 95 | predicates []toolbox.Predicate 96 | } 97 | 98 | func (p *sqlCriteriaPredicate) Apply(source interface{}) bool { 99 | var sourceMap, ok = source.(map[string]interface{}) 100 | if !ok { 101 | return false 102 | } 103 | result := true 104 | var logicalPredicate toolbox.Predicate 105 | 106 | for i := 0; i < len(p.Criteria); i++ { 107 | criterion := p.Criteria[i] 108 | value := sourceMap[toolbox.AsString(criterion.LeftOperand)] 109 | predicate := p.predicates[i] 110 | result = predicate.Apply(value) 111 | if criterion.Inverse { 112 | result = !result 113 | } 114 | if logicalPredicate != nil { 115 | result = logicalPredicate.Apply(result) 116 | } 117 | if p.LogicalOperator != "" { 118 | if strings.ToLower(p.LogicalOperator) == "and" && !result { 119 | //shortcut 120 | break 121 | } 122 | logicalPredicate = NewBooleanPredicate(result, p.LogicalOperator) 123 | } 124 | } 125 | return result 126 | } 127 | 128 | //NewSQLCriteriaPredicate create a new sql criteria predicate, it takes binding parameters iterator, and actual criteria. 129 | func NewSQLCriteriaPredicate(parameters toolbox.Iterator, sqlCriteria *SQLCriteria) (toolbox.Predicate, error) { 130 | var predicates = make([]toolbox.Predicate, 0) 131 | 132 | for i := 0; i < len(sqlCriteria.Criteria); i++ { 133 | criterion := sqlCriteria.Criteria[i] 134 | predicate, err := NewSQLCriterionPredicate(criterion, parameters) 135 | if err != nil { 136 | return nil, err 137 | } 138 | predicates = append(predicates, predicate) 139 | } 140 | return &sqlCriteriaPredicate{SQLCriteria: sqlCriteria, predicates: predicates}, nil 141 | } 142 | -------------------------------------------------------------------------------- /sql_predicate_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/dsc" 8 | "github.com/viant/toolbox" 9 | ) 10 | 11 | func TestNewCriterionPredicate(t *testing.T) { 12 | { //Like case 13 | parameters := toolbox.NewSliceIterator([]string{"abc%"}) 14 | predicate, err := dsc.NewSQLCriterionPredicate(&dsc.SQLCriterion{Operator: "Like", RightOperand: "?"}, parameters) 15 | assert.Nil(t, err) 16 | { 17 | assert.True(t, predicate.Apply("ABC")) 18 | assert.False(t, predicate.Apply("AB")) 19 | 20 | } 21 | } 22 | { //between case 23 | parameters := toolbox.NewSliceIterator([]string{"1", "10"}) 24 | predicate, err := dsc.NewSQLCriterionPredicate(&dsc.SQLCriterion{Operator: "BETWEEN", RightOperands: []interface{}{"?", "?"}}, parameters) 25 | assert.Nil(t, err) 26 | { 27 | assert.True(t, predicate.Apply(5)) 28 | assert.False(t, predicate.Apply(12)) 29 | 30 | } 31 | } 32 | 33 | { //error no operator 34 | parameters := toolbox.NewSliceIterator([]string{}) 35 | _, err := dsc.NewSQLCriterionPredicate(&dsc.SQLCriterion{Operator: "", RightOperands: []interface{}{"?"}}, parameters) 36 | assert.NotNil(t, err) 37 | } 38 | 39 | { //between error case 40 | parameters := toolbox.NewSliceIterator([]string{"1"}) 41 | _, err := dsc.NewSQLCriterionPredicate(&dsc.SQLCriterion{Operator: "BETWEEN", RightOperands: []interface{}{"?", "?"}}, parameters) 42 | assert.NotNil(t, err) 43 | } 44 | { //like error case 45 | parameters := toolbox.NewSliceIterator([]string{}) 46 | _, err := dsc.NewSQLCriterionPredicate(&dsc.SQLCriterion{Operator: "Like", RightOperand: "?"}, parameters) 47 | assert.NotNil(t, err) 48 | } 49 | { //in error case 50 | parameters := toolbox.NewSliceIterator([]string{}) 51 | _, err := dsc.NewSQLCriterionPredicate(&dsc.SQLCriterion{Operator: "in", RightOperands: []interface{}{"?"}}, parameters) 52 | assert.NotNil(t, err) 53 | } 54 | { //default error case 55 | parameters := toolbox.NewSliceIterator([]string{}) 56 | _, err := dsc.NewSQLCriterionPredicate(&dsc.SQLCriterion{Operator: "=", RightOperand: "?"}, parameters) 57 | assert.NotNil(t, err) 58 | } 59 | 60 | } 61 | 62 | func TestNewCriteriaPredicate(t *testing.T) { 63 | 64 | parameters := []interface{}{"abc%", 123} 65 | iterator := toolbox.NewSliceIterator(parameters) 66 | predicate, err := dsc.NewSQLCriteriaPredicate(iterator, 67 | &dsc.SQLCriteria{ 68 | LogicalOperator: "or", 69 | Criteria: []*dsc.SQLCriterion{ 70 | {LeftOperand: "column1", Operator: "Like", RightOperand: "?"}, 71 | {LeftOperand: "column2", Operator: "=", RightOperand: "?"}, 72 | }, 73 | }, 74 | ) 75 | assert.Nil(t, err) 76 | { 77 | assert.False(t, predicate.Apply( 78 | map[string]interface{}{ 79 | "column1": "AB", 80 | "column2": 12, 81 | }, 82 | )) 83 | assert.True(t, predicate.Apply( 84 | map[string]interface{}{ 85 | "column1": "ABc", 86 | "column2": 12, 87 | }, 88 | )) 89 | assert.True(t, predicate.Apply( 90 | map[string]interface{}{ 91 | "column1": "AB", 92 | "column2": 123, 93 | }, 94 | )) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /sql_scanner.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import "github.com/viant/toolbox" 4 | 5 | type SQLScanner struct { 6 | query *QueryStatement 7 | columns []string 8 | types []ColumnType 9 | columnTypes []ColumnType 10 | converter toolbox.Converter 11 | Values map[string]interface{} 12 | } 13 | 14 | func (s *SQLScanner) ColumnTypes() ([]ColumnType, error) { 15 | return s.columnTypes, nil 16 | } 17 | 18 | func (s *SQLScanner) Columns() ([]string, error) { 19 | return s.columns, nil 20 | } 21 | 22 | func (s *SQLScanner) Scan(destinations ...interface{}) error { 23 | var columns, _ = s.Columns() 24 | if len(destinations) == 1 { 25 | if toolbox.IsMap(destinations[0]) { 26 | aMap := toolbox.AsMap(destinations[0]) 27 | for k, v := range s.Values { 28 | aMap[k] = v 29 | } 30 | return nil 31 | } 32 | } 33 | for i, dest := range destinations { 34 | if dest == nil { 35 | continue 36 | } 37 | if value, found := s.Values[columns[i]]; found { 38 | err := s.converter.AssignConverted(dest, value) 39 | if err != nil { 40 | return err 41 | } 42 | } 43 | } 44 | return nil 45 | } 46 | 47 | //NewSQLScannerWithTypes create a scanner with type 48 | func NewSQLScannerWithTypes(query *QueryStatement, config *Config, columns []string, types []ColumnType) *SQLScanner { 49 | converter := *toolbox.NewColumnConverter(config.GetDateLayout()) 50 | if len(columns) == 0 { 51 | columns = query.ColumnNames() 52 | } 53 | 54 | return &SQLScanner{ 55 | query: query, 56 | types: types, 57 | columns: columns, 58 | converter: converter, 59 | } 60 | } 61 | 62 | //NewSQLScanner creates a new sql scanner 63 | func NewSQLScanner(query *QueryStatement, config *Config, columns []string) *SQLScanner { 64 | return NewSQLScannerWithTypes(query, config, columns, make([]ColumnType, 0)) 65 | } 66 | -------------------------------------------------------------------------------- /table.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import "fmt" 4 | 5 | //TableDescriptor represents a table details. 6 | type TableDescriptor struct { 7 | Table string 8 | Autoincrement bool 9 | PkColumns []string 10 | Columns []string 11 | ColumnTypes map[string]string 12 | Nullables map[string]bool 13 | OrderColumns []string 14 | Schema []map[string]interface{} //Schema to be interpreted by NoSQL drivers for create table operation . 15 | SchemaURL string //url with JSON to the TableDescriptor.Schema. 16 | FromQuery string //If table is query base then specify FromQuery 17 | FromQueryAlias string 18 | } 19 | 20 | func (t *TableDescriptor) From() string { 21 | if t.FromQuery != "" { 22 | if t.FromQueryAlias == "" { 23 | t.FromQueryAlias = "t" 24 | } 25 | return fmt.Sprintf("(%v) AS %v", t.FromQuery, t.FromQueryAlias) 26 | } 27 | return t.Table 28 | } 29 | 30 | //TableDescriptorRegistry represents a registry to store table descriptors by table name. 31 | type TableDescriptorRegistry interface { 32 | //Has checks if descriptor is defined for the table. 33 | Has(table string) bool 34 | 35 | //Get returns a table descriptor for passed in table, it calls panic if descriptor is not found, to avoid it please always use Has check. 36 | Get(table string) *TableDescriptor 37 | 38 | //Register registers a table descriptor. 39 | Register(descriptor *TableDescriptor) error 40 | 41 | //Tables returns all registered tables. 42 | Tables() []string 43 | } 44 | -------------------------------------------------------------------------------- /table_descriptor.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "sync" 7 | 8 | "github.com/viant/toolbox" 9 | "strings" 10 | ) 11 | 12 | type commonTableDescriptorRegistry struct { 13 | sync.RWMutex 14 | manager Manager 15 | registry map[string]*TableDescriptor 16 | } 17 | 18 | func (r *commonTableDescriptorRegistry) Has(table string) bool { 19 | r.RLock() 20 | defer r.RUnlock() 21 | _, found := r.registry[table] 22 | return found 23 | } 24 | 25 | func (r *commonTableDescriptorRegistry) getDescriptor(table string) *TableDescriptor { 26 | dbConfig := r.manager.Config() 27 | dialect := GetDatastoreDialect(dbConfig.DriverName) 28 | datastore, _ := dialect.GetCurrentDatastore(r.manager) 29 | key := dialect.GetKeyName(r.manager, datastore, table) 30 | isAutoincrement := dialect.IsAutoincrement(r.manager, datastore, table) 31 | descriptor := &TableDescriptor{ 32 | Table: table, 33 | Autoincrement: isAutoincrement, 34 | PkColumns: []string{}, 35 | Columns: []string{}, 36 | } 37 | if key != "" { 38 | descriptor.PkColumns = strings.Split(key, ",") 39 | } 40 | columns, _ := dialect.GetColumns(r.manager, datastore, table) 41 | for _, column := range columns { 42 | descriptor.Columns = append(descriptor.Columns, column.Name()) 43 | } 44 | return descriptor 45 | } 46 | 47 | func (r *commonTableDescriptorRegistry) Get(table string) *TableDescriptor { 48 | r.RLock() 49 | if descriptor, found := r.registry[table]; found { 50 | r.RUnlock() 51 | return descriptor 52 | } 53 | r.RUnlock() 54 | var result = r.getDescriptor(table) 55 | _ = r.Register(result) 56 | return result 57 | } 58 | 59 | func (r *commonTableDescriptorRegistry) Register(descriptor *TableDescriptor) error { 60 | if descriptor.Table == "" { 61 | return fmt.Errorf("table name was not set %v", descriptor) 62 | } 63 | for i, column := range descriptor.Columns { 64 | if column == "" { 65 | return fmt.Errorf("columns[%d] was empty %v %v", i, descriptor.Table, descriptor.Columns) 66 | } 67 | } 68 | for i, column := range descriptor.PkColumns { 69 | if column == "" { 70 | return fmt.Errorf("pkColumns[%d] was empty %v %v", i, descriptor.Table, descriptor.Columns) 71 | } 72 | } 73 | r.Lock() 74 | defer r.Unlock() 75 | r.registry[descriptor.Table] = descriptor 76 | return nil 77 | } 78 | 79 | func (r *commonTableDescriptorRegistry) Tables() []string { 80 | r.RLock() 81 | defer r.RUnlock() 82 | var result = make([]string, 0) 83 | for key := range r.registry { 84 | result = append(result, key) 85 | } 86 | return result 87 | } 88 | 89 | func newTableDescriptorRegistry() *commonTableDescriptorRegistry { 90 | return &commonTableDescriptorRegistry{registry: make(map[string]*TableDescriptor)} 91 | } 92 | 93 | //newTableDescriptorRegistry returns a new newTableDescriptorRegistry 94 | func NewTableDescriptorRegistry() TableDescriptorRegistry { 95 | return newTableDescriptorRegistry() 96 | } 97 | 98 | //HasSchema check if table desciptor has defined schema. 99 | func (d *TableDescriptor) HasSchema() bool { 100 | return len(d.SchemaURL) > 0 || d.Schema != nil 101 | } 102 | 103 | //NewTableDescriptor creates a new table descriptor for passed in instance, it can use the following tags:"column", "dateLayout","dateFormat", "autoincrement", "primaryKey", "sequence", "transient" 104 | func NewTableDescriptor(table string, instance interface{}) (*TableDescriptor, error) { 105 | targetType := toolbox.DiscoverTypeByKind(instance, reflect.Struct) 106 | var autoincrement bool 107 | var pkColumns = make([]string, 0) 108 | var columns = make([]string, 0) 109 | columnToFieldMap := toolbox.NewFieldSettingByKey(targetType, "column") 110 | 111 | for key := range columnToFieldMap { 112 | mapping, _ := columnToFieldMap[key] 113 | column, ok := mapping["column"] 114 | if !ok { 115 | column = mapping["fieldName"] 116 | } 117 | if _, ok := mapping["autoincrement"]; ok { 118 | pkColumns = append(pkColumns, column) 119 | autoincrement = true 120 | continue 121 | } 122 | } 123 | 124 | for key := range columnToFieldMap { 125 | mapping, _ := columnToFieldMap[key] 126 | column, ok := mapping["column"] 127 | if !ok { 128 | column = mapping["fieldName"] 129 | } 130 | 131 | columns = append(columns, column) 132 | if _, ok := mapping["primaryKey"]; ok { 133 | if !toolbox.HasSliceAnyElements(pkColumns, column) { 134 | pkColumns = append(pkColumns, column) 135 | } 136 | continue 137 | } 138 | if key == "id" { 139 | if !toolbox.HasSliceAnyElements(pkColumns, column) { 140 | pkColumns = append(pkColumns, column) 141 | } 142 | continue 143 | } 144 | } 145 | 146 | return &TableDescriptor{ 147 | Table: table, 148 | Autoincrement: autoincrement, 149 | Columns: columns, 150 | PkColumns: pkColumns, 151 | }, nil 152 | } 153 | -------------------------------------------------------------------------------- /table_descriptor_test.go: -------------------------------------------------------------------------------- 1 | package dsc_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/viant/dsc" 9 | ) 10 | 11 | type User1 struct { 12 | Name string `column:"name"` 13 | DateOfBirth time.Time `column:"date" dateFormat:"2006-01-02 15:04:05.000000"` 14 | Id int `autoincrement:"true"` 15 | Other string `transient:"true"` 16 | } 17 | 18 | func TestTableDescriptor(t *testing.T) { 19 | 20 | descriptor, err := dsc.NewTableDescriptor("users", (*User1)(nil)) 21 | assert.Nil(t, err) 22 | assert.Equal(t, "users", descriptor.Table) 23 | assert.Equal(t, "Id", descriptor.PkColumns[0]) 24 | assert.Equal(t, true, descriptor.Autoincrement) 25 | assert.Equal(t, 3, len(descriptor.Columns)) 26 | 27 | assert.False(t, descriptor.HasSchema()) 28 | 29 | } 30 | func TestTableDescriptorRegistry(t *testing.T) { 31 | descriptor, err := dsc.NewTableDescriptor("users", (*User1)(nil)) 32 | assert.Nil(t, err) 33 | registry := dsc.NewTableDescriptorRegistry() 34 | assert.False(t, registry.Has("users")) 35 | registry.Register(descriptor) 36 | assert.True(t, registry.Has("users")) 37 | assert.Equal(t, []string{"users"}, registry.Tables()) 38 | } 39 | -------------------------------------------------------------------------------- /table_test.go: -------------------------------------------------------------------------------- 1 | package dsc 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestTableDescriptor_From(t *testing.T) { 9 | 10 | var useCases = []struct { 11 | description string 12 | table *TableDescriptor 13 | expect string 14 | }{ 15 | { 16 | description: "from table", 17 | table: &TableDescriptor{Table: "table1"}, 18 | expect: "table1", 19 | }, 20 | { 21 | description: "from query - default alias", 22 | table: &TableDescriptor{Table: "table1", FromQuery: "SELECT * FROM table1"}, 23 | expect: "(SELECT * FROM table1) AS t", 24 | }, 25 | { 26 | description: "from query with alias", 27 | table: &TableDescriptor{Table: "table1", FromQuery: "SELECT * FROM table1", FromQueryAlias: "newTable"}, 28 | expect: "(SELECT * FROM table1) AS newTable", 29 | }, 30 | } 31 | 32 | for _, useCase := range useCases { 33 | actual := useCase.table.From() 34 | assert.Equal(t, useCase.expect, actual, useCase.description) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /test/file_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "DriverName": "ndjson", 3 | "Descriptor": "[url]", 4 | "Parameters": { 5 | "url": "test:///test/", 6 | "dateFormat": "yyyy-MM-dd hh:mm:ss" 7 | } 8 | } -------------------------------------------------------------------------------- /test/store.json: -------------------------------------------------------------------------------- 1 | { 2 | "DriverName": "sqlite3", 3 | "Descriptor": "[url]", 4 | "Parameters": { 5 | "url": "./test/foo.db" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /test/test1_expect_users.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"id":1, "username":"Dudi", "active":true, "salary":12400, "comments":"abc","last_access_time": "2016-03-01 03:10:00"}, 3 | {"id":2, "username":"Rudi", "active":true, "salary":12600, "comments":"def","last_access_time": "2016-03-01 05:10:00"}, 4 | {"id":4, "username":"Vudi", "active":true, "salary":12800, "comments":"xyz","last_access_time": "2016-03-01 08:43:00"} 5 | ] -------------------------------------------------------------------------------- /test/traveler.csv: -------------------------------------------------------------------------------- 1 | id,name 2 | 1,Bob 3 | 2,John 4 | 3,Darek 5 | 4,Tom -------------------------------------------------------------------------------- /test/travelers1.json: -------------------------------------------------------------------------------- 1 | { "id": 1, "name": "Rob", "visitedCities": [{"visits": 3,"city": "Warsaw"}, {"visits": 4,"city": "Berlin"}],"mostLikedCity": {"visits": 4, "city": "Berlin"},"achievements": ["abc1","cde"], "LastVisitTime": "2016-03-01 03:10:00"} 2 | { "id": 2, "name": "Vodi", "visitedCities": [{"visits": 2,"city": "Cracow"}, {"visits": 1,"city": "London"}],"achievements": ["abc2","cde"]} 3 | { "id": 3, "name": "Dodi", "visitedCities": [{"visits": 1,"city": "Paris"}, {"visits": 3,"city": "Berlin"}], "achievements": ["abc3","cde"]} 4 | { "id": 4, "name": "Bogi", "visitedCities": [{"visits": 6,"city": "Paris"}, {"visits": 2,"city": "London"}],"mostLikedCity": {"visits": 6, "city": "Paris"},"achievements": ["abc4","cde"]} -------------------------------------------------------------------------------- /test/travelers2.json: -------------------------------------------------------------------------------- 1 | {"Achievements":["z","g"],"Id":20,"LastVisitTime":"2019-01-19 08:12:14","MostLikedCity":{"City":"Moscow","Souvenirs":["s3","sN"],"Visits":3},"Name":"Robin"} 2 | --------------------------------------------------------------------------------