├── .gitignore ├── .golangci.yml ├── .pre-commit-config.yaml ├── .travis.yml ├── README.md ├── client.go ├── connection.go ├── connection_test.go ├── driver.go ├── driver_test.go ├── dsn.go ├── dsn_test.go ├── errors.go ├── errors_test.go ├── go.mod ├── go.sum ├── index.html ├── instance.go ├── instance_test.go ├── job.go ├── job_test.go ├── log.go ├── result.go ├── rows.go ├── rows_test.go ├── task.go ├── task_test.go └── tunnel.go /.gitignore: -------------------------------------------------------------------------------- 1 | nohup.out 2 | *~ 3 | *.vim 4 | *~ 5 | *.test 6 | /Gopkg.lock 7 | /Gopkg.toml 8 | /vendor/ 9 | *.swp 10 | .idea/ 11 | *.vim 12 | *.log 13 | /logs 14 | sql/my_dnn_model 15 | .DS_Store 16 | coverage.out 17 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | # timeout for analysis, e.g. 30s, 5m, default is 1m 3 | deadline: 5m 4 | tests: false 5 | # which dirs to skip: they won't be analyzed; 6 | # can use regexp here: generated.*, regexp is applied on full path; 7 | # default value is empty list, but next dirs are always skipped independently 8 | # from this option's value: 9 | # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ 10 | skip-dirs: 11 | - genfiles$ 12 | - vendor$ 13 | - testdata$ 14 | - testing$ 15 | 16 | # which files to skip: they will be analyzed, but issues from them 17 | # won't be reported. Default value is empty list, but there is 18 | # no need to include all autogenerated files, we confidently recognize 19 | # autogenerated files. If it's not please let us know. 20 | skip-files: 21 | - ".*\\.pb\\.go" 22 | - ".*\\.gen\\.go" 23 | 24 | linters: 25 | enable-all: true 26 | disable: 27 | - depguard 28 | - dupl 29 | - gochecknoglobals 30 | - gochecknoinits 31 | - goconst 32 | - gocyclo 33 | - gosec 34 | - nakedret 35 | - prealloc 36 | - scopelint 37 | - structcheck 38 | - maligned 39 | - lll 40 | - golint 41 | - unparam 42 | - deadcode 43 | - varcheck 44 | - errcheck 45 | - funlen 46 | - whitespace 47 | - gocognit 48 | - godox 49 | - wsl 50 | - stylecheck 51 | fast: false 52 | 53 | linters-settings: 54 | errcheck: 55 | # report about not checking of errors in type assetions: `a := b.(MyStruct)`; 56 | # default is false: such cases aren't reported by default. 57 | check-type-assertions: false 58 | 59 | # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; 60 | # default is false: such cases aren't reported by default. 61 | check-blank: false 62 | govet: 63 | # report about shadowed variables 64 | check-shadowing: false 65 | golint: 66 | # minimal confidence for issues, default is 0.8 67 | min-confidence: 0.0 68 | gofmt: 69 | # simplify code: gofmt with `-s` option, true by default 70 | simplify: true 71 | goimports: 72 | # put imports beginning with prefix after 3rd-party packages; 73 | # it's a comma-separated list of prefixes 74 | local-prefixes: istio.io/ 75 | maligned: 76 | # print struct with more effective memory layout or not, false by default 77 | suggest-new: true 78 | misspell: 79 | # Correct spellings using locale preferences for US or UK. 80 | # Default is to use a neutral variety of English. 81 | # Setting locale to US will correct the British spelling of 'colour' to 'color'. 82 | locale: US 83 | lll: 84 | # max line length, lines longer will be reported. Default is 120. 85 | # '\t' is counted as 1 character by default, and can be changed with the tab-width option 86 | line-length: 120 87 | # tab width in spaces. Default to 1. 88 | tab-width: 1 89 | funlen: 90 | lines: 120 91 | statements: 40 92 | unused: 93 | # treat code as a program (not a library) and report unused exported identifiers; default is false. 94 | # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: 95 | # if it's called for subdir of a project it can't find funcs usages. All text editor integrations 96 | # with golangci-lint call it on a directory with the changed file. 97 | check-exported: false 98 | unparam: 99 | # call graph construction algorithm (cha, rta). In general, use cha for libraries, 100 | # and rta for programs with main packages. Default is cha. 101 | algo: cha 102 | 103 | # Inspect exported functions, default is false. Set to true if no external program/library imports your code. 104 | # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: 105 | # if it's called for subdir of a project it can't find external interfaces. All text editor integrations 106 | # with golangci-lint call it on a directory with the changed file. 107 | check-exported: false 108 | 109 | gocritic: 110 | enabled-checks: 111 | - appendCombine 112 | - argOrder 113 | - assignOp 114 | - badCond 115 | - boolExprSimplify 116 | - builtinShadow 117 | - captLocal 118 | - caseOrder 119 | - codegenComment 120 | - commentedOutCode 121 | - commentedOutImport 122 | - defaultCaseOrder 123 | - deprecatedComment 124 | - docStub 125 | - dupArg 126 | - dupBranchBody 127 | - dupCase 128 | - dupSubExpr 129 | - elseif 130 | - emptyFallthrough 131 | - equalFold 132 | - flagDeref 133 | - flagName 134 | - hexLiteral 135 | - indexAlloc 136 | - initClause 137 | - methodExprCall 138 | - nilValReturn 139 | - octalLiteral 140 | - offBy1 141 | - rangeExprCopy 142 | - regexpMust 143 | - sloppyLen 144 | - stringXbytes 145 | - switchTrue 146 | - typeAssertChain 147 | - typeSwitchVar 148 | - typeUnparen 149 | - underef 150 | - unlambda 151 | - unnecessaryBlock 152 | - unslice 153 | - valSwap 154 | - weakCond 155 | - yodaStyleExpr 156 | 157 | # Unused 158 | # - appendAssign 159 | # - commentFormatting 160 | # - emptyStringTest 161 | # - exitAfterDefer 162 | # - ifElseChain 163 | # - hugeParam 164 | # - importShadow 165 | # - nestingReduce 166 | # - paramTypeCombine 167 | # - ptrToRefParam 168 | # - rangeValCopy 169 | # - singleCaseSwitch 170 | # - sloppyReassign 171 | # - unlabelStmt 172 | # - unnamedResult 173 | # - wrapperFunc 174 | 175 | issues: 176 | # List of regexps of issue texts to exclude, empty list by default. 177 | # But independently from this option we use default exclude patterns, 178 | # it can be disabled by `exclude-use-default: false`. To list all 179 | # excluded by default patterns execute `golangci-lint run --help` 180 | exclude: 181 | - composite literal uses unkeyed fields 182 | 183 | exclude-rules: 184 | # Exclude some linters from running on test files. 185 | - path: _test\.go$|^tests/|^samples/ 186 | linters: 187 | - errcheck 188 | - maligned 189 | 190 | # Independently from option `exclude` we use default exclude patterns, 191 | # it can be disabled by this option. To list all 192 | # excluded by default patterns execute `golangci-lint run --help`. 193 | # Default value for this option is true. 194 | exclude-use-default: false 195 | 196 | # Maximum issues count per one linter. Set to 0 to disable. Default is 50. 197 | max-per-linter: 0 198 | 199 | # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. 200 | max-same-issues: 0 201 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | - repo: git://github.com/dnephin/pre-commit-golang 2 | rev: master 3 | hooks: 4 | - id: go-fmt 5 | - id: golangci-lint 6 | args: 7 | - --fix 8 | - --color=always 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go_import_path: sqlflow.org/gomaxcompute 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MaxCompute Go Driver 2 | 3 | [![Build Status](https://travis-ci.org/sql-machine-learning/gomaxcompute.svg?branch=develop)](https://travis-ci.org/sql-machine-learning/gomaxcompute) [![GoDoc](https://godoc.org/github.com/sql-machine-learning/gomaxcompute?status.svg)](https://godoc.org/github.com/sql-machine-learning/gomaxcompute) [![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE) 4 | 5 | 6 | [MaxCompute](https://www.alibabacloud.com/product/maxcompute), also known as ODPS, is a distributed storage service and SQL engine provided by [Alibaba Cloud](https://www.alibabacloud.com/). This repository contains a Go [SQLDriver](https://github.com/golang/go/wiki/SQLDrivers) of MaxCompute. If you are going to write a Go program that calls the standard library `database/sql` to access MaxCompute databases, you could use this driver. 7 | 8 | This project is in its early stage. Your issues and pull requests are very welcome! 9 | 10 | 11 | ## What This Is and Isn't 12 | 13 | This project is a driver that helps Go's standard database API talking to MaxCompute server. It has the following features: 14 | 15 | - In pure Go. Not a wrapper of any C/C++ library. 16 | - Connect to MaxCompute through its [HTTP interface](http://repo.aliyun.com/api-doc/). 17 | - Improve I/O throughput using MaxCompute's [tunnel service](https://www.alibabacloud.com/help/doc-detail/27833.htm). 18 | 19 | Alibaba Cloud open sourced some client SDKs of MaxCompute: 20 | 21 | - Java: https://github.com/aliyun/aliyun-odps-java-sdk 22 | - Python: https://github.com/aliyun/aliyun-odps-python-sdk 23 | 24 | This project is not an SDK. 25 | 26 | Alibaba Cloud also provides ODBC/JDBC drivers: 27 | 28 | - https://github.com/aliyun/aliyun-odps-jdbc 29 | 30 | This project is a Go's `database/sql` driver. 31 | 32 | 33 | ## How to Use 34 | 35 | Please make sure you have Go 1.6 or high release. 36 | 37 | You can clone the source code by running the following command. 38 | 39 | ```go 40 | go get -u sqlflow.org/gomaxcompute 41 | ``` 42 | 43 | Here is a simple example: 44 | 45 | ```go 46 | package main 47 | 48 | import ( 49 | "database/sql" 50 | "sqlflow.org/gomaxcompute" 51 | ) 52 | 53 | func assertNoError(e error) { 54 | if e != nil { 55 | panic(e) 56 | } 57 | } 58 | 59 | func main() { 60 | config := gomaxcompute.Config{ 61 | AccessID: "", 62 | AccessKey: "", 63 | Endpoint: "", 64 | Project: ""} 65 | db, e := sql.Open("maxcompute", config.FormatDSN()) 66 | assertNoError(e) 67 | defer db.Close() 68 | 69 | const sql = `SELECT 70 | cast('1' AS BIGINT) AS a, 71 | cast(TRUE AS BOOLEAN) AS b, 72 | cast('hi' AS STRING) AS c, 73 | cast('3.14' AS DOUBLE) AS d, 74 | cast('2017-11-11 03:12:11' AS DATETIME) AS e, 75 | cast('100.01' AS DECIMAL) AS f;` 76 | rows, e := db.Query(sql) 77 | assertNoError(e) 78 | defer rows.Close() 79 | 80 | for rows.Next() { 81 | // do your stuff 82 | } 83 | } 84 | ``` 85 | 86 | Please be aware that to connect to a MaxCompute database, the user needs to provide 87 | 88 | 1. the access ID 89 | 1. the access key 90 | 1. the endpoint pointing to the MaxCompute service 91 | 1. a project, which is something similar to a database in MySQL. 92 | 93 | ## Acknowledgement 94 | 95 | Our respect and thanks to Ruohang Feng, who wrote a Go SDK for MaxCompute when he worked in Alibaba, for his warm help that enabled this project. 96 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "bytes" 5 | "crypto/hmac" 6 | "crypto/sha1" 7 | "encoding/base64" 8 | "encoding/json" 9 | "encoding/xml" 10 | "fmt" 11 | "io/ioutil" 12 | "net/http" 13 | "net/url" 14 | "sort" 15 | "strconv" 16 | "strings" 17 | "time" 18 | 19 | "github.com/pkg/errors" 20 | ) 21 | 22 | var ( 23 | errNilBody = errors.New("nil body") 24 | requestGMT, _ = time.LoadLocation("GMT") 25 | ) 26 | 27 | const currentProject = "curr_project" 28 | 29 | // http response error 30 | type responseError struct { 31 | Code string `json:"Code"` 32 | Message string `json:"Message"` 33 | } 34 | 35 | type pair struct { 36 | k string 37 | v string 38 | } 39 | 40 | // optional: Body, Header 41 | func (conn *odpsConn) request(method, resource string, body []byte, header ...pair) (res *http.Response, err error) { 42 | return conn.requestEndpoint(conn.Endpoint, method, resource, body, header...) 43 | } 44 | 45 | func (conn *odpsConn) requestEndpoint(endpoint, method, resource string, body []byte, header ...pair) (res *http.Response, err error) { 46 | var req *http.Request 47 | url := endpoint + resource 48 | if body != nil { 49 | if req, err = http.NewRequest(method, url, bytes.NewBuffer(body)); err != nil { 50 | return nil, errors.WithStack(err) 51 | } 52 | req.Header.Set("Content-Length", strconv.Itoa(len(body))) 53 | } else if req, err = http.NewRequest(method, url, nil); err != nil { 54 | return nil, errors.WithStack(err) 55 | } 56 | 57 | req.Header.Set("x-odps-user-agent", "gomaxcompute/0.0.1") 58 | req.Header.Set("Content-Type", "application/xml") 59 | 60 | if dateStr := req.Header.Get("Date"); dateStr == "" { 61 | gmtTime := time.Now().In(requestGMT).Format(time.RFC1123) 62 | req.Header.Set("Date", gmtTime) 63 | } 64 | // overwrite with user-provide header 65 | if header != nil || len(header) == 0 { 66 | for _, arg := range header { 67 | req.Header.Set(arg.k, arg.v) 68 | } 69 | } 70 | 71 | // fill curr_project 72 | if req.URL.Query().Get(currentProject) == "" { 73 | req.URL.Query().Set(currentProject, conn.Project) 74 | } 75 | conn.sign(req) 76 | log.Debug("--------------------------------") 77 | log.Debugf("request.url: %v", req.URL.String()) 78 | log.Debugf("request.header: %v", req.Header) 79 | return conn.Do(req) 80 | } 81 | 82 | // signature 83 | func (conn *odpsConn) sign(r *http.Request) { 84 | var msg, auth bytes.Buffer 85 | msg.WriteString(r.Method) 86 | msg.WriteByte('\n') 87 | // common header 88 | msg.WriteString(r.Header.Get("Content-MD5")) 89 | msg.WriteByte('\n') 90 | msg.WriteString(r.Header.Get("Content-Type")) 91 | msg.WriteByte('\n') 92 | msg.WriteString(r.Header.Get("Date")) 93 | msg.WriteByte('\n') 94 | // canonical header 95 | for k, v := range r.Header { 96 | lowerK := strings.ToLower(k) 97 | if strings.HasPrefix(lowerK, "x-odps-") { 98 | msg.WriteString(lowerK) 99 | msg.WriteByte(':') 100 | msg.WriteString(strings.Join(v, ",")) 101 | msg.WriteByte('\n') 102 | } 103 | } 104 | 105 | // canonical resource 106 | var canonicalResource bytes.Buffer 107 | epURL, _ := url.Parse(conn.Endpoint) 108 | if strings.HasPrefix(r.URL.Path, epURL.Path) { 109 | canonicalResource.WriteString(r.URL.Path[len(epURL.Path):]) 110 | } else { 111 | canonicalResource.WriteString(r.URL.Path) 112 | } 113 | if urlParams := r.URL.Query(); len(urlParams) > 0 { 114 | // query parameters need to be hashed in alphabet order 115 | keys := make([]string, len(urlParams)) 116 | i := 0 117 | for k := range urlParams { 118 | keys[i] = k 119 | i++ 120 | } 121 | sort.Strings(keys) 122 | 123 | first := true 124 | for _, k := range keys { 125 | if first { 126 | canonicalResource.WriteByte('?') 127 | first = false 128 | } else { 129 | canonicalResource.WriteByte('&') 130 | } 131 | canonicalResource.WriteString(k) 132 | v := urlParams[k] 133 | if len(v) > 0 && v[0] != "" { 134 | canonicalResource.WriteByte('=') 135 | canonicalResource.WriteString(v[0]) 136 | } 137 | } 138 | } 139 | msg.WriteString(canonicalResource.String()) 140 | 141 | hasher := hmac.New(sha1.New, []byte(conn.AccessKey)) 142 | hasher.Write(msg.Bytes()) 143 | auth.WriteString("ODPS ") 144 | auth.WriteString(conn.AccessID) 145 | auth.WriteByte(':') 146 | auth.WriteString(base64.StdEncoding.EncodeToString(hasher.Sum(nil))) 147 | r.Header.Set("Authorization", auth.String()) 148 | } 149 | 150 | func (cred *Config) resource(resource string, args ...pair) string { 151 | if len(args) == 0 { 152 | return fmt.Sprintf("/projects/%s%s", cred.Project, resource) 153 | } 154 | 155 | ps := url.Values{} 156 | for _, i := range args { 157 | ps.Add(i.k, i.v) 158 | } 159 | return fmt.Sprintf("/projects/%s%s?%s", cred.Project, resource, ps.Encode()) 160 | } 161 | 162 | func parseResponse(rsp *http.Response) ([]byte, error) { 163 | if rsp == nil || rsp.Body == nil { 164 | return nil, errNilBody 165 | } 166 | log.Debugf("response code: %v", rsp.StatusCode) 167 | 168 | defer rsp.Body.Close() 169 | body, err := ioutil.ReadAll(rsp.Body) 170 | if err != nil { 171 | return nil, err 172 | } 173 | 174 | if rsp.StatusCode >= 400 { 175 | return nil, parseResponseError(rsp.StatusCode, body) 176 | } 177 | return body, err 178 | } 179 | 180 | func parseResponseError(statusCode int, body []byte) error { 181 | re := responseError{} 182 | if err := json.Unmarshal(body, &re); err != nil { 183 | ie := instanceError{} 184 | if err := xml.Unmarshal(body, &ie); err != nil { 185 | return errors.WithStack(fmt.Errorf("response error %d: %s", statusCode, string(body))) 186 | } 187 | 188 | code, err := parseErrorCode(ie.Message.CDATA) 189 | if err != nil { 190 | return errors.WithStack(fmt.Errorf("response error %d: %s", statusCode, string(body))) 191 | } 192 | return &MaxcomputeError{code, ie.Message.CDATA} 193 | } 194 | return errors.WithStack(fmt.Errorf("response error %d: %s. %s", statusCode, re.Code, re.Message)) 195 | } 196 | -------------------------------------------------------------------------------- /connection.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "database/sql/driver" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "time" 10 | 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | const ( 15 | waitInteveralMs = 1000 16 | tunnelHTTPProtocal = "http" 17 | terminated = "Terminated" 18 | methodGet = "GET" 19 | methodPost = "POST" 20 | ) 21 | 22 | type odpsConn struct { 23 | *http.Client 24 | *Config 25 | } 26 | 27 | // ODPS does not support transaction 28 | func (*odpsConn) Begin() (driver.Tx, error) { 29 | return nil, nil 30 | } 31 | 32 | // ODPS does not support Prepare 33 | func (*odpsConn) Prepare(query string) (driver.Stmt, error) { 34 | panic("Not implemented") 35 | } 36 | 37 | // Goodps accesses server by restful, so Close() do nth. 38 | func (*odpsConn) Close() error { 39 | return nil 40 | } 41 | 42 | // Implements database/sql/driver.Execer. Notice result is nil 43 | func (conn *odpsConn) Exec(query string, args []driver.Value) (driver.Result, error) { 44 | log.Debug("--------------------------------------------------------------------------------") 45 | log.Debugf("Exec:[%s]", query) 46 | ins, err := conn.wait(query, args) 47 | if err != nil { 48 | return nil, err 49 | } 50 | _, err = conn.getInstanceResult(ins) 51 | if err != nil { 52 | return nil, err 53 | } 54 | // FIXME(weiguo): precise result 55 | return &odpsResult{-1, -1}, nil 56 | } 57 | 58 | // Implements database/sql/driver.Queryer 59 | func (conn *odpsConn) Query(query string, args []driver.Value) (driver.Rows, error) { 60 | log.Debug("--------------------------------------------------------------------------------") 61 | log.Debugf("Query:[%s]", query) 62 | 63 | ins, err := conn.wait(query, args) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | // check if instance success 69 | res, err := conn.getInstanceResult(ins) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | // get tunnel server 75 | tunnelServer, err := conn.getTunnelServer() 76 | if err != nil { 77 | return nil, err 78 | } 79 | // get meta by tunnel 80 | meta, err := conn.getResultMeta(ins, tunnelServer) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | return newRows(meta, res) 86 | } 87 | 88 | func (conn *odpsConn) getResultMeta(instance, tunnelServer string) (*resultMeta, error) { 89 | endpoint := fmt.Sprintf("%s://%s", tunnelHTTPProtocal, tunnelServer) 90 | rsc := fmt.Sprintf("/projects/%s/instances/%s", conn.Project, instance) 91 | params := url.Values{} 92 | params.Add(currentProject, conn.Project) 93 | params.Add("downloads", "") 94 | url := rsc + "?" + params.Encode() 95 | 96 | rsp, err := conn.requestEndpoint(endpoint, methodPost, url, nil) 97 | if err != nil { 98 | return nil, err 99 | } 100 | body, err := parseResponse(rsp) 101 | if err != nil { 102 | return nil, err 103 | } 104 | 105 | meta := resultMeta{} 106 | if err := json.Unmarshal(body, &meta); err != nil { 107 | return nil, errors.WithStack(fmt.Errorf("%v %s", err, string(body))) 108 | } 109 | log.Debugf("meta: %+v", meta) 110 | 111 | return &meta, nil 112 | } 113 | 114 | func (conn *odpsConn) getTunnelServer() (string, error) { 115 | rsp, err := conn.request(methodGet, conn.resource("/tunnel"), nil) 116 | if err != nil { 117 | return "", err 118 | } 119 | 120 | url, err := parseResponse(rsp) 121 | if err != nil { 122 | return "", err 123 | } 124 | return string(url), nil 125 | } 126 | 127 | func (conn *odpsConn) wait(query string, args []driver.Value) (string, error) { 128 | if len(args) > 0 { 129 | query = fmt.Sprintf(query, args) 130 | } 131 | 132 | ins, err := conn.createInstance(newSQLJob(query, conn.QueryHints)) 133 | if err != nil { 134 | return "", err 135 | } 136 | if err := conn.poll(ins, waitInteveralMs); err != nil { 137 | return "", err 138 | } 139 | return ins, nil 140 | } 141 | 142 | func (conn *odpsConn) poll(instanceID string, interval int) error { 143 | du := time.Duration(interval) * time.Millisecond 144 | for { 145 | status, err := conn.getInstanceStatus(instanceID) 146 | if err != nil { 147 | return err 148 | } 149 | if status == terminated { 150 | break 151 | } 152 | time.Sleep(du) 153 | } 154 | return nil 155 | } 156 | -------------------------------------------------------------------------------- /connection_test.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "math/rand" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestQuery(t *testing.T) { 14 | a := assert.New(t) 15 | db, err := sql.Open("maxcompute", cfg4test.FormatDSN()) 16 | a.NoError(err) 17 | 18 | queries := []string{ 19 | `SELECT * FROM gomaxcompute_test LIMIT 1`, 20 | `SELECT * FROM gomaxcompute_test LIMIT 1;`, 21 | } 22 | for _, query := range queries { 23 | _, err = db.Query(query) 24 | a.NoError(err) 25 | } 26 | } 27 | 28 | func TestExec(t *testing.T) { 29 | a := assert.New(t) 30 | db, err := sql.Open("maxcompute", cfg4test.FormatDSN()) 31 | a.NoError(err) 32 | 33 | tn := fmt.Sprintf("unitest%d", rand.Int()) 34 | _, err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s(shop_name STRING);", tn)) 35 | a.NoError(err) 36 | _, err = db.Exec(fmt.Sprintf("DROP TABLE %s;", tn)) 37 | a.NoError(err) 38 | } 39 | 40 | func TestQueryBase64(t *testing.T) { 41 | a := assert.New(t) 42 | db, err := sql.Open("maxcompute", cfg4test.FormatDSN()) 43 | a.NoError(err) 44 | 45 | row, err := db.Query(`SELECT CAST("\001" AS string) AS a;`) 46 | a.NoError(err) 47 | for row.Next() { 48 | var s string 49 | row.Scan(&s) 50 | a.Equal("\001", s) 51 | } 52 | } 53 | 54 | func TestBadQuery(t *testing.T) { 55 | a := assert.New(t) 56 | db, err := sql.Open("maxcompute", cfg4test.FormatDSN()) 57 | a.NoError(err) 58 | 59 | // Table not found 60 | tn := fmt.Sprintf("unitest%d", rand.Int()) 61 | _, err = db.Query(fmt.Sprintf("SELECT * FROM %s;", tn)) 62 | a.Error(err) 63 | a.True(strings.Contains(err.Error(), "Table not found")) 64 | } 65 | 66 | func TestBadExec(t *testing.T) { 67 | a := assert.New(t) 68 | db, err := sql.Open("maxcompute", cfg4test.FormatDSN()) 69 | a.NoError(err) 70 | 71 | // Table not found 72 | tn := fmt.Sprintf("unitest%d", rand.Int()) 73 | _, err = db.Exec(fmt.Sprintf("DROP TABLE %s;", tn)) 74 | a.Error(err) 75 | a.True(strings.Contains(err.Error(), "Table not found")) 76 | } 77 | 78 | func TestInvalidSyntax(t *testing.T) { 79 | a := assert.New(t) 80 | db, e := sql.Open("maxcompute", cfg4test.FormatDSN()) 81 | a.NoError(e) 82 | 83 | { 84 | _, err := db.Query(`SLEECT CAST("\001" AS string) AS a;`) 85 | a.NotNil(err) 86 | me, ok := err.(*MaxcomputeError) 87 | a.True(ok) 88 | a.Equal(`0130161`, me.Code) 89 | a.Equal(`ODPS-0130161:[1,1] Parse exception - invalid token 'SLEECT'`, me.Error()) 90 | } 91 | 92 | { 93 | _, err := db.Exec(`DROP ;`) 94 | a.NotNil(err) 95 | me, ok := err.(*MaxcomputeError) 96 | a.True(ok) 97 | a.Equal(`0130161`, me.Code) 98 | a.Equal(`ODPS-0130161:[1,6] Parse exception - invalid token ';'`, me.Error()) 99 | } 100 | 101 | { 102 | _, err := db.Query(`SELECT * FROM i_dont_exist;`) 103 | a.NotNil(err) 104 | me, ok := err.(*MaxcomputeError) 105 | a.True(ok) 106 | a.Equal(`0130131`, me.Code) 107 | a.Equal(`ODPS-0130131:[1,15] Table not found - table gomaxcompute_driver_w7u.i_dont_exist cannot be resolved`, me.Error()) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /driver.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "database/sql" 5 | "database/sql/driver" 6 | "net/http" 7 | ) 8 | 9 | // register driver 10 | func init() { 11 | sql.Register("maxcompute", &Driver{}) 12 | } 13 | 14 | // impls database/sql/driver.Driver 15 | type Driver struct{} 16 | 17 | func (d Driver) Open(dsn string) (driver.Conn, error) { 18 | cfg, err := ParseDSN(dsn) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return &odpsConn{&http.Client{}, cfg}, nil 23 | } 24 | -------------------------------------------------------------------------------- /driver_test.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "database/sql" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | var cfg4test = &Config{ 12 | AccessID: os.Getenv("ODPS_ACCESS_ID"), 13 | AccessKey: os.Getenv("ODPS_ACCESS_KEY"), 14 | Project: os.Getenv("ODPS_PROJECT"), 15 | Endpoint: os.Getenv("ODPS_ENDPOINT"), 16 | QueryHints: map[string]string{"odps.sql.mapper.split_size": "16"}, 17 | } 18 | 19 | func TestSQLOpen(t *testing.T) { 20 | a := assert.New(t) 21 | db, err := sql.Open("maxcompute", cfg4test.FormatDSN()) 22 | defer db.Close() 23 | a.NoError(err) 24 | } 25 | 26 | func TestQuerySettings(t *testing.T) { 27 | a := assert.New(t) 28 | db, err := sql.Open("maxcompute", cfg4test.FormatDSN()) 29 | a.NoError(err) 30 | _, err = db.Query("SELECT * FROM gomaxcompute_test LIMIT;") 31 | a.NoError(err) 32 | } 33 | -------------------------------------------------------------------------------- /dsn.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "regexp" 7 | "strings" 8 | ) 9 | 10 | var ( 11 | // Regexp syntax: https://github.com/google/re2/wiki/Syntax 12 | reDSN = regexp.MustCompile(`^([a-zA-Z0-9_-]+):([=a-zA-Z0-9_-]+)@([:a-zA-Z0-9/_.-]+)\?([^/]+)$`) 13 | ) 14 | 15 | const HINT_PREFIX = "hint_" 16 | 17 | type Config struct { 18 | AccessID string 19 | AccessKey string 20 | Project string 21 | Endpoint string 22 | QueryHints map[string]string 23 | } 24 | 25 | func ParseDSN(dsn string) (*Config, error) { 26 | sub := reDSN.FindStringSubmatch(dsn) 27 | if len(sub) != 5 { 28 | return nil, fmt.Errorf("dsn %s doesn't match access_id:access_key@url?curr_project=project&scheme=http|https", dsn) 29 | } 30 | id, key, endpointURL := sub[1], sub[2], sub[3] 31 | 32 | var schemeArgs []string 33 | var currProjArgs []string 34 | var ok bool 35 | queryHints := make(map[string]string) 36 | 37 | querys, err := url.ParseQuery(sub[4]) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | if schemeArgs, ok = querys["scheme"]; !ok || len(schemeArgs) != 1 { 43 | return nil, fmt.Errorf("dsn %s should have one scheme argument", dsn) 44 | } 45 | if currProjArgs, ok = querys[currentProject]; !ok || len(currProjArgs) != 1 { 46 | return nil, fmt.Errorf("dsn %s should have one current_project argument", dsn) 47 | } 48 | 49 | for k, v := range querys { 50 | // The query args such as hints_odps.sql.mapper.split_size=16 51 | // would be converted to the maxcompute query hints: {"odps.sql.mapper.split_size": "16"} 52 | if strings.HasPrefix(k, HINT_PREFIX) { 53 | queryHints[k[5:]] = v[0] 54 | } 55 | } 56 | 57 | if schemeArgs[0] != "http" && schemeArgs[0] != "https" { 58 | return nil, fmt.Errorf("dsn %s 's scheme is neither http nor https", dsn) 59 | } 60 | 61 | config := &Config{ 62 | AccessID: id, 63 | AccessKey: key, 64 | Project: currProjArgs[0], 65 | Endpoint: schemeArgs[0] + "://" + endpointURL, 66 | QueryHints: queryHints} 67 | 68 | return config, nil 69 | } 70 | 71 | func (cfg *Config) FormatDSN() string { 72 | pair := strings.Split(cfg.Endpoint, "://") 73 | if len(pair) != 2 { 74 | return "" 75 | } 76 | scheme, endpointURL := pair[0], pair[1] 77 | dsnFormt := fmt.Sprintf("%s:%s@%s?curr_project=%s&scheme=%s", 78 | cfg.AccessID, cfg.AccessKey, endpointURL, cfg.Project, scheme) 79 | if len(cfg.QueryHints) != 0 { 80 | for k, v := range cfg.QueryHints { 81 | dsnFormt = fmt.Sprintf("%s&hint_%s=%v", dsnFormt, k, v) 82 | } 83 | } 84 | return dsnFormt 85 | } 86 | -------------------------------------------------------------------------------- /dsn_test.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestConfig_ParseDSN(t *testing.T) { 10 | a := assert.New(t) 11 | 12 | correct := "access_id:access_key@service.com/api?curr_project=test_ci&scheme=http&hint_odps.sql.mapper.split_size=16" 13 | config, err := ParseDSN(correct) 14 | a.NoError(err) 15 | a.Equal("access_id", config.AccessID) 16 | a.Equal("access_key", config.AccessKey) 17 | a.Equal("test_ci", config.Project) 18 | a.Equal("http://service.com/api", config.Endpoint) 19 | a.Equal("16", config.QueryHints["odps.sql.mapper.split_size"]) 20 | 21 | badDSN := []string{ 22 | "", // empty 23 | ":access_key@service.com/api?curr_project=test_ci&scheme=http", // missing access_id 24 | "access_idaccess_key@service.com/api?curr_project=test_ci&scheme=http", // missing : 25 | "access_id:@service.com/api?curr_project=test_ci&scheme=http", // missing @ 26 | "access_id:access_key@?curr_project=test_ci&scheme=http", // missing endpoint 27 | "access_id:access_key@service.com/apicurr_project=test_ci&scheme=http", // missing ? 28 | "access_id:access_key@service.com/api?scheme=http", // missing curr_project 29 | "access_id:access_key@service.com/api?curr_project=test_ci", // missing scheme 30 | "access_id:access_key@service.com/api?curr_project=test_ci&scheme=whatever", // invalid scheme 31 | "access_id:access_key@service.com/api?curr_project=test_ci&scheeeeeee=http", // invalid name 32 | } 33 | for _, dsn := range badDSN { 34 | _, err = ParseDSN(dsn) 35 | a.Error(err) 36 | } 37 | 38 | goodDSN := []string{ 39 | "64pxmm:oRdhg=@127.0.0.1:8002/api?curr_project=test_ci&scheme=http", 40 | } 41 | for _, dsn := range goodDSN { 42 | _, err = ParseDSN(dsn) 43 | a.NoError(err) 44 | } 45 | } 46 | 47 | func TestConfig_FormatDSN(t *testing.T) { 48 | a := assert.New(t) 49 | config := Config{ 50 | AccessID: "access_id", 51 | AccessKey: "access_key", 52 | Project: "test_ci", 53 | Endpoint: "http://service.com/api", 54 | QueryHints: map[string]string{"odps.sql.mapper.split_size": "16"}} 55 | a.Equal("access_id:access_key@service.com/api?curr_project="+ 56 | "test_ci&scheme=http&hint_odps.sql.mapper.split_size=16", config.FormatDSN()) 57 | } 58 | 59 | func TestConfig_ParseAndFormatRoundTrip(t *testing.T) { 60 | a := assert.New(t) 61 | dsn := "access_id:access_key@service.com/api?curr_project=test_ci&scheme=http" 62 | 63 | config, err := ParseDSN(dsn) 64 | a.NoError(err) 65 | a.Equal(dsn, config.FormatDSN()) 66 | } 67 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | ) 7 | 8 | var ( 9 | reErrorCode = regexp.MustCompile(`^.*ODPS-([0-9]+):.*$`) 10 | ) 11 | 12 | func parseErrorCode(message string) (string, error) { 13 | sub := reErrorCode.FindStringSubmatch(message) 14 | if len(sub) != 2 { 15 | return "", fmt.Errorf("fail parse error: %s", message) 16 | } 17 | 18 | return sub[1], nil 19 | } 20 | 21 | // MaxcomputeError is an error type which represents a single Maxcompute error 22 | // Please refer to https://www.alibabacloud.com/help/doc-detail/64654.htm 23 | // for the list of SQL common error code 24 | type MaxcomputeError struct { 25 | Code string 26 | Message string 27 | } 28 | 29 | func (e *MaxcomputeError) Error() string { 30 | return e.Message 31 | } 32 | -------------------------------------------------------------------------------- /errors_test.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestParseErrorCode(t *testing.T) { 9 | a := assert.New(t) 10 | 11 | code, err := parseErrorCode(`ParseError: {ODPS-0130161:[1,6] Parse exception - invalid token ';'}`) 12 | a.Nil(err) 13 | a.Equal(`0130161`, code) 14 | 15 | code, err = parseErrorCode(`ODPS-0130131:[1,15] Table not found - table gomaxcompute_driver_w7u.i_dont_exist cannot be resolved`) 16 | a.Nil(err) 17 | a.Equal(`0130131`, code) 18 | } 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module sqlflow.org/gomaxcompute 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/google/uuid v1.3.0 7 | github.com/pkg/errors v0.9.1 8 | github.com/sirupsen/logrus v1.4.2 9 | github.com/stretchr/testify v1.4.0 10 | ) 11 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 5 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 6 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 7 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 8 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 9 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 10 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 11 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 12 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 13 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 14 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 15 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 16 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 17 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 18 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 19 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= 20 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 21 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 22 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 23 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 24 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 25 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 9 | 10 | 11 |

Redirecting to https://github.com/sql-machine-learning/gomaxcompute

12 | 13 | 14 | -------------------------------------------------------------------------------- /instance.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/xml" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | type instanceStatus struct { 13 | XMLName xml.Name `xml:"Instance"` 14 | Status string `xml:"Status"` 15 | } 16 | 17 | type Result struct { 18 | Content string `xml:",cdata"` 19 | Transform string `xml:"Transform,attr"` 20 | Format string `xml:"Format,attr"` 21 | } 22 | 23 | type instanceResult struct { 24 | XMLName xml.Name `xml:"Instance"` 25 | Result Result `xml:"Tasks>Task>Result"` 26 | } 27 | 28 | type instanceErrorMessage struct { 29 | CDATA string `xml:",cdata"` 30 | } 31 | 32 | type instanceError struct { 33 | XMLName xml.Name `xml:"Error"` 34 | Code string `xml:"Code"` 35 | Message instanceErrorMessage `xml:"Message"` 36 | RequestId string `xml:"RequestId"` 37 | HostId string `xml:"HostId"` 38 | } 39 | 40 | // instance types：SQL 41 | func (conn *odpsConn) createInstance(job *odpsJob) (string, error) { 42 | if job == nil { 43 | return "", errors.New("nil job") 44 | } 45 | 46 | // Create 47 | res, err := conn.request(methodPost, conn.resource("/instances"), job.XML()) 48 | if err != nil { 49 | return "", err 50 | } 51 | if _, err = parseResponse(res); err != nil && err != errNilBody { 52 | return "", err 53 | } 54 | 55 | // Parse response header "Location" to get instance ID 56 | ins := location2InstanceID(res.Header.Get("Location")) 57 | if ins == "" { 58 | return "", errors.New("no instance id") 59 | } 60 | return ins, nil 61 | } 62 | 63 | // parse instance id 64 | func location2InstanceID(location string) string { 65 | pieces := strings.Split(location, "/") 66 | if len(pieces) < 2 { 67 | return "" 68 | } 69 | return pieces[len(pieces)-1] 70 | } 71 | 72 | // instance status：Running/Suspended/Terminated 73 | func (conn *odpsConn) getInstanceStatus(instanceID string) (string, error) { 74 | res, err := conn.request(methodGet, conn.resource("/instances/"+instanceID), nil) 75 | if err != nil { 76 | return "", err 77 | } 78 | body, err := parseResponse(res) 79 | if err != nil { 80 | return "", err 81 | } 82 | 83 | var is instanceStatus 84 | err = xml.Unmarshal(body, &is) 85 | if err != nil { 86 | return "", err 87 | } 88 | return is.Status, nil 89 | } 90 | 91 | // getInstanceResult is valid while instance status is `Terminated` 92 | // notice: records up to 10000 by limitation, and result type is string 93 | func (conn *odpsConn) getInstanceResult(instanceID string) (string, error) { 94 | rsc := conn.resource("/instances/"+instanceID, pair{k: "result"}) 95 | rsp, err := conn.request(methodGet, rsc, nil) 96 | if err != nil { 97 | return "", err 98 | } 99 | body, err := parseResponse(rsp) 100 | if err != nil { 101 | return "", err 102 | } 103 | return decodeInstanceResult(body) 104 | } 105 | 106 | func decodeInstanceResult(result []byte) (string, error) { 107 | var ir instanceResult 108 | if err := xml.Unmarshal(result, &ir); err != nil { 109 | return "", err 110 | } 111 | 112 | if ir.Result.Format == "text" { 113 | log.Debug(ir.Result.Content) 114 | // ODPS errors are text begin with "ODPS-" 115 | if strings.HasPrefix(ir.Result.Content, "ODPS-") { 116 | code, err := parseErrorCode(ir.Result.Content) 117 | if err != nil { 118 | return "", errors.WithStack(errors.New(ir.Result.Content)) 119 | } 120 | return "", &MaxcomputeError{code, ir.Result.Content} 121 | } 122 | // FIXME(tony): the result non-query statement usually in text format. 123 | // Go's database/sql API only supports lastId and affectedRows. 124 | return "", nil 125 | } 126 | 127 | if ir.Result.Format != "csv" { 128 | return "", errors.WithStack(fmt.Errorf("unsupported format %v", ir.Result.Format)) 129 | } 130 | 131 | switch ir.Result.Transform { 132 | case "": 133 | return ir.Result.Content, nil 134 | case "Base64": 135 | content, err := base64.StdEncoding.DecodeString(ir.Result.Content) 136 | if err != nil { 137 | return "", err 138 | } 139 | return string(content), err 140 | default: 141 | return "", errors.WithStack(fmt.Errorf("unsupported transform %v", ir.Result.Transform)) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /instance_test.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/xml" 6 | "fmt" 7 | "github.com/stretchr/testify/assert" 8 | "testing" 9 | ) 10 | 11 | func TestDecodeInstanceError(t *testing.T) { 12 | a := assert.New(t) 13 | 14 | s := ` 15 | 16 | ParseError 17 | 18 | 5583C0893F7EC2D353 19 | odps.aliyun.com 20 | ` 21 | 22 | var ie instanceError 23 | a.NoError(xml.Unmarshal([]byte(s), &ie)) 24 | a.Equal(`ParseError`, ie.Code) 25 | a.Equal(`ODPS-0130161:[1,1] Parse exception - invalid token 'SLEECT'`, ie.Message.CDATA) 26 | a.Equal(`5583C0893F7EC2D353`, ie.RequestId) 27 | a.Equal(`odps.aliyun.com`, ie.HostId) 28 | } 29 | 30 | func TestDecodeInstanceResult(t *testing.T) { 31 | a := assert.New(t) 32 | 33 | s := ` 34 | 35 | 36 | 37 | AnonymousSQLTask 38 | 39 | 40 | 41 | ` 42 | 43 | { 44 | data := "abc123!?$*&()'-=@~" 45 | sEnc := base64.StdEncoding.EncodeToString([]byte(data)) 46 | content, err := decodeInstanceResult([]byte(fmt.Sprintf(s, "Transform=\"Base64\"", "Format=\"csv\"", sEnc))) 47 | a.NoError(err) 48 | a.Equal(data, content) 49 | } 50 | 51 | { 52 | data := "1,2,3" 53 | content, err := decodeInstanceResult([]byte(fmt.Sprintf(s, "", "Format=\"csv\"", data))) 54 | a.NoError(err) 55 | a.Equal(data, content) 56 | } 57 | 58 | { 59 | _, err := decodeInstanceResult([]byte(fmt.Sprintf(s, "", "Format=\"text\"", "1,2,3"))) 60 | a.NoError(err) 61 | } 62 | 63 | { 64 | _, err := decodeInstanceResult([]byte(fmt.Sprintf(s, "Transform=\"zip\"", "Format=\"csv\"", "1,2,3"))) 65 | a.Error(err) // unsupported transform zip 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /job.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "encoding/xml" 5 | "strconv" 6 | ) 7 | 8 | // musts: Priority, Tasks 9 | type odpsJob struct { 10 | xml.Marshaler 11 | Name string `xml:"Name"` 12 | Comment string `xml:"Comment"` 13 | Priority int `xml:"Priority"` 14 | Tasks map[string]odpsTask `xml:"Tasks"` 15 | } 16 | 17 | func newJob(tasks ...odpsTask) *odpsJob { 18 | taskMap := make(map[string]odpsTask, len(tasks)) 19 | for _, t := range tasks { 20 | taskMap[t.GetName()] = t 21 | } 22 | return &odpsJob{ 23 | Priority: 1, 24 | Tasks: taskMap, 25 | } 26 | } 27 | 28 | func newSQLJob(sql string, hints map[string]string) *odpsJob { 29 | return newJob(newAnonymousSQLTask(sql, hints)) 30 | } 31 | 32 | func (j *odpsJob) MarshalXML(e *xml.Encoder, start xml.StartElement) (err error) { 33 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Instance"}}) 34 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Job"}}) 35 | 36 | // optional job name 37 | if j.Name != "" { 38 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Comment"}}) 39 | e.EncodeToken(xml.CharData(j.Name)) 40 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Comment"}}) 41 | } 42 | 43 | // optional job comment 44 | if j.Comment != "" { 45 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Comment"}}) 46 | e.EncodeToken(xml.CharData(j.Comment)) 47 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Comment"}}) 48 | } 49 | 50 | // Job.Priority 51 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Priority"}}) 52 | e.EncodeToken(xml.CharData(strconv.Itoa(j.Priority))) 53 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Priority"}}) 54 | 55 | // Job.Tasks 56 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Tasks"}}) 57 | for _, task := range j.Tasks { 58 | if err = e.EncodeElement(task, xml.StartElement{Name: xml.Name{Local: "Task"}}); err != nil { 59 | return 60 | } 61 | } 62 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Tasks"}}) 63 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Job"}}) 64 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Instance"}}) 65 | return e.Flush() 66 | } 67 | 68 | func (j *odpsJob) SetName(name string) { 69 | j.Name = name 70 | } 71 | 72 | func (j *odpsJob) SetComment(comment string) { 73 | j.Comment = comment 74 | } 75 | 76 | func (j *odpsJob) SetPriority(priority int) { 77 | j.Priority = priority 78 | } 79 | 80 | func (j *odpsJob) SetTasks(ts ...odpsTask) { 81 | taskMap := make(map[string]odpsTask, len(ts)) 82 | for _, t := range ts { 83 | taskMap[t.GetName()] = t 84 | } 85 | j.Tasks = taskMap 86 | } 87 | 88 | func (j *odpsJob) AddTask(t odpsTask) { 89 | j.Tasks[t.GetName()] = t 90 | } 91 | 92 | func (j *odpsJob) XML() []byte { 93 | body, _ := xml.MarshalIndent(j, " ", " ") 94 | return body 95 | } 96 | -------------------------------------------------------------------------------- /job_test.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "encoding/xml" 5 | "testing" 6 | 7 | "github.com/google/uuid" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestJob_GenCreateInstanceXml(t *testing.T) { 12 | a := assert.New(t) 13 | task := newAnonymousSQLTask("SELECT 1;", map[string]string{ 14 | "uuid": uuid.NewString(), 15 | "settings": `{"odps.sql.udf.strict.mode": "true"}`, 16 | }) 17 | job := newJob(task) 18 | if job == nil { 19 | t.Error("fail to create new job") 20 | } 21 | 22 | _, err := xml.MarshalIndent(job, "", " ") 23 | a.NoError(err) 24 | } 25 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path" 8 | 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | var log *logrus.Entry 13 | 14 | func getEnv(key, fallback string) string { 15 | value := os.Getenv(key) 16 | if len(value) == 0 { 17 | return fallback 18 | } 19 | return value 20 | } 21 | 22 | func init() { 23 | logDir := getEnv("GOMAXCOMPUTE_log_dir", "") 24 | logLevel := getEnv("GOMAXCOMPUTE_log_level", "info") 25 | 26 | ll, e := logrus.ParseLevel(logLevel) 27 | if e != nil { 28 | ll = logrus.InfoLevel 29 | } 30 | var f io.Writer 31 | if logDir != "" { 32 | e = os.MkdirAll(logDir, 0744) 33 | if e != nil { 34 | log.Panicf("create log directory failed: %v", e) 35 | } 36 | 37 | f, e = os.Create(path.Join(logDir, fmt.Sprintf("gomaxcompute-%d.log", os.Getpid()))) 38 | if e != nil { 39 | log.Panicf("open log file failed: %v", e) 40 | } 41 | } else { 42 | f = os.Stdout 43 | } 44 | 45 | lg := logrus.New() 46 | lg.SetOutput(f) 47 | lg.SetLevel(ll) 48 | log = lg.WithFields(logrus.Fields{"driver": "odps"}) 49 | } 50 | -------------------------------------------------------------------------------- /result.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | // odpsResult implements https://golang.org/pkg/database/sql/driver/#Result 4 | type odpsResult struct { 5 | affectedRows int64 6 | insertId int64 7 | } 8 | 9 | func (res *odpsResult) LastInsertId() (int64, error) { 10 | return res.insertId, nil 11 | } 12 | 13 | func (res *odpsResult) RowsAffected() (int64, error) { 14 | return res.affectedRows, nil 15 | } 16 | -------------------------------------------------------------------------------- /rows.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "bytes" 5 | "database/sql/driver" 6 | "encoding/csv" 7 | "reflect" 8 | ) 9 | 10 | var builtinString = reflect.TypeOf(string("")) 11 | 12 | type odpsRows struct { 13 | meta *resultMeta 14 | reader *csv.Reader 15 | header []string 16 | headerLen int 17 | } 18 | 19 | func (rs *odpsRows) Close() error { 20 | return nil 21 | } 22 | 23 | func (rs *odpsRows) Columns() []string { 24 | return rs.header 25 | } 26 | 27 | // Notice: odps using `\N` to denote nil, even for not-string field 28 | func (rs *odpsRows) Next(dst []driver.Value) error { 29 | records, err := rs.reader.Read() 30 | if err != nil { 31 | return err 32 | } 33 | for i := 0; i < rs.headerLen; i++ { 34 | dst[i] = records[i] 35 | } 36 | return nil 37 | } 38 | 39 | func (rs *odpsRows) ColumnTypeScanType(i int) reflect.Type { 40 | return builtinString 41 | } 42 | 43 | func (rs *odpsRows) ColumnTypeDatabaseTypeName(i int) string { 44 | return rs.meta.Schema.Columns[i].Type 45 | } 46 | 47 | func newRows(m *resultMeta, res string) (*odpsRows, error) { 48 | rd := csv.NewReader(bytes.NewBufferString(res)) 49 | // hr equas to [m.Schema.Columns.Name] 50 | hr, err := rd.Read() 51 | if err != nil { 52 | return nil, err 53 | } 54 | return &odpsRows{meta: m, reader: rd, header: hr, headerLen: len(hr)}, nil 55 | } 56 | -------------------------------------------------------------------------------- /rows_test.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestNext(t *testing.T) { 13 | a := assert.New(t) 14 | 15 | db, err := sql.Open("maxcompute", cfg4test.FormatDSN()) 16 | a.NoError(err) 17 | 18 | const sql = `SELECT * from gomaxcompute_test;` 19 | rows, err := db.Query(sql) 20 | defer rows.Close() 21 | a.NoError(err) 22 | 23 | cols, _ := rows.Columns() 24 | cnt := len(cols) 25 | values := make([]interface{}, cnt) 26 | row := make([]interface{}, cnt) 27 | for rows.Next() { 28 | cts, err := rows.ColumnTypes() 29 | a.NoError(err) 30 | 31 | for i, ct := range cts { 32 | v, err := createByType(ct.ScanType()) 33 | a.NoError(err) 34 | values[i] = v 35 | } 36 | err = rows.Scan(values...) 37 | a.NoError(err) 38 | 39 | for i, val := range values { 40 | v, err := parseVal(val) 41 | a.NoError(err) 42 | row[i] = v 43 | } 44 | } 45 | } 46 | 47 | func createByType(rt reflect.Type) (interface{}, error) { 48 | switch rt { 49 | case builtinString: 50 | return new(string), nil 51 | default: 52 | return nil, fmt.Errorf("unrecognized column scan type %v", rt) 53 | } 54 | } 55 | 56 | func parseVal(val interface{}) (interface{}, error) { 57 | switch v := val.(type) { 58 | case *string: 59 | return *v, nil 60 | default: 61 | return nil, fmt.Errorf("unrecogized type %v", v) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /task.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "encoding/xml" 5 | "strings" 6 | 7 | "github.com/google/uuid" 8 | ) 9 | 10 | const anonymousSQLTask = "AnonymousSQLTask" 11 | 12 | type odpsTask interface { 13 | GetName() string 14 | GetComment() string 15 | xml.Marshaler 16 | } 17 | 18 | type cdata struct { 19 | String string `xml:",cdata"` 20 | } 21 | 22 | type odpsSQLTask struct { 23 | // default: AnonymousSQLTask 24 | Name string 25 | Query string 26 | Comment string 27 | // uuid settings 28 | Config map[string]string 29 | } 30 | 31 | func (s odpsSQLTask) MarshalXML(e *xml.Encoder, start xml.StartElement) error { 32 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "SQL"}}) 33 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Name"}}) 34 | e.EncodeToken(xml.CharData(anonymousSQLTask)) 35 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Name"}}) 36 | 37 | if s.Comment != "" { 38 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Comment"}}) 39 | e.EncodeToken(xml.CharData(s.Comment)) 40 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Comment"}}) 41 | } 42 | 43 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Config"}}) 44 | for key, value := range s.Config { 45 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Property"}}) 46 | e.EncodeToken(xml.StartElement{Name: xml.Name{Local: "Name"}}) 47 | e.EncodeToken(xml.CharData(key)) 48 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Name"}}) 49 | e.EncodeElement(cdata{value}, xml.StartElement{Name: xml.Name{Local: "Value"}}) 50 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Property"}}) 51 | } 52 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "Config"}}) 53 | e.EncodeElement(cdata{s.Query}, xml.StartElement{Name: xml.Name{Local: "Query"}}) 54 | e.EncodeToken(xml.EndElement{Name: xml.Name{Local: "SQL"}}) 55 | return e.Flush() 56 | } 57 | 58 | func (s *odpsSQLTask) GetName() string { 59 | return s.Name 60 | } 61 | 62 | func (s *odpsSQLTask) GetComment() string { 63 | return s.Comment 64 | } 65 | 66 | func newAnonymousSQLTask(query string, config map[string]string) odpsTask { 67 | return newSQLTask(anonymousSQLTask, query, config) 68 | } 69 | 70 | func newSQLTask(name, query string, config map[string]string) odpsTask { 71 | if config == nil { 72 | config = map[string]string{ 73 | "uuid": uuid.NewString(), 74 | "settings": `{"odps.sql.udf.strict.mode": "true"}`, 75 | } 76 | } else if _, ok := config["uuid"]; !ok { 77 | config["uuid"] = uuid.NewString() 78 | } 79 | // maxcompute sql ends with a ';' 80 | query = strings.TrimSpace(query) 81 | if n := len(query); n > 0 && query[n-1] != ';' { 82 | query += ";" 83 | } 84 | 85 | return &odpsSQLTask{ 86 | Name: name, 87 | Query: query, 88 | Config: config, 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /task_test.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | import ( 4 | "encoding/xml" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestSQLTask_MarshalXML(t *testing.T) { 11 | a := assert.New(t) 12 | task := newAnonymousSQLTask("SELECT 1;", nil) 13 | _, err := xml.Marshal(task) 14 | a.NoError(err) 15 | } 16 | -------------------------------------------------------------------------------- /tunnel.go: -------------------------------------------------------------------------------- 1 | package gomaxcompute 2 | 3 | type resultMeta struct { 4 | DownloadID string `json:"DownloadID"` 5 | RecordCount int64 `json:"RecordCount"` 6 | Schema struct { 7 | IsVirtualView bool `json:"IsVirtualView"` 8 | Columns []struct { 9 | Comment string `json:"comment"` 10 | Name string `json:"name"` 11 | Nullable bool `json:"nullable"` 12 | Type string `json:"type"` 13 | } `json:"columns"` 14 | } `json:"Schema"` 15 | // not parsed: PartitionKeys []string `json:"partitionKeys"` 16 | Status string `json:"Status"` 17 | } 18 | --------------------------------------------------------------------------------