├── .gitignore ├── Godeps ├── Makefile ├── CHANGELOG.md ├── LICENSE ├── command └── query │ ├── customFunc.go │ ├── bq.go │ ├── customFunc_test.go │ ├── bq_test.go │ ├── query.go │ ├── query_test.go │ ├── decorator.go │ └── decorator_test.go ├── wercker.yml ├── main.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .godeps 2 | -------------------------------------------------------------------------------- /Godeps: -------------------------------------------------------------------------------- 1 | github.com/codegangsta/cli v1.2.0 2 | github.com/k0kubun/pp v2.1.0 3 | github.com/smartystreets/goconvey 627707e8db4a4e52f4e1fbbb4e10d98e79a3c946 4 | github.com/jacobsa/oglematchers 4fc24f97b5b74022c2a3f4ca7eed57ca29083d3e 5 | github.com/dustin/go-humanize c128122e0b9b93799aef8181a537e5d8fd7081d6 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DEBUG_FLAG = $(if $(DEBUG),-v) 2 | GOPATH_ENV="$(PWD)/.godeps:$(PWD)" 3 | GOBIN_ENV="$(PWD)/.godeps/bin" 4 | GOPKG_ENV="$(PWD)/.godeps/pkg/darwin_amd64/github.com/yoheimuta/dbq" 5 | 6 | deps: 7 | wget -qO- https://raw.githubusercontent.com/pote/gpm/v1.2.3/bin/gpm | GOPATH=$(GOPATH_ENV) bash 8 | 9 | check: 10 | goimports -l main.go command/ 11 | vet main.go; vet command/ 12 | golint ./... 13 | 14 | fix: 15 | goimports -w main.go command/ 16 | 17 | test: 18 | GOPATH=$(GOPATH_ENV) go test $(DEBUG_FLAG) ./... 19 | 20 | install: 21 | rm -r $(GOPKG_ENV); GOBIN=$(GOBIN_ENV) GOPATH=$(GOPATH_ENV) go install 22 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## [v0.0.3](https://github.com/yoheimuta/dbq/tree/v0.0.3) (2015-07-19) 4 | 5 | [Full Changelog](https://github.com/yoheimuta/dbq/compare/v0.0.2...v0.0.3) 6 | 7 | **Implemented enhancements:** 8 | 9 | - Output human readable result of dryRun [\#1](https://github.com/yoheimuta/dbq/issues/1) 10 | 11 | **Merged pull requests:** 12 | 13 | - Output human readable result of dryRun [\#2](https://github.com/yoheimuta/dbq/pull/2) ([yoheimuta](https://github.com/yoheimuta)) 14 | 15 | ## [v0.0.2](https://github.com/yoheimuta/dbq/tree/v0.0.2) (2015-07-13) 16 | 17 | [Full Changelog](https://github.com/yoheimuta/dbq/compare/v0.0.1...v0.0.2) 18 | 19 | ## [v0.0.1](https://github.com/yoheimuta/dbq/tree/v0.0.1) (2015-07-12) 20 | 21 | 22 | 23 | \* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 yoheimuta 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /command/query/customFunc.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | ) 8 | 9 | var dateAddFuncRule = regexp.MustCompile("_tz\\((.*?)\\)") 10 | 11 | // CustomFunc handles Custom Functions that are applied to the statement 12 | type CustomFunc struct { 13 | args Args 14 | } 15 | 16 | // NewCustomFunc initializes the CustomFunc struct. 17 | func NewCustomFunc(args Args) *CustomFunc { 18 | customFunc := &CustomFunc{ 19 | args: args, 20 | } 21 | return customFunc 22 | } 23 | 24 | // Apply is a facade method that runs custom functions 25 | func (c *CustomFunc) Apply(statement string) string { 26 | return c._tz(statement) 27 | } 28 | 29 | func (c *CustomFunc) _tz(statement string) string { 30 | if c.args.tz == 0 { 31 | if isVerbose { 32 | fmt.Printf("Skip to replace _tz(): tz=0\n") 33 | } 34 | return statement 35 | } 36 | 37 | replaced := statement 38 | matches := dateAddFuncRule.FindAllStringSubmatch(statement, -1) 39 | 40 | for _, match := range matches { 41 | if len(match) < 2 { 42 | if isVerbose { 43 | fmt.Printf("Skip to replace _tz(): statement=%v\n", statement) 44 | } 45 | return statement 46 | } 47 | 48 | date := match[1] 49 | old := fmt.Sprintf("_tz(%v)", date) 50 | new := fmt.Sprintf("DATE_ADD('%v', %v, 'HOUR')", date, c.args.tz) 51 | replaced = strings.Replace(replaced, old, new, 1) 52 | } 53 | return replaced 54 | } 55 | -------------------------------------------------------------------------------- /wercker.yml: -------------------------------------------------------------------------------- 1 | box: tcnksm/gox 2 | # Build definition 3 | build: 4 | # The steps that will be executed on build 5 | steps: 6 | # Sets the go workspace and places you package 7 | # at the right place in the workspace tree 8 | - setup-go-workspace 9 | 10 | # Gets the packages 11 | - install-packages: 12 | packages: wget ruby zip 13 | 14 | # Gets the go dependencies 15 | - script: 16 | name: go get 17 | code: | 18 | wget -qO- https://raw.githubusercontent.com/pote/gpm/v1.3.2/bin/gpm | bash 19 | 20 | # Checks source code 21 | - script: 22 | name: go static analysis 23 | code: | 24 | go get golang.org/x/tools/cmd/goimports 25 | go get golang.org/x/tools/cmd/vet 26 | go get github.com/golang/lint/golint 27 | goimports -l main.go command/ | xargs -r false 28 | go vet ./... 29 | golint ./... | xargs -r false 30 | 31 | # Tests the project 32 | - script: 33 | name: go test 34 | code: | 35 | go test -v ./... 36 | 37 | # Builds binaries 38 | - tcnksm/gox 39 | - tcnksm/zip: 40 | input: ${WERCKER_OUTPUT_DIR}/pkg 41 | output: ${WERCKER_OUTPUT_DIR}/dist 42 | 43 | after-steps: 44 | 45 | # Slack integration 46 | - wantedly/pretty-slack-notify: 47 | webhook_url: $SLACK_WEBHOOKS_URL 48 | channel: tech-notification 49 | 50 | # Deploy definition 51 | deploy: 52 | steps: 53 | 54 | # GitHub Releases 55 | - tcnksm/ghr: 56 | token: $GITHUB_TOKEN 57 | input: dist 58 | version: v0.0.3 59 | replace: true 60 | -------------------------------------------------------------------------------- /command/query/bq.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "os/exec" 7 | "regexp" 8 | "strconv" 9 | 10 | "github.com/dustin/go-humanize" 11 | ) 12 | 13 | var bytesRule = regexp.MustCompile("process (\\d*?) bytes of data") 14 | 15 | // Bq is an Executor that run the bq command. 16 | type Bq struct{} 17 | 18 | // NewBq initializes the Bq struct. 19 | func NewBq() *Bq { 20 | bq := &Bq{} 21 | return bq 22 | } 23 | 24 | var execCommand = exec.Command 25 | 26 | // Query runs the bq query command. 27 | func (b *Bq) Query(statement string) (out string) { 28 | args := b.buildArgs(statement) 29 | cmd := execCommand("bq", args...) 30 | output, _ := cmd.CombinedOutput() 31 | return string(output) 32 | } 33 | 34 | // GetHumanReadbleInfo changes the raw result to human readable one. 35 | func (b *Bq) GetHumanReadbleInfo(result string) (info string) { 36 | matches := bytesRule.FindStringSubmatch(result) 37 | 38 | if len(matches) < 2 { 39 | return "Failed to change the result of dryRun to human readable one: result=" + result 40 | } 41 | 42 | bytes, err := strconv.ParseInt(matches[1], 10, 64) 43 | if err != nil { 44 | return "Failed to change the result of dryRun to human readable one: result=" + result + ": match=" + matches[1] 45 | } 46 | 47 | info = fmt.Sprintf("- %v bytes equal to %v bytes\n", bytes, humanize.Comma(bytes)) 48 | info += fmt.Sprintf("- %v bytes equal to %v\n", bytes, humanize.IBytes(uint64(bytes))) 49 | 50 | tibibytes := float64(bytes) / math.Pow(1024, 4) 51 | costs := 5.0 52 | info += fmt.Sprintf("- %v bytes equal to $%.5f (= %.5f TiB * $%v)\n", bytes, tibibytes*costs, tibibytes, costs) 53 | return info 54 | } 55 | 56 | func (b *Bq) buildArgs(statement string) (args []string) { 57 | args = append(args, "query") 58 | 59 | if isDryRun { 60 | cflags := "--dry_run" 61 | args = append(args, cflags) 62 | } 63 | 64 | args = append(args, statement) 65 | return args 66 | } 67 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/yoheimuta/dbq/command/query" 7 | 8 | "github.com/codegangsta/cli" 9 | ) 10 | 11 | func main() { 12 | app := cli.NewApp() 13 | app.Name = "dbq" 14 | app.Usage = "CLI tool to decorate bigquery table" 15 | app.Version = "0.0.3" 16 | app.EnableBashCompletion = true 17 | app.Commands = []cli.Command{ 18 | { 19 | Name: "query", 20 | ShortName: "q", 21 | Usage: "Run bq query with complementing table range decorator", 22 | Flags: []cli.Flag{ 23 | cli.Float64Flag{ 24 | Name: "beforeHour", 25 | Value: 3, 26 | Usage: "a decimal to specify the hour ago, relative to the current time", 27 | }, 28 | cli.StringFlag{ 29 | Name: "startDate", 30 | Value: "", 31 | Usage: "a datetime to specify date range with end flag", 32 | }, 33 | cli.StringFlag{ 34 | Name: "endDate", 35 | Value: "", 36 | Usage: "a datetime to specify date range with start flag", 37 | }, 38 | cli.Float64Flag{ 39 | Name: "tz", 40 | Value: 0, 41 | Usage: "a decimal of hour or -hour to add to start and end datetime, considering timezone", 42 | }, 43 | cli.Float64Flag{ 44 | Name: "buffer", 45 | Value: 1, 46 | Usage: "a decimal of hour to add to start and end datetime, it's heuristic value", 47 | }, 48 | cli.StringFlag{ 49 | Name: "gflags", 50 | Value: "", 51 | Usage: "no support. Use onlyStatement instead", 52 | }, 53 | cli.StringFlag{ 54 | Name: "cflags", 55 | Value: "", 56 | Usage: "no support. Use onlyStatement instead", 57 | }, 58 | cli.BoolFlag{ 59 | Name: "verbose", 60 | Usage: "a flag to output verbosely", 61 | }, 62 | cli.BoolFlag{ 63 | Name: "dryRun", 64 | Usage: "a flag to run without any changes", 65 | }, 66 | cli.BoolFlag{ 67 | Name: "onlyStatement", 68 | Usage: "a flag to output only a decorated statement", 69 | }, 70 | }, 71 | Action: query.Run, 72 | }, 73 | } 74 | app.Run(os.Args) 75 | } 76 | -------------------------------------------------------------------------------- /command/query/customFunc_test.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/smartystreets/goconvey/convey" 7 | ) 8 | 9 | func TestCustomFunc(t *testing.T) { 10 | Convey("NewCustomFunc", t, func() { 11 | So(func() { NewCustomFunc(Args{}) }, ShouldNotPanic) 12 | }) 13 | 14 | Convey("When the statement includes the placeholder of _tz", t, func() { 15 | 16 | Convey("When the placeholder of _tz appears once", func() { 17 | statement := "SELECT * FROM [account.table@] WHERE col='val' and _tz(2015-07-08 17:00:00) <= time ORDER BY time DESC" 18 | 19 | Convey("The argument of tz is 0, which means UTC", func() { 20 | cf := NewCustomFunc(Args{tz: 0}) 21 | So(cf.Apply(statement), ShouldEqual, statement) 22 | }) 23 | 24 | Convey("The argument of tz is -9, which means JST", func() { 25 | cf := NewCustomFunc(Args{tz: -9}) 26 | So(cf.Apply(statement), ShouldNotEqual, statement) 27 | 28 | expected := "SELECT * FROM [account.table@] WHERE col='val' and DATE_ADD('2015-07-08 17:00:00', -9, 'HOUR') <= time ORDER BY time DESC" 29 | So(cf.Apply(statement), ShouldEqual, expected) 30 | }) 31 | }) 32 | 33 | Convey("When the placeholder of _tz appears twice", func() { 34 | statement := "SELECT * FROM [account.table@] WHERE col='val' and _tz(2015-07-08 17:00:00) <= time and time <= _tz(2015-07-08 18:00:00) ORDER BY time DESC" 35 | 36 | Convey("The argument of tz is 0, which means UTC", func() { 37 | cf := NewCustomFunc(Args{tz: 0}) 38 | So(cf.Apply(statement), ShouldEqual, statement) 39 | }) 40 | 41 | Convey("The argument of tz is -9, which means JST", func() { 42 | cf := NewCustomFunc(Args{tz: -9}) 43 | So(cf.Apply(statement), ShouldNotEqual, statement) 44 | 45 | expected := "SELECT * FROM [account.table@] WHERE col='val' and DATE_ADD('2015-07-08 17:00:00', -9, 'HOUR') <= time and time <= DATE_ADD('2015-07-08 18:00:00', -9, 'HOUR') ORDER BY time DESC" 46 | So(cf.Apply(statement), ShouldEqual, expected) 47 | }) 48 | }) 49 | }) 50 | 51 | Convey("When the statement doesn't include placeholder of _tz", t, func() { 52 | statement := "SELECT * FROM [account.table@] WHERE col='val' and DATE_ADD('2015-07-08 17:00:00', -9, 'HOUR') ORDER BY time DESC" 53 | cf := NewCustomFunc(Args{tz: -9}) 54 | So(cf.Apply(statement), ShouldEqual, statement) 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /command/query/bq_test.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "regexp" 8 | "testing" 9 | 10 | . "github.com/smartystreets/goconvey/convey" 11 | ) 12 | 13 | func fakeExecCommand(command string, args ...string) *exec.Cmd { 14 | cs := []string{"-test.run=TestHelperProcess", "--", command} 15 | cs = append(cs, args...) 16 | cmd := exec.Command(os.Args[0], cs...) 17 | cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} 18 | return cmd 19 | } 20 | 21 | const mockReturnVal = "RESULT" 22 | 23 | func TestBq(t *testing.T) { 24 | Convey("NewBq", t, func() { 25 | So(func() { NewBq() }, ShouldNotPanic) 26 | }) 27 | 28 | Convey("When the bq command executes the statement", t, func() { 29 | execCommand = fakeExecCommand 30 | defer func() { execCommand = exec.Command }() 31 | 32 | actual := NewBq().Query("SELECT * FROM [account.table]") 33 | So(regexp.MustCompile(mockReturnVal).MatchString(actual), ShouldBeTrue) 34 | }) 35 | 36 | Convey("When the arguments are built", t, func() { 37 | b := NewBq() 38 | statement := "SELECT * FROM [account.table]" 39 | 40 | Convey("When the dryRun flag is off", func() { 41 | actual := b.buildArgs(statement) 42 | So(actual[0], ShouldEqual, "query") 43 | So(actual[1], ShouldEqual, statement) 44 | }) 45 | 46 | Convey("When the dryRun flag is on", func() { 47 | isDryRun = true 48 | actual := b.buildArgs(statement) 49 | So(actual[0], ShouldEqual, "query") 50 | So(actual[1], ShouldEqual, "--dry_run") 51 | So(actual[2], ShouldEqual, statement) 52 | isDryRun = false 53 | }) 54 | }) 55 | 56 | Convey("When the result of dryRun query is changed to humanreadable one", t, func() { 57 | b := NewBq() 58 | result := "Query successfully validated. Assuming the tables are not modified, running this query will process 8133291239 bytes of data." 59 | 60 | info := b.GetHumanReadbleInfo(result) 61 | 62 | line1 := "- 8133291239 bytes equal to 8,133,291,239 bytes" 63 | line2 := "- 8133291239 bytes equal to 7.6GiB" 64 | line3 := "- 8133291239 bytes equal to $0.03699 (= 0.00740 TiB * $5)" 65 | So(info, ShouldEqual, fmt.Sprintf("%v\n%v\n%v\n", line1, line2, line3)) 66 | }) 67 | 68 | } 69 | 70 | // TestHelperProcess isn't a real test. It's used as a helper process 71 | func TestHelperProcess(t *testing.T) { 72 | if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { 73 | return 74 | } 75 | defer os.Exit(0) 76 | 77 | fmt.Fprintf(os.Stdout, mockReturnVal) 78 | } 79 | -------------------------------------------------------------------------------- /command/query/query.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/codegangsta/cli" 7 | ) 8 | 9 | var isVerbose bool 10 | var isDryRun bool 11 | var onlyStatement bool 12 | 13 | // Args represents Arguments of the query command 14 | type Args struct { 15 | beforeHour float64 16 | startDate string 17 | endDate string 18 | tz float64 19 | buffer float64 20 | } 21 | 22 | // Run is a facade method of the query command 23 | func Run(c *cli.Context) { 24 | if len(c.Args()) != 1 { 25 | fmt.Println("Not Found a statement") 26 | return 27 | } 28 | statement := c.Args()[0] 29 | 30 | isVerbose = c.Bool("verbose") 31 | isDryRun = c.Bool("dryRun") 32 | onlyStatement = c.Bool("onlyStatement") 33 | 34 | args := Args{ 35 | beforeHour: c.Float64("beforeHour"), 36 | startDate: c.String("startDate"), 37 | endDate: c.String("endDate"), 38 | tz: c.Float64("tz"), 39 | buffer: c.Float64("buffer"), 40 | } 41 | 42 | q := newQuery(statement, args) 43 | output, err := q.query() 44 | if err != nil { 45 | fmt.Printf("Failed to run the command\n:error=%v\n", err) 46 | return 47 | } 48 | if output != "" { 49 | fmt.Printf(output) 50 | } 51 | } 52 | 53 | // Query is an Implementation that decorates the statement and run the bq query 54 | type Query struct { 55 | deco *Decorator 56 | bq *Bq 57 | } 58 | 59 | var newQuery = func(statement string, args Args) *Query { 60 | return &Query{ 61 | deco: NewDecorator(statement, args), 62 | bq: NewBq(), 63 | } 64 | } 65 | 66 | func (q Query) query() (output string, err error) { 67 | if onlyStatement { 68 | return q.printStmt() 69 | } 70 | 71 | if isDryRun { 72 | return q.dryRun() 73 | } 74 | 75 | return q.run() 76 | } 77 | 78 | func (q Query) printStmt() (output string, err error) { 79 | dStmt, err := q.deco.Apply() 80 | if err != nil { 81 | return "", err 82 | } 83 | 84 | return dStmt, nil 85 | } 86 | 87 | func (q Query) dryRun() (output string, err error) { 88 | // Displays the result without decarator 89 | raw := q.deco.Revert() 90 | res := q.bq.Query(raw) 91 | info := q.bq.GetHumanReadbleInfo(res) 92 | fmt.Printf("Raw: %v\n%v\n%v\n", raw, res, info) 93 | 94 | // Displays the result with decorator 95 | dStmt, err := q.deco.Apply() 96 | if err != nil { 97 | return "", err 98 | } 99 | res = q.bq.Query(dStmt) 100 | info = q.bq.GetHumanReadbleInfo(res) 101 | fmt.Printf("Decorated: %v\n%v\n%v\n", dStmt, res, info) 102 | return "", nil 103 | } 104 | 105 | func (q Query) run() (output string, err error) { 106 | dStmt, err := q.deco.Apply() 107 | if err != nil { 108 | return "", err 109 | } 110 | 111 | fmt.Printf("Decorated: %v\n", dStmt) 112 | return q.bq.Query(dStmt), nil 113 | } 114 | -------------------------------------------------------------------------------- /command/query/query_test.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "flag" 5 | "os/exec" 6 | "regexp" 7 | "testing" 8 | 9 | "github.com/codegangsta/cli" 10 | . "github.com/smartystreets/goconvey/convey" 11 | ) 12 | 13 | func TestRun(t *testing.T) { 14 | Convey("When the arguements are parsed", t, func() { 15 | expectedStmt := "SELECT * FROM [account.table]" 16 | expectedBeforeHour := 1.0 17 | expectedStart := "2015-07-08 17:00:00" 18 | expectedEnd := "2015-07-08 18:00:00" 19 | expectedHadd := -9.0 20 | expectedBuffer := 1.0 21 | 22 | prevNewQuery := newQuery 23 | newQuery = func(statement string, args Args) *Query { 24 | So(statement, ShouldEqual, expectedStmt) 25 | 26 | So(args.beforeHour, ShouldEqual, expectedBeforeHour) 27 | So(args.startDate, ShouldEqual, expectedStart) 28 | So(args.endDate, ShouldEqual, expectedEnd) 29 | So(args.tz, ShouldEqual, expectedHadd) 30 | So(args.buffer, ShouldEqual, expectedBuffer) 31 | So(isVerbose, ShouldBeFalse) 32 | So(isDryRun, ShouldBeTrue) 33 | So(onlyStatement, ShouldBeTrue) 34 | 35 | return &Query{ 36 | deco: NewDecorator(statement, args), 37 | bq: NewBq(), 38 | } 39 | } 40 | defer func() { newQuery = prevNewQuery }() 41 | 42 | set := flag.NewFlagSet("test", 0) 43 | set.Parse([]string{expectedStmt}) 44 | set.Float64("beforeHour", expectedBeforeHour, "") 45 | set.String("startDate", expectedStart, "") 46 | set.String("endDate", expectedEnd, "") 47 | set.Float64("tz", expectedHadd, "") 48 | set.Float64("buffer", expectedBuffer, "") 49 | set.Bool("verbose", false, "") 50 | set.Bool("dryRun", true, "") 51 | set.Bool("onlyStatement", true, "") 52 | c := cli.NewContext(nil, set, nil) 53 | 54 | Run(c) 55 | }) 56 | } 57 | 58 | func TestQuery(t *testing.T) { 59 | statement := "SELECT * FROM [account.table@] WHERE _tz(2015-07-08 17:00:00) <= time" 60 | args := Args{ 61 | beforeHour: 3.0, 62 | tz: -9.0, 63 | buffer: 1.0, 64 | } 65 | q := newQuery(statement, args) 66 | 67 | Convey("When the onlyStatement flag is on", t, func() { 68 | onlyStatement = true 69 | actual, err := q.query() 70 | 71 | expected := "SELECT * FROM [account.table@-10800000-] WHERE DATE_ADD('2015-07-08 17:00:00', -9, 'HOUR') <= time" 72 | So(actual, ShouldEqual, expected) 73 | So(err, ShouldBeNil) 74 | }) 75 | 76 | Convey("When the isDryRun flag is on", t, func() { 77 | execCommand = fakeExecCommand 78 | defer func() { execCommand = exec.Command }() 79 | 80 | onlyStatement = false 81 | isDryRun = true 82 | actual, err := q.query() 83 | 84 | So(actual, ShouldEqual, "") 85 | So(err, ShouldBeNil) 86 | }) 87 | 88 | Convey("When the onlyStatement and isDryRun flag are off", t, func() { 89 | execCommand = fakeExecCommand 90 | defer func() { execCommand = exec.Command }() 91 | 92 | onlyStatement = false 93 | isDryRun = false 94 | actual, err := q.query() 95 | 96 | So(regexp.MustCompile(mockReturnVal).MatchString(actual), ShouldBeTrue) 97 | So(err, ShouldBeNil) 98 | }) 99 | } 100 | -------------------------------------------------------------------------------- /command/query/decorator.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "time" 7 | ) 8 | 9 | var tableRule = regexp.MustCompile("@") 10 | 11 | // Decorator transforms the no-decorated statement into the decorated one 12 | type Decorator struct { 13 | statement string 14 | args Args 15 | customFunc *CustomFunc 16 | } 17 | 18 | // NewDecorator initializes the Decorator struct. 19 | func NewDecorator(statement string, args Args) *Decorator { 20 | cf := NewCustomFunc(args) 21 | 22 | decorator := &Decorator{ 23 | statement: statement, 24 | args: args, 25 | customFunc: cf, 26 | } 27 | return decorator 28 | } 29 | 30 | // Apply is a facade method that transforms the no-decorated statement into the decorated one 31 | func (d *Decorator) Apply() (decorated string, err error) { 32 | stmt := d.customFunc.Apply(d.statement) 33 | 34 | if d.args.startDate != "" { 35 | startMSec, endMSec, err := d.getRangeMSec() 36 | if err != nil { 37 | return "", err 38 | } 39 | decorated = d.useAbs(stmt, startMSec, endMSec) 40 | } else { 41 | beforeHour := d.args.beforeHour 42 | if beforeHour <= 0 { 43 | beforeHour = 3.0 44 | } 45 | beforeMSec := int(beforeHour * 60 * 60 * 1000) 46 | decorated = d.useRel(stmt, beforeMSec) 47 | } 48 | 49 | if stmt == decorated { 50 | return "", fmt.Errorf("Failed to decorated table: input=%s", stmt) 51 | } 52 | return decorated, nil 53 | } 54 | 55 | // Revert transforms the statement with @ into the one without @ 56 | func (d *Decorator) Revert() (raw string) { 57 | stmt := d.customFunc.Apply(d.statement) 58 | 59 | // input: tableName@ 60 | // output: tableName 61 | raw = tableRule.ReplaceAllString(stmt, "") 62 | return raw 63 | } 64 | 65 | func (d *Decorator) getRangeMSec() (startMSec int64, endMSec int64, err error) { 66 | tz := d.args.tz 67 | buffer := d.args.buffer 68 | 69 | // startTime: convert the formatted string into the unixtime ms 70 | startTime, err := time.Parse("2006-01-02 15:04:05", d.args.startDate) 71 | if err != nil { 72 | return 0, 0, err 73 | } 74 | startTime = startTime.Add(time.Duration((tz+buffer*-1)*60) * time.Minute) 75 | startMSec = startTime.Unix() * 1000 76 | 77 | // endTime: convert the formatted string into the unixtime ms 78 | end := d.args.endDate 79 | if end == "" { 80 | return startMSec, 0, nil 81 | } 82 | endTime, err := time.Parse("2006-01-02 15:04:05", end) 83 | if err != nil { 84 | return 0, 0, err 85 | } 86 | endTime = endTime.Add(time.Duration((tz+buffer)*60) * time.Minute) 87 | endMSec = endTime.Unix() * 1000 88 | return startMSec, endMSec, nil 89 | } 90 | 91 | func (d *Decorator) useAbs(statement string, startMSec int64, endMSec int64) string { 92 | var replaced string 93 | if endMSec == 0 { 94 | replaced = fmt.Sprintf("@%d-", startMSec) 95 | } else { 96 | replaced = fmt.Sprintf("@%d-%d", startMSec, endMSec) 97 | } 98 | 99 | // input: tableName@ 100 | // output: tableName@1435997334864-1436001613000 101 | decorated := tableRule.ReplaceAllString(statement, replaced) 102 | return decorated 103 | } 104 | 105 | func (d *Decorator) useRel(statement string, beforeMSec int) string { 106 | // input: tableName@ 107 | // output: tableName@-3600000- 108 | decorated := tableRule.ReplaceAllString(statement, fmt.Sprintf("@-%d-", beforeMSec)) 109 | return decorated 110 | } 111 | -------------------------------------------------------------------------------- /command/query/decorator_test.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/smartystreets/goconvey/convey" 7 | ) 8 | 9 | func TestDecorator(t *testing.T) { 10 | Convey("NewDecorator", t, func() { 11 | So(func() { NewDecorator("", Args{}) }, ShouldNotPanic) 12 | }) 13 | 14 | Convey("Apply", t, func() { 15 | 16 | Convey("When the statement doesn't include the placeholder of decorator", func() { 17 | statement := "SELECT * FROM [account.table]" 18 | 19 | d := NewDecorator(statement, Args{startDate: "2015-07-08 17:00:00"}) 20 | actual, err := d.Apply() 21 | 22 | So(actual, ShouldBeBlank) 23 | So(err, ShouldNotBeNil) 24 | }) 25 | 26 | Convey("When the statement includes the placeholder of decorator", func() { 27 | statement := "SELECT * FROM [account.table@]" 28 | 29 | Convey("The argument includes the startDate", func() { 30 | 31 | Convey("The argument of startDate is formatted date string", func() { 32 | startDate := "2015-07-08 17:00:00" 33 | 34 | Convey("The argument of tz is 0, which means UTC", func() { 35 | d := NewDecorator(statement, Args{startDate: startDate, buffer: 1}) 36 | actual, err := d.Apply() 37 | 38 | expected := "SELECT * FROM [account.table@1436371200000-]" 39 | So(actual, ShouldEqual, expected) 40 | So(err, ShouldBeNil) 41 | }) 42 | 43 | Convey("The argument of tz is -9, which means JST", func() { 44 | d := NewDecorator(statement, Args{startDate: startDate, tz: -9, buffer: 1}) 45 | actual, err := d.Apply() 46 | 47 | expected := "SELECT * FROM [account.table@1436338800000-]" 48 | So(actual, ShouldEqual, expected) 49 | So(err, ShouldBeNil) 50 | }) 51 | }) 52 | 53 | Convey("The argument of startDate is not formatted date string", func() { 54 | startDate := "2015/07/08 170000" 55 | 56 | d := NewDecorator(statement, Args{startDate: startDate}) 57 | actual, err := d.Apply() 58 | 59 | So(actual, ShouldBeBlank) 60 | So(err, ShouldNotBeNil) 61 | }) 62 | }) 63 | 64 | Convey("The argument includes the startDate and endDate", func() { 65 | 66 | Convey("The argument of startDate and endDate are formatted date string", func() { 67 | startDate := "2015-07-08 17:00:00" 68 | endDate := "2015-07-08 18:00:00" 69 | 70 | Convey("The argument of tz is 0, which means UTC", func() { 71 | d := NewDecorator(statement, Args{startDate: startDate, endDate: endDate, buffer: 1}) 72 | actual, err := d.Apply() 73 | 74 | expected := "SELECT * FROM [account.table@1436371200000-1436382000000]" 75 | So(actual, ShouldEqual, expected) 76 | So(err, ShouldBeNil) 77 | }) 78 | 79 | Convey("The argument of tz is -9, which means JST", func() { 80 | d := NewDecorator(statement, Args{startDate: startDate, endDate: endDate, tz: -9, buffer: 1}) 81 | actual, err := d.Apply() 82 | 83 | expected := "SELECT * FROM [account.table@1436338800000-1436349600000]" 84 | So(actual, ShouldEqual, expected) 85 | So(err, ShouldBeNil) 86 | }) 87 | }) 88 | 89 | Convey("The argument of endDate is not formatted date string", func() { 90 | startDate := "2015-07-08 17:00:00" 91 | endDate := "2015/07/08 180000" 92 | 93 | d := NewDecorator(statement, Args{startDate: startDate, endDate: endDate}) 94 | actual, err := d.Apply() 95 | 96 | So(actual, ShouldBeBlank) 97 | So(err, ShouldNotBeNil) 98 | }) 99 | }) 100 | 101 | Convey("The argument includes the beforeHour", func() { 102 | 103 | Convey("The argument of beforeHour is greater than 0", func() { 104 | d := NewDecorator(statement, Args{beforeHour: 3.0, buffer: 1}) 105 | actual, err := d.Apply() 106 | 107 | expected := "SELECT * FROM [account.table@-10800000-]" 108 | So(actual, ShouldEqual, expected) 109 | So(err, ShouldBeNil) 110 | }) 111 | 112 | Convey("The argument of beforeHour is 0", func() { 113 | d := NewDecorator(statement, Args{beforeHour: 1.0, buffer: 1}) 114 | actual, err := d.Apply() 115 | 116 | expected := "SELECT * FROM [account.table@-3600000-]" 117 | So(actual, ShouldEqual, expected) 118 | So(err, ShouldBeNil) 119 | }) 120 | }) 121 | }) 122 | 123 | Convey("When the statement includes the placeholders of decorator and customFunc", func() { 124 | statement := "SELECT * FROM [account.table@] WHERE _tz(2015-07-08 17:00:00) <= time" 125 | d := NewDecorator(statement, Args{beforeHour: 3.0, tz: -9.0, buffer: 1}) 126 | actual, err := d.Apply() 127 | 128 | expected := "SELECT * FROM [account.table@-10800000-] WHERE DATE_ADD('2015-07-08 17:00:00', -9, 'HOUR') <= time" 129 | So(actual, ShouldEqual, expected) 130 | So(err, ShouldBeNil) 131 | }) 132 | }) 133 | 134 | Convey("Revert", t, func() { 135 | statement := "SELECT * FROM [account.table@] WHERE _tz(2015-07-08 17:00:00) <= time ORDER BY time DESC" 136 | d := NewDecorator(statement, Args{tz: -9}) 137 | 138 | expected := "SELECT * FROM [account.table] WHERE DATE_ADD('2015-07-08 17:00:00', -9, 'HOUR') <= time ORDER BY time DESC" 139 | So(d.Revert(), ShouldEqual, expected) 140 | }) 141 | } 142 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dbq # 2 | 3 | [![GitHub release](http://img.shields.io/github/release/yoheimuta/go-from-gist-to-issue.svg?style=flat-square)][release] 4 | [![Wercker](http://img.shields.io/wercker/ci/54393fe184570fc622001411.svg?style=flat-square)][wercker] 5 | [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)][license] 6 | 7 | [release]: https://github.com/yoheimuta/dbq/releases 8 | [wercker]: https://app.wercker.com/project/bykey/d232745d4f00166be66404af31469036 9 | [license]: https://github.com/yoheimuta/dbq/blob/master/LICENSE 10 | 11 | CLI tool to easily Decorate BigQuery table name 12 | 13 | ## Description 14 | 15 | `dbq` enables you to use [Table Range Decorators](https://cloud.google.com/bigquery/table-decorators) to perform a more cost-effective query to [BigQuery](https://cloud.google.com/bigquery/what-is-bigquery) without complex calculation. 16 | 17 | - `dbq` supports both Relative value and Absolute value 18 | - `dbq` also supports timezone calculation. 19 | 20 | `dbq` will cut down on considerable data processed and spending :bowtie: :moneybag: 21 | 22 | - [BigQuery offers on-demand pricing for queries](https://cloud.google.com/bigquery/pricing) and 1 TB of data processed costs $5. 23 | - For example, a query to a 6TB table costs $30. Any SQL statements (ex. with LIMIT 1) don't affect the pricing. 24 | 25 | ## Installation 26 | 27 | To install dbq, please use go get. 28 | 29 | ``` 30 | $ go get github.com/yoheimuta/dbq 31 | ... 32 | $ dbq help 33 | ... 34 | ``` 35 | 36 | Or you can download a binary from [github relases page](https://github.com/yoheimuta/dbq/releases) and place it in $PATH directory. 37 | 38 | ## Requirements 39 | 40 | - [bq-command-line-tool](https://cloud.google.com/bigquery/bq-command-line-tool) 41 | 42 | ## Usage 43 | 44 | ```ruby 45 | # no-option equal to `--beforeHour=3` 46 | $ dbq query "SELECT * FROM [foo.bar@]" 47 | 48 | # equal to SELECT * FROM [foo.bar@-10800000-] 49 | $ dbq query "SELECT * FROM [foo.bar@]" --beforeHour=3 50 | 51 | # equal to SELECT * FROM [foo.bar@1436371200000-] 52 | $ dbq query "SELECT * FROM [foo.bar@]" --startDate="2015-07-08 17:00:00" 53 | 54 | # equal to SELECT * FROM [foo.bar@1436371200000-1436382000000] 55 | $ dbq query "SELECT * FROM [foo.bar@]" --startDate="2015-07-08 17:00:00" --endDate="2015-07-08 18:00:00" 56 | 57 | # equal to SELECT * FROM [foo.bar@1436338800000-] 58 | $ dbq query "SELECT * FROM [foo.bar@]" --startDate="2015-07-08 17:00:00" --tz="-9" 59 | 60 | # equal to SELECT * FROM [foo.bar@1436338800000-] WHERE DATE_ADD('2015-07-08 17:00:00', -9, 'HOUR') <= time and time <= DATE_ADD('2015-07-08 18:00:00', -9, 'HOUR') 61 | $ dbq query "SELECT * FROM [foo.bar@] WHERE _tz(2015-07-08 17:00:00) <= time and time <= _tz(2015-07-08 18:00:00)" --startDate="2015-07-08 17:00:00" --tz="-9" 62 | ``` 63 | 64 | ### Placeholders 65 | 66 | - `@` will be replaced with `@-` 67 | - required 68 | - `_tz(datetime)` will be replaced with `DATE_ADD('datetime', tz value, 'HOUR')` 69 | - optional 70 | 71 | ## DryRun 72 | 73 | The option of `dryRun` shows how much cut down full scan bytes, so I strongly recommend to use this option before running any queries. 74 | 75 | - A query with no table decorator will process 6.0 TiB, then costs `6.0 * $5 = $30`. 76 | - A query with table decorator will process 110 GiB, then costs `0.1 * $5 = $0.5`. `dbq` will save `$29.5`. 77 | 78 | ```ruby 79 | $ dbq query "SELECT * FROM [foo.bar@]" --dryRun 80 | Raw: SELECT * FROM [foo.bar] 81 | Query successfully validated. Assuming the tables are not modified, running this query will process 6630178173385 bytes of data. 82 | - 6630178173385 bytes equal to 6,630,178,173,385 bytes 83 | - 6630178173385 bytes equal to 6.0TiB 84 | - 6630178173385 bytes equal to $30.15056 (= 6.03011 TiB * $5) 85 | 86 | Decorated: SELECT * FROM [foo.bar@-10800000-] 87 | Query successfully validated. Assuming the tables are not modified, running this query will process 117636313873 bytes of data. 88 | - 117636313873 bytes equal to 117,636,313,873 bytes 89 | - 117636313873 bytes equal to 110GiB 90 | - 117636313873 bytes equal to $0.53495 (= 0.10699 TiB * $5) 91 | ``` 92 | 93 | ## Options 94 | 95 | ``` 96 | $ dbq help query 97 | NAME: 98 | query - Run bq query with complementing table range decorator 99 | 100 | USAGE: 101 | command query [command options] [arguments...] 102 | 103 | DESCRIPTION: 104 | 105 | 106 | OPTIONS: 107 | --beforeHour '3' a decimal to specify the hour ago, relative to the current time 108 | --startDate a datetime to specify date range with end flag 109 | --endDate a datetime to specify date range with start flag 110 | --tz '0' a decimal of hour or -hour to add to start and end datetime, considering timezone 111 | --buffer '1' a decimal of hour to add to start and end datetime, it's heuristic value 112 | --gflags no support. Use onlyStatement instead 113 | --cflags no support. Use onlyStatement instead 114 | --verbose a flag to output verbosely 115 | --dryRun a flag to run without any changes 116 | --onlyStatement a flag to output only a decorated statement 117 | ``` 118 | 119 | ## CHANGELOG 120 | 121 | See [CHANGELOG](CHANGELOG.md) 122 | --------------------------------------------------------------------------------