├── merge_test.go ├── merge.go ├── LICENSE ├── README.md ├── qson.go └── qson_test.go /merge_test.go: -------------------------------------------------------------------------------- 1 | package qson 2 | 3 | import "testing" 4 | 5 | func TestMergeSlice(t *testing.T) { 6 | a := []interface{}{"a"} 7 | b := []interface{}{"b"} 8 | actual := mergeSlice(a, b) 9 | if len(actual) != 2 { 10 | t.Errorf("Expected size to be 2.") 11 | } 12 | if actual[0] != "a" { 13 | t.Errorf("Expected index 0 to have value a. Actual: %s", actual[0]) 14 | } 15 | if actual[1] != "b" { 16 | t.Errorf("Expected index 1 to have value b. Actual: %s", actual[1]) 17 | } 18 | } 19 | 20 | func TestMergeMap(t *testing.T) { 21 | a := map[string]interface{}{ 22 | "a": "b", 23 | } 24 | b := map[string]interface{}{ 25 | "b": "c", 26 | } 27 | actual := mergeMap(a, b) 28 | if len(actual) != 2 { 29 | t.Errorf("Expected size to be 2.") 30 | } 31 | if actual["a"] != "b" { 32 | t.Errorf("Expected key \"a\" to have value b. Actual: %s", actual["a"]) 33 | } 34 | if actual["b"] != "c" { 35 | t.Errorf("Expected key \"b\" to have value c. Actual: %s", actual["b"]) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /merge.go: -------------------------------------------------------------------------------- 1 | package qson 2 | 3 | // merge merges a with b if they are either both slices 4 | // or map[string]interface{} types. Otherwise it returns b. 5 | func merge(a interface{}, b interface{}) interface{} { 6 | switch aT := a.(type) { 7 | case map[string]interface{}: 8 | return mergeMap(aT, b.(map[string]interface{})) 9 | case []interface{}: 10 | return mergeSlice(aT, b.([]interface{})) 11 | default: 12 | return b 13 | } 14 | } 15 | 16 | // mergeMap merges a with b, attempting to merge any nested 17 | // values in nested maps but eventually overwriting anything 18 | // in a that can't be merged with whatever is in b. 19 | func mergeMap(a map[string]interface{}, b map[string]interface{}) map[string]interface{} { 20 | for bK, bV := range b { 21 | if _, ok := a[bK]; ok { 22 | a[bK] = merge(a[bK], bV) 23 | } else { 24 | a[bK] = bV 25 | } 26 | } 27 | return a 28 | } 29 | 30 | // mergeSlice merges a with b and returns the result. 31 | func mergeSlice(a []interface{}, b []interface{}) []interface{} { 32 | a = append(a, b...) 33 | return a 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Jon Calhoun 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # qson 2 | 3 | ***WARNING** This isn't actively maintained, but it should work as a proof of concept for anyone who is interested in seeing how you might write a library like this for your own projects.* 4 | 5 | Convert URL query strings into JSON so that you can more easily parse them into structs/maps in Go. 6 | 7 | I wrote this to help someone in the Gopher Slack, so it isn't really 100% complete but it should be stable enough to use in most production environments and work well as a starting point if you need something more custom. 8 | 9 | If you end up using the package, feel free to submit any bugs and feature requests and I'll try to get to those updated time permitting. 10 | 11 | ## Usage 12 | 13 | You can either turn a URL query param into a JSON byte array, or unmarshal that directly into a Go object. 14 | 15 | Transforming the URL query param into a JSON byte array: 16 | 17 | ```go 18 | import "github.com/joncalhoun/qson" 19 | 20 | func main() { 21 | b, err := qson.ToJSON("bar%5Bone%5D%5Btwo%5D=2&bar[one][red]=112") 22 | if err != nil { 23 | panic(err) 24 | } 25 | fmt.Println(string(b)) 26 | // Should output: {"bar":{"one":{"red":112,"two":2}}} 27 | } 28 | ``` 29 | 30 | Or unmarshalling directly into a Go object using JSON struct tags: 31 | 32 | ```go 33 | import "github.com/joncalhoun/qson" 34 | 35 | type unmarshalT struct { 36 | A string `json:"a"` 37 | B unmarshalB `json:"b"` 38 | } 39 | type unmarshalB struct { 40 | C int `json:"c"` 41 | } 42 | 43 | func main() { 44 | var out unmarshalT 45 | query := "a=xyz&b[c]=456" 46 | err := Unmarshal(&out, query) 47 | if err != nil { 48 | t.Error(err) 49 | } 50 | // out should equal 51 | // unmarshalT{ 52 | // A: "xyz", 53 | // B: unmarshalB{ 54 | // C: 456, 55 | // }, 56 | // } 57 | } 58 | ``` 59 | 60 | To get a query string like in the two previous examples you can use the `RawQuery` field on the [net/url.URL](https://golang.org/pkg/net/url/#URL) type. 61 | -------------------------------------------------------------------------------- /qson.go: -------------------------------------------------------------------------------- 1 | // Package qson implmenets decoding of URL query params 2 | // into JSON and Go values (using JSON struct tags). 3 | // 4 | // See https://golang.org/pkg/encoding/json/ for more 5 | // details on JSON struct tags. 6 | package qson 7 | 8 | import ( 9 | "encoding/json" 10 | "errors" 11 | "net/url" 12 | "regexp" 13 | "strings" 14 | ) 15 | 16 | var ( 17 | // ErrInvalidParam is returned when invalid data is provided to the ToJSON or Unmarshal function. 18 | // Specifically, this will be returned when there is no equals sign present in the URL query parameter. 19 | ErrInvalidParam error = errors.New("qson: invalid url query param provided") 20 | 21 | bracketSplitter *regexp.Regexp 22 | ) 23 | 24 | func init() { 25 | bracketSplitter = regexp.MustCompile("\\[|\\]") 26 | } 27 | 28 | // Unmarshal will take a dest along with URL 29 | // query params and attempt to first turn the query params 30 | // into JSON and then unmarshal those into the dest variable 31 | // 32 | // BUG(joncalhoun): If a URL query param value is something 33 | // like 123 but is expected to be parsed into a string this 34 | // will currently result in an error because the JSON 35 | // transformation will assume this is intended to be an int. 36 | // This should only affect the Unmarshal function and 37 | // could likely be fixed, but someone will need to submit a 38 | // PR if they want that fixed. 39 | func Unmarshal(dst interface{}, query string) error { 40 | b, err := ToJSON(query) 41 | if err != nil { 42 | return err 43 | } 44 | return json.Unmarshal(b, dst) 45 | } 46 | 47 | // ToJSON will turn a query string like: 48 | // cat=1&bar%5Bone%5D%5Btwo%5D=2&bar[one][red]=112 49 | // Into a JSON object with all the data merged as nicely as 50 | // possible. Eg the example above would output: 51 | // {"bar":{"one":{"two":2,"red":112}}} 52 | func ToJSON(query string) ([]byte, error) { 53 | var ( 54 | builder interface{} = make(map[string]interface{}) 55 | ) 56 | params := strings.Split(query, "&") 57 | for _, part := range params { 58 | tempMap, err := queryToMap(part) 59 | if err != nil { 60 | return nil, err 61 | } 62 | builder = merge(builder, tempMap) 63 | } 64 | return json.Marshal(builder) 65 | } 66 | 67 | // queryToMap turns something like a[b][c]=4 into 68 | // map[string]interface{}{ 69 | // "a": map[string]interface{}{ 70 | // "b": map[string]interface{}{ 71 | // "c": 4, 72 | // }, 73 | // }, 74 | // } 75 | func queryToMap(param string) (map[string]interface{}, error) { 76 | rawKey, rawValue, err := splitKeyAndValue(param) 77 | if err != nil { 78 | return nil, err 79 | } 80 | rawValue, err = url.QueryUnescape(rawValue) 81 | if err != nil { 82 | return nil, err 83 | } 84 | rawKey, err = url.QueryUnescape(rawKey) 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | pieces := bracketSplitter.Split(rawKey, -1) 90 | key := pieces[0] 91 | 92 | // If len==1 then rawKey has no [] chars and we can just 93 | // decode this as key=value into {key: value} 94 | if len(pieces) == 1 { 95 | var value interface{} 96 | // First we try parsing it as an int, bool, null, etc 97 | err = json.Unmarshal([]byte(rawValue), &value) 98 | if err != nil { 99 | // If we got an error we try wrapping the value in 100 | // quotes and processing it as a string 101 | err = json.Unmarshal([]byte("\""+rawValue+"\""), &value) 102 | if err != nil { 103 | // If we can't decode as a string we return the err 104 | return nil, err 105 | } 106 | } 107 | return map[string]interface{}{ 108 | key: value, 109 | }, nil 110 | } 111 | 112 | // If len > 1 then we have something like a[b][c]=2 113 | // so we need to turn this into {"a": {"b": {"c": 2}}} 114 | // To do this we break our key into two pieces: 115 | // a and b[c] 116 | // and then we set {"a": queryToMap("b[c]", value)} 117 | ret := make(map[string]interface{}, 0) 118 | ret[key], err = queryToMap(buildNewKey(rawKey) + "=" + rawValue) 119 | if err != nil { 120 | return nil, err 121 | } 122 | 123 | // When URL params have a set of empty brackets (eg a[]=1) 124 | // it is assumed to be an array. This will get us the 125 | // correct value for the array item and return it as an 126 | // []interface{} so that it can be merged properly. 127 | if pieces[1] == "" { 128 | temp := ret[key].(map[string]interface{}) 129 | ret[key] = []interface{}{temp[""]} 130 | } 131 | return ret, nil 132 | } 133 | 134 | // buildNewKey will take something like: 135 | // origKey = "bar[one][two]" 136 | // pieces = [bar one two ] 137 | // and return "one[two]" 138 | func buildNewKey(origKey string) string { 139 | pieces := bracketSplitter.Split(origKey, -1) 140 | ret := origKey[len(pieces[0])+1:] 141 | ret = ret[:len(pieces[1])] + ret[len(pieces[1])+1:] 142 | return ret 143 | } 144 | 145 | // splitKeyAndValue splits a URL param at the last equal 146 | // sign and returns the two strings. If no equal sign is 147 | // found, the ErrInvalidParam error is returned. 148 | func splitKeyAndValue(param string) (string, string, error) { 149 | li := strings.LastIndex(param, "=") 150 | if li == -1 { 151 | return "", "", ErrInvalidParam 152 | } 153 | return param[:li], param[li+1:], nil 154 | } 155 | -------------------------------------------------------------------------------- /qson_test.go: -------------------------------------------------------------------------------- 1 | package qson 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func ExampleUnmarshal() { 9 | type Ex struct { 10 | A string `json:"a"` 11 | B struct { 12 | C int `json:"c"` 13 | } `json:"b"` 14 | } 15 | var ex Ex 16 | if err := Unmarshal(&ex, "a=xyz&b[c]=456"); err != nil { 17 | panic(err) 18 | } 19 | fmt.Printf("%+v\n", ex) 20 | // Output: {A:xyz B:{C:456}} 21 | } 22 | 23 | type unmarshalT struct { 24 | A string `json:"a"` 25 | B unmarshalB `json:"b"` 26 | } 27 | type unmarshalB struct { 28 | C int `json:"c"` 29 | D string `json:"D"` 30 | } 31 | 32 | func TestUnmarshal(t *testing.T) { 33 | query := "a=xyz&b[c]=456" 34 | expected := unmarshalT{ 35 | A: "xyz", 36 | B: unmarshalB{ 37 | C: 456, 38 | }, 39 | } 40 | var actual unmarshalT 41 | err := Unmarshal(&actual, query) 42 | if err != nil { 43 | t.Error(err) 44 | } 45 | if expected != actual { 46 | t.Errorf("Expected: %+v Actual: %+v", expected, actual) 47 | } 48 | } 49 | 50 | func ExampleToJSON() { 51 | b, err := ToJSON("a=xyz&b[c]=456") 52 | if err != nil { 53 | panic(err) 54 | } 55 | fmt.Printf(string(b)) 56 | // Output: {"a":"xyz","b":{"c":456}} 57 | } 58 | 59 | func TestToJSONNested(t *testing.T) { 60 | query := "bar%5Bone%5D%5Btwo%5D=2&bar[one][red]=112" 61 | expected := `{"bar":{"one":{"red":112,"two":2}}}` 62 | actual, err := ToJSON(query) 63 | if err != nil { 64 | t.Error(err) 65 | } 66 | actualStr := string(actual) 67 | if actualStr != expected { 68 | t.Errorf("Expected: %s Actual: %s", expected, actualStr) 69 | } 70 | } 71 | 72 | func TestToJSONPlain(t *testing.T) { 73 | query := "cat=1&dog=2" 74 | expected := `{"cat":1,"dog":2}` 75 | actual, err := ToJSON(query) 76 | if err != nil { 77 | t.Error(err) 78 | } 79 | actualStr := string(actual) 80 | if actualStr != expected { 81 | t.Errorf("Expected: %s Actual: %s", expected, actualStr) 82 | } 83 | } 84 | 85 | func TestToJSONSlice(t *testing.T) { 86 | query := "cat[]=1&cat[]=34" 87 | expected := `{"cat":[1,34]}` 88 | actual, err := ToJSON(query) 89 | if err != nil { 90 | t.Error(err) 91 | } 92 | actualStr := string(actual) 93 | if actualStr != expected { 94 | t.Errorf("Expected: %s Actual: %s", expected, actualStr) 95 | } 96 | } 97 | 98 | func TestToJSONBig(t *testing.T) { 99 | query := "distinct_id=763_1495187301909_3495×tamp=1495187523&event=product_add_cart¶ms%5BproductRefId%5D=8284563078¶ms%5Bapps%5D%5B%5D=precommend¶ms%5Bapps%5D%5B%5D=bsales¶ms%5Bsource%5D=item¶ms%5Boptions%5D%5Bsegment%5D=cart_recommendation¶ms%5Boptions%5D%5Btype%5D=up_sell¶ms%5BtimeExpire%5D=1495187599642¶ms%5Brecommend_system_product_source%5D=item¶ms%5Bproduct_id%5D=8284563078¶ms%5Bvariant_id%5D=27661944134¶ms%5Bsku%5D=00483332%20(black)¶ms%5Bsources%5D%5B%5D=product_recommendation¶ms%5Bcart_token%5D=dc2c336a009edf2762128e65806dfb1d¶ms%5Bquantity%5D=1¶ms%5Bnew_popup_upsell_mobile%5D=false¶ms%5BclientDevice%5D=desktop¶ms%5BclientIsMobile%5D=false¶ms%5BclientIsSmallScreen%5D=false¶ms%5Bnew_popup_crossell_mobile%5D=false&api_key=14c5b7dacea9157029265b174491d340" 100 | expected := `{"api_key":"14c5b7dacea9157029265b174491d340","distinct_id":"763_1495187301909_3495","event":"product_add_cart","params":{"apps":["precommend","bsales"],"cart_token":"dc2c336a009edf2762128e65806dfb1d","clientDevice":"desktop","clientIsMobile":false,"clientIsSmallScreen":false,"new_popup_crossell_mobile":false,"new_popup_upsell_mobile":false,"options":{"segment":"cart_recommendation","type":"up_sell"},"productRefId":8284563078,"product_id":8284563078,"quantity":1,"recommend_system_product_source":"item","sku":"00483332 (black)","source":"item","sources":["product_recommendation"],"timeExpire":1495187599642,"variant_id":27661944134},"timestamp":1495187523}` 101 | actual, err := ToJSON(query) 102 | if err != nil { 103 | t.Error(err) 104 | } 105 | actualStr := string(actual) 106 | if actualStr != expected { 107 | t.Errorf("Expected: %s Actual: %s", expected, actualStr) 108 | } 109 | } 110 | 111 | func TestToJSONDuplicateKey(t *testing.T) { 112 | query := "cat=1&cat=2" 113 | expected := `{"cat":2}` 114 | actual, err := ToJSON(query) 115 | if err != nil { 116 | t.Error(err) 117 | } 118 | actualStr := string(actual) 119 | if actualStr != expected { 120 | t.Errorf("Expected: %s Actual: %s", expected, actualStr) 121 | } 122 | } 123 | 124 | func TestSplitKeyAndValue(t *testing.T) { 125 | param := "a[dog][=cat]=123" 126 | eKey, eValue := "a[dog][=cat]", "123" 127 | aKey, aValue, err := splitKeyAndValue(param) 128 | if err != nil { 129 | t.Error(err) 130 | } 131 | if eKey != aKey { 132 | t.Errorf("Keys do not match. Expected: %s Actual: %s", eKey, aKey) 133 | } 134 | if eValue != aValue { 135 | t.Errorf("Values do not match. Expected: %s Actual: %s", eValue, aValue) 136 | } 137 | } 138 | 139 | func TestEncodedAmpersand(t *testing.T) { 140 | query := "a=xyz&b[d]=ben%26jerry" 141 | expected := unmarshalT{ 142 | A: "xyz", 143 | B: unmarshalB{ 144 | D: "ben&jerry", 145 | }, 146 | } 147 | var actual unmarshalT 148 | err := Unmarshal(&actual, query) 149 | if err != nil { 150 | t.Error(err) 151 | } 152 | if expected != actual { 153 | t.Errorf("Expected: %+v Actual: %+v", expected, actual) 154 | } 155 | } 156 | 157 | func TestEncodedAmpersand2(t *testing.T) { 158 | query := "filter=parent%3Dflow12345%26request%3Dreq12345&meta.limit=20&meta.offset=0" 159 | expected := map[string]interface{}{"filter": "parent=flow12345&request=req12345", "meta.limit": float64(20), "meta.offset": float64(0)} 160 | actual := make(map[string]interface{}) 161 | err := Unmarshal(&actual, query) 162 | if err != nil { 163 | t.Error(err) 164 | } 165 | for k, v := range actual { 166 | if nv, ok := expected[k]; !ok || nv != v { 167 | t.Errorf("Expected: %+v Actual: %+v", expected, actual) 168 | } 169 | } 170 | } 171 | --------------------------------------------------------------------------------