├── .gitignore ├── .idea ├── .gitignore ├── modules.xml ├── re-txt.iml └── vcs.xml ├── README.md ├── etc.go ├── examples ├── example.csv ├── example.hcl ├── example.json ├── example.toml └── example.yaml ├── go.mod ├── go.sum ├── handlers ├── handlers.go └── text │ ├── etc.go │ ├── handlers.go │ └── init.go └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/re-txt.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | re-txt 2 | ========== 3 | > reformates a text file from a structure to another, i.e: convert from json to yaml, toml to json, ... etc 4 | 5 | Supported Source Formats 6 | ========================= 7 | - json 8 | - yaml 9 | - hcl 10 | - toml 11 | - csv 12 | 13 | Supported Target Formats 14 | ========================= 15 | - json 16 | - yaml 17 | - toml 18 | 19 | Examples 20 | ========= 21 | 22 | ```bash 23 | # json to yaml using flag 24 | $ re-txt --src example.json json2yaml 25 | 26 | # json to yaml using stdin pipe 27 | $ cat example.json | re-txt json2yaml 28 | 29 | # csv to yaml by first converting to json 30 | $ cat example.csv | re-txt csv2json | re-txt json2yaml 31 | 32 | # csv to yaml by first converting to json 33 | # also merge multiple json files and convert the piped result & them into yaml 34 | $ cat example.csv | re-txt csv2json | re-txt --dest ./result.yaml --src=another1.json --src=another2.json json2yaml 35 | ``` 36 | 37 | Installations 38 | ============== 39 | - from source: `go get github.com/alash3al/re-txt` 40 | - binary download: go to [there](https://github.com/alash3al/re-txt/releases) and download the binary which support your env -------------------------------------------------------------------------------- /etc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | 8 | "github.com/alash3al/re-txt/handlers" 9 | "github.com/urfave/cli/v2" 10 | ) 11 | 12 | // wrapHandler wrap our handler into cli.ActionFunc 13 | func wrapHandler(fn handlers.HandlerFunc) cli.ActionFunc { 14 | return func(c *cli.Context) error { 15 | sources := [][]byte{} 16 | 17 | for _, src := range c.StringSlice("src") { 18 | srcBytes, err := ioutil.ReadFile(src) 19 | if err != nil { 20 | return err 21 | } 22 | sources = append(sources, srcBytes) 23 | } 24 | 25 | { 26 | stdin := os.Stdin 27 | stat, err := stdin.Stat() 28 | if err != nil { 29 | return err 30 | } 31 | 32 | if (stat.Mode() & os.ModeDevice) == 0 { 33 | stdinBytes, err := ioutil.ReadAll(stdin) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | sources = append(sources, stdinBytes) 39 | } 40 | } 41 | 42 | result, err := fn(&handlers.Context{ 43 | Context: c, 44 | Input: sources, 45 | }) 46 | 47 | if err != nil { 48 | return err 49 | } 50 | 51 | if c.String("dest") != "" { 52 | return ioutil.WriteFile(c.String("dest"), result, 0755) 53 | } else { 54 | fmt.Println(string(result)) 55 | } 56 | 57 | return nil 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /examples/example.csv: -------------------------------------------------------------------------------- 1 | album, year, US_peak_chart_post 2 | The White Stripes, 1999, - 3 | De Stijl, 2000, - 4 | White Blood Cells, 2001, 61 5 | Elephant, 2003, 6 6 | Get Behind Me Satan, 2005, 3 7 | Icky Thump, 2007, 2 8 | Under Great White Northern Lights, 2010, 11 9 | Live in Mississippi, 2011, - 10 | Live at the Gold Dollar, 2012, - 11 | Nine Miles from the White City, 2013, - -------------------------------------------------------------------------------- /examples/example.hcl: -------------------------------------------------------------------------------- 1 | resource "aws_instance" "example" { 2 | ami = "abc123" 3 | 4 | service { 5 | key = "value" 6 | } 7 | } -------------------------------------------------------------------------------- /examples/example.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test", 3 | "value": "ok" 4 | } -------------------------------------------------------------------------------- /examples/example.toml: -------------------------------------------------------------------------------- 1 | # This is a TOML document. Boom. 2 | 3 | title = "TOML Example" 4 | 5 | [owner] 6 | name = "Tom Preston-Werner" 7 | organization = "GitHub" 8 | bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." 9 | dob = 1979-05-27T07:32:00Z # First class dates? Why not? 10 | 11 | [database] 12 | server = "192.168.1.1" 13 | ports = [ 8001, 8001, 8002 ] 14 | connection_max = 5000 15 | enabled = true 16 | 17 | [servers] 18 | 19 | # You can indent as you please. Tabs or spaces. TOML don't care. 20 | [servers.alpha] 21 | ip = "10.0.0.1" 22 | dc = "eqdc10" 23 | 24 | [servers.beta] 25 | ip = "10.0.0.2" 26 | dc = "eqdc10" 27 | 28 | [clients] 29 | data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it 30 | 31 | # Line breaks are OK when inside arrays 32 | hosts = [ 33 | "alpha", 34 | "omega" 35 | ] -------------------------------------------------------------------------------- /examples/example.yaml: -------------------------------------------------------------------------------- 1 | - martin: 2 | name: Martin D'vloper 3 | job: Developer 4 | skills: 5 | - python 6 | - perl 7 | - pascal 8 | - tabitha: 9 | name: Tabitha Bitumen 10 | job: Developer 11 | skills: 12 | - lisp 13 | - fortran 14 | - erlang -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/alash3al/re-txt 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/BurntSushi/toml v0.3.1 7 | github.com/hashicorp/hcl v1.0.0 8 | github.com/urfave/cli/v2 v2.3.0 9 | gopkg.in/yaml.v2 v2.2.3 10 | ) 11 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 4 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 8 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 9 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 10 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 11 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 12 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 13 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 14 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 15 | github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= 16 | github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= 17 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 18 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 19 | gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= 20 | gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 21 | -------------------------------------------------------------------------------- /handlers/handlers.go: -------------------------------------------------------------------------------- 1 | // Package handlers contains the global handlers registry 2 | package handlers 3 | 4 | import ( 5 | "bytes" 6 | "sync" 7 | 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | var ( 12 | handlers = []*Handler{} 13 | 14 | l = &sync.RWMutex{} 15 | ) 16 | 17 | type Handler struct { 18 | Command cli.Command 19 | Action HandlerFunc 20 | } 21 | 22 | type Context struct { 23 | *cli.Context 24 | Input [][]byte 25 | } 26 | 27 | func (c *Context) MergeInputs() []byte { 28 | result := []byte{} 29 | 30 | for _, b := range c.Input { 31 | result = append(result, b...) 32 | } 33 | 34 | return result 35 | } 36 | 37 | func (c *Context) MergeInputsAsJSON() []byte { 38 | if len(c.Input) > 1 { 39 | return append(append([]byte("["), bytes.Join(c.Input, []byte(","))...), []byte("]")...) 40 | } 41 | 42 | return c.MergeInputs() 43 | } 44 | 45 | type HandlerFunc func(*Context) ([]byte, error) 46 | 47 | // Handle register a new handler 48 | func Handle(h *Handler) { 49 | l.Lock() 50 | defer l.Unlock() 51 | 52 | handlers = append(handlers, h) 53 | } 54 | 55 | // Handlers retruns a list with all registered handlers 56 | func Handlers() []*Handler { 57 | l.RLock() 58 | defer l.RUnlock() 59 | 60 | result := make([]*Handler, len(handlers)) 61 | 62 | copy(result, handlers) 63 | 64 | return result 65 | } 66 | -------------------------------------------------------------------------------- /handlers/text/etc.go: -------------------------------------------------------------------------------- 1 | package text 2 | 3 | func normalizeInterface(i interface{}) interface{} { 4 | switch x := i.(type) { 5 | case map[interface{}]interface{}: 6 | m2 := map[string]interface{}{} 7 | for k, v := range x { 8 | m2[k.(string)] = normalizeInterface(v) 9 | } 10 | return m2 11 | case []interface{}: 12 | for i, v := range x { 13 | x[i] = normalizeInterface(v) 14 | } 15 | } 16 | return i 17 | } 18 | -------------------------------------------------------------------------------- /handlers/text/handlers.go: -------------------------------------------------------------------------------- 1 | // Package text converts from text format to another 2 | package text 3 | 4 | import ( 5 | "bytes" 6 | "encoding/csv" 7 | "encoding/json" 8 | "strings" 9 | 10 | "github.com/BurntSushi/toml" 11 | "github.com/alash3al/re-txt/handlers" 12 | "github.com/hashicorp/hcl" 13 | "gopkg.in/yaml.v2" 14 | ) 15 | 16 | func yaml2json(c *handlers.Context) (jsonBytes []byte, err error) { 17 | var data interface{} 18 | 19 | if err := yaml.Unmarshal(c.MergeInputs(), &data); err != nil { 20 | return nil, err 21 | } 22 | 23 | if c.Bool("pretty") { 24 | jsonBytes, err = json.MarshalIndent(normalizeInterface(data), "", " ") 25 | } else { 26 | jsonBytes, err = json.Marshal(normalizeInterface(data)) 27 | } 28 | 29 | return jsonBytes, err 30 | } 31 | 32 | func json2yaml(c *handlers.Context) ([]byte, error) { 33 | var data interface{} 34 | 35 | if err := json.Unmarshal(c.MergeInputsAsJSON(), &data); err != nil { 36 | return nil, err 37 | } 38 | 39 | return yaml.Marshal(normalizeInterface(data)) 40 | } 41 | 42 | func json2toml(c *handlers.Context) ([]byte, error) { 43 | var data interface{} 44 | 45 | if err := json.Unmarshal(c.MergeInputsAsJSON(), &data); err != nil { 46 | return nil, err 47 | } 48 | 49 | var writer bytes.Buffer 50 | 51 | if err := toml.NewEncoder(&writer).Encode(normalizeInterface(data)); err != nil { 52 | return nil, err 53 | } 54 | 55 | return writer.Bytes(), nil 56 | } 57 | 58 | func toml2json(c *handlers.Context) (jsonBytes []byte, err error) { 59 | var data interface{} 60 | 61 | if err := toml.Unmarshal(c.MergeInputs(), &data); err != nil { 62 | return nil, err 63 | } 64 | 65 | if c.Bool("pretty") { 66 | jsonBytes, err = json.MarshalIndent(normalizeInterface(data), "", " ") 67 | } else { 68 | jsonBytes, err = json.Marshal(normalizeInterface(data)) 69 | } 70 | 71 | return jsonBytes, err 72 | } 73 | 74 | func hcl2json(c *handlers.Context) (jsonBytes []byte, err error) { 75 | var data interface{} 76 | 77 | if err := hcl.Unmarshal(c.MergeInputs(), &data); err != nil { 78 | return nil, err 79 | } 80 | 81 | if c.Bool("pretty") { 82 | jsonBytes, err = json.MarshalIndent(normalizeInterface(data), "", " ") 83 | } else { 84 | jsonBytes, err = json.Marshal(normalizeInterface(data)) 85 | } 86 | 87 | return jsonBytes, err 88 | } 89 | 90 | func csv2json(c *handlers.Context) (jsonBytes []byte, err error) { 91 | data := []map[string]interface{}{} 92 | 93 | r := csv.NewReader(bytes.NewBuffer(c.MergeInputs())) 94 | r.Comma = rune(c.String("separator")[0]) 95 | r.Comment = rune(c.String("comment")[0]) 96 | r.TrimLeadingSpace = c.Bool("trim-leading-space") 97 | r.LazyQuotes = c.Bool("lazy-quotes") 98 | 99 | header := strings.Split(c.String("header"), string(r.Comma)) 100 | records, err := r.ReadAll() 101 | if err != nil { 102 | return nil, err 103 | } 104 | 105 | for ri, record := range records { 106 | if ri == 0 && c.Bool("contains-header") { 107 | continue 108 | } 109 | 110 | item := map[string]interface{}{} 111 | for i, v := range header { 112 | if len(record)-1 < i { 113 | continue 114 | } 115 | 116 | item[v] = record[i] 117 | } 118 | 119 | data = append(data, item) 120 | } 121 | 122 | if c.Bool("pretty") { 123 | jsonBytes, err = json.MarshalIndent(data, "", " ") 124 | } else { 125 | jsonBytes, err = json.Marshal(data) 126 | } 127 | 128 | return jsonBytes, err 129 | } 130 | -------------------------------------------------------------------------------- /handlers/text/init.go: -------------------------------------------------------------------------------- 1 | package text 2 | 3 | import ( 4 | "github.com/alash3al/re-txt/handlers" 5 | "github.com/urfave/cli/v2" 6 | ) 7 | 8 | func init() { 9 | prettyFlag := &cli.BoolFlag{ 10 | Name: "pretty", 11 | Aliases: []string{"p"}, 12 | Value: true, 13 | } 14 | 15 | handlers.Handle(&handlers.Handler{ 16 | Command: cli.Command{ 17 | Name: "yaml2json", 18 | Usage: "convert from yaml to json", 19 | Aliases: []string{"y2j"}, 20 | Category: "Text Converters", 21 | Flags: []cli.Flag{ 22 | prettyFlag, 23 | }, 24 | }, 25 | 26 | Action: handlers.HandlerFunc(yaml2json), 27 | }) 28 | 29 | handlers.Handle(&handlers.Handler{ 30 | Command: cli.Command{ 31 | Name: "json2yaml", 32 | Usage: "convert from json to yaml", 33 | Aliases: []string{"j2y"}, 34 | Category: "Text Converters", 35 | }, 36 | 37 | Action: handlers.HandlerFunc(json2yaml), 38 | }) 39 | 40 | handlers.Handle(&handlers.Handler{ 41 | Command: cli.Command{ 42 | Name: "toml2json", 43 | Usage: "convert from toml to json", 44 | Aliases: []string{"t2j"}, 45 | Category: "Text Converters", 46 | Flags: []cli.Flag{ 47 | prettyFlag, 48 | }, 49 | }, 50 | 51 | Action: handlers.HandlerFunc(toml2json), 52 | }) 53 | 54 | handlers.Handle(&handlers.Handler{ 55 | Command: cli.Command{ 56 | Name: "json2toml", 57 | Usage: "convert from json to toml", 58 | Aliases: []string{"j2t"}, 59 | Category: "Text Converters", 60 | }, 61 | 62 | Action: handlers.HandlerFunc(json2toml), 63 | }) 64 | 65 | handlers.Handle(&handlers.Handler{ 66 | Command: cli.Command{ 67 | Name: "hcl2json", 68 | Usage: "convert from hcl to json", 69 | Aliases: []string{"h2j"}, 70 | Category: "Text Converters", 71 | Flags: []cli.Flag{ 72 | prettyFlag, 73 | }, 74 | }, 75 | 76 | Action: handlers.HandlerFunc(hcl2json), 77 | }) 78 | 79 | handlers.Handle(&handlers.Handler{ 80 | Command: cli.Command{ 81 | Name: "csv2json", 82 | Usage: "convert from csv to json", 83 | Aliases: []string{"c2j"}, 84 | Category: "Text Converters", 85 | Flags: []cli.Flag{ 86 | prettyFlag, 87 | &cli.StringFlag{ 88 | Name: "header", 89 | Required: true, 90 | }, 91 | &cli.BoolFlag{ 92 | Name: "contains-header", 93 | Required: true, 94 | Value: false, 95 | }, 96 | &cli.StringFlag{ 97 | Name: "separator", 98 | Aliases: []string{"s"}, 99 | Value: ",", 100 | }, 101 | &cli.StringFlag{ 102 | Name: "comment", 103 | Aliases: []string{"c"}, 104 | Value: "#", 105 | }, 106 | &cli.BoolFlag{ 107 | Name: "trim-leading-space", 108 | Value: true, 109 | }, 110 | &cli.BoolFlag{ 111 | Name: "lazy-quotes", 112 | Value: true, 113 | }, 114 | }, 115 | }, 116 | 117 | Action: handlers.HandlerFunc(csv2json), 118 | }) 119 | } 120 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/alash3al/re-txt/handlers" 8 | "github.com/urfave/cli/v2" 9 | 10 | _ "github.com/alash3al/re-txt/handlers/text" 11 | ) 12 | 13 | func main() { 14 | app := &cli.App{ 15 | Name: "re-txt", 16 | Usage: "convert anything to anything", 17 | EnableBashCompletion: true, 18 | Commands: []*cli.Command{}, 19 | Flags: []cli.Flag{ 20 | &cli.StringSliceFlag{ 21 | Name: "src", 22 | Aliases: []string{"s"}, 23 | Usage: "the source filename", 24 | }, 25 | &cli.StringFlag{ 26 | Name: "dest", 27 | Aliases: []string{"d"}, 28 | Usage: "the destenation filename", 29 | }, 30 | }, 31 | } 32 | 33 | for _, h := range handlers.Handlers() { 34 | h.Command.Action = wrapHandler(h.Action) 35 | app.Commands = append(app.Commands, &h.Command) 36 | } 37 | 38 | if err := app.Run(os.Args); err != nil { 39 | log.Fatal(err.Error()) 40 | } 41 | } 42 | --------------------------------------------------------------------------------