├── .gitignore ├── Justfile ├── LICENSE ├── README.md ├── cmd ├── root.go └── version.go ├── go.mod ├── go.sum ├── img └── social_banner.jpg ├── main.go └── pkg ├── constants └── constants.go ├── db ├── from_csv.go ├── from_json.go ├── from_jsonl.go ├── load_sql.go ├── query_gen.go ├── to_csv.go ├── to_json.go ├── to_jsonl.go ├── to_sqlite.go └── utils.go ├── file_types ├── file_types.go ├── file_types_test.go └── resolve.go ├── logger ├── log.go └── panic.go └── utils ├── json.go ├── map.go ├── pipe.go ├── slice.go └── types.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/go 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=go 3 | 4 | ### Go ### 5 | # If you prefer the allow list template instead of the deny list, see community template: 6 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 7 | # 8 | # Binaries for programs and plugins 9 | *.exe 10 | *.exe~ 11 | *.dll 12 | *.so 13 | *.dylib 14 | 15 | # Test binary, built with `go test -c` 16 | *.test 17 | 18 | # Output of the go coverage tool, specifically when used with LiteIDE 19 | *.out 20 | 21 | # Dependency directories (remove the comment below to include it) 22 | # vendor/ 23 | 24 | # Go workspace file 25 | go.work 26 | 27 | # End of https://www.toptal.com/developers/gitignore/api/go 28 | bin 29 | dist 30 | *.db 31 | *.jsonl 32 | *.json 33 | *.csv 34 | **.DS_Store** 35 | -------------------------------------------------------------------------------- /Justfile: -------------------------------------------------------------------------------- 1 | set dotenv-load 2 | 3 | DATE := `date +"%Y-%m-%d_%H:%M:%S"` 4 | GIT_COMMIT := `git rev-parse HEAD` 5 | VERSION_TAG := `git describe --tags --abbrev=0` 6 | LD_FLAGS := "-X github.com/chand1012/sq/cmd.buildDate=" + DATE + " -X github.com/chand1012/sq/cmd.commitHash=" + GIT_COMMIT + " -X github.com/chand1012/sq/cmd.tag=" + VERSION_TAG 7 | # EXEC_EXT := `[[ "$(uname -o)" == "Msys" ]] && echo ".exe"` # uncomment on windows 8 | EXEC_EXT := "" # comment out on windows 9 | 10 | default: 11 | just --list --unsorted 12 | 13 | tidy: 14 | go mod tidy 15 | 16 | build: 17 | go build -ldflags "{{LD_FLAGS}}" -v -o bin/sq{{EXEC_EXT}} 18 | 19 | gen-test path prompt="Write unit tests for all functions in the given file.": 20 | #!/bin/bash 21 | NEW_FILE=$(echo {{path}} | sed 's/\.go/_test.go/') 22 | otto edit $NEW_FILE -c {{path}} -g "{{prompt}}" 23 | 24 | add command: 25 | cobra-cli add {{command}} 26 | 27 | test: 28 | go test -v ./... 29 | 30 | cobra-docs: 31 | rm docs/*.md 32 | go run docs/gen_docs.go 33 | 34 | install: build 35 | rm -rf $GOPATH/bin/sq 36 | cp bin/sq $GOPATH/bin 37 | 38 | clean: 39 | rm -rf bin 40 | rm -rf dist 41 | 42 | crossbuild: clean 43 | #!/bin/bash 44 | 45 | # Set the name of the output binary and Go package 46 | BINARY_NAME="sq" 47 | GO_PACKAGE="github.com/chand1012/sq" 48 | 49 | mkdir -p dist 50 | 51 | # Build for M1 Mac (Apple Silicon) 52 | echo "Building for M1 Mac (Apple Silicon)" 53 | env GOOS=darwin GOARCH=arm64 go build -ldflags "{{LD_FLAGS}}" -o "${BINARY_NAME}" "${GO_PACKAGE}" 54 | zip "${BINARY_NAME}_darwin_arm64.zip" "${BINARY_NAME}" 55 | rm "${BINARY_NAME}" 56 | mv "${BINARY_NAME}_darwin_arm64.zip" dist/ 57 | 58 | # Build for AMD64 Mac (Intel) 59 | echo "Building for AMD64 Mac (Intel)" 60 | env GOOS=darwin GOARCH=amd64 go build -ldflags "{{LD_FLAGS}}" -o "${BINARY_NAME}" "${GO_PACKAGE}" 61 | zip "${BINARY_NAME}_darwin_amd64.zip" "${BINARY_NAME}" 62 | rm "${BINARY_NAME}" 63 | mv "${BINARY_NAME}_darwin_amd64.zip" dist/ 64 | 65 | # Build for AMD64 Windows 66 | echo "Building for AMD64 Windows" 67 | env GOOS=windows GOARCH=amd64 go build -ldflags "{{LD_FLAGS}}" -o "${BINARY_NAME}.exe" "${GO_PACKAGE}" 68 | zip "${BINARY_NAME}_windows_amd64.zip" "${BINARY_NAME}.exe" 69 | rm "${BINARY_NAME}.exe" 70 | mv "${BINARY_NAME}_windows_amd64.zip" dist/ 71 | 72 | # Build for AMD64 Linux 73 | echo "Building for AMD64 Linux" 74 | env GOOS=linux GOARCH=amd64 go build -ldflags "{{LD_FLAGS}}" -o "${BINARY_NAME}" "${GO_PACKAGE}" 75 | tar czvf "${BINARY_NAME}_linux_amd64.tar.gz" "${BINARY_NAME}" 76 | rm "${BINARY_NAME}" 77 | mv "${BINARY_NAME}_linux_amd64.tar.gz" dist/ 78 | 79 | # Build for ARM64 Linux 80 | echo "Building for ARM64 Linux" 81 | env GOOS=linux GOARCH=arm64 go build -ldflags "{{LD_FLAGS}}" -o "${BINARY_NAME}" "${GO_PACKAGE}" 82 | tar czvf "${BINARY_NAME}_linux_arm64.tar.gz" "${BINARY_NAME}" 83 | rm "${BINARY_NAME}" 84 | mv "${BINARY_NAME}_linux_arm64.tar.gz" dist/ 85 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2024 Chandler Lofland 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the “Software”), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

sq

2 |

Convert and query JSON, JSONL, CSV, and SQLite with ease!

3 | 4 | `sq` is a simple yet powerful command line tool for query and converting from and to SQLite, CSV, JSON, and JSONL files, heavily inspired by [ `jq` ](https://jqlang.github.io/jq/). 5 | 6 | ## Features 7 | 8 | * Convert between SQLite, CSV, JSON, and JSONL files. 9 | * Query CSV, JSON, and JSONL using real SQLite queries. 10 | * Allows for rapid scripting and conversion of data. 11 | * Pipe in data or read from files. 12 | 13 | ## Installation 14 | 15 | Download a prebuilt binary from the [releases page](https://github.com/chand1012/sq/releases) and add it to your PATH, or use `go install` to install the latest version. 16 | 17 | ```bash 18 | go install github.com/chand1012/sq@latest 19 | ``` 20 | 21 | ## Examples 22 | 23 | Query some orders from a CSV file. Column names for CSV files are converted to lower case and spaces are replaced with underscores. 24 | 25 | ```bash 26 | $ sq -r orders.csv 'SELECT country FROM sq WHERE seller_amount > 20 NOT NULL;' 27 | United States of America 28 | United States of America 29 | United States of America 30 | United States of America 31 | United States of America 32 | United States of America 33 | United Kingdom 34 | Canada 35 | United States of America 36 | United States of America 37 | United States of America 38 | United States of America 39 | United States of America 40 | Canada 41 | United States of America 42 | Canada 43 | Canada 44 | Australia 45 | United States of America 46 | ... 47 | ``` 48 | 49 | Download and query some JSONL datasets. 50 | 51 | ```bash 52 | $ curl https://raw.githubusercontent.com/TimeSurgeLabs/llm-finetuning/4e934ce602f34f62f4d803c40cd1e7825d216192/data/fingpt-sentiment-1k.jsonl | sq 'SELECT * FROM sq WHERE output = "positive";' -f jsonl > positive.jsonl 53 | ``` 54 | 55 | You can even use it with `jq` ! 56 | 57 | ```bash 58 | $ curl https://api.gogopool.com/stakers | jq '.stakers' | sq -t stakers 'SELECT stakerAddr,avaxValidating FROM stakers WHERE avaxValidating > 0;' -f json > stakers.json 59 | ``` 60 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2024 Chandler 3 | */ 4 | package cmd 5 | 6 | import ( 7 | "database/sql" 8 | "errors" 9 | "os" 10 | "strings" 11 | 12 | "github.com/spf13/cobra" 13 | 14 | "github.com/chand1012/sq/pkg/constants" 15 | "github.com/chand1012/sq/pkg/db" 16 | "github.com/chand1012/sq/pkg/file_types" 17 | "github.com/chand1012/sq/pkg/logger" 18 | "github.com/chand1012/sq/pkg/utils" 19 | ) 20 | 21 | var log = logger.DefaultLogger 22 | 23 | var inputFilePath string 24 | var tableName string 25 | var quiet bool 26 | var outputFormat string // csv, json, jsonl, sqlite 27 | var outputFilePath string 28 | var columnNames bool 29 | var verbose bool 30 | 31 | // rootCmd represents the base command when called without any subcommands 32 | var rootCmd = &cobra.Command{ 33 | Use: "sq", 34 | Short: "Convert and query JSON, JSONL, CSV, and SQLite with ease!", 35 | Long: `Like jq, but for SQL! Simply pipe in your data or specify a file and run your SQL queries!`, 36 | Run: run, 37 | Args: cobra.MaximumNArgs(1), 38 | PreRun: prerun, 39 | } 40 | 41 | func Execute() { 42 | err := rootCmd.Execute() 43 | if err != nil { 44 | os.Exit(1) 45 | } 46 | } 47 | 48 | func init() { 49 | rootCmd.Flags().StringVarP(&inputFilePath, "read", "r", "", "Input file path.") 50 | rootCmd.Flags().StringVarP(&tableName, "table", "t", constants.TableName, "Table name for non-SQL input.") 51 | rootCmd.Flags().StringVarP(&outputFormat, "format", "f", "", "Output format. Must be one of csv, json, jsonl, or sqlite") 52 | rootCmd.Flags().StringVarP(&outputFilePath, "output", "o", "", "Output file path. Required for sqlite output.") 53 | rootCmd.Flags().BoolVarP(&columnNames, "columns", "c", false, "Print the columns names and exit.") 54 | rootCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Print verbose output. Prints full stack trace for debugging.") 55 | rootCmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "Execute the query and exit without printing anything. For use in scripts where input file is an existing SQLite database.") 56 | } 57 | 58 | func run(cmd *cobra.Command, args []string) { 59 | var err error 60 | var d *sql.DB 61 | // var filePath string 62 | var content string 63 | var query string 64 | // tread the input as bytes 65 | // no matter where it came from 66 | // or what it is 67 | var input []byte 68 | 69 | if len(args) > 0 { 70 | query = args[0] 71 | } else { 72 | query = "SELECT * FROM " + tableName + ";" 73 | } 74 | 75 | log.Debugf("Query: %s", query) 76 | 77 | // if the input file path is not empty 78 | // load the bytes from the file 79 | if inputFilePath != "" { 80 | log.Debugf("Input file path: %s", inputFilePath) 81 | d, _, err = db.LoadFile(inputFilePath) 82 | if err != nil { 83 | log.Debug("Input file is not a SQLite database, reading as a regular file") 84 | file, err := os.ReadFile(inputFilePath) 85 | if err != nil { 86 | logger.HandlePanic(log, err, verbose) 87 | } 88 | input = file 89 | } 90 | } else { 91 | log.Debug("Reading from stdin") 92 | // check stdin 93 | input, err = utils.ReadStdin() 94 | if err != nil { 95 | logger.HandlePanic(log, err, verbose) 96 | } 97 | } 98 | 99 | log.Debugf("Read %d bytes from input", len(input)) 100 | 101 | // if the database hasn't been loaded 102 | if d == nil { 103 | // resolve the type 104 | 105 | // if input is empty, panic 106 | if len(input) == 0 { 107 | logger.HandlePanic(log, errors.New("input is empty"), verbose) 108 | } 109 | 110 | fType := file_types.Resolve(input) 111 | log.Debugf("Resolved file type: %s", fType.String()) 112 | 113 | switch fType { 114 | case file_types.SQLite: 115 | d, _, err = db.LoadStdin(input) 116 | case file_types.JSONL: 117 | d, _, err = db.FromJSONL(input, tableName) 118 | case file_types.JSON: 119 | d, _, err = db.FromJSON(input, tableName) 120 | case file_types.CSV: 121 | d, _, err = db.FromCSV(input, tableName) 122 | default: 123 | logger.HandlePanic(log, errors.New("unsupported file type"), verbose) 124 | } 125 | 126 | if outputFormat == "" { 127 | outputFormat = fType.String() 128 | } else { 129 | switch outputFormat { 130 | case "json", "jsonl", "csv", "sqlite": 131 | // do nothing 132 | default: 133 | logger.HandlePanic(log, errors.New("unsupported output format"), verbose) 134 | } 135 | } 136 | } 137 | 138 | if err != nil { 139 | logger.HandlePanic(log, err, verbose) 140 | } 141 | log.Debug("Loaded file") 142 | defer d.Close() 143 | 144 | if columnNames { 145 | log.Debug("Printing column names and exiting") 146 | columns, err := db.GetColumnNames(d, tableName) 147 | if err != nil { 148 | logger.HandlePanic(log, err, verbose) 149 | } 150 | os.Stdout.WriteString(strings.Join(columns, ",") + "\n") 151 | os.Exit(0) 152 | } 153 | 154 | log.Debug("Executing query") 155 | // run the query 156 | rows, err := d.Query(query) 157 | if err != nil { 158 | logger.HandlePanic(log, err, verbose) 159 | } 160 | defer rows.Close() 161 | 162 | // if quiet is true, exit without printing anything 163 | if quiet { 164 | os.Exit(0) 165 | } 166 | 167 | log.Debugf("Outputting rows to format %s", outputFormat) 168 | 169 | switch outputFormat { 170 | case "json": 171 | content, err = db.RowsToJSON(rows) 172 | case "jsonl": 173 | content, err = db.RowsToJSONL(rows) 174 | case "csv": 175 | content, err = db.RowsToCSV(rows) 176 | case "sqlite": 177 | if outputFilePath == "" { 178 | logger.HandlePanic(log, errors.New("output file path required for sqlite output"), verbose) 179 | } 180 | err = db.RowsToSQLite(rows, tableName, outputFilePath) 181 | if err != nil { 182 | logger.HandlePanic(log, err, verbose) 183 | } 184 | os.Exit(0) 185 | default: 186 | logger.HandlePanic(log, errors.New("unsupported output format"), verbose) 187 | } 188 | 189 | if err != nil { 190 | logger.HandlePanic(log, err, verbose) 191 | } 192 | 193 | if outputFilePath != "" { 194 | log.Debugf("Writing to file: %s", outputFilePath) 195 | err = os.WriteFile(outputFilePath, []byte(content), 0644) 196 | if err != nil { 197 | logger.HandlePanic(log, err, verbose) 198 | } 199 | } else { 200 | os.Stdout.WriteString(content) 201 | } 202 | } 203 | 204 | // sets up the logger and output format 205 | func prerun(cmd *cobra.Command, args []string) { 206 | if verbose { 207 | log = logger.VerboseLogger 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2024 TimeSurgeLabs 3 | */ 4 | package cmd 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | var commitHash string 14 | var buildDate string 15 | var tag string 16 | 17 | // versionCmd represents the version command 18 | var versionCmd = &cobra.Command{ 19 | Use: "version", 20 | Short: "Prints version information.", 21 | Long: `Prints version information.`, 22 | Run: func(cmd *cobra.Command, args []string) { 23 | fmt.Println("sq") 24 | fmt.Println("commit hash:", commitHash) 25 | fmt.Println("build date:", strings.ReplaceAll(buildDate, "_", " ")) 26 | fmt.Println("version:", tag) 27 | }, 28 | } 29 | 30 | func init() { 31 | rootCmd.AddCommand(versionCmd) 32 | } 33 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/chand1012/sq 2 | 3 | go 1.22.0 4 | 5 | require ( 6 | github.com/charmbracelet/log v0.3.1 7 | github.com/glebarez/go-sqlite v1.22.0 8 | github.com/spf13/cobra v1.8.0 9 | ) 10 | 11 | require ( 12 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 13 | github.com/charmbracelet/lipgloss v0.9.1 // indirect 14 | github.com/dustin/go-humanize v1.0.1 // indirect 15 | github.com/go-logfmt/logfmt v0.6.0 // indirect 16 | github.com/google/uuid v1.5.0 // indirect 17 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 18 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 19 | github.com/mattn/go-isatty v0.0.20 // indirect 20 | github.com/mattn/go-runewidth v0.0.15 // indirect 21 | github.com/muesli/reflow v0.3.0 // indirect 22 | github.com/muesli/termenv v0.15.2 // indirect 23 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect 24 | github.com/rivo/uniseg v0.2.0 // indirect 25 | github.com/spf13/pflag v1.0.5 // indirect 26 | golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect 27 | golang.org/x/sys v0.17.0 // indirect 28 | modernc.org/libc v1.37.6 // indirect 29 | modernc.org/mathutil v1.6.0 // indirect 30 | modernc.org/memory v1.7.2 // indirect 31 | modernc.org/sqlite v1.28.0 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= 2 | github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= 3 | github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= 4 | github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= 5 | github.com/charmbracelet/log v0.3.1 h1:TjuY4OBNbxmHWSwO3tosgqs5I3biyY8sQPny/eCMTYw= 6 | github.com/charmbracelet/log v0.3.1/go.mod h1:OR4E1hutLsax3ZKpXbgUqPtTjQfrh1pG3zwHGWuuq8g= 7 | github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 11 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 12 | github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= 13 | github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= 14 | github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= 15 | github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= 16 | github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= 17 | github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= 18 | github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= 19 | github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 20 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 21 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 22 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 23 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 24 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 25 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 26 | github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 27 | github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= 28 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 29 | github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= 30 | github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= 31 | github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= 32 | github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= 33 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 34 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 35 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= 36 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 37 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 38 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 39 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 40 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 41 | github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= 42 | github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= 43 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 44 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 45 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 46 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 47 | golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= 48 | golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= 49 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 50 | golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= 51 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 52 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 53 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 54 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 55 | modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw= 56 | modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE= 57 | modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= 58 | modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= 59 | modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= 60 | modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= 61 | modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ= 62 | modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= 63 | -------------------------------------------------------------------------------- /img/social_banner.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chand1012/sq/48dd33e2b9ee34960366205b78d35cfd56da7fd3/img/social_banner.jpg -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2024 Chandler 3 | 4 | */ 5 | package main 6 | 7 | import "github.com/chand1012/sq/cmd" 8 | 9 | func main() { 10 | cmd.Execute() 11 | } 12 | -------------------------------------------------------------------------------- /pkg/constants/constants.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const TableName = "sq" 4 | -------------------------------------------------------------------------------- /pkg/db/from_csv.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "encoding/csv" 7 | 8 | _ "github.com/glebarez/go-sqlite" 9 | 10 | "github.com/chand1012/sq/pkg/constants" 11 | "github.com/chand1012/sq/pkg/utils" 12 | ) 13 | 14 | func FromCSV(b []byte, tableName string) (*sql.DB, string, error) { 15 | if tableName == "" { 16 | tableName = constants.TableName 17 | } 18 | // create a new temp database 19 | db, tempName, err := createTempDB() 20 | if err != nil { 21 | return nil, "", err 22 | } 23 | 24 | // load the csv using the csv reader 25 | csvReader := csv.NewReader(bytes.NewReader(b)) 26 | // get the headers for column names 27 | headers, err := csvReader.Read() 28 | if err != nil { 29 | return nil, tempName, err 30 | } 31 | // preprocess the column names 32 | headers = processColumnNames(headers) 33 | // also get the first row for type inference 34 | firstRow, err := csvReader.Read() 35 | if err != nil { 36 | return nil, tempName, err 37 | } 38 | columnTypes := make([]string, len(headers)) 39 | for i, v := range firstRow { 40 | // attempt to infer the column type 41 | columnTypes[i] = guessType(v) 42 | } 43 | 44 | // construct the query that will create the table 45 | createTableQuery := genCreateTableQuery(tableName, headers, columnTypes) 46 | 47 | // execute the query to create the table 48 | _, err = db.Exec(createTableQuery) 49 | if err != nil { 50 | return nil, tempName, err 51 | } 52 | 53 | // construct the query that will insert the data 54 | insertQuery := genInsertQuery(tableName, headers) 55 | 56 | // prepare the insert statement 57 | stmt, err := db.Prepare(insertQuery) 58 | if err != nil { 59 | return nil, tempName, err 60 | } 61 | 62 | // iterate over the csv and insert the data 63 | for { 64 | record, err := csvReader.Read() 65 | if err != nil { 66 | break 67 | } 68 | // convert the records into a slice of any 69 | _, err = stmt.Exec(utils.StringSliceToAnySlice(record)...) 70 | if err != nil { 71 | return nil, tempName, err 72 | } 73 | } 74 | 75 | return db, tempName, nil 76 | } 77 | -------------------------------------------------------------------------------- /pkg/db/from_json.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "reflect" 7 | 8 | _ "github.com/glebarez/go-sqlite" 9 | 10 | "github.com/chand1012/sq/pkg/constants" 11 | "github.com/chand1012/sq/pkg/utils" 12 | ) 13 | 14 | func FromJSON(b []byte, tableName string) (*sql.DB, string, error) { 15 | 16 | if tableName == "" { 17 | tableName = constants.TableName 18 | } 19 | 20 | db, tempName, err := createTempDB() 21 | if err != nil { 22 | return nil, tempName, err 23 | } 24 | 25 | // convert it to a slice of maps 26 | var data []map[string]any 27 | err = json.Unmarshal(b, &data) 28 | if err != nil { 29 | return nil, tempName, err 30 | } 31 | 32 | // map to store the column types 33 | typeMap := make(map[string]string) 34 | // loop through all the records and infer the column types 35 | // if a column type's second inference is different from the first, it's type is set to text 36 | for _, record := range data { 37 | for k, v := range record { 38 | if typeMap[k] == "" { 39 | typeMap[k] = reflectType(v) 40 | } else if typeMap[k] != reflectType(v) { 41 | typeMap[k] = "TEXT" 42 | } 43 | } 44 | } 45 | 46 | columns, types := utils.BreakOutMap(typeMap) 47 | 48 | // // preprocess the column names 49 | // columns = processColumnNames(columns) 50 | 51 | createQuery := genCreateTableQuery(tableName, columns, types) 52 | 53 | _, err = db.Exec(createQuery) 54 | if err != nil { 55 | return nil, tempName, err 56 | } 57 | 58 | insertQuery := genInsertQuery(tableName, columns) 59 | 60 | stmt, err := db.Prepare(insertQuery) 61 | if err != nil { 62 | return nil, tempName, err 63 | } 64 | 65 | for _, record := range data { 66 | var values []any 67 | for _, column := range columns { 68 | // if the column is not present in the record, insert a NULL 69 | value := record[column] 70 | if value == nil { 71 | values = append(values, nil) 72 | continue 73 | } 74 | // Use reflect to determine if the type is a map or a slice 75 | valType := reflect.TypeOf(value) 76 | if valType.Kind() == reflect.Map || valType.Kind() == reflect.Slice { 77 | // Marshal to JSON 78 | jsonValue, err := json.Marshal(value) 79 | if err != nil { 80 | return nil, tempName, err 81 | } 82 | values = append(values, string(jsonValue)) 83 | continue // Continue to the next column after appending 84 | } else if valType.Kind() == reflect.Bool { 85 | // cast to bool 86 | boolVal := value.(bool) 87 | if boolVal { 88 | values = append(values, "true") 89 | } else { 90 | values = append(values, "false") 91 | } 92 | } else { 93 | values = append(values, value) 94 | } 95 | } 96 | _, err = stmt.Exec(values...) 97 | if err != nil { 98 | return nil, tempName, err 99 | } 100 | } 101 | 102 | return db, tempName, nil 103 | } 104 | -------------------------------------------------------------------------------- /pkg/db/from_jsonl.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "encoding/json" 7 | 8 | _ "github.com/glebarez/go-sqlite" 9 | 10 | "github.com/chand1012/sq/pkg/constants" 11 | "github.com/chand1012/sq/pkg/utils" 12 | ) 13 | 14 | func FromJSONL(b []byte, tableName string) (*sql.DB, string, error) { 15 | if tableName == "" { 16 | tableName = constants.TableName 17 | } 18 | 19 | db, tempName, err := createTempDB() 20 | if err != nil { 21 | return nil, "", err 22 | } 23 | 24 | // separate the jsonl into lines 25 | lines := bytes.Split(b, []byte("\n")) 26 | 27 | typeMap := make(map[string]string) 28 | // loop through each lines and get all the keys 29 | for _, line := range lines { 30 | if len(line) == 0 { 31 | continue 32 | } 33 | // convert each line to a map 34 | var data map[string]any 35 | err = json.Unmarshal(line, &data) 36 | if err != nil { 37 | return nil, tempName, err 38 | } 39 | // loop through the map and get all the keys 40 | for k, v := range data { 41 | if typeMap[k] == "" { 42 | typeMap[k] = reflectType(v) 43 | } else if typeMap[k] != reflectType(v) { 44 | typeMap[k] = "TEXT" 45 | } 46 | } 47 | } 48 | 49 | columns, types := utils.BreakOutMap(typeMap) 50 | 51 | // // preprocess the column names 52 | // columns = processColumnNames(columns) 53 | 54 | createQuery := genCreateTableQuery(tableName, columns, types) 55 | 56 | _, err = db.Exec(createQuery) 57 | if err != nil { 58 | return nil, tempName, err 59 | } 60 | 61 | insertQuery := genInsertQuery(tableName, columns) 62 | 63 | stmt, err := db.Prepare(insertQuery) 64 | if err != nil { 65 | return nil, tempName, err 66 | } 67 | 68 | // iterate over the jsonl and insert the data 69 | for _, line := range lines { 70 | // convert the records into a slice of any 71 | var data map[string]any 72 | err = json.Unmarshal(line, &data) 73 | if len(line) == 0 { 74 | continue 75 | } 76 | if err != nil { 77 | return nil, tempName, err 78 | } 79 | // convert the map to a slice of any 80 | args := make([]any, len(columns)) 81 | for i, col := range columns { 82 | args[i] = data[col] 83 | } 84 | _, err = stmt.Exec(args...) 85 | if err != nil { 86 | return nil, tempName, err 87 | } 88 | } 89 | 90 | return db, tempName, nil 91 | } 92 | -------------------------------------------------------------------------------- /pkg/db/load_sql.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "os" 7 | 8 | _ "github.com/glebarez/go-sqlite" 9 | ) 10 | 11 | func createTempDB() (*sql.DB, string, error) { 12 | db, err := sql.Open("sqlite", ":memory:") 13 | if err != nil { 14 | return nil, "", err 15 | } 16 | 17 | return db, "", nil 18 | } 19 | 20 | // check if the file is a valid sqlite db 21 | func IsValidDB(fileName string) bool { 22 | db, err := sql.Open("sqlite", fileName) 23 | if err != nil { 24 | return false 25 | } 26 | defer db.Close() 27 | 28 | // try to query the db 29 | _, err = db.Query("SELECT * FROM sqlite_master") 30 | return err == nil 31 | } 32 | 33 | // load a sql file from bytes 34 | func LoadStdin(bytes []byte) (*sql.DB, string, error) { 35 | tmpFile, err := os.CreateTemp(os.TempDir(), "sq-*.sql") 36 | if err != nil { 37 | return nil, "", err 38 | } 39 | defer tmpFile.Close() 40 | 41 | _, err = tmpFile.Write(bytes) 42 | if err != nil { 43 | return nil, "", err 44 | } 45 | 46 | db, err := sql.Open("sqlite", tmpFile.Name()) 47 | if err != nil { 48 | return nil, "", err 49 | } 50 | 51 | return db, tmpFile.Name(), nil 52 | } 53 | 54 | func LoadFile(fileName string) (*sql.DB, string, error) { 55 | if !IsValidDB(fileName) { 56 | return nil, "", errors.New("file is not a valid SQLite database") 57 | } 58 | db, err := sql.Open("sqlite", fileName) 59 | if err != nil { 60 | return nil, fileName, err 61 | } 62 | 63 | return db, fileName, nil 64 | } 65 | -------------------------------------------------------------------------------- /pkg/db/query_gen.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | func genCreateTableQuery(tableName string, headers []string, columnTypes []string) string { 4 | var createTableQuery string 5 | createTableQuery = "CREATE TABLE " + tableName + " (" 6 | for i, header := range headers { 7 | createTableQuery += "'" + header + "' " + columnTypes[i] 8 | if i != len(headers)-1 { 9 | createTableQuery += ", " 10 | } 11 | } 12 | createTableQuery += ");" 13 | return createTableQuery 14 | } 15 | 16 | func genInsertQuery(tableName string, headers []string) string { 17 | var insertQuery string 18 | insertQuery = "INSERT INTO " + tableName + " (" 19 | for i, header := range headers { 20 | insertQuery += "'" + header + "'" 21 | if i != len(headers)-1 { 22 | insertQuery += ", " 23 | } 24 | } 25 | insertQuery += ") VALUES (" 26 | for i := range headers { 27 | insertQuery += "?" 28 | if i != len(headers)-1 { 29 | insertQuery += ", " 30 | } 31 | } 32 | insertQuery += ");" 33 | return insertQuery 34 | } 35 | -------------------------------------------------------------------------------- /pkg/db/to_csv.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "reflect" 7 | "strings" 8 | 9 | "database/sql" 10 | ) 11 | 12 | func ToCSV(db *sql.DB, tableName string) (string, error) { 13 | // Query table data 14 | query := fmt.Sprintf("SELECT * FROM %s", tableName) 15 | rows, err := db.Query(query) 16 | if err != nil { 17 | return "", err 18 | } 19 | defer rows.Close() 20 | 21 | return RowsToCSV(rows) 22 | } 23 | 24 | func RowsToCSV(rows *sql.Rows) (string, error) { 25 | var buffer strings.Builder 26 | writer := csv.NewWriter(&buffer) 27 | 28 | // Get column names 29 | cols, err := rows.Columns() 30 | if err != nil { 31 | return "", err 32 | } 33 | 34 | // Write header row 35 | err = writer.Write(cols) 36 | if err != nil { 37 | return "", err 38 | } 39 | 40 | // Write data rows 41 | for rows.Next() { 42 | // Scan row data into interface slice 43 | scanValues := make([]interface{}, len(cols)) 44 | scanPointers := make([]interface{}, len(cols)) 45 | for i := range scanValues { 46 | scanPointers[i] = &scanValues[i] 47 | } 48 | err = rows.Scan(scanPointers...) 49 | if err != nil { 50 | return "", err 51 | } 52 | 53 | // Convert scanned values to strings 54 | row := make([]string, len(cols)) 55 | for i, v := range scanValues { 56 | val := reflect.ValueOf(v) 57 | if val.Kind() == reflect.String { 58 | row[i] = val.String() 59 | } else { 60 | row[i] = fmt.Sprint(v) 61 | } 62 | } 63 | 64 | // Write row to CSV 65 | err = writer.Write(row) 66 | if err != nil { 67 | return "", err 68 | } 69 | } 70 | 71 | writer.Flush() 72 | return buffer.String(), nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/db/to_json.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "fmt" 7 | "strings" 8 | ) 9 | 10 | func ToJSON(db *sql.DB, tableName string) (string, error) { 11 | // Query table data 12 | query := fmt.Sprintf("SELECT * FROM %s", tableName) 13 | rows, err := db.Query(query) 14 | if err != nil { 15 | return "", err 16 | } 17 | defer rows.Close() 18 | 19 | return RowsToJSON(rows) 20 | } 21 | 22 | func RowsToJSON(rows *sql.Rows) (string, error) { 23 | var data []map[string]interface{} 24 | 25 | // Get column names 26 | cols, err := rows.Columns() 27 | if err != nil { 28 | return "", err 29 | } 30 | 31 | // Iterate through rows and build data slice 32 | for rows.Next() { 33 | // Scan row data 34 | scanValues := make([]interface{}, len(cols)) 35 | scanPointers := make([]interface{}, len(cols)) 36 | for i := range scanValues { 37 | scanPointers[i] = &scanValues[i] 38 | } 39 | err := rows.Scan(scanPointers...) 40 | if err != nil { 41 | return "", err 42 | } 43 | 44 | // Create a map to store row data, excluding null values 45 | rowMap := map[string]interface{}{} 46 | for i, col := range cols { 47 | if scanValues[i] != nil { 48 | rowMap[col] = scanValues[i] 49 | // try to cast it to a string 50 | if s, ok := scanValues[i].(string); ok { 51 | rowMap[col] = s 52 | // if the string starts and ends with either [] or {} 53 | // then we need to attempt to unmarshal it 54 | if (strings.HasPrefix(rowMap[col].(string), "[") && strings.HasSuffix(rowMap[col].(string), "]")) || 55 | (strings.HasPrefix(rowMap[col].(string), "{") && strings.HasSuffix(rowMap[col].(string), "}")) { 56 | var v interface{} 57 | err := json.Unmarshal([]byte(rowMap[col].(string)), &v) 58 | if err == nil { 59 | rowMap[col] = v 60 | } 61 | } 62 | if rowMap[col] == "true" || rowMap[col] == "false" { 63 | rowMap[col] = rowMap[col].(string) == "true" 64 | } 65 | if b, ok := rowMap[col].(bool); ok { 66 | rowMap[col] = b 67 | } 68 | } 69 | } 70 | } 71 | 72 | // Append row data to the slice 73 | data = append(data, rowMap) 74 | } 75 | 76 | // Marshal data slice to JSON 77 | jsonData, err := json.Marshal(data) 78 | if err != nil { 79 | return "", err 80 | } 81 | 82 | return string(jsonData), nil 83 | } 84 | -------------------------------------------------------------------------------- /pkg/db/to_jsonl.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "database/sql" 9 | ) 10 | 11 | func ToJSONL(db *sql.DB, tableName string) (string, error) { 12 | // Query table data 13 | query := fmt.Sprintf("SELECT * FROM %s", tableName) 14 | rows, err := db.Query(query) 15 | if err != nil { 16 | return "", err 17 | } 18 | defer rows.Close() 19 | 20 | return RowsToJSONL(rows) 21 | } 22 | 23 | func RowsToJSONL(rows *sql.Rows) (string, error) { 24 | var buffer bytes.Buffer 25 | 26 | // Get column names 27 | cols, err := rows.Columns() 28 | if err != nil { 29 | return "", err 30 | } 31 | 32 | // Iterate through rows and build JSONL data 33 | for rows.Next() { 34 | // Scan row data 35 | scanValues := make([]interface{}, len(cols)) 36 | scanPointers := make([]interface{}, len(cols)) 37 | for i := range scanValues { 38 | scanPointers[i] = &scanValues[i] 39 | } 40 | err := rows.Scan(scanPointers...) 41 | if err != nil { 42 | return "", err 43 | } 44 | 45 | // Create a map to store row data, excluding null values 46 | rowMap := map[string]interface{}{} 47 | for i, col := range cols { 48 | if scanValues[i] != nil { 49 | rowMap[col] = scanValues[i] 50 | } 51 | } 52 | 53 | // Marshal row data to JSON 54 | jsonData, err := json.Marshal(rowMap) 55 | if err != nil { 56 | return "", err 57 | } 58 | 59 | // Append JSON data with newline character 60 | buffer.WriteString(string(jsonData)) 61 | buffer.WriteByte('\n') 62 | } 63 | 64 | return buffer.String(), nil 65 | } 66 | -------------------------------------------------------------------------------- /pkg/db/to_sqlite.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "io" 6 | ) 7 | 8 | func createDB(fileName string) (*sql.DB, string, error) { 9 | db, err := sql.Open("sqlite", fileName) 10 | if err != nil { 11 | return nil, fileName, err 12 | } 13 | 14 | return db, fileName, nil 15 | } 16 | 17 | func getNextRow(rows *sql.Rows) ([]any, error) { 18 | columns, err := rows.Columns() 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | values := make([]interface{}, len(columns)) 24 | valuePtrs := make([]interface{}, len(columns)) 25 | 26 | for i := range columns { 27 | valuePtrs[i] = &values[i] 28 | } 29 | 30 | // call next to get the next row 31 | if !rows.Next() { 32 | return nil, io.EOF 33 | } 34 | 35 | err = rows.Scan(valuePtrs...) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | return values, nil 41 | } 42 | 43 | func RowsToSQLite(rows *sql.Rows, tableName string, fileName string) error { 44 | d, _, err := createDB(fileName) 45 | if err != nil { 46 | return err 47 | } 48 | defer d.Close() 49 | columns, err := rows.Columns() 50 | if err != nil { 51 | return err 52 | } 53 | 54 | firstRow, err := getNextRow(rows) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | var columnTypes []string 60 | for _, value := range firstRow { 61 | t := reflectType(value) 62 | columnTypes = append(columnTypes, t) 63 | } 64 | 65 | createQuery := genCreateTableQuery(tableName, columns, columnTypes) 66 | _, err = d.Exec(createQuery) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | insertQuery := genInsertQuery(tableName, columns) 72 | stmt, err := d.Prepare(insertQuery) 73 | if err != nil { 74 | return err 75 | } 76 | // we already pulled the first row, so we need to insert it 77 | _, err = stmt.Exec(firstRow...) 78 | for { 79 | row, err := getNextRow(rows) 80 | if err != nil { 81 | if err == io.EOF { 82 | break 83 | } 84 | return err 85 | } 86 | if row == nil { 87 | break 88 | } 89 | 90 | _, err = stmt.Exec(row...) 91 | if err != nil { 92 | return err 93 | } 94 | } 95 | 96 | return err 97 | } 98 | -------------------------------------------------------------------------------- /pkg/db/utils.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | func guessType(s string) string { 11 | // assume all numbers are floats 12 | if _, err := strconv.ParseFloat(s, 64); err == nil { 13 | return "REAL" 14 | } 15 | 16 | // Otherwise, it's a string 17 | return "TEXT" 18 | } 19 | 20 | // same as above, just takes generics and uses reflection 21 | func reflectType(v any) string { 22 | switch v.(type) { 23 | case float32, float64, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: 24 | return "REAL" 25 | default: 26 | return "TEXT" 27 | } 28 | } 29 | 30 | // makes the column names all lowercase and replaces spaces with underscores 31 | func processColumnNames(columnNames []string) []string { 32 | var processedColumnNames []string 33 | for _, name := range columnNames { 34 | processedColumnNames = append(processedColumnNames, strings.ReplaceAll(strings.ToLower(name), " ", "_")) 35 | } 36 | return processedColumnNames 37 | } 38 | 39 | func GetColumnNames(db *sql.DB, tableName string) ([]string, error) { 40 | // Build the query to retrieve column names 41 | query := "SELECT * FROM '" + tableName + "' LIMIT 1" 42 | 43 | // Execute the query 44 | rows, err := db.Query(query) 45 | if err != nil { 46 | return nil, fmt.Errorf("error querying column names: %w", err) 47 | } 48 | defer rows.Close() 49 | 50 | // Extract and print each column name 51 | columns, err := rows.Columns() 52 | if err != nil { 53 | return nil, fmt.Errorf("error getting column names: %w", err) 54 | } 55 | 56 | // Return the column names 57 | return columns, nil 58 | } 59 | -------------------------------------------------------------------------------- /pkg/file_types/file_types.go: -------------------------------------------------------------------------------- 1 | package file_types 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "encoding/json" 7 | "unicode/utf8" 8 | ) 9 | 10 | func IsSQLiteFile(data []byte) bool { 11 | // SQLite database file header (magic string) 12 | const sqliteHeader = "SQLite format 3\x00" 13 | // Convert the first 16 bytes of data to a string for comparison. 14 | // Ensure data has at least 16 bytes to avoid slicing beyond its length. 15 | if len(data) < len(sqliteHeader) { 16 | return false 17 | } 18 | header := string(data[:len(sqliteHeader)]) 19 | return header == sqliteHeader 20 | } 21 | 22 | func IsValidJSON(b []byte) bool { 23 | if !utf8.Valid(b) { 24 | return false 25 | } 26 | var js json.RawMessage 27 | return json.Unmarshal(b, &js) == nil 28 | } 29 | 30 | func IsValidJSONL(b []byte) bool { 31 | if !utf8.Valid(b) { 32 | return false 33 | } 34 | 35 | lines := bytes.Split(b, []byte("\n")) 36 | for _, line := range lines { 37 | // Skip empty lines 38 | if len(line) == 0 { 39 | continue 40 | } 41 | if !IsValidJSON(line) { 42 | return false 43 | } 44 | } 45 | return true // All lines are valid JSON 46 | } 47 | 48 | func IsValidCSV(b []byte) bool { 49 | if !utf8.Valid(b) { 50 | return false 51 | } 52 | // Create a new reader to consume the byte slice as CSV 53 | r := csv.NewReader(bytes.NewReader(b)) 54 | 55 | // Attempt to read all records to ensure the CSV format is correct 56 | // We're not interested in the records themselves, just the format validation 57 | _, err := r.ReadAll() 58 | return err == nil 59 | } 60 | -------------------------------------------------------------------------------- /pkg/file_types/file_types_test.go: -------------------------------------------------------------------------------- 1 | package file_types 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestIsSQLiteFile(t *testing.T) { 8 | testCases := []struct { 9 | data []byte 10 | expected bool 11 | }{ 12 | {[]byte("SQLite format 3\x00"), true}, 13 | {[]byte(""), false}, 14 | } 15 | 16 | for _, testCase := range testCases { 17 | actual := IsSQLiteFile(testCase.data) 18 | if actual != testCase.expected { 19 | t.Errorf("Expected %v, but got %v", testCase.expected, actual) 20 | } 21 | } 22 | } 23 | 24 | func TestIsValidJSON(t *testing.T) { 25 | testCases := []struct { 26 | data []byte 27 | expected bool 28 | }{ 29 | {[]byte("{\"key\": \"value\"}"), true}, 30 | {[]byte("{"), false}, 31 | } 32 | 33 | for _, testCase := range testCases { 34 | actual := IsValidJSON(testCase.data) 35 | if actual != testCase.expected { 36 | t.Errorf("Expected %v, but got %v", testCase.expected, actual) 37 | } 38 | } 39 | } 40 | 41 | func TestIsValidJSONL(t *testing.T) { 42 | testCases := []struct { 43 | data []byte 44 | expected bool 45 | }{ 46 | {[]byte("{\"key\": \"value\"}\n{\"key2\": \"value2\"}"), true}, 47 | {[]byte("{\"key\": \"value\"}\n{\"key2\": "), false}, 48 | } 49 | 50 | for _, testCase := range testCases { 51 | actual := IsValidJSONL(testCase.data) 52 | if actual != testCase.expected { 53 | t.Errorf("Expected %v, but got %v", testCase.expected, actual) 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /pkg/file_types/resolve.go: -------------------------------------------------------------------------------- 1 | package file_types 2 | 3 | import ( 4 | "path/filepath" 5 | "strings" 6 | ) 7 | 8 | // enum of the valid types 9 | // json, csv, and jsonl for now 10 | type FileType int 11 | 12 | const ( 13 | JSON FileType = iota 14 | CSV 15 | JSONL 16 | SQLite 17 | Unknown 18 | ) 19 | 20 | func Resolve(b []byte) FileType { 21 | if IsSQLiteFile(b) { 22 | return SQLite 23 | } 24 | if IsValidCSV(b) { 25 | return CSV 26 | } 27 | if IsValidJSON(b) { 28 | return JSON 29 | } 30 | if IsValidJSONL(b) { 31 | return JSONL 32 | } 33 | return Unknown 34 | } 35 | 36 | func ResolveByPath(fileName string) FileType { 37 | // Get the file extension (lowercase) 38 | ext := strings.ToLower(filepath.Ext(fileName)) 39 | 40 | switch ext { 41 | case ".csv": 42 | return CSV 43 | case ".sqlite", ".db", ".sqlite3", ".db3", ".sdb", ".dat": 44 | return SQLite 45 | case ".json": 46 | return JSON 47 | case ".jsonl": 48 | return JSONL 49 | default: 50 | return Unknown 51 | } 52 | } 53 | 54 | func (ft FileType) String() string { 55 | switch ft { 56 | case JSON: 57 | return "json" 58 | case CSV: 59 | return "csv" 60 | case JSONL: 61 | return "jsonl" 62 | case SQLite: 63 | return "sqlite" 64 | default: 65 | return "unknown" 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /pkg/logger/log.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "os" 5 | 6 | l "github.com/charmbracelet/log" 7 | ) 8 | 9 | var DefaultLogger = l.NewWithOptions(os.Stderr, l.Options{ 10 | Level: l.InfoLevel, 11 | ReportTimestamp: false, 12 | }) 13 | 14 | var VerboseLogger = l.NewWithOptions(os.Stderr, l.Options{ 15 | Level: l.DebugLevel, 16 | ReportTimestamp: false, 17 | }) 18 | -------------------------------------------------------------------------------- /pkg/logger/panic.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "os" 5 | 6 | l "github.com/charmbracelet/log" 7 | ) 8 | 9 | func HandlePanic(log *l.Logger, err error, verbose bool) { 10 | log.Error(err.Error()) 11 | if verbose { 12 | panic(err) 13 | } 14 | os.Exit(1) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/utils/json.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | func GetJSONKeys(j any) ([]string, error) { 8 | var keys []string 9 | switch t := j.(type) { 10 | case map[string]interface{}: 11 | for key := range t { 12 | keys = append(keys, key) 13 | } 14 | default: 15 | return nil, errors.New("not a map, no keys") 16 | } 17 | return keys, nil 18 | } 19 | -------------------------------------------------------------------------------- /pkg/utils/map.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func BreakOutMap(m map[string]string) (keys []string, values []string) { 4 | for k, v := range m { 5 | keys = append(keys, k) 6 | values = append(values, v) 7 | } 8 | return keys, values 9 | } 10 | -------------------------------------------------------------------------------- /pkg/utils/pipe.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | func ReadStdin() ([]byte, error) { 9 | bytes, err := io.ReadAll(os.Stdin) 10 | if err != nil { 11 | return nil, err // Return the error to the caller 12 | } 13 | return bytes, nil // Return the bytes to the caller 14 | } 15 | -------------------------------------------------------------------------------- /pkg/utils/slice.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | // checks if a string is in a slice of strings 4 | func StringInSlice(s string, slice []string) bool { 5 | for _, v := range slice { 6 | if s == v { 7 | return true 8 | } 9 | } 10 | return false 11 | } 12 | 13 | func DedupeStringSlice(s []string) []string { 14 | keys := make(map[string]bool) 15 | var list []string 16 | for _, entry := range s { 17 | if _, value := keys[entry]; !value { 18 | keys[entry] = true 19 | list = append(list, entry) 20 | } 21 | } 22 | return list 23 | } 24 | -------------------------------------------------------------------------------- /pkg/utils/types.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func StringSliceToAnySlice(s []string) []any { 4 | var i []any 5 | for _, v := range s { 6 | i = append(i, v) 7 | } 8 | return i 9 | } 10 | --------------------------------------------------------------------------------