├── .github └── workflows │ └── ci.yaml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── aggregation └── strings.go ├── arrow ├── arrow.py ├── bool.bin ├── float.bin ├── int.bin ├── mixed.bin └── string.bin ├── benchmark_test.go ├── cmd └── qfgenerate │ └── main.go ├── config ├── csv │ └── config.go ├── doc.go ├── eval │ ├── config.go │ └── context.go ├── groupby │ └── config.go ├── newqf │ └── config.go ├── rolling │ └── config.go └── sql │ └── config.go ├── contrib └── gonum │ └── qplot │ ├── config.go │ ├── doc.go │ ├── helpers.go │ ├── plotter.go │ ├── plotter_funcs.go │ ├── plotter_test.go │ ├── qplot.go │ ├── qplot_test.go │ └── testdata │ ├── GlobalTemperatures.csv │ └── GlobalTemperatures.png ├── doc.go ├── examples_test.go ├── expression.go ├── filter.go ├── filter └── filter.go ├── filter_test.go ├── function ├── bool.go ├── doc.go ├── float.go ├── int.go └── string.go ├── go.mod ├── go.sum ├── grouper.go ├── internal ├── bcolumn │ ├── aggregations.go │ ├── column.go │ ├── column_gen.go │ ├── doc_gen.go │ ├── filters.go │ ├── filters_gen.go │ └── generator.go ├── column │ └── column.go ├── ecolumn │ ├── bitset.go │ ├── column.go │ ├── doc_gen.go │ ├── filters.go │ ├── filters_gen.go │ ├── generator.go │ └── view.go ├── fastcsv │ ├── LICENSE │ ├── README.md │ ├── csv.go │ ├── csv_test.go │ └── testdata │ │ ├── addquotes.go │ │ ├── fl_insurance.csv │ │ └── fl_insurance_quoted.csv ├── fcolumn │ ├── aggregations.go │ ├── column.go │ ├── column_gen.go │ ├── doc_gen.go │ ├── filters.go │ ├── filters_gen.go │ └── generator.go ├── grouper │ └── grouper.go ├── hash │ ├── memhash.go │ └── memhash_test.go ├── icolumn │ ├── aggregations.go │ ├── column.go │ ├── column_gen.go │ ├── doc_gen.go │ ├── filters.go │ ├── filters_gen.go │ └── generator.go ├── index │ └── index.go ├── io │ ├── csv.go │ ├── json.go │ └── sql │ │ ├── coerce.go │ │ ├── column.go │ │ ├── column_test.go │ │ ├── reader.go │ │ ├── stmt.go │ │ ├── stmt_test.go │ │ └── types.go ├── maps │ └── maps.go ├── math │ ├── float │ │ └── float.go │ └── integer │ │ └── int.go ├── ncolumn │ └── column.go ├── qframe │ └── generator │ │ └── generator.go ├── ryu │ ├── LICENSE │ ├── README.md │ ├── ryu.go │ ├── ryu32.go │ ├── ryu64.go │ └── tables.go ├── scolumn │ ├── column.go │ ├── doc_gen.go │ ├── filters.go │ ├── filters_gen.go │ ├── generator.go │ └── view.go ├── sort │ ├── GO-LICENCE │ └── sorter.go ├── strings │ ├── convert.go │ ├── match.go │ ├── name.go │ ├── pointer.go │ ├── serialize.go │ └── set.go └── template │ ├── column.go │ ├── docs.go │ ├── filters.go │ ├── generate.go │ └── placeholders.go ├── qerrors └── error.go ├── qframe.go ├── qframe_gen.go ├── qframe_sql_test.go ├── qframe_test.go └── types ├── aliases.go ├── markers.go └── types.go /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | 11 | strategy: 12 | matrix: 13 | go-version: ["1.21.x"] 14 | 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v2 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: ${{ matrix.go-version }} 23 | id: go 24 | 25 | - name: Install dependencies 26 | run: make dev-deps 27 | 28 | - name: Run tests 29 | run: make ci 30 | 31 | - name: Update coverage report 32 | uses: ncruces/go-coverage-report@v0 33 | with: 34 | report: true 35 | chart: true 36 | amend: true 37 | reuse-go: true 38 | if: | 39 | github.event_name == 'push' 40 | continue-on-error: true 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/*/ 2 | go-qcache 3 | *.test 4 | *.out 5 | *.prof 6 | .idea 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ### 2023-11-19 v0.4.0 2 | * Modernize codebase, update to Go 1.20+. 3 | * Fix #44, ignore when reading quoted fields. 4 | 5 | ### 2019-10-13 v0.3.0 6 | * Backwards incompatible change of errors package name to qerrors to support code generator. 7 | * Some performance improvements to grouping since v0.2.0. 8 | 9 | ### 2018-09-09 v0.2.0 10 | SQL and plotting support! Thanks a lot to @kevinschoon for adding this! 11 | * Add support for reading from/writing to SQL databases. Thanks @kevinschoon for this! 12 | * Add support for plotting using Gonum plot. Thanks @kevinschoon for this! 13 | * Rename `ToJson/FromJson/ToCsv/FromCsv` -> `ToJSON/FromJSON/ToCSV/FromCSV` for 14 | consistency with stdlib. Thanks @sbinet for this! 15 | * qframe.Expr1 and qframe.Expr2 have been merged to one function qframe.Expr. 16 | * Minor bug fix in CSV reading where reads that return io.EOF together with 17 | data previously did not work. 18 | 19 | ### 2018-05-23 v0.1.0 20 | * Initial release 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 Tobias Gustafsson 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | generate: 2 | # Build and install generator binary first 3 | go generate github.com/tobgu/qframe/... || true 4 | go install github.com/tobgu/qframe/cmd/qfgenerate 5 | go generate github.com/tobgu/qframe/... 6 | 7 | test: generate 8 | go test ./... 9 | 10 | lint: 11 | ~/go/bin/golangci-lint run ./... 12 | 13 | ci: test lint 14 | 15 | fmt: generate 16 | go fmt ./... 17 | 18 | vet: generate 19 | go vet ./... 20 | 21 | cov: generate 22 | go test github.com/tobgu/qframe/... -coverprofile=coverage.out -coverpkg=all 23 | go tool cover -html=coverage.out 24 | 25 | deps: 26 | go get -t ./... 27 | 28 | dev-deps: deps 29 | go install github.com/mauricelam/genny@latest 30 | mkdir -p ~/go/bin 31 | curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ~/go/bin v1.55.2 32 | 33 | qplot_examples: 34 | cd contrib/gonum/qplot/examples \ 35 | && go run temperatures.go 36 | -------------------------------------------------------------------------------- /aggregation/strings.go: -------------------------------------------------------------------------------- 1 | package aggregation 2 | 3 | import "strings" 4 | 5 | // StrJoin creates a function that joins a slice of strings into 6 | // a single string using the provided separator. 7 | // It is provided as an example and can be used in aggregations 8 | // on string and enum columns. 9 | func StrJoin(sep string) func([]*string) *string { 10 | return func(input []*string) *string { 11 | s := make([]string, 0, len(input)) 12 | for _, sPtr := range input { 13 | if sPtr != nil { 14 | s = append(s, *sPtr) 15 | } 16 | } 17 | 18 | result := strings.Join(s, sep) 19 | return &result 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /arrow/arrow.py: -------------------------------------------------------------------------------- 1 | # Utility script for cross language test of arrow format. 2 | # 3 | # Requires that pyarrow is installed: 4 | # pip install pyarrow 5 | # 6 | # Run: 7 | # python arrow.py 8 | 9 | import pyarrow as pa 10 | 11 | def write_data(data_dict, file_name): 12 | keys = sorted(data_dict.keys()) 13 | data = [pa.array(data_dict[k]) for k in keys] 14 | batch = pa.RecordBatch.from_arrays(data, keys) 15 | writer = pa.RecordBatchStreamWriter(file_name, batch.schema) 16 | writer.write(batch) 17 | writer.close() 18 | 19 | def read_data(file_name): 20 | reader = pa.RecordBatchStreamReader(file_name) 21 | table = reader.read_all() 22 | print(str(table.to_pydict())) 23 | 24 | 25 | write_data({'f0': [True, False, True]}, 'bool.bin') 26 | write_data({'f0': [1.5, 2.5, None]}, 'float.bin') 27 | write_data({'f0': ['foo', 'bar', None]}, 'string.bin') 28 | write_data({'f0': [1, 2, 3]}, 'int.bin') 29 | write_data({'f0': [1, 2, 3], 30 | 'f1': [1.5, 2.5, None], 31 | 'f2': [True, False, True], 32 | 'f3': ['foo', 'bar', None]}, 'mixed.bin') 33 | 34 | read_data('mixed.bin') 35 | 36 | # TODO: dictionary/enum 37 | # TODO: corner cases, empty arrays for example 38 | # TODO: Test with tables/columns as well -------------------------------------------------------------------------------- /arrow/bool.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tobgu/qframe/edb23855dc466ccea4b89d245aa06adc94cba431/arrow/bool.bin -------------------------------------------------------------------------------- /arrow/float.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tobgu/qframe/edb23855dc466ccea4b89d245aa06adc94cba431/arrow/float.bin -------------------------------------------------------------------------------- /arrow/int.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tobgu/qframe/edb23855dc466ccea4b89d245aa06adc94cba431/arrow/int.bin -------------------------------------------------------------------------------- /arrow/mixed.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tobgu/qframe/edb23855dc466ccea4b89d245aa06adc94cba431/arrow/mixed.bin -------------------------------------------------------------------------------- /arrow/string.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tobgu/qframe/edb23855dc466ccea4b89d245aa06adc94cba431/arrow/string.bin -------------------------------------------------------------------------------- /cmd/qfgenerate/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | "go/format" 8 | "os" 9 | 10 | bgenerator "github.com/tobgu/qframe/internal/bcolumn" 11 | egenerator "github.com/tobgu/qframe/internal/ecolumn" 12 | fgenerator "github.com/tobgu/qframe/internal/fcolumn" 13 | igenerator "github.com/tobgu/qframe/internal/icolumn" 14 | qfgenerator "github.com/tobgu/qframe/internal/qframe/generator" 15 | sgenerator "github.com/tobgu/qframe/internal/scolumn" 16 | "github.com/tobgu/qframe/qerrors" 17 | ) 18 | 19 | /* 20 | Simple code generator used in various places to reduce code duplication 21 | */ 22 | 23 | func main() { 24 | dstFile := flag.String("dst-file", "", "File that the code should be generated to") 25 | source := flag.String("source", "", "Which package that code should be generated for") 26 | flag.Parse() 27 | 28 | if *dstFile == "" { 29 | panic("Destination file must be given") 30 | } 31 | 32 | generators := map[string]func() (*bytes.Buffer, error){ 33 | "idoc": igenerator.GenerateDoc, 34 | "ifilter": igenerator.GenerateFilters, 35 | "fdoc": fgenerator.GenerateDoc, 36 | "ffilter": fgenerator.GenerateFilters, 37 | "bdoc": bgenerator.GenerateDoc, 38 | "bfilter": bgenerator.GenerateFilters, 39 | "edoc": egenerator.GenerateDoc, 40 | "efilter": egenerator.GenerateFilters, 41 | "sdoc": sgenerator.GenerateDoc, 42 | "sfilter": sgenerator.GenerateFilters, 43 | "qframe": qfgenerator.GenerateQFrame, 44 | } 45 | 46 | generator, ok := generators[*source] 47 | if !ok { 48 | panic(fmt.Sprintf("Unknown source: \"%s\"", *source)) 49 | } 50 | 51 | buf, err := generator() 52 | if err != nil { 53 | panic(err) 54 | } 55 | 56 | if err := writeFile(buf, *dstFile); err != nil { 57 | panic(err) 58 | } 59 | } 60 | 61 | func writeFile(buf *bytes.Buffer, file string) error { 62 | if file == "" { 63 | return qerrors.New("writeFile", "Output file must be specified") 64 | } 65 | 66 | f, err := os.Create(file) 67 | if err != nil { 68 | return err 69 | } 70 | defer f.Close() 71 | 72 | // The equivalent of "go fmt" before writing content 73 | src := buf.Bytes() 74 | fmtSrc, err := format.Source(src) 75 | if err != nil { 76 | os.Stdout.WriteString(string(src)) 77 | return qerrors.Propagate("Format error", err) 78 | } 79 | 80 | _, err = f.Write(fmtSrc) 81 | return err 82 | } 83 | -------------------------------------------------------------------------------- /config/csv/config.go: -------------------------------------------------------------------------------- 1 | package csv 2 | 3 | import ( 4 | qfio "github.com/tobgu/qframe/internal/io" 5 | "github.com/tobgu/qframe/types" 6 | ) 7 | 8 | // Config holds configuration for reading CSV files into QFrames. 9 | // It should be considered a private implementation detail and should never be 10 | // referenced or used directly outside of the QFrame code. To manipulate it 11 | // use the functions returning ConfigFunc below. 12 | type Config qfio.CSVConfig 13 | 14 | // ConfigFunc is a function that operates on a Config object. 15 | type ConfigFunc func(*Config) 16 | 17 | // NewConfig creates a new Config object. 18 | // This function should never be called from outside QFrame. 19 | func NewConfig(ff []ConfigFunc) Config { 20 | conf := Config{Delimiter: ','} 21 | for _, f := range ff { 22 | f(&conf) 23 | } 24 | return conf 25 | } 26 | 27 | // EmptyNull configures if empty strings should be considered as empty strings (default) or null. 28 | // 29 | // emptyNull - If set to true empty string will be translated to null. 30 | func EmptyNull(emptyNull bool) ConfigFunc { 31 | return func(c *Config) { 32 | c.EmptyNull = emptyNull 33 | } 34 | } 35 | 36 | // MissingColumnNameAlias sets the name to be used for empty columns name with given string 37 | func MissingColumnNameAlias(MissingColumnNameAlias string) ConfigFunc { 38 | return func(c *Config) { 39 | c.MissingColumnNameAlias = MissingColumnNameAlias 40 | } 41 | } 42 | 43 | // RenameDuplicateColumns configures if duplicate column names should have the column index appended to the column name to resolve the conflict. 44 | func RenameDuplicateColumns(RenameDuplicateColumns bool) ConfigFunc { 45 | return func(c *Config) { 46 | c.RenameDuplicateColumns = RenameDuplicateColumns 47 | } 48 | } 49 | 50 | // IgnoreEmptyLines configures if a line without any characters should be ignored or interpreted 51 | // as a zero length string. 52 | // 53 | // IgnoreEmptyLines - If set to true empty lines will not produce any data. 54 | func IgnoreEmptyLines(ignoreEmptyLines bool) ConfigFunc { 55 | return func(c *Config) { 56 | c.IgnoreEmptyLines = ignoreEmptyLines 57 | } 58 | } 59 | 60 | // Delimiter configures the delimiter/separator between columns. 61 | // Only byte representable delimiters are supported. Default is ','. 62 | // 63 | // delimiter - The delimiter to use. 64 | func Delimiter(delimiter byte) ConfigFunc { 65 | return func(c *Config) { 66 | c.Delimiter = delimiter 67 | } 68 | } 69 | 70 | // Types is used set types for certain columns. 71 | // If types are not given a best effort attempt will be done to auto detected the type. 72 | // 73 | // typs - map column name -> type name. For a list of type names see package qframe/types. 74 | func Types(typs map[string]string) ConfigFunc { 75 | return func(c *Config) { 76 | c.Types = make(map[string]types.DataType, len(typs)) 77 | for k, v := range typs { 78 | c.Types[k] = types.DataType(v) 79 | } 80 | } 81 | } 82 | 83 | // EnumValues is used to list the possible values and internal order of these values for an enum column. 84 | // 85 | // values - map column name -> list of valid values. 86 | // 87 | // Enum columns that do not specify the values are automatically assigned values based on the content 88 | // of the column. The ordering between these values is undefined. It hence doesn't make much sense to 89 | // sort a QFrame on an enum column unless the ordering has been specified. 90 | // 91 | // Note that the column must be listed as having an enum type (using Types above) for this option to take effect. 92 | func EnumValues(values map[string][]string) ConfigFunc { 93 | return func(c *Config) { 94 | c.EnumVals = make(map[string][]string) 95 | for k, v := range values { 96 | c.EnumVals[k] = v 97 | } 98 | } 99 | } 100 | 101 | // RowCountHint can be used to provide an indication of the number of rows 102 | // in the CSV. In some cases this will help allocating buffers more efficiently 103 | // and improve import times. 104 | // 105 | // rowCount - The number of rows. 106 | func RowCountHint(rowCount int) ConfigFunc { 107 | return func(c *Config) { 108 | c.RowCountHint = rowCount 109 | } 110 | } 111 | 112 | // Headers can be used to specify the header names for a CSV file without header. 113 | // 114 | // header - Slice with column names. 115 | func Headers(headers []string) ConfigFunc { 116 | return func(c *Config) { 117 | c.Headers = headers 118 | } 119 | } 120 | 121 | // ToConfig holds configuration for writing CSV files 122 | type ToConfig qfio.ToCsvConfig 123 | 124 | // ToConfigFunc is a function that operates on a ToConfig object. 125 | type ToConfigFunc func(*ToConfig) 126 | 127 | // NewConfig creates a new ToConfig object. 128 | // This function should never be called from outside QFrame. 129 | func NewToConfig(ff []ToConfigFunc) ToConfig { 130 | conf := ToConfig{Header: true} //Default 131 | for _, f := range ff { 132 | f(&conf) 133 | } 134 | return conf 135 | } 136 | 137 | // Header indicates whether or not the CSV file should be written with a header. 138 | // Default is true. 139 | func Header(header bool) ToConfigFunc { 140 | return func(c *ToConfig) { 141 | c.Header = header 142 | } 143 | } 144 | 145 | // Columns holds the order to write CSV columns. 146 | func Columns(cols []string) ToConfigFunc { 147 | return func(c *ToConfig) { 148 | c.Columns = cols 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /config/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package config acts as a base package for different configuration options used when creating or working with QFrames. 3 | 4 | Most of the configs use "functional options" as presented here: 5 | https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis 6 | 7 | While it is a nice way to overcome the lack of keyword arguments in Go to be able to extend function 8 | signatures in a backwards compatible way the problem with this method (as I see it) is that the 9 | discoverability of existing config options is lacking. 10 | 11 | This structure hopes to help (a bit) in fixing that by having a separate package for each configuration 12 | type. That way the function listing of each package conveys all options available for that particular 13 | config option. 14 | */ 15 | package config 16 | -------------------------------------------------------------------------------- /config/eval/config.go: -------------------------------------------------------------------------------- 1 | package eval 2 | 3 | // Config holds configuration for evaluating expressions on QFrames. 4 | // It should be considered a private implementation detail and should never be 5 | // referenced or used directly outside of the QFrame code. To manipulate it 6 | // use the functions returning ConfigFunc below. 7 | type Config struct { 8 | Ctx *Context 9 | } 10 | 11 | // ConfigFunc is a function that operates on a Config object. 12 | type ConfigFunc func(*Config) 13 | 14 | // NewConfig creates a new Config object. 15 | // This function should never be called from outside QFrame. 16 | func NewConfig(ff []ConfigFunc) Config { 17 | result := Config{} 18 | for _, f := range ff { 19 | f(&result) 20 | } 21 | 22 | if result.Ctx == nil { 23 | result.Ctx = NewDefaultCtx() 24 | } 25 | 26 | return result 27 | } 28 | 29 | // EvalContext sets the evaluation context to use. 30 | func EvalContext(ctx *Context) ConfigFunc { 31 | return func(c *Config) { 32 | c.Ctx = ctx 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /config/eval/context.go: -------------------------------------------------------------------------------- 1 | package eval 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "reflect" 7 | "strings" 8 | 9 | "github.com/tobgu/qframe/function" 10 | qfstrings "github.com/tobgu/qframe/internal/strings" 11 | "github.com/tobgu/qframe/qerrors" 12 | "github.com/tobgu/qframe/types" 13 | ) 14 | 15 | type functionsByArgCount struct { 16 | singleArgs map[string]interface{} 17 | doubleArgs map[string]interface{} 18 | } 19 | 20 | type functionsByArgType map[types.FunctionType]functionsByArgCount 21 | 22 | // ArgCount is the number of arguments passed to a function to be evaluated. 23 | type ArgCount byte 24 | 25 | const ( 26 | ArgCountOne ArgCount = iota 27 | ArgCountTwo 28 | ) 29 | 30 | // String returns a string representation of the ArgCount 31 | func (c ArgCount) String() string { 32 | switch c { 33 | case ArgCountOne: 34 | return "Single argument" 35 | case ArgCountTwo: 36 | return "Double argument" 37 | default: 38 | return "Unknown argument count" 39 | } 40 | } 41 | 42 | // Context describes the context in which an expression is executed. 43 | // It maps function names to actual functions. 44 | type Context struct { 45 | functions functionsByArgType 46 | } 47 | 48 | // NewDefaultCtx creates a default context containing a base set of functions. 49 | // It can be used as is or enhanced with other/more functions. See the source code 50 | // for the current set of functions. 51 | func NewDefaultCtx() *Context { 52 | return &Context{ 53 | functionsByArgType{ 54 | types.FunctionTypeFloat: functionsByArgCount{ 55 | singleArgs: map[string]interface{}{ 56 | "abs": math.Abs, 57 | "str": function.StrF, 58 | "int": function.IntF, 59 | }, 60 | doubleArgs: map[string]interface{}{ 61 | "+": function.PlusF, 62 | "-": function.MinusF, 63 | "*": function.MulF, 64 | "/": function.DivF, 65 | }, 66 | }, 67 | types.FunctionTypeInt: functionsByArgCount{ 68 | singleArgs: map[string]interface{}{ 69 | "abs": function.AbsI, 70 | "str": function.StrI, 71 | "bool": function.BoolI, 72 | "float": function.FloatI, 73 | }, 74 | doubleArgs: map[string]interface{}{ 75 | "+": function.PlusI, 76 | "-": function.MinusI, 77 | "*": function.MulI, 78 | "/": function.DivI, 79 | }, 80 | }, 81 | types.FunctionTypeBool: functionsByArgCount{ 82 | singleArgs: map[string]interface{}{ 83 | "!": function.NotB, 84 | "str": function.StrB, 85 | "int": function.IntB, 86 | }, 87 | doubleArgs: map[string]interface{}{ 88 | "&": function.AndB, 89 | "|": function.OrB, 90 | "!=": function.XorB, 91 | "nand": function.NandB, 92 | }, 93 | }, 94 | types.FunctionTypeString: functionsByArgCount{ 95 | singleArgs: map[string]interface{}{ 96 | "upper": function.UpperS, 97 | "lower": function.LowerS, 98 | "str": function.StrS, 99 | "len": function.LenS, 100 | }, 101 | doubleArgs: map[string]interface{}{ 102 | "+": function.ConcatS, 103 | }, 104 | }, 105 | }, 106 | } 107 | } 108 | 109 | // GetFunc returns a reference to a function matching the given function type, argument count and name. 110 | // If no matching function is found in the context the second return value is set to false. 111 | func (ctx *Context) GetFunc(typ types.FunctionType, ac ArgCount, name string) (interface{}, bool) { 112 | if typ == types.FunctionTypeUndefined { 113 | // This is a special case for functions on columns with undefined type. These columns 114 | // always of zero and the function will never be executed. 115 | return nil, true 116 | } 117 | 118 | var fn interface{} 119 | var ok bool 120 | if ac == ArgCountOne { 121 | fn, ok = ctx.functions[typ].singleArgs[name] 122 | } else { 123 | fn, ok = ctx.functions[typ].doubleArgs[name] 124 | } 125 | 126 | return fn, ok 127 | } 128 | 129 | func (ctx *Context) setFunc(typ types.FunctionType, ac ArgCount, name string, fn interface{}) { 130 | if ac == ArgCountOne { 131 | ctx.functions[typ].singleArgs[name] = fn 132 | } else { 133 | ctx.functions[typ].doubleArgs[name] = fn 134 | } 135 | } 136 | 137 | // SetFunc inserts a function into the context under the given name. 138 | func (ctx *Context) SetFunc(name string, fn interface{}) error { 139 | if err := qfstrings.CheckName(name); err != nil { 140 | return qerrors.Propagate("SetFunc", err) 141 | } 142 | 143 | // Since there's such a flexibility in the function types that can be 144 | // used and there is no static typing to support it this function 145 | // acts as the gate keeper for adding new functions. 146 | var ac ArgCount 147 | var typ types.FunctionType 148 | switch fn.(type) { 149 | // Int 150 | case func(int, int) int: 151 | ac, typ = ArgCountTwo, types.FunctionTypeInt 152 | case func(int) int, func(int) bool, func(int) float64, func(int) *string: 153 | ac, typ = ArgCountOne, types.FunctionTypeInt 154 | 155 | // Float 156 | case func(float64, float64) float64: 157 | ac, typ = ArgCountTwo, types.FunctionTypeFloat 158 | case func(float64) float64, func(float64) int, func(float64) bool, func(float64) *string: 159 | ac, typ = ArgCountOne, types.FunctionTypeFloat 160 | 161 | // Bool 162 | case func(bool, bool) bool: 163 | ac, typ = ArgCountTwo, types.FunctionTypeBool 164 | case func(bool) bool, func(bool) int, func(bool) float64, func(bool) *string: 165 | ac, typ = ArgCountOne, types.FunctionTypeBool 166 | 167 | // String 168 | case func(*string, *string) *string: 169 | ac, typ = ArgCountTwo, types.FunctionTypeString 170 | case func(*string) *string, func(*string) int, func(*string) float64, func(*string) bool: 171 | ac, typ = ArgCountOne, types.FunctionTypeString 172 | 173 | default: 174 | return qerrors.New("SetFunc", "invalid function type for function \"%s\": %v", name, reflect.TypeOf(fn)) 175 | } 176 | 177 | ctx.setFunc(typ, ac, name, fn) 178 | return nil 179 | } 180 | 181 | func (ctx *Context) String() string { 182 | result := "" 183 | for fnType, funcs := range ctx.functions { 184 | result += fmt.Sprintf("\n%s\n%s", fnType, strings.Repeat("-", len(fnType.String()))) 185 | result += "\n Single arg\n" 186 | for funcName := range funcs.singleArgs { 187 | result += " " + funcName + "\n" 188 | } 189 | 190 | result += "\n Double arg\n" 191 | for funcName := range funcs.doubleArgs { 192 | result += " " + funcName + "\n" 193 | } 194 | } 195 | 196 | return result 197 | } 198 | -------------------------------------------------------------------------------- /config/groupby/config.go: -------------------------------------------------------------------------------- 1 | package groupby 2 | 3 | // Config holds configuration for group by operations on QFrames. 4 | // It should be considered a private implementation detail and should never be 5 | // referenced or used directly outside of the QFrame code. To manipulate it 6 | // use the functions returning ConfigFunc below. 7 | type Config struct { 8 | Columns []string 9 | GroupByNull bool 10 | // dropNulls? 11 | } 12 | 13 | // ConfigFunc is a function that operates on a Config object. 14 | type ConfigFunc func(c *Config) 15 | 16 | // NewConfig creates a new Config object. 17 | // This function should never be called from outside QFrame. 18 | func NewConfig(configFns []ConfigFunc) Config { 19 | var config Config 20 | for _, f := range configFns { 21 | f(&config) 22 | } 23 | 24 | return config 25 | } 26 | 27 | // Columns sets the columns by which the data should be grouped. 28 | // Leaving this configuration option out will group on all columns in the QFrame. 29 | // 30 | // The order of columns does not matter from a functional point of view but 31 | // it may impact execution time a bit. For optimal performance order columns 32 | // according to type with the following priority: 33 | // 1. int 34 | // 2. float 35 | // 3. enum/bool 36 | // 4. string 37 | func Columns(columns ...string) ConfigFunc { 38 | return func(c *Config) { 39 | c.Columns = columns 40 | } 41 | } 42 | 43 | // Null configures if Na/nulls should be grouped together or not. 44 | // Default is false (eg. don't group null/NaN). 45 | func Null(b bool) ConfigFunc { 46 | return func(c *Config) { 47 | c.GroupByNull = b 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /config/newqf/config.go: -------------------------------------------------------------------------------- 1 | package newqf 2 | 3 | // Config holds configuration for creating new QFrames using the New constructor. 4 | // It should be considered a private implementation detail and should never be 5 | // referenced or used directly outside of the QFrame code. To manipulate it 6 | // use the functions returning ConfigFunc below. 7 | type Config struct { 8 | ColumnOrder []string 9 | EnumColumns map[string][]string 10 | } 11 | 12 | // ConfigFunc is a function that operates on a Config object. 13 | type ConfigFunc func(c *Config) 14 | 15 | // NewConfig creates a new Config object. 16 | // This function should never be called from outside QFrame. 17 | func NewConfig(fns []ConfigFunc) *Config { 18 | // TODO: This function returns a pointer while most of the other returns values. Decide which way to do it. 19 | config := &Config{} 20 | for _, fn := range fns { 21 | fn(config) 22 | } 23 | return config 24 | } 25 | 26 | // ColumnOrder provides the order in which columns are displayed, etc. 27 | func ColumnOrder(columns ...string) ConfigFunc { 28 | return func(c *Config) { 29 | c.ColumnOrder = make([]string, len(columns)) 30 | copy(c.ColumnOrder, columns) 31 | } 32 | } 33 | 34 | // Enums lists columns that should be considered enums. 35 | // The map key specifies the columns name, the value if there is a fixed set of 36 | // values and their internal ordering. If value is nil or empty list the values 37 | // will be derived from the columns content and the ordering unspecified. 38 | func Enums(columns map[string][]string) ConfigFunc { 39 | return func(c *Config) { 40 | c.EnumColumns = make(map[string][]string) 41 | for k, v := range columns { 42 | c.EnumColumns[k] = v 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /config/rolling/config.go: -------------------------------------------------------------------------------- 1 | package rolling 2 | 3 | import "github.com/tobgu/qframe/qerrors" 4 | 5 | // DataValue can be any of int/float/*string/bool, eg. any type that a column may take. 6 | type DataValue = interface{} 7 | 8 | // IntervalFunc is a function taking two parameters of the same DataValue and returning boolean stating if 9 | // the two values are part of the same interval or not. 10 | // 11 | // For example, x and y within one unit from each other (with x assumed to be <= y): 12 | type IntervalFunc = interface{} 13 | 14 | // It should be considered a private implementation detail and should never be 15 | // referenced or used directly outside of the QFrame code. To manipulate it 16 | // use the functions returning ConfigFunc below. 17 | type Config struct { 18 | PadValue DataValue 19 | IntervalColName string 20 | IntervalFunc IntervalFunc 21 | WindowSize int 22 | Position string // center/start/end 23 | } 24 | 25 | // ConfigFunc is a function that operates on a Config object. 26 | type ConfigFunc func(c *Config) 27 | 28 | func NewConfig(ff []ConfigFunc) (Config, error) { 29 | c := Config{ 30 | WindowSize: 1, 31 | Position: "center", 32 | } 33 | 34 | for _, fn := range ff { 35 | fn(&c) 36 | } 37 | 38 | if c.WindowSize <= 0 { 39 | return c, qerrors.New("Rolling config", "Window size must be positive, was %d", c.WindowSize) 40 | } 41 | 42 | if c.Position != "center" && c.Position != "start" && c.Position != "end" { 43 | return c, qerrors.New("Rolling config", "Position must be center/start/end, was %s", c.Position) 44 | } 45 | 46 | if c.IntervalFunc != nil && c.WindowSize != 1 { 47 | return c, qerrors.New("Rolling config", "Cannot set both interval function and window size") 48 | } 49 | 50 | return c, nil 51 | } 52 | 53 | // PadValue can be used to set the value to use in the beginning and/or end of the column to fill out any values 54 | // where fewer than WindowSize values are available. 55 | func PadValue(v DataValue) ConfigFunc { 56 | return func(c *Config) { 57 | c.PadValue = v 58 | } 59 | } 60 | 61 | // IntervalFunction can be used to set a dynamic interval based on the content of another column. 62 | // QFrame will include all rows from the start row of the window until (but not including) the first row that is 63 | // not part of the interval according to 'fn'. The first parameter passed to 'fn' is always the value at the start 64 | // of the window. 65 | // 66 | // For example, lets say that you have a time series with millisecond resolution integer timestamps in column 'ts' 67 | // and values in column 'value' that you would like to compute a rolling average over a minute for. 68 | // 69 | // In this case: 70 | // col = "ts", fn = func(tsStart, tsEnd int) bool { return tsEnd < tsStart + int(time.Minute / time.Millisecond)} 71 | func IntervalFunction(colName string, fn IntervalFunc) ConfigFunc { 72 | return func(c *Config) { 73 | c.IntervalColName = colName 74 | c.IntervalFunc = fn 75 | } 76 | } 77 | 78 | // WindowSize is used to set the size of the Window. By default this is 1. 79 | func WindowSize(s int) ConfigFunc { 80 | return func(c *Config) { 81 | c.WindowSize = s 82 | } 83 | } 84 | 85 | // Position is used to set where in window the resulting value should be inserted. 86 | // Valid values: start/center/end 87 | // Default value: center 88 | func Position(p string) ConfigFunc { 89 | return func(c *Config) { 90 | c.Position = p 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /config/sql/config.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | qsqlio "github.com/tobgu/qframe/internal/io/sql" 5 | ) 6 | 7 | type coerceType int 8 | 9 | const ( 10 | _ coerceType = iota 11 | // Int64ToBool casts an int64 type into a bool, 12 | // useful for handling SQLite INT -> BOOL. 13 | Int64ToBool 14 | StringToFloat 15 | ) 16 | 17 | // CoercePair casts the scanned value in Column 18 | // to another type. 19 | type CoercePair struct { 20 | Column string 21 | Type coerceType 22 | } 23 | 24 | func coerceFunc(cType coerceType) qsqlio.CoerceFunc { 25 | switch cType { 26 | case Int64ToBool: 27 | return qsqlio.Int64ToBool 28 | case StringToFloat: 29 | return qsqlio.StringToFloat 30 | } 31 | return nil 32 | } 33 | 34 | // Config holds configuration parameters for reading/writing to/from a SQL DB. 35 | type Config qsqlio.SQLConfig 36 | 37 | // ConfigFunc manipulates a Config object. 38 | type ConfigFunc func(*Config) 39 | 40 | // NewConfig creates a new config object. 41 | func NewConfig(ff []ConfigFunc) Config { 42 | conf := Config{} 43 | for _, f := range ff { 44 | f(&conf) 45 | } 46 | return conf 47 | } 48 | 49 | // Query is a Raw SQL statement which must return 50 | // appropriate types which can be inferred 51 | // and loaded into a new QFrame. 52 | func Query(query string) ConfigFunc { 53 | return func(c *Config) { 54 | c.Query = query 55 | } 56 | } 57 | 58 | // Table is the name of the table to be used 59 | // for generating an INSERT statement. 60 | func Table(table string) ConfigFunc { 61 | return func(c *Config) { 62 | c.Table = table 63 | } 64 | } 65 | 66 | // Postgres configures the query builder 67 | // to generate SQL that is compatible with 68 | // PostgreSQL. See github.com/lib/pq 69 | func Postgres() ConfigFunc { 70 | return func(c *Config) { 71 | EscapeChar('"')(c) 72 | Incrementing()(c) 73 | } 74 | } 75 | 76 | // SQLite configures the query builder to 77 | // generate SQL that is compatible with 78 | // SQLite3. See github.com/mattn/go-sqlite3 79 | func SQLite() ConfigFunc { 80 | return func(c *Config) { 81 | EscapeChar('"')(c) 82 | } 83 | } 84 | 85 | // MySQL configures the query builder to 86 | // generate SQL that is compatible with MySQL/MariaDB 87 | // See github.com/go-sql-driver/mysql 88 | func MySQL() ConfigFunc { 89 | return func(c *Config) { 90 | EscapeChar('`')(c) 91 | } 92 | } 93 | 94 | // Incrementing indicates the PostgreSQL variant 95 | // of parameter markers will be used, e.g. $1..$2. 96 | // The default style is ?..?. 97 | func Incrementing() ConfigFunc { 98 | return func(c *Config) { 99 | c.Incrementing = true 100 | } 101 | } 102 | 103 | // EscapeChar is a rune which column and table 104 | // names will be escaped with. PostgreSQL and SQLite 105 | // both accept double quotes "" while MariaDB/MySQL 106 | // only accept backticks. 107 | func EscapeChar(r rune) ConfigFunc { 108 | return func(c *Config) { 109 | c.EscapeChar = r 110 | } 111 | } 112 | 113 | // Coerce accepts a map of column names that 114 | // will be cast explicitly into the desired type. 115 | func Coerce(pairs ...CoercePair) ConfigFunc { 116 | return func(c *Config) { 117 | c.CoerceMap = map[string]qsqlio.CoerceFunc{} 118 | for _, pair := range pairs { 119 | c.CoerceMap[pair.Column] = coerceFunc(pair.Type) 120 | } 121 | } 122 | } 123 | 124 | // Precision sets the precision float64 types will 125 | // be rounded to when read from SQL. 126 | func Precision(i int) ConfigFunc { 127 | return func(c *Config) { 128 | c.Precision = i 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /contrib/gonum/qplot/config.go: -------------------------------------------------------------------------------- 1 | package qplot 2 | 3 | import ( 4 | "gonum.org/v1/plot" 5 | "gonum.org/v1/plot/vg" 6 | ) 7 | 8 | // FormatType indicates the output format 9 | // for the plot. 10 | type FormatType string 11 | 12 | const ( 13 | SVG = FormatType("svg") 14 | PNG = FormatType("png") 15 | ) 16 | 17 | // Config specifies the QPlot configuration. 18 | type Config struct { 19 | Plotters []PlotterFunc 20 | Width vg.Length 21 | Height vg.Length 22 | Format FormatType 23 | PlotConfig func(*plot.Plot) 24 | } 25 | 26 | // ConfigFunc is a functional option for configuring QPlot. 27 | type ConfigFunc func(*Config) 28 | 29 | // NewConfig returns a new QPlot config. 30 | func NewConfig(fns ...ConfigFunc) Config { 31 | cfg := Config{ 32 | // Defaults 33 | Format: PNG, 34 | Width: 245 * vg.Millimeter, 35 | Height: 127 * vg.Millimeter, 36 | } 37 | for _, fn := range fns { 38 | fn(&cfg) 39 | } 40 | return cfg 41 | } 42 | 43 | // Plotter appends a PlotterFunc to the plot. 44 | func Plotter(fn PlotterFunc) ConfigFunc { 45 | return func(cfg *Config) { 46 | cfg.Plotters = append(cfg.Plotters, fn) 47 | } 48 | } 49 | 50 | // Format sets the output format of the plot. 51 | func Format(format FormatType) ConfigFunc { 52 | return func(cfg *Config) { 53 | cfg.Format = format 54 | } 55 | } 56 | 57 | // PlotConfig is an optional function 58 | // which configures a plot.Plot prior 59 | // to serialization. 60 | func PlotConfig(fn func(*plot.Plot)) ConfigFunc { 61 | return func(cfg *Config) { 62 | cfg.PlotConfig = fn 63 | } 64 | } 65 | 66 | // Height sets the height of the plot. 67 | func Height(height vg.Length) ConfigFunc { 68 | return func(cfg *Config) { 69 | cfg.Height = height 70 | } 71 | } 72 | 73 | // Width sets the width of the plot. 74 | func Width(width vg.Length) ConfigFunc { 75 | return func(cfg *Config) { 76 | cfg.Width = width 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /contrib/gonum/qplot/doc.go: -------------------------------------------------------------------------------- 1 | // package qplot provides compatibility between QFrame and gonum.org/v1/plot 2 | package qplot 3 | -------------------------------------------------------------------------------- /contrib/gonum/qplot/helpers.go: -------------------------------------------------------------------------------- 1 | package qplot 2 | 3 | import ( 4 | "github.com/tobgu/qframe" 5 | "github.com/tobgu/qframe/types" 6 | ) 7 | 8 | // isNumCol checks to see if column contains a numeric 9 | // type and may be plotted. 10 | func isNumCol(col string, qf qframe.QFrame) bool { 11 | cType, ok := qf.ColumnTypeMap()[col] 12 | if !ok { 13 | return false 14 | } 15 | switch cType { 16 | case types.Float: 17 | return true 18 | case types.Int: 19 | return true 20 | } 21 | return false 22 | } 23 | -------------------------------------------------------------------------------- /contrib/gonum/qplot/plotter_funcs.go: -------------------------------------------------------------------------------- 1 | package qplot 2 | 3 | import ( 4 | "gonum.org/v1/plot" 5 | "gonum.org/v1/plot/plotter" 6 | "gonum.org/v1/plot/vg" 7 | 8 | "github.com/tobgu/qframe/qerrors" 9 | ) 10 | 11 | // PlotterFunc returns a plot.Plotter. 12 | type PlotterFunc func(plt *plot.Plot) (plot.Plotter, error) 13 | 14 | // LineConfig is an optional function which 15 | // configures a Line after creation. 16 | type LineConfig func(*plot.Plot, *plotter.Line) 17 | 18 | // LinePlotter returns a new PlotterFunc that plots a line 19 | func LinePlotter(xyer plotter.XYer, cfg LineConfig) PlotterFunc { 20 | return func(plt *plot.Plot) (plot.Plotter, error) { 21 | pltr, err := plotter.NewLine(xyer) 22 | if err != nil { 23 | return nil, qerrors.Propagate("LinePlotter", err) 24 | } 25 | if cfg != nil { 26 | cfg(plt, pltr) 27 | } 28 | return pltr, nil 29 | } 30 | } 31 | 32 | // BarConfig is an optional function which 33 | // configures a BarChart after creation. 34 | type BarConfig func(*plot.Plot, *plotter.BarChart) 35 | 36 | // BarPlotter returns a new PlotterFunc that plots a bar 37 | func BarPlotter(valuer plotter.Valuer, width vg.Length, cfg BarConfig) PlotterFunc { 38 | return func(plt *plot.Plot) (plot.Plotter, error) { 39 | pltr, err := plotter.NewBarChart(valuer, width) 40 | if err != nil { 41 | return nil, qerrors.Propagate("BarPlotter", err) 42 | } 43 | if cfg != nil { 44 | cfg(plt, pltr) 45 | } 46 | return pltr, nil 47 | } 48 | } 49 | 50 | // HistogramConfig is an optional function which 51 | // configures a Histogram after creation. 52 | type HistogramConfig func(*plot.Plot, *plotter.Histogram) 53 | 54 | // HistogramPlotter returns a new PlotterFunc that plots a histogram 55 | func HistogramPlotter(xyer plotter.XYer, n int, cfg HistogramConfig) PlotterFunc { 56 | return func(plt *plot.Plot) (plot.Plotter, error) { 57 | pltr, err := plotter.NewHistogram(xyer, n) 58 | if err != nil { 59 | return nil, qerrors.Propagate("HistogramPlotter", err) 60 | } 61 | if cfg != nil { 62 | cfg(plt, pltr) 63 | } 64 | return pltr, nil 65 | } 66 | } 67 | 68 | // PolygonConfig is an optional function which 69 | // configures a Polygon after creation. 70 | type PolygonConfig func(*plot.Plot, *plotter.Polygon) 71 | 72 | // PolygonPlotter returns a new PlotterFunc that plots a polygon 73 | func PolygonPlotter(xyer plotter.XYer, cfg PolygonConfig) PlotterFunc { 74 | return func(plt *plot.Plot) (plot.Plotter, error) { 75 | pltr, err := plotter.NewPolygon(xyer) 76 | if err != nil { 77 | return nil, qerrors.Propagate("PolygonPlotter", err) 78 | } 79 | if cfg != nil { 80 | cfg(plt, pltr) 81 | } 82 | return pltr, nil 83 | } 84 | } 85 | 86 | // ScatterConfig is an optional function which 87 | // configures a Scatter after creation. 88 | type ScatterConfig func(*plot.Plot, *plotter.Scatter) 89 | 90 | // ScatterPlotter returns a new PlotterFunc that plots a Scatter. 91 | func ScatterPlotter(xyer plotter.XYer, cfg ScatterConfig) PlotterFunc { 92 | return func(plt *plot.Plot) (plot.Plotter, error) { 93 | pltr, err := plotter.NewScatter(xyer) 94 | if err != nil { 95 | return nil, qerrors.Propagate("ScatterPlotter", err) 96 | } 97 | if cfg != nil { 98 | cfg(plt, pltr) 99 | } 100 | return pltr, nil 101 | } 102 | } 103 | 104 | // BoxPlotConfig is an optional function which 105 | // configures a BoxPlot after creation. 106 | type BoxPlotConfig func(*plot.Plot, *plotter.BoxPlot) 107 | 108 | // BoxPlot returns a new PlotterFunc that plots a BoxPlot. 109 | func BoxPlot(w vg.Length, loc float64, values plotter.Valuer, cfg BoxPlotConfig) PlotterFunc { 110 | return func(plt *plot.Plot) (plot.Plotter, error) { 111 | pltr, err := plotter.NewBoxPlot(w, loc, values) 112 | if err != nil { 113 | return nil, qerrors.Propagate("BoxPlot", err) 114 | } 115 | if cfg != nil { 116 | cfg(plt, pltr) 117 | } 118 | return pltr, nil 119 | } 120 | } 121 | 122 | // LabelsConfig is an optional function which 123 | // configures a Labels after creation. 124 | type LabelsConfig func(*plot.Plot, *plotter.Labels) 125 | 126 | // Labels returns a new PlotterFunc that plots a plotter.Labels. 127 | func Labels(labeller XYLabeller, cfg LabelsConfig) PlotterFunc { 128 | return func(plt *plot.Plot) (plot.Plotter, error) { 129 | pltr, err := plotter.NewLabels(labeller) 130 | if err != nil { 131 | return nil, qerrors.Propagate("Labels", err) 132 | } 133 | if cfg != nil { 134 | cfg(plt, pltr) 135 | } 136 | return pltr, nil 137 | } 138 | } 139 | 140 | // QuartConfig is an optional function which 141 | // configures a QuartPlot after creation. 142 | type QuartConfig func(*plot.Plot, *plotter.QuartPlot) 143 | 144 | // QuartPlot returns a new PlotterFunc that plots a QuartPlot. 145 | func QuartPlot(loc float64, values plotter.Valuer, cfg QuartConfig) PlotterFunc { 146 | return func(plt *plot.Plot) (plot.Plotter, error) { 147 | pltr, err := plotter.NewQuartPlot(loc, values) 148 | if err != nil { 149 | return nil, qerrors.Propagate("QuartPlot", err) 150 | } 151 | if cfg != nil { 152 | cfg(plt, pltr) 153 | } 154 | return pltr, nil 155 | } 156 | } 157 | 158 | // satisfies NewErrorBars function interface 159 | type errorBars struct { 160 | XYer 161 | YErrorer 162 | XErrorer 163 | } 164 | 165 | // YErrorBarsConfig is an optional function which 166 | // configures a YErrorBars after creation. 167 | type YErrorBarsConfig func(*plot.Plot, *plotter.YErrorBars) 168 | 169 | // YErrorBars returns a new PlotterFunc that plots a YErrorBars. 170 | func YErrorBars(xyer XYer, yerr YErrorer, cfg YErrorBarsConfig) PlotterFunc { 171 | return func(plt *plot.Plot) (plot.Plotter, error) { 172 | pltr, err := plotter.NewYErrorBars(errorBars{XYer: xyer, YErrorer: yerr}) 173 | if err != nil { 174 | return nil, qerrors.Propagate("YErrorBars", err) 175 | } 176 | if cfg != nil { 177 | cfg(plt, pltr) 178 | } 179 | return pltr, nil 180 | } 181 | } 182 | 183 | // XErrorBarsConfig is an optional function which 184 | // configures a XErrorBars after creation. 185 | type XErrorBarsConfig func(*plot.Plot, *plotter.XErrorBars) 186 | 187 | // XErrorBars returns a new PlotterFunc that plots a XErrorBars. 188 | func XErrorBars(xyer XYer, xerr XErrorer, cfg XErrorBarsConfig) PlotterFunc { 189 | return func(plt *plot.Plot) (plot.Plotter, error) { 190 | pltr, err := plotter.NewXErrorBars(errorBars{XYer: xyer, XErrorer: xerr}) 191 | if err != nil { 192 | return nil, qerrors.Propagate("XErrorBars", err) 193 | } 194 | if cfg != nil { 195 | cfg(plt, pltr) 196 | } 197 | return pltr, nil 198 | } 199 | } 200 | 201 | // TODO - These don't really make sense to include 202 | // in the API but can easily be added with a custom PlotterFunc 203 | // plotter.Function 204 | // plotter.HeatMap 205 | // plotter.Grid 206 | // plotter.Image 207 | // plotter.Sankey 208 | -------------------------------------------------------------------------------- /contrib/gonum/qplot/plotter_test.go: -------------------------------------------------------------------------------- 1 | package qplot_test 2 | 3 | import ( 4 | "gonum.org/v1/plot/plotter" 5 | 6 | "github.com/tobgu/qframe/contrib/gonum/qplot" 7 | ) 8 | 9 | var ( 10 | _ plotter.XYer = (*qplot.XYer)(nil) 11 | _ plotter.XYZer = (*qplot.XYZer)(nil) 12 | _ plotter.Labeller = (*qplot.Labeller)(nil) 13 | _ plotter.XYLabeller = (*qplot.XYLabeller)(nil) 14 | _ plotter.YErrorer = (*qplot.YErrorer)(nil) 15 | _ plotter.XErrorer = (*qplot.XErrorer)(nil) 16 | ) 17 | -------------------------------------------------------------------------------- /contrib/gonum/qplot/qplot.go: -------------------------------------------------------------------------------- 1 | package qplot 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | 7 | "gonum.org/v1/plot" 8 | 9 | "github.com/tobgu/qframe/qerrors" 10 | ) 11 | 12 | // QPlot is a abstraction over Gonum's plotting interface 13 | // for a less verbose experience in interactive environments 14 | // such as Jypter notebooks. 15 | type QPlot struct { 16 | Config 17 | } 18 | 19 | // NewQPlot returns a new QPlot. 20 | func NewQPlot(cfg Config) QPlot { 21 | return QPlot{Config: cfg} 22 | } 23 | 24 | // WriteTo writes a plot to an io.Writer 25 | func (qp QPlot) WriteTo(writer io.Writer) error { 26 | plt, err := plot.New() 27 | if err != nil { 28 | return err 29 | } 30 | for _, fn := range qp.Plotters { 31 | pltr, err := fn(plt) 32 | if err != nil { 33 | return qerrors.Propagate("WriteTo", err) 34 | } 35 | plt.Add(pltr) 36 | } 37 | if qp.PlotConfig != nil { 38 | qp.PlotConfig(plt) 39 | } 40 | w, err := plt.WriterTo(qp.Width, qp.Height, string(qp.Format)) 41 | if err != nil { 42 | return err 43 | } 44 | _, err = w.WriteTo(writer) 45 | return err 46 | } 47 | 48 | // Bytes returns a plot in the configured FormatType. 49 | func (qp QPlot) Bytes() ([]byte, error) { 50 | buf := bytes.NewBuffer(nil) 51 | err := qp.WriteTo(buf) 52 | if err != nil { 53 | return nil, qerrors.Propagate("Bytes", err) 54 | } 55 | return buf.Bytes(), nil 56 | } 57 | 58 | // MustBytes returns a plot in the configured FormatType 59 | // and panics if it encounters an error. 60 | func (qp QPlot) MustBytes() []byte { 61 | raw, err := qp.Bytes() 62 | if err != nil { 63 | panic(qerrors.Propagate("MustBytes", err)) 64 | } 65 | return raw 66 | } 67 | -------------------------------------------------------------------------------- /contrib/gonum/qplot/qplot_test.go: -------------------------------------------------------------------------------- 1 | package qplot_test 2 | 3 | import ( 4 | "crypto/sha256" 5 | "math" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "gonum.org/v1/gonum/stat" 11 | 12 | "gonum.org/v1/plot" 13 | "gonum.org/v1/plot/plotter" 14 | "gonum.org/v1/plot/plotutil" 15 | 16 | "github.com/tobgu/qframe" 17 | "github.com/tobgu/qframe/contrib/gonum/qplot" 18 | ) 19 | 20 | func panicOnErr(err error) { 21 | if err != nil { 22 | panic(err) 23 | } 24 | } 25 | 26 | // SlidingWindow returns a function that finds 27 | // the average of n time periods. 28 | func SlidingWindow(n int) func(float64) float64 { 29 | var buf []float64 30 | return func(value float64) float64 { 31 | if len(buf) < n { 32 | buf = append(buf, value) 33 | return value 34 | } 35 | buf = append(buf[1:], value) 36 | return stat.Mean(buf, nil) 37 | } 38 | } 39 | 40 | func ExampleQPlot() { 41 | fp, err := os.Open("testdata/GlobalTemperatures.csv") 42 | panicOnErr(err) 43 | defer fp.Close() 44 | 45 | qf := qframe.ReadCSV(fp) 46 | // Filter out any missing values 47 | qf = qf.Filter(qframe.Filter{ 48 | Column: "LandAndOceanAverageTemperature", 49 | Comparator: func(f float64) bool { return !math.IsNaN(f) }, 50 | }) 51 | // QFrame does not yet have native support for timeseries 52 | // data so we convert the timestamp to epoch time. 53 | qf = qf.Apply(qframe.Instruction{ 54 | Fn: func(ts *string) int { 55 | tm, err := time.Parse("2006-01-02", *ts) 56 | if err != nil { 57 | panic(err) 58 | } 59 | return int(tm.Unix()) 60 | }, 61 | SrcCol1: "dt", 62 | DstCol: "time", 63 | }) 64 | // Compute the average of the last 2 years of temperatures. 65 | window := SlidingWindow(24) 66 | qf = qf.Apply(qframe.Instruction{ 67 | Fn: func(value float64) float64 { 68 | return window(value) 69 | }, 70 | SrcCol1: "LandAndOceanAverageTemperature", 71 | DstCol: "SMA", 72 | }) 73 | 74 | // Create a new configuration 75 | cfg := qplot.NewConfig( 76 | // Configure the base Plot 77 | qplot.PlotConfig( 78 | func(plt *plot.Plot) { 79 | plt.Add(plotter.NewGrid()) 80 | plt.Title.Text = "Global Land & Ocean Temperatures" 81 | plt.X.Label.Text = "Time" 82 | plt.Y.Label.Text = "Temperature" 83 | }, 84 | ), 85 | // Plot each recorded temperature as a scatter plot 86 | qplot.Plotter( 87 | qplot.ScatterPlotter( 88 | qplot.MustNewXYer("time", "LandAndOceanAverageTemperature", qf), 89 | func(plt *plot.Plot, line *plotter.Scatter) { 90 | plt.Legend.Add("Temperature", line) 91 | line.Color = plotutil.Color(2) 92 | }, 93 | )), 94 | // Plot the SMA as a line 95 | qplot.Plotter( 96 | qplot.LinePlotter( 97 | qplot.MustNewXYer("time", "SMA", qf), 98 | func(plt *plot.Plot, line *plotter.Line) { 99 | plt.Legend.Add("SMA", line) 100 | line.Color = plotutil.Color(1) 101 | }, 102 | )), 103 | ) 104 | // Create a new QPlot 105 | qp := qplot.NewQPlot(cfg) 106 | // Write the plot to disk 107 | panicOnErr(os.WriteFile("testdata/GlobalTemperatures.png", qp.MustBytes(), 0644)) 108 | } 109 | 110 | func getHash(t *testing.T, path string) [32]byte { 111 | raw, err := os.ReadFile(path) 112 | panicOnErr(err) 113 | return sha256.Sum256(raw) 114 | } 115 | 116 | func TestQPlot(t *testing.T) { 117 | original := getHash(t, "testdata/GlobalTemperatures.png") 118 | ExampleQPlot() 119 | modified := getHash(t, "testdata/GlobalTemperatures.png") 120 | if original != modified { 121 | t.Errorf("output image has changed") 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /contrib/gonum/qplot/testdata/GlobalTemperatures.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tobgu/qframe/edb23855dc466ccea4b89d245aa06adc94cba431/contrib/gonum/qplot/testdata/GlobalTemperatures.png -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package qframe holds the main QFrame implementation and acts as an entry point to QFrame. 3 | */ 4 | package qframe 5 | -------------------------------------------------------------------------------- /filter/filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import "fmt" 4 | 5 | const ( 6 | // Gt = Greater than. 7 | Gt = ">" 8 | 9 | // Gte = Greater than equals. 10 | Gte = ">=" 11 | 12 | // Eq = Equals. 13 | Eq = "=" 14 | 15 | // Neq = Not equals. 16 | Neq = "!=" 17 | 18 | // Lt = Less than. 19 | Lt = "<" 20 | 21 | // Lte = Less than equals. 22 | Lte = "<=" 23 | 24 | // In = In given set. 25 | In = "in" 26 | 27 | // Nin = Not in given set. 28 | Nin = "not in" 29 | 30 | // IsNull = Is null. 31 | IsNull = "isnull" 32 | 33 | // IsNotNull = IsNotNull. 34 | IsNotNull = "isnotnull" 35 | ) 36 | 37 | // Inverse is a mapping from one comparator to its inverse. 38 | var Inverse = map[string]string{ 39 | Gt: Lte, 40 | Gte: Lt, 41 | Eq: Neq, 42 | Lt: Gte, 43 | Lte: Gt, 44 | In: Nin, 45 | Nin: In, 46 | IsNotNull: IsNull, 47 | IsNull: IsNotNull, 48 | } 49 | 50 | // Filter represents a filter to apply to a QFrame. 51 | // 52 | // Example using a built in comparator on a float column: 53 | // Filter{Comparator: ">", Column: "COL1", Arg: 1.2} 54 | // 55 | // Same example as above but with a custom function: 56 | // Filter{Comparator: func(f float64) bool { return f > 1.2 }, Column: "COL1"} 57 | type Filter struct { 58 | // Comparator may be a string referring to a built in or a function taking an argument matching the 59 | // column type and returning a bool bool. 60 | // 61 | // IMPORTANT: For pointer and reference types you must not assume that the data passed argument 62 | // to this function is valid after the function returns. If you plan to keep it around you need 63 | // to take a copy of the data. 64 | Comparator interface{} 65 | 66 | // Column is the name to filter by 67 | Column string 68 | 69 | // Arg is passed as argument to built in functions. 70 | Arg interface{} 71 | 72 | // Inverse can be set to true to negate the filter. 73 | Inverse bool 74 | } 75 | 76 | // String returns a string representation of the filter. 77 | func (f Filter) String() string { 78 | arg := f.Arg 79 | if s, ok := f.Arg.(string); ok { 80 | arg = fmt.Sprintf(`"%s"`, s) 81 | } 82 | 83 | s := fmt.Sprintf(`["%v", "%s", %v]`, f.Comparator, f.Column, arg) 84 | if f.Inverse { 85 | return fmt.Sprintf(`["!", %s]`, s) 86 | } 87 | return s 88 | } 89 | -------------------------------------------------------------------------------- /filter_test.go: -------------------------------------------------------------------------------- 1 | package qframe_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/tobgu/qframe" 8 | ) 9 | 10 | func f(column string, comparator string, arg interface{}) qframe.Filter { 11 | return qframe.Filter{Column: column, Comparator: comparator, Arg: arg} 12 | } 13 | 14 | func notf(column string, comparator string, arg interface{}) qframe.Filter { 15 | filter := f(column, comparator, arg) 16 | filter.Inverse = true 17 | return filter 18 | } 19 | 20 | func and(clauses ...qframe.FilterClause) qframe.AndClause { 21 | return qframe.And(clauses...) 22 | } 23 | 24 | func or(clauses ...qframe.FilterClause) qframe.OrClause { 25 | return qframe.Or(clauses...) 26 | } 27 | 28 | func not(clause qframe.FilterClause) qframe.NotClause { 29 | return qframe.Not(clause) 30 | } 31 | 32 | func TestFilter_Success(t *testing.T) { 33 | input := qframe.New(map[string]interface{}{ 34 | "COL1": []int{1, 2, 3, 4, 5}, 35 | }) 36 | 37 | eq := func(x int) qframe.FilterClause { 38 | return f("COL1", "=", x) 39 | } 40 | 41 | table := []struct { 42 | name string 43 | clause qframe.FilterClause 44 | expected []int 45 | }{ 46 | { 47 | "Single filter", 48 | f("COL1", ">", 3), 49 | []int{4, 5}, 50 | }, 51 | { 52 | "Simple or", 53 | or(f("COL1", ">", 3), f("COL1", "<", 2)), 54 | []int{1, 4, 5}, 55 | }, 56 | { 57 | "Simple and", 58 | and(f("COL1", "<", 3), f("COL1", ">", 1)), 59 | []int{2}, 60 | }, 61 | { 62 | "Or with nested and", 63 | or( 64 | and(f("COL1", "<", 3), f("COL1", ">", 1)), 65 | eq(5)), 66 | []int{2, 5}, 67 | }, 68 | { 69 | "Or with nested and, reverse clause", 70 | or(eq(5), 71 | and(f("COL1", "<", 3), f("COL1", ">", 1))), 72 | []int{2, 5}, 73 | }, 74 | { 75 | "Or with mixed nested or and clause", 76 | or(eq(1), or(eq(3), eq(4)), eq(5)), 77 | []int{1, 3, 4, 5}, 78 | }, 79 | { 80 | "Nested single clause", 81 | or(and(eq(4))), 82 | []int{4}, 83 | }, 84 | { 85 | "Not start", 86 | not(or(eq(1), eq(2))), 87 | []int{3, 4, 5}, 88 | }, 89 | { 90 | "Not end", 91 | not(or(eq(4), eq(5))), 92 | []int{1, 2, 3}, 93 | }, 94 | { 95 | "Not mixed", 96 | not(or(eq(4), eq(2))), 97 | []int{1, 3, 5}, 98 | }, 99 | { 100 | "Not empty", 101 | not(eq(6)), 102 | []int{1, 2, 3, 4, 5}, 103 | }, 104 | { 105 | "Not full", 106 | not(f("COL1", "<", 6)), 107 | []int{}, 108 | }, 109 | } 110 | 111 | for _, tc := range table { 112 | t.Run(fmt.Sprintf("Filter %s", tc.name), func(t *testing.T) { 113 | assertNotErr(t, tc.clause.Err()) 114 | out := input.Filter(tc.clause) 115 | assertNotErr(t, out.Err) 116 | assertEquals(t, qframe.New(map[string]interface{}{"COL1": tc.expected}), out) 117 | }) 118 | } 119 | } 120 | 121 | func TestFilter_ErrorColumnDoesNotExist(t *testing.T) { 122 | input := qframe.New(map[string]interface{}{ 123 | "COL1": []int{1, 2, 3, 4, 5}, 124 | }) 125 | 126 | colGt3 := f("COL", ">", 3) 127 | col1Gt3 := f("COL1", ">", 3) 128 | 129 | table := []qframe.FilterClause{ 130 | colGt3, 131 | or(col1Gt3, colGt3), 132 | and(col1Gt3, colGt3), 133 | and(col1Gt3, and(col1Gt3, colGt3)), 134 | or(and(col1Gt3, colGt3), col1Gt3), 135 | or(and(col1Gt3, col1Gt3), colGt3), 136 | } 137 | 138 | for i, c := range table { 139 | t.Run(fmt.Sprintf("Filter %d", i), func(t *testing.T) { 140 | out := input.Filter(c) 141 | assertErr(t, out.Err, "unknown column") 142 | }) 143 | } 144 | } 145 | 146 | func TestFilter_String(t *testing.T) { 147 | table := []struct { 148 | clause qframe.FilterClause 149 | expected string 150 | }{ 151 | {f("COL1", ">", 3), `[">", "COL1", 3]`}, 152 | {f("COL1", ">", "3"), `[">", "COL1", "3"]`}, 153 | {not(f("COL1", ">", 3)), `["!", [">", "COL1", 3]]`}, 154 | {notf("COL1", ">", 3), `["!", [">", "COL1", 3]]`}, 155 | {and(f("COL1", ">", 3)), `["and", [">", "COL1", 3]]`}, 156 | {or(f("COL1", ">", 3)), `["or", [">", "COL1", 3]]`}, 157 | { 158 | and(f("COL1", ">", 3), f("COL2", ">", 3)), 159 | `["and", [">", "COL1", 3], [">", "COL2", 3]]`, 160 | }, 161 | { 162 | or(f("COL1", ">", 3), f("COL2", ">", 3)), 163 | `["or", [">", "COL1", 3], [">", "COL2", 3]]`, 164 | }, 165 | } 166 | 167 | for _, tc := range table { 168 | t.Run(fmt.Sprintf("String %s", tc.expected), func(t *testing.T) { 169 | assertNotErr(t, tc.clause.Err()) 170 | if tc.expected != tc.clause.String() { 171 | t.Errorf("%s != %s", tc.expected, tc.clause.String()) 172 | } 173 | }) 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /function/bool.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import "strconv" 4 | 5 | // NotB returns the inverse of x 6 | func NotB(x bool) bool { 7 | return !x 8 | } 9 | 10 | // AndB returns the logical conjunction of x and y. 11 | func AndB(x, y bool) bool { 12 | return x && y 13 | } 14 | 15 | // OrB returns the logical disjunction of x and y. 16 | func OrB(x, y bool) bool { 17 | return x || y 18 | } 19 | 20 | // XorB returns the exclusive disjunction of x and y 21 | func XorB(x, y bool) bool { 22 | return (x && !y) || (!x && y) 23 | } 24 | 25 | // NandB returns the inverse logical conjunction of x and b. 26 | func NandB(x, y bool) bool { 27 | return !AndB(x, y) 28 | } 29 | 30 | // StrB returns the string representation of x. 31 | func StrB(x bool) *string { 32 | result := strconv.FormatBool(x) 33 | return &result 34 | } 35 | 36 | // IntB casts x to int. true => 1 and false => 0. 37 | func IntB(x bool) int { 38 | if x { 39 | return 1 40 | } 41 | 42 | return 0 43 | } 44 | -------------------------------------------------------------------------------- /function/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package function contains example functions that can be used in QFrame.Apply and QFrame.Eval. 3 | */ 4 | package function 5 | -------------------------------------------------------------------------------- /function/float.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import "fmt" 4 | 5 | // PlusF returns x + y. 6 | func PlusF(x, y float64) float64 { 7 | return x + y 8 | } 9 | 10 | // MinusF returns x - y. 11 | func MinusF(x, y float64) float64 { 12 | return x - y 13 | } 14 | 15 | // MulF returns x * y. 16 | func MulF(x, y float64) float64 { 17 | return x * y 18 | } 19 | 20 | // DivF returns x / y. y == 0 will cause panic. 21 | func DivF(x, y float64) float64 { 22 | return x / y 23 | } 24 | 25 | // StrF returns the string representation of x. 26 | func StrF(x float64) *string { 27 | result := fmt.Sprintf("%f", x) 28 | return &result 29 | } 30 | 31 | // IntF casts x to int. 32 | func IntF(x float64) int { 33 | return int(x) 34 | } 35 | -------------------------------------------------------------------------------- /function/int.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import "strconv" 4 | 5 | // AbsI returns the absolute value of x. 6 | func AbsI(x int) int { 7 | if x < 0 { 8 | return -x 9 | } 10 | return x 11 | } 12 | 13 | // PlusI returns x + y. 14 | func PlusI(x, y int) int { 15 | return x + y 16 | } 17 | 18 | // MinusI returns x - y. 19 | func MinusI(x, y int) int { 20 | return x - y 21 | } 22 | 23 | // MulI returns x * y. 24 | func MulI(x, y int) int { 25 | return x * y 26 | } 27 | 28 | // DivI returns x / y. y == 0 will cause panic. 29 | func DivI(x, y int) int { 30 | return x / y 31 | } 32 | 33 | // StrI returns the string representation of x. 34 | func StrI(x int) *string { 35 | result := strconv.Itoa(x) 36 | return &result 37 | } 38 | 39 | // FloatI casts x to float. 40 | func FloatI(x int) float64 { 41 | return float64(x) 42 | } 43 | 44 | // BoolI returns bool representation of x. x == 0 => false, all other values result in true. 45 | func BoolI(x int) bool { 46 | return x != 0 47 | } 48 | -------------------------------------------------------------------------------- /function/string.go: -------------------------------------------------------------------------------- 1 | package function 2 | 3 | import "strings" 4 | 5 | func nilSafe(f func(string) string) func(*string) *string { 6 | return func(s *string) *string { 7 | if s == nil { 8 | return nil 9 | } 10 | 11 | result := f(*s) 12 | return &result 13 | } 14 | } 15 | 16 | // UpperS returns the upper case representation of s. 17 | var UpperS = nilSafe(strings.ToUpper) 18 | 19 | // LowerS returns the lower case representation of s. 20 | var LowerS = nilSafe(strings.ToLower) 21 | 22 | // StrS returns s. 23 | // 24 | // This may appear useless but this can be used to convert enum columns to string 25 | // columns so that the two can be used as input to other functions. It is 26 | // currently not possible to combine enum and string as input. 27 | func StrS(s *string) *string { 28 | return s 29 | } 30 | 31 | // LenS returns the length of s. 32 | func LenS(s *string) int { 33 | if s == nil { 34 | return 0 35 | } 36 | 37 | return len(*s) 38 | } 39 | 40 | // ConcatS returns the concatenation of x and y. 41 | func ConcatS(x, y *string) *string { 42 | if x == nil { 43 | return y 44 | } 45 | 46 | if y == nil { 47 | return x 48 | } 49 | 50 | result := *x + *y 51 | return &result 52 | } 53 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tobgu/qframe 2 | 3 | require ( 4 | github.com/mauricelam/genny v0.0.0-20190320071652-0800202903e5 5 | gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4 6 | gonum.org/v1/plot v0.0.0-20180905080458-5f3c436ce602 7 | ) 8 | 9 | require ( 10 | github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af // indirect 11 | github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect 12 | github.com/jung-kurt/gofpdf v1.0.0 // indirect 13 | github.com/llgcode/draw2d v0.0.0-20180817132918-587a55234ca2 // indirect 14 | golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 // indirect 15 | gonum.org/v1/netlib v0.0.0-20180816165226-ebcc3d2662d3 // indirect 16 | ) 17 | 18 | go 1.20 19 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af h1:wVe6/Ea46ZMeNkQjjBW6xcqyQA/j5e0D6GytH95g0gQ= 2 | github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/go-gl/gl v0.0.0-20180407155706-68e253793080/go.mod h1:482civXOzJJCPzJ4ZOX/pwvXBWSnzD4OKMdH4ClKGbk= 5 | github.com/go-gl/glfw v0.0.0-20180426074136-46a8d530c326/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= 6 | github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= 7 | github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= 8 | github.com/jung-kurt/gofpdf v1.0.0 h1:EroSdlP9BOoL5ssLYf3uLJXhCQMMM2fFxCJDKA3RhnA= 9 | github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= 10 | github.com/llgcode/draw2d v0.0.0-20180817132918-587a55234ca2 h1:3xDkT1Tbsw2yDtKWUrROAlr15+dzp76kwucDvAPPnQo= 11 | github.com/llgcode/draw2d v0.0.0-20180817132918-587a55234ca2/go.mod h1:mVa0dA29Db2S4LVqDYLlsePDzRJLDfdhVZiI15uY0FA= 12 | github.com/llgcode/ps v0.0.0-20150911083025-f1443b32eedb h1:61ndUreYSlWFeCY44JxDDkngVoI7/1MVhEl98Nm0KOk= 13 | github.com/llgcode/ps v0.0.0-20150911083025-f1443b32eedb/go.mod h1:1l8ky+Ew27CMX29uG+a2hNOKpeNYEQjjtiALiBlFQbY= 14 | github.com/mauricelam/genny v0.0.0-20190320071652-0800202903e5 h1:PnFl95tWh3j7c5DebZG/TGsBJvbnHvPjK4lzltouI4Y= 15 | github.com/mauricelam/genny v0.0.0-20190320071652-0800202903e5/go.mod h1:i2AazGGunAlAR5u0zXGYVmIT7nnwE6j9lwKSMx7N6ko= 16 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 17 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 18 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 19 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 20 | golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 21 | golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f h1:9kQ594xxPWRNKfTOnPjPcgrIJ19zM3ic57aI7PbMyAA= 22 | golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 23 | golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI= 24 | golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= 25 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 26 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 27 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 28 | golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 29 | golang.org/x/tools v0.0.0-20190319232107-3f1ed9edd1b4/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 30 | gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4 h1:nYxTaCPaVoJbxx+vMVnsFb6kw5+6aJCx52m/lmM/Vog= 31 | gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= 32 | gonum.org/v1/netlib v0.0.0-20180816165226-ebcc3d2662d3 h1:4rNS4o1ZJGkBicts7UTCEjuP5+FXNKYqkMibESESFu8= 33 | gonum.org/v1/netlib v0.0.0-20180816165226-ebcc3d2662d3/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= 34 | gonum.org/v1/plot v0.0.0-20180905080458-5f3c436ce602 h1:vweNa1DjFigVk8LA0oMwfyi6WdNX+ox4vjGHvRqyHYY= 35 | gonum.org/v1/plot v0.0.0-20180905080458-5f3c436ce602/go.mod h1:VIQWjXleEHakKVLjfhAAXUy3mq0NuXvobpOBf0ZBZro= 36 | rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= 37 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 38 | -------------------------------------------------------------------------------- /grouper.go: -------------------------------------------------------------------------------- 1 | package qframe 2 | 3 | import ( 4 | "github.com/tobgu/qframe/internal/grouper" 5 | "github.com/tobgu/qframe/internal/icolumn" 6 | "github.com/tobgu/qframe/internal/index" 7 | "github.com/tobgu/qframe/qerrors" 8 | "github.com/tobgu/qframe/types" 9 | ) 10 | 11 | // GroupStats contains internal statistics for grouping. 12 | // Clients should not depend on this for any type of decision making. It is strictly "for info". 13 | // The layout may change if the underlying grouping mechanisms change. 14 | type GroupStats grouper.GroupStats 15 | 16 | // Grouper contains groups of rows produced by the QFrame.GroupBy function. 17 | type Grouper struct { 18 | indices []index.Int 19 | groupedColumns []string 20 | columns []namedColumn 21 | columnsByName map[string]namedColumn 22 | Err error 23 | Stats GroupStats 24 | } 25 | 26 | // Aggregation represents a function to apply to a column. 27 | type Aggregation struct { 28 | // Fn is the aggregation function to apply. 29 | // 30 | // IMPORTANT: For pointer and reference types you must not assume that the data passed argument 31 | // to this function is valid after the function returns. If you plan to keep it around you need 32 | // to take a copy of the data. 33 | Fn types.SliceFuncOrBuiltInId 34 | 35 | // Column is the name of the column to apply the aggregation to. 36 | Column string 37 | 38 | // As can be used to specify the destination column name, if not given defaults to the 39 | // value of Column. 40 | As string 41 | } 42 | 43 | // Aggregate applies the given aggregations to all row groups in the Grouper. 44 | // 45 | // Time complexity O(m*n) where m = number of aggregations, n = number of rows. 46 | func (g Grouper) Aggregate(aggs ...Aggregation) QFrame { 47 | if g.Err != nil { 48 | return QFrame{Err: g.Err} 49 | } 50 | 51 | // Loop over all groups and pick the first row in each of the groups. 52 | // This index will be used to populate the grouped by columns below. 53 | firstElementIx := make(index.Int, len(g.indices)) 54 | for i, ix := range g.indices { 55 | firstElementIx[i] = ix[0] 56 | } 57 | 58 | newColumnsByName := make(map[string]namedColumn, len(g.groupedColumns)+len(aggs)) 59 | newColumns := make([]namedColumn, 0, len(g.groupedColumns)+len(aggs)) 60 | for i, colName := range g.groupedColumns { 61 | col := g.columnsByName[colName] 62 | col.pos = i 63 | col.Column = col.Subset(firstElementIx) 64 | newColumnsByName[colName] = col 65 | newColumns = append(newColumns, col) 66 | } 67 | 68 | var err error 69 | for _, agg := range aggs { 70 | col, ok := g.columnsByName[agg.Column] 71 | if !ok { 72 | return QFrame{Err: qerrors.New("Aggregate", unknownCol(agg.Column))} 73 | } 74 | 75 | newColumnName := agg.Column 76 | if agg.As != "" { 77 | newColumnName = agg.As 78 | } 79 | col.name = newColumnName 80 | 81 | _, ok = newColumnsByName[newColumnName] 82 | if ok { 83 | return QFrame{Err: qerrors.New( 84 | "Aggregate", 85 | "cannot aggregate on column that is part of group by or is already an aggregate: %s", newColumnName)} 86 | } 87 | 88 | if agg.Fn == "count" { 89 | // Special convenience case for "count" which would normally require a cast from 90 | // any other type of column to int before being executed. 91 | counts := make([]int, len(g.indices)) 92 | for i, ix := range g.indices { 93 | counts[i] = len(ix) 94 | } 95 | 96 | col.Column = icolumn.New(counts) 97 | } else { 98 | col.Column, err = col.Aggregate(g.indices, agg.Fn) 99 | if err != nil { 100 | return QFrame{Err: qerrors.Propagate("Aggregate", err)} 101 | } 102 | } 103 | 104 | newColumnsByName[newColumnName] = col 105 | newColumns = append(newColumns, col) 106 | } 107 | 108 | return QFrame{columns: newColumns, columnsByName: newColumnsByName, index: index.NewAscending(uint32(len(g.indices)))} 109 | } 110 | 111 | // QFrames returns a slice of QFrame where each frame represents the content of one group. 112 | // 113 | // Time complexity O(n) where n = number of groups. 114 | func (g Grouper) QFrames() ([]QFrame, error) { 115 | if g.Err != nil { 116 | return nil, g.Err 117 | } 118 | 119 | baseFrame := QFrame{columns: g.columns, columnsByName: g.columnsByName, index: index.Int{}} 120 | result := make([]QFrame, len(g.indices)) 121 | for i, ix := range g.indices { 122 | result[i] = baseFrame.withIndex(ix) 123 | } 124 | return result, nil 125 | } 126 | -------------------------------------------------------------------------------- /internal/bcolumn/aggregations.go: -------------------------------------------------------------------------------- 1 | package bcolumn 2 | 3 | var aggregations = map[string]func([]bool) bool{ 4 | "majority": majority, 5 | } 6 | 7 | func majority(b []bool) bool { 8 | tCount, fCount := 0, 0 9 | for _, x := range b { 10 | if x { 11 | tCount++ 12 | } else { 13 | fCount++ 14 | } 15 | } 16 | 17 | return tCount > fCount 18 | } 19 | -------------------------------------------------------------------------------- /internal/bcolumn/column.go: -------------------------------------------------------------------------------- 1 | package bcolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/internal/column" 5 | "github.com/tobgu/qframe/internal/hash" 6 | "github.com/tobgu/qframe/internal/index" 7 | "github.com/tobgu/qframe/qerrors" 8 | "github.com/tobgu/qframe/types" 9 | "reflect" 10 | "strconv" 11 | ) 12 | 13 | func (c Comparable) Compare(i, j uint32) column.CompareResult { 14 | x, y := c.data[i], c.data[j] 15 | if x == y { 16 | return column.Equal 17 | } 18 | 19 | if x { 20 | return c.gtValue 21 | } 22 | 23 | return c.ltValue 24 | } 25 | 26 | func (c Comparable) Hash(i uint32, seed uint64) uint64 { 27 | if c.data[i] { 28 | b := [1]byte{1} 29 | return hash.HashBytes(b[:], seed) 30 | } 31 | 32 | b := [1]byte{0} 33 | return hash.HashBytes(b[:], seed) 34 | } 35 | 36 | func (c Column) DataType() types.DataType { 37 | return types.Bool 38 | } 39 | 40 | func (c Column) StringAt(i uint32, _ string) string { 41 | return strconv.FormatBool(c.data[i]) 42 | } 43 | 44 | func (c Column) AppendByteStringAt(buf []byte, i uint32) []byte { 45 | return strconv.AppendBool(buf, c.data[i]) 46 | } 47 | 48 | func (c Column) ByteSize() int { 49 | // Slice header + data 50 | return 2*8 + cap(c.data) 51 | } 52 | 53 | func (c Column) Equals(index index.Int, other column.Column, otherIndex index.Int) bool { 54 | otherI, ok := other.(Column) 55 | if !ok { 56 | return false 57 | } 58 | 59 | for ix, x := range index { 60 | if c.data[x] != otherI.data[otherIndex[ix]] { 61 | return false 62 | } 63 | } 64 | 65 | return true 66 | } 67 | 68 | func (c Column) filterBuiltIn(index index.Int, comparator string, comparatee interface{}, bIndex index.Bool) error { 69 | switch t := comparatee.(type) { 70 | case bool: 71 | compFunc, ok := filterFuncs[comparator] 72 | if !ok { 73 | return qerrors.New("filter bool", "invalid comparison operator for bool, %v", comparator) 74 | } 75 | compFunc(index, c.data, t, bIndex) 76 | case Column: 77 | compFunc, ok := filterFuncs2[comparator] 78 | if !ok { 79 | return qerrors.New("filter bool", "invalid comparison operator for bool, %v", comparator) 80 | } 81 | compFunc(index, c.data, t.data, bIndex) 82 | default: 83 | return qerrors.New("filter bool", "invalid comparison value type %v", reflect.TypeOf(comparatee)) 84 | } 85 | return nil 86 | } 87 | 88 | func (c Column) filterCustom1(index index.Int, fn func(bool) bool, bIndex index.Bool) { 89 | for i, x := range bIndex { 90 | if !x { 91 | bIndex[i] = fn(c.data[index[i]]) 92 | } 93 | } 94 | } 95 | 96 | func (c Column) filterCustom2(index index.Int, fn func(bool, bool) bool, comparatee interface{}, bIndex index.Bool) error { 97 | otherC, ok := comparatee.(Column) 98 | if !ok { 99 | return qerrors.New("filter bool", "expected comparatee to be bool column, was %v", reflect.TypeOf(comparatee)) 100 | } 101 | 102 | for i, x := range bIndex { 103 | if !x { 104 | bIndex[i] = fn(c.data[index[i]], otherC.data[index[i]]) 105 | } 106 | } 107 | 108 | return nil 109 | } 110 | 111 | func (c Column) Filter(index index.Int, comparator interface{}, comparatee interface{}, bIndex index.Bool) error { 112 | var err error 113 | switch t := comparator.(type) { 114 | case string: 115 | err = c.filterBuiltIn(index, t, comparatee, bIndex) 116 | case func(bool) bool: 117 | c.filterCustom1(index, t, bIndex) 118 | case func(bool, bool) bool: 119 | err = c.filterCustom2(index, t, comparatee, bIndex) 120 | default: 121 | err = qerrors.New("filter bool", "invalid filter type %v", reflect.TypeOf(comparator)) 122 | } 123 | return err 124 | } 125 | 126 | func (c Column) FunctionType() types.FunctionType { 127 | return types.FunctionTypeBool 128 | } 129 | 130 | func (c Column) Append(cols ...column.Column) (column.Column, error) { 131 | // TODO Append 132 | return nil, qerrors.New("Append", "Not implemented yet") 133 | } 134 | -------------------------------------------------------------------------------- /internal/bcolumn/column_gen.go: -------------------------------------------------------------------------------- 1 | // Code generated by genny. DO NOT EDIT. 2 | // This file was automatically generated by genny. 3 | // Any changes will be lost if this file is regenerated. 4 | // see https://github.com/mauricelam/genny 5 | 6 | package bcolumn 7 | 8 | // Code generated from template/column.go DO NOT EDIT 9 | 10 | import ( 11 | "fmt" 12 | 13 | "github.com/tobgu/qframe/config/rolling" 14 | 15 | "github.com/tobgu/qframe/internal/column" 16 | "github.com/tobgu/qframe/internal/index" 17 | "github.com/tobgu/qframe/qerrors" 18 | ) 19 | 20 | type Column struct { 21 | data []bool 22 | } 23 | 24 | func New(d []bool) Column { 25 | return Column{data: d} 26 | } 27 | 28 | func NewConst(val bool, count int) Column { 29 | var nullVal bool 30 | data := make([]bool, count) 31 | if val != nullVal { 32 | for i := range data { 33 | data[i] = val 34 | } 35 | } 36 | 37 | return Column{data: data} 38 | } 39 | 40 | func (c Column) fnName(name string) string { 41 | return fmt.Sprintf("%s.%s", c.DataType(), name) 42 | } 43 | 44 | // Apply single argument function. The result may be a column 45 | // of a different type than the current column. 46 | func (c Column) Apply1(fn interface{}, ix index.Int) (interface{}, error) { 47 | switch t := fn.(type) { 48 | case func(bool) int: 49 | result := make([]int, len(c.data)) 50 | for _, i := range ix { 51 | result[i] = t(c.data[i]) 52 | } 53 | return result, nil 54 | case func(bool) float64: 55 | result := make([]float64, len(c.data)) 56 | for _, i := range ix { 57 | result[i] = t(c.data[i]) 58 | } 59 | return result, nil 60 | case func(bool) bool: 61 | result := make([]bool, len(c.data)) 62 | for _, i := range ix { 63 | result[i] = t(c.data[i]) 64 | } 65 | return result, nil 66 | case func(bool) *string: 67 | result := make([]*string, len(c.data)) 68 | for _, i := range ix { 69 | result[i] = t(c.data[i]) 70 | } 71 | return result, nil 72 | default: 73 | return nil, qerrors.New(c.fnName("Apply1"), "cannot apply type %#v to column", fn) 74 | } 75 | } 76 | 77 | // Apply double argument function to two columns. Both columns must have the 78 | // same type. The resulting column will have the same type as this column. 79 | func (c Column) Apply2(fn interface{}, s2 column.Column, ix index.Int) (column.Column, error) { 80 | ss2, ok := s2.(Column) 81 | if !ok { 82 | return Column{}, qerrors.New(c.fnName("Apply2"), "invalid column type: %s", s2.DataType()) 83 | } 84 | 85 | t, ok := fn.(func(bool, bool) bool) 86 | if !ok { 87 | return Column{}, qerrors.New("Apply2", "invalid function type: %#v", fn) 88 | } 89 | 90 | result := make([]bool, len(c.data)) 91 | for _, i := range ix { 92 | result[i] = t(c.data[i], ss2.data[i]) 93 | } 94 | 95 | return New(result), nil 96 | } 97 | 98 | func (c Column) subset(index index.Int) Column { 99 | data := make([]bool, len(index)) 100 | for i, ix := range index { 101 | data[i] = c.data[ix] 102 | } 103 | 104 | return Column{data: data} 105 | } 106 | 107 | func (c Column) Subset(index index.Int) column.Column { 108 | return c.subset(index) 109 | } 110 | 111 | func (c Column) Comparable(reverse, equalNull, nullLast bool) column.Comparable { 112 | result := Comparable{data: c.data, ltValue: column.LessThan, gtValue: column.GreaterThan, nullLtValue: column.LessThan, nullGtValue: column.GreaterThan, equalNullValue: column.NotEqual} 113 | if reverse { 114 | result.ltValue, result.nullLtValue, result.gtValue, result.nullGtValue = 115 | result.gtValue, result.nullGtValue, result.ltValue, result.nullLtValue 116 | } 117 | 118 | if nullLast { 119 | result.nullLtValue, result.nullGtValue = result.nullGtValue, result.nullLtValue 120 | } 121 | 122 | if equalNull { 123 | result.equalNullValue = column.Equal 124 | } 125 | 126 | return result 127 | } 128 | 129 | func (c Column) String() string { 130 | return fmt.Sprintf("%v", c.data) 131 | } 132 | 133 | func (c Column) Len() int { 134 | return len(c.data) 135 | } 136 | 137 | func (c Column) Aggregate(indices []index.Int, fn interface{}) (column.Column, error) { 138 | var actualFn func([]bool) bool 139 | var ok bool 140 | 141 | switch t := fn.(type) { 142 | case string: 143 | actualFn, ok = aggregations[t] 144 | if !ok { 145 | return nil, qerrors.New(c.fnName("Aggregate"), "aggregation function %c is not defined for column", fn) 146 | } 147 | case func([]bool) bool: 148 | actualFn = t 149 | default: 150 | return nil, qerrors.New(c.fnName("Aggregate"), "invalid aggregation function type: %v", t) 151 | } 152 | 153 | data := make([]bool, 0, len(indices)) 154 | var buf []bool 155 | for _, ix := range indices { 156 | subS := c.subsetWithBuf(ix, &buf) 157 | data = append(data, actualFn(subS.data)) 158 | } 159 | 160 | return Column{data: data}, nil 161 | } 162 | 163 | func (c Column) subsetWithBuf(index index.Int, buf *[]bool) Column { 164 | if cap(*buf) < len(index) { 165 | *buf = make([]bool, 0, len(index)) 166 | } 167 | 168 | data := (*buf)[:0] 169 | for _, ix := range index { 170 | data = append(data, c.data[ix]) 171 | } 172 | 173 | return Column{data: data} 174 | } 175 | 176 | func (c Column) View(ix index.Int) View { 177 | return View{data: c.data, index: ix} 178 | } 179 | 180 | func (c Column) Rolling(fn interface{}, ix index.Int, config rolling.Config) (column.Column, error) { 181 | return c, nil 182 | } 183 | 184 | type Comparable struct { 185 | data []bool 186 | ltValue column.CompareResult 187 | nullLtValue column.CompareResult 188 | gtValue column.CompareResult 189 | nullGtValue column.CompareResult 190 | equalNullValue column.CompareResult 191 | } 192 | 193 | // View is a view into a column that allows access to individual elements by index. 194 | type View struct { 195 | data []bool 196 | index index.Int 197 | } 198 | 199 | // ItemAt returns the value at position i. 200 | func (v View) ItemAt(i int) bool { 201 | return v.data[v.index[i]] 202 | } 203 | 204 | // Len returns the column length. 205 | func (v View) Len() int { 206 | return len(v.index) 207 | } 208 | 209 | // Slice returns a slice containing a copy of the column data. 210 | func (v View) Slice() []bool { 211 | // TODO: This forces an alloc, as an alternative a slice could be taken 212 | // as input that can be (re)used by the client. Are there use cases 213 | // where this would actually make sense? 214 | result := make([]bool, v.Len()) 215 | for i, j := range v.index { 216 | result[i] = v.data[j] 217 | } 218 | return result 219 | } 220 | -------------------------------------------------------------------------------- /internal/bcolumn/doc_gen.go: -------------------------------------------------------------------------------- 1 | package bcolumn 2 | 3 | // Code generated from template/... DO NOT EDIT 4 | 5 | func Doc() string { 6 | return "\n Built in filters\n" + 7 | " !=\n" + 8 | " =\n" + 9 | 10 | "\n Built in aggregations\n" + 11 | " majority\n" + 12 | "\n" 13 | } 14 | -------------------------------------------------------------------------------- /internal/bcolumn/filters.go: -------------------------------------------------------------------------------- 1 | package bcolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/filter" 5 | "github.com/tobgu/qframe/internal/index" 6 | ) 7 | 8 | var filterFuncs = map[string]func(index.Int, []bool, bool, index.Bool){ 9 | filter.Eq: eq, 10 | filter.Neq: neq, 11 | } 12 | 13 | var filterFuncs2 = map[string]func(index.Int, []bool, []bool, index.Bool){ 14 | filter.Eq: eq2, 15 | filter.Neq: neq2, 16 | } 17 | -------------------------------------------------------------------------------- /internal/bcolumn/filters_gen.go: -------------------------------------------------------------------------------- 1 | package bcolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/internal/index" 5 | ) 6 | 7 | // Code generated from template/... DO NOT EDIT 8 | 9 | func eq(index index.Int, column []bool, comp bool, bIndex index.Bool) { 10 | for i, x := range bIndex { 11 | if !x { 12 | bIndex[i] = column[index[i]] == comp 13 | } 14 | } 15 | } 16 | 17 | func neq(index index.Int, column []bool, comp bool, bIndex index.Bool) { 18 | for i, x := range bIndex { 19 | if !x { 20 | bIndex[i] = column[index[i]] != comp 21 | } 22 | } 23 | } 24 | 25 | func eq2(index index.Int, column []bool, compCol []bool, bIndex index.Bool) { 26 | for i, x := range bIndex { 27 | if !x { 28 | pos := index[i] 29 | bIndex[i] = column[pos] == compCol[pos] 30 | } 31 | } 32 | } 33 | 34 | func neq2(index index.Int, column []bool, compCol []bool, bIndex index.Bool) { 35 | for i, x := range bIndex { 36 | if !x { 37 | pos := index[i] 38 | bIndex[i] = column[pos] != compCol[pos] 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /internal/bcolumn/generator.go: -------------------------------------------------------------------------------- 1 | package bcolumn 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/tobgu/qframe/filter" 7 | "github.com/tobgu/qframe/internal/maps" 8 | "github.com/tobgu/qframe/internal/template" 9 | ) 10 | 11 | //go:generate qfgenerate -source=bfilter -dst-file=filters_gen.go 12 | //go:generate qfgenerate -source=bdoc -dst-file=doc_gen.go 13 | 14 | func spec(name, operator, templateStr string) template.Spec { 15 | return template.Spec{ 16 | Name: name, 17 | Template: templateStr, 18 | Values: map[string]interface{}{"name": name, "dataType": "bool", "operator": operator}} 19 | } 20 | 21 | func colConstComparison(name, operator string) template.Spec { 22 | return spec(name, operator, template.BasicColConstComparison) 23 | } 24 | 25 | func colColComparison(name, operator string) template.Spec { 26 | return spec(name, operator, template.BasicColColComparison) 27 | } 28 | 29 | func GenerateFilters() (*bytes.Buffer, error) { 30 | // If adding more filters here make sure to also add a reference to them 31 | // in the corresponding filter map so that they can be looked up. 32 | return template.GenerateFilters("bcolumn", []template.Spec{ 33 | colConstComparison("eq", "=="), // Go eq ("==") differs from qframe eq ("=") 34 | colConstComparison("neq", filter.Neq), 35 | colColComparison("eq2", "=="), // Go eq ("==") differs from qframe eq ("=") 36 | colColComparison("neq2", filter.Neq), 37 | }) 38 | } 39 | 40 | func GenerateDoc() (*bytes.Buffer, error) { 41 | return template.GenerateDocs( 42 | "bcolumn", 43 | maps.StringKeys(filterFuncs, filterFuncs2), 44 | maps.StringKeys(aggregations)) 45 | } 46 | -------------------------------------------------------------------------------- /internal/column/column.go: -------------------------------------------------------------------------------- 1 | package column 2 | 3 | import ( 4 | "fmt" 5 | "github.com/tobgu/qframe/config/rolling" 6 | 7 | "github.com/tobgu/qframe/internal/index" 8 | "github.com/tobgu/qframe/types" 9 | ) 10 | 11 | type Column interface { 12 | fmt.Stringer 13 | Filter(index index.Int, comparator interface{}, comparatee interface{}, bIndex index.Bool) error 14 | Subset(index index.Int) Column 15 | Append(cols ...Column) (Column, error) 16 | Equals(index index.Int, other Column, otherIndex index.Int) bool 17 | Comparable(reverse, equalNull, nullLast bool) Comparable 18 | Aggregate(indices []index.Int, fn interface{}) (Column, error) 19 | StringAt(i uint32, naRep string) string 20 | AppendByteStringAt(buf []byte, i uint32) []byte 21 | ByteSize() int 22 | Len() int 23 | 24 | Apply1(fn interface{}, ix index.Int) (interface{}, error) 25 | Apply2(fn interface{}, s2 Column, ix index.Int) (Column, error) 26 | 27 | Rolling(fn interface{}, ix index.Int, config rolling.Config) (Column, error) 28 | 29 | FunctionType() types.FunctionType 30 | DataType() types.DataType 31 | } 32 | 33 | type CompareResult byte 34 | 35 | const ( 36 | LessThan CompareResult = iota 37 | GreaterThan 38 | Equal 39 | 40 | // Used when comparing null with null 41 | NotEqual 42 | ) 43 | 44 | type Comparable interface { 45 | Compare(i, j uint32) CompareResult 46 | Hash(i uint32, seed uint64) uint64 47 | } 48 | -------------------------------------------------------------------------------- /internal/ecolumn/bitset.go: -------------------------------------------------------------------------------- 1 | package ecolumn 2 | 3 | import "fmt" 4 | 5 | // Helper type for multi value filtering 6 | type bitset [4]uint64 7 | 8 | func (s *bitset) set(val enumVal) { 9 | s[val>>6] |= 1 << (val & 0x3F) 10 | } 11 | 12 | func (s *bitset) isSet(val enumVal) bool { 13 | return s[val>>6]&(1<<(val&0x3F)) > 0 14 | } 15 | 16 | func (s *bitset) String() string { 17 | return fmt.Sprintf("%X %X %X %X", s[3], s[2], s[1], s[0]) 18 | } 19 | -------------------------------------------------------------------------------- /internal/ecolumn/doc_gen.go: -------------------------------------------------------------------------------- 1 | package ecolumn 2 | 3 | // Code generated from template/... DO NOT EDIT 4 | 5 | func Doc() string { 6 | return "\n Built in filters\n" + 7 | " !=\n" + 8 | " <\n" + 9 | " <=\n" + 10 | " =\n" + 11 | " >\n" + 12 | " >=\n" + 13 | " ilike\n" + 14 | " in\n" + 15 | " isnotnull\n" + 16 | " isnull\n" + 17 | " like\n" + 18 | 19 | "\n Built in aggregations\n" + 20 | "\n" 21 | } 22 | -------------------------------------------------------------------------------- /internal/ecolumn/filters.go: -------------------------------------------------------------------------------- 1 | package ecolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/filter" 5 | "github.com/tobgu/qframe/internal/index" 6 | qfstrings "github.com/tobgu/qframe/internal/strings" 7 | "github.com/tobgu/qframe/qerrors" 8 | ) 9 | 10 | var filterFuncs0 = map[string]func(index.Int, []enumVal, index.Bool){ 11 | filter.IsNull: isNull, 12 | filter.IsNotNull: isNotNull, 13 | } 14 | 15 | var filterFuncs1 = map[string]func(index.Int, []enumVal, enumVal, index.Bool){ 16 | filter.Gt: gt, 17 | filter.Gte: gte, 18 | filter.Lt: lt, 19 | filter.Lte: lte, 20 | filter.Eq: eq, 21 | filter.Neq: neq, 22 | } 23 | 24 | var filterFuncs2 = map[string]func(index.Int, []enumVal, []enumVal, index.Bool){ 25 | filter.Gt: gt2, 26 | filter.Gte: gte2, 27 | filter.Lt: lt2, 28 | filter.Lte: lte2, 29 | filter.Eq: eq2, 30 | filter.Neq: neq2, 31 | } 32 | 33 | var multiFilterFuncs = map[string]func(comparatee string, values []string) (*bitset, error){ 34 | "like": like, 35 | "ilike": ilike, 36 | } 37 | 38 | var multiInputFilterFuncs = map[string]func(comparatee qfstrings.StringSet, values []string) *bitset{ 39 | "in": in, 40 | } 41 | 42 | func like(comp string, values []string) (*bitset, error) { 43 | return filterLike(comp, values, true) 44 | } 45 | 46 | func ilike(comp string, values []string) (*bitset, error) { 47 | return filterLike(comp, values, false) 48 | } 49 | 50 | func filterLike(comp string, values []string, caseSensitive bool) (*bitset, error) { 51 | matcher, err := qfstrings.NewMatcher(comp, caseSensitive) 52 | if err != nil { 53 | return nil, qerrors.Propagate("enum like", err) 54 | } 55 | 56 | bset := &bitset{} 57 | for i, v := range values { 58 | if matcher.Matches(v) { 59 | bset.set(enumVal(i)) 60 | } 61 | } 62 | 63 | return bset, nil 64 | } 65 | 66 | func in(comp qfstrings.StringSet, values []string) *bitset { 67 | bset := &bitset{} 68 | for i, v := range values { 69 | if comp.Contains(v) { 70 | bset.set(enumVal(i)) 71 | } 72 | } 73 | 74 | return bset 75 | } 76 | 77 | func neq(index index.Int, column []enumVal, comparatee enumVal, bIndex index.Bool) { 78 | for i, x := range bIndex { 79 | if !x { 80 | enum := column[index[i]] 81 | bIndex[i] = enum.isNull() || enum.compVal() != comparatee.compVal() 82 | } 83 | } 84 | } 85 | 86 | func neq2(index index.Int, col, col2 []enumVal, bIndex index.Bool) { 87 | for i, x := range bIndex { 88 | if !x { 89 | enum, enum2 := col[index[i]], col2[index[i]] 90 | bIndex[i] = enum.isNull() || enum2.isNull() || enum.compVal() != enum2.compVal() 91 | } 92 | } 93 | } 94 | 95 | func isNull(index index.Int, col []enumVal, bIndex index.Bool) { 96 | for i, x := range bIndex { 97 | if !x { 98 | enum := col[index[i]] 99 | bIndex[i] = enum.isNull() 100 | } 101 | } 102 | } 103 | 104 | func isNotNull(index index.Int, col []enumVal, bIndex index.Bool) { 105 | for i, x := range bIndex { 106 | if !x { 107 | enum := col[index[i]] 108 | bIndex[i] = !enum.isNull() 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /internal/ecolumn/filters_gen.go: -------------------------------------------------------------------------------- 1 | package ecolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/internal/index" 5 | ) 6 | 7 | // Code generated from template/... DO NOT EDIT 8 | 9 | func lt(index index.Int, column []enumVal, comparatee enumVal, bIndex index.Bool) { 10 | for i, x := range bIndex { 11 | if !x { 12 | enum := column[index[i]] 13 | bIndex[i] = !enum.isNull() && enum.compVal() < comparatee.compVal() 14 | } 15 | } 16 | } 17 | 18 | func lte(index index.Int, column []enumVal, comparatee enumVal, bIndex index.Bool) { 19 | for i, x := range bIndex { 20 | if !x { 21 | enum := column[index[i]] 22 | bIndex[i] = !enum.isNull() && enum.compVal() <= comparatee.compVal() 23 | } 24 | } 25 | } 26 | 27 | func gt(index index.Int, column []enumVal, comparatee enumVal, bIndex index.Bool) { 28 | for i, x := range bIndex { 29 | if !x { 30 | enum := column[index[i]] 31 | bIndex[i] = !enum.isNull() && enum.compVal() > comparatee.compVal() 32 | } 33 | } 34 | } 35 | 36 | func gte(index index.Int, column []enumVal, comparatee enumVal, bIndex index.Bool) { 37 | for i, x := range bIndex { 38 | if !x { 39 | enum := column[index[i]] 40 | bIndex[i] = !enum.isNull() && enum.compVal() >= comparatee.compVal() 41 | } 42 | } 43 | } 44 | 45 | func eq(index index.Int, column []enumVal, comparatee enumVal, bIndex index.Bool) { 46 | for i, x := range bIndex { 47 | if !x { 48 | enum := column[index[i]] 49 | bIndex[i] = !enum.isNull() && enum.compVal() == comparatee.compVal() 50 | } 51 | } 52 | } 53 | 54 | func lt2(index index.Int, col, col2 []enumVal, bIndex index.Bool) { 55 | for i, x := range bIndex { 56 | if !x { 57 | enum, enum2 := col[index[i]], col2[index[i]] 58 | bIndex[i] = !enum.isNull() && !enum2.isNull() && enum.compVal() < enum2.compVal() 59 | } 60 | } 61 | } 62 | 63 | func lte2(index index.Int, col, col2 []enumVal, bIndex index.Bool) { 64 | for i, x := range bIndex { 65 | if !x { 66 | enum, enum2 := col[index[i]], col2[index[i]] 67 | bIndex[i] = !enum.isNull() && !enum2.isNull() && enum.compVal() <= enum2.compVal() 68 | } 69 | } 70 | } 71 | 72 | func gt2(index index.Int, col, col2 []enumVal, bIndex index.Bool) { 73 | for i, x := range bIndex { 74 | if !x { 75 | enum, enum2 := col[index[i]], col2[index[i]] 76 | bIndex[i] = !enum.isNull() && !enum2.isNull() && enum.compVal() > enum2.compVal() 77 | } 78 | } 79 | } 80 | 81 | func gte2(index index.Int, col, col2 []enumVal, bIndex index.Bool) { 82 | for i, x := range bIndex { 83 | if !x { 84 | enum, enum2 := col[index[i]], col2[index[i]] 85 | bIndex[i] = !enum.isNull() && !enum2.isNull() && enum.compVal() >= enum2.compVal() 86 | } 87 | } 88 | } 89 | 90 | func eq2(index index.Int, col, col2 []enumVal, bIndex index.Bool) { 91 | for i, x := range bIndex { 92 | if !x { 93 | enum, enum2 := col[index[i]], col2[index[i]] 94 | bIndex[i] = !enum.isNull() && !enum2.isNull() && enum.compVal() == enum2.compVal() 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /internal/ecolumn/generator.go: -------------------------------------------------------------------------------- 1 | package ecolumn 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/tobgu/qframe/filter" 7 | "github.com/tobgu/qframe/internal/maps" 8 | "github.com/tobgu/qframe/internal/template" 9 | ) 10 | 11 | //go:generate qfgenerate -source=efilter -dst-file=filters_gen.go 12 | //go:generate qfgenerate -source=edoc -dst-file=doc_gen.go 13 | 14 | const basicColConstComparison = ` 15 | func {{.name}}(index index.Int, column []enumVal, comparatee enumVal, bIndex index.Bool) { 16 | for i, x := range bIndex { 17 | if !x { 18 | enum := column[index[i]] 19 | bIndex[i] = !enum.isNull() && enum.compVal() {{.operator}} comparatee.compVal() 20 | } 21 | } 22 | } 23 | ` 24 | 25 | const basicColColComparison = ` 26 | func {{.name}}(index index.Int, col, col2 []enumVal, bIndex index.Bool) { 27 | for i, x := range bIndex { 28 | if !x { 29 | enum, enum2 := col[index[i]], col2[index[i]] 30 | bIndex[i] = !enum.isNull() && !enum2.isNull() && enum.compVal() {{.operator}} enum2.compVal() 31 | } 32 | } 33 | } 34 | ` 35 | 36 | func spec(name, operator, templateStr string) template.Spec { 37 | return template.Spec{ 38 | Name: name, 39 | Template: templateStr, 40 | Values: map[string]interface{}{"name": name, "operator": operator}} 41 | } 42 | 43 | func colConstComparison(name, operator string) template.Spec { 44 | return spec(name, operator, basicColConstComparison) 45 | } 46 | 47 | func colColComparison(name, operator string) template.Spec { 48 | return spec(name, operator, basicColColComparison) 49 | } 50 | 51 | func GenerateFilters() (*bytes.Buffer, error) { 52 | // If adding more filters here make sure to also add a reference to them 53 | // in the corresponding filter map so that they can be looked up. 54 | return template.GenerateFilters("ecolumn", []template.Spec{ 55 | colConstComparison("lt", filter.Lt), 56 | colConstComparison("lte", filter.Lte), 57 | colConstComparison("gt", filter.Gt), 58 | colConstComparison("gte", filter.Gte), 59 | colConstComparison("eq", "=="), // Go eq ("==") differs from qframe eq ("=") 60 | colColComparison("lt2", filter.Lt), 61 | colColComparison("lte2", filter.Lte), 62 | colColComparison("gt2", filter.Gt), 63 | colColComparison("gte2", filter.Gte), 64 | colColComparison("eq2", "=="), // Go eq ("==") differs from qframe eq ("=") 65 | }) 66 | } 67 | 68 | func GenerateDoc() (*bytes.Buffer, error) { 69 | return template.GenerateDocs( 70 | "ecolumn", 71 | maps.StringKeys(filterFuncs0, filterFuncs1, filterFuncs2, multiFilterFuncs, multiInputFilterFuncs), 72 | maps.StringKeys()) 73 | } 74 | -------------------------------------------------------------------------------- /internal/ecolumn/view.go: -------------------------------------------------------------------------------- 1 | package ecolumn 2 | 3 | import "github.com/tobgu/qframe/internal/index" 4 | 5 | type View struct { 6 | column Column 7 | index index.Int 8 | } 9 | 10 | func (v View) ItemAt(i int) *string { 11 | return v.column.stringPtrAt(v.index[i]) 12 | } 13 | 14 | func (v View) Len() int { 15 | return len(v.index) 16 | } 17 | 18 | func (v View) Slice() []*string { 19 | result := make([]*string, v.Len()) 20 | for i := range v.index { 21 | result[i] = v.ItemAt(i) 22 | } 23 | 24 | return result 25 | } 26 | -------------------------------------------------------------------------------- /internal/fastcsv/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Craig Weber 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /internal/fastcsv/README.md: -------------------------------------------------------------------------------- 1 | This is copy of https://bitbucket.org/weberc2/fastcsv/ which has 2 | been enhanced with a proposed bug fix with the patch attached to 3 | issue https://bitbucket.org/weberc2/fastcsv/issues/6/some-fields-quoted-fails 4 | applied. 5 | 6 | It also contains support for configurable (ascii) delimiter. 7 | 8 | The original licence (MIT) can be found in the LICENCE file in the 9 | current directory. -------------------------------------------------------------------------------- /internal/fastcsv/csv_test.go: -------------------------------------------------------------------------------- 1 | package fastcsv 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "fmt" 7 | "io" 8 | "os" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | func toStrings(bs [][]byte) []string { 14 | strs := make([]string, 0, len(bs)) 15 | for _, b := range bs { 16 | strs = append(strs, string(b)) 17 | } 18 | return strs 19 | } 20 | 21 | func quote(strs []string) []string { 22 | out := make([]string, 0, len(strs)) 23 | for _, s := range strs { 24 | out = append(out, fmt.Sprintf("\"%s\"", s)) 25 | } 26 | return out 27 | } 28 | 29 | func compareLine(line [][]byte, wanted ...string) error { 30 | if len(line) != len(wanted) { 31 | return fmt.Errorf( 32 | "Wanted [%s]; got [%s]", 33 | strings.Join(quote(wanted), ", "), 34 | strings.Join(quote(toStrings(line)), ", "), 35 | ) 36 | } 37 | for i, s := range toStrings(line) { 38 | if s != wanted[i] { 39 | return fmt.Errorf( 40 | "Mismatch at item %d; wanted '%s'; got '%s'", 41 | i, 42 | wanted[i], 43 | s, 44 | ) 45 | } 46 | } 47 | return nil 48 | } 49 | 50 | func TestRead(t *testing.T) { 51 | testCases := []struct { 52 | Title string 53 | Input string 54 | Wanted [][]string 55 | BufferCap int 56 | }{{ 57 | Title: "OneRow", 58 | Input: "abc,def,ghi", 59 | Wanted: [][]string{{"abc", "def", "ghi"}}, 60 | }, { 61 | Title: "MultipleLines", 62 | Input: "abc,def\n1234,56", 63 | Wanted: [][]string{{"abc", "def"}, {"1234", "56"}}, 64 | }, { 65 | Title: "QuotedField", 66 | Input: "\"abc\",\"123\",\"456\"", 67 | Wanted: [][]string{{"abc", "123", "456"}}, 68 | }, { 69 | Title: "QuotedFieldMultipleLines", 70 | Input: "\"abc\",\"123\"\n\"def\",\"456\"", 71 | Wanted: [][]string{{"abc", "123"}, {"def", "456"}}, 72 | }, { 73 | Title: "SomeQuoted", 74 | Input: `hello,"hello2",hello3`, 75 | Wanted: [][]string{{"hello", "hello2", "hello3"}}, 76 | }, { 77 | Title: "QuotedFieldsWithComma", 78 | Input: "\"a,b,c\",\"d,e,f\"", 79 | Wanted: [][]string{{"a,b,c", "d,e,f"}}, 80 | }, { 81 | Title: "QuotedFieldsWithNewLine", 82 | Input: "\"a\nb\nc\"", 83 | Wanted: [][]string{{"a\nb\nc"}}, 84 | }, { 85 | Title: "QuotedFieldsWithEscapedQuotes", 86 | Input: "\"a\"\"b\"", 87 | Wanted: [][]string{{"a\"b"}}, 88 | }, { 89 | Title: "QuotedFieldsWithConsecutiveEscapedQuotes", 90 | Input: "\"\"\"\"\"a\"\"\"\"\"", 91 | Wanted: [][]string{{"\"\"a\"\""}}, 92 | }, { 93 | Title: "QuotedFieldsWithEscapeQuotesAndMultipleLines", 94 | Input: "abc,\"1\"\"\n2\"", 95 | Wanted: [][]string{{"abc", "1\"\n2"}}, 96 | }, { 97 | Title: "QuotedFieldsWithConsecutiveEscapedQuotesAndMultipleLines", 98 | Input: "abc,\"\"\"\"\"a\"\"\"\"\nb\"", 99 | Wanted: [][]string{{"abc", "\"\"a\"\"\nb"}}, 100 | }, { 101 | Title: "QuotedFieldsWithLinesLongerThanBuffer", 102 | Input: "\"abc\",\"def\",\"ghi\"", 103 | Wanted: [][]string{{"abc", "def", "ghi"}}, 104 | BufferCap: 4, 105 | }, { 106 | Title: "TrailingNewline", 107 | Input: "a,b,c\n", 108 | Wanted: [][]string{{"a", "b", "c"}}, 109 | }, { 110 | Title: "EmptyMiddleLine", 111 | Input: "a,b\n\nc,d", 112 | Wanted: [][]string{{"a", "b"}, {""}, {"c", "d"}}, 113 | }, { 114 | Title: "CRLF", 115 | Input: "a,b,c\r\nd,e,f", 116 | Wanted: [][]string{{"a", "b", "c"}, {"d", "e", "f"}}, 117 | }, { 118 | Title: "CRLF with quote in last column", 119 | Input: "\"a\"\r\n\"b\"", 120 | Wanted: [][]string{{"a"}, {"b"}}, 121 | }, { 122 | Title: "CRLF with quote in last column with EOF", 123 | Input: "\"a\"\r\n", 124 | Wanted: [][]string{{"a"}}, 125 | }} 126 | 127 | for _, testCase := range testCases { 128 | t.Run(testCase.Title, func(t *testing.T) { 129 | r := Reader{ 130 | fields: fields{ 131 | // initialize with a deliberately small buffer so we get 132 | // good coverage of i/o buffering 133 | buffer: bufferedReader{ 134 | r: strings.NewReader(testCase.Input), 135 | data: make([]byte, 0, testCase.BufferCap), 136 | }, 137 | delimiter: ',', 138 | }, 139 | fieldsBuffer: make([][]byte, 0, 16), 140 | } 141 | for i, wantedLine := range testCase.Wanted { 142 | fields, err := r.Read() 143 | if err != nil { 144 | t.Fatalf("Unexpected error on line %d: %v", i+1, err) 145 | } 146 | if err := compareLine(fields, wantedLine...); err != nil { 147 | t.Fatalf("Mismatch on line %d: %v", i+1, err) 148 | } 149 | } 150 | if _, err := r.Read(); err != io.EOF { 151 | t.Fatal("Wanted io.EOF; got:", err) 152 | } 153 | }) 154 | } 155 | } 156 | 157 | func BenchmarkRead(b *testing.B) { 158 | data, err := os.ReadFile("testdata/fl_insurance.csv") 159 | if err != nil { 160 | b.Fatal(err) 161 | } 162 | quotedData, err := os.ReadFile("testdata/fl_insurance_quoted.csv") 163 | if err != nil { 164 | b.Fatal(err) 165 | } 166 | 167 | b.Run("StdCSV", func(b *testing.B) { 168 | for i := 0; i < b.N; i++ { 169 | r := csv.NewReader(bytes.NewReader(data)) 170 | for { 171 | if _, err := r.Read(); err != nil { 172 | if err == io.EOF { 173 | break 174 | } 175 | b.Fatal(err) 176 | } 177 | } 178 | } 179 | }) 180 | b.Run("FastCSV", func(b *testing.B) { 181 | for i := 0; i < b.N; i++ { 182 | r := NewReader(bytes.NewReader(data), ',') 183 | for { 184 | if _, err := r.Read(); err != nil { 185 | if err == io.EOF { 186 | break 187 | } 188 | b.Fatal(err) 189 | } 190 | } 191 | } 192 | }) 193 | b.Run("StdCSVQuoted", func(b *testing.B) { 194 | for i := 0; i < b.N; i++ { 195 | r := csv.NewReader(bytes.NewReader(quotedData)) 196 | for { 197 | if _, err := r.Read(); err != nil { 198 | if err == io.EOF { 199 | break 200 | } 201 | b.Fatal(err) 202 | } 203 | } 204 | } 205 | }) 206 | b.Run("FastCSVQuoted", func(b *testing.B) { 207 | for i := 0; i < b.N; i++ { 208 | r := NewReader(bytes.NewReader(quotedData), ',') 209 | for { 210 | if _, err := r.Read(); err != nil { 211 | if err == io.EOF { 212 | break 213 | } 214 | b.Fatal(err) 215 | } 216 | } 217 | } 218 | }) 219 | } 220 | -------------------------------------------------------------------------------- /internal/fastcsv/testdata/addquotes.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func main() { 12 | r := csv.NewReader(os.Stdin) 13 | for { 14 | row, err := r.Read() 15 | if err != nil { 16 | if err != io.EOF { 17 | panic(err) 18 | } 19 | break 20 | } 21 | for i := range row { 22 | row[i] = fmt.Sprintf("\"%s\"", row[i]) 23 | } 24 | fmt.Fprintln(os.Stdout, strings.Join(row, ",")) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /internal/fcolumn/aggregations.go: -------------------------------------------------------------------------------- 1 | package fcolumn 2 | 3 | import "math" 4 | 5 | var aggregations = map[string]func([]float64) float64{ 6 | "max": max, 7 | "min": min, 8 | "sum": sum, 9 | "avg": avg, 10 | } 11 | 12 | func sum(values []float64) float64 { 13 | result := 0.0 14 | for _, v := range values { 15 | result += v 16 | } 17 | return result 18 | } 19 | 20 | func avg(values []float64) float64 { 21 | result := 0.0 22 | for _, v := range values { 23 | result += v 24 | } 25 | 26 | return result / float64(len(values)) 27 | } 28 | 29 | func max(values []float64) float64 { 30 | result := values[0] 31 | for _, v := range values[1:] { 32 | result = math.Max(result, v) 33 | } 34 | return result 35 | } 36 | 37 | func min(values []float64) float64 { 38 | result := values[0] 39 | for _, v := range values[1:] { 40 | result = math.Min(result, v) 41 | } 42 | return result 43 | } 44 | -------------------------------------------------------------------------------- /internal/fcolumn/column.go: -------------------------------------------------------------------------------- 1 | package fcolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/internal/ryu" 5 | "math" 6 | "math/rand" 7 | "reflect" 8 | "strconv" 9 | "unsafe" 10 | 11 | "github.com/tobgu/qframe/internal/column" 12 | "github.com/tobgu/qframe/internal/hash" 13 | "github.com/tobgu/qframe/internal/index" 14 | "github.com/tobgu/qframe/qerrors" 15 | "github.com/tobgu/qframe/types" 16 | ) 17 | 18 | func (c Column) DataType() types.DataType { 19 | return types.Float 20 | } 21 | 22 | func (c Column) StringAt(i uint32, naRep string) string { 23 | value := c.data[i] 24 | if math.IsNaN(value) { 25 | return naRep 26 | } 27 | return strconv.FormatFloat(c.data[i], 'f', -1, 64) 28 | } 29 | 30 | func (c Column) AppendByteStringAt(buf []byte, i uint32) []byte { 31 | value := c.data[i] 32 | if math.IsNaN(value) { 33 | return append(buf, "null"...) 34 | } 35 | 36 | return ryu.AppendFloat64f(buf, value) 37 | } 38 | 39 | func (c Column) ByteSize() int { 40 | // Slice header + data 41 | return 2*8 + 8*cap(c.data) 42 | } 43 | 44 | func (c Column) Equals(index index.Int, other column.Column, otherIndex index.Int) bool { 45 | otherI, ok := other.(Column) 46 | if !ok { 47 | return false 48 | } 49 | 50 | for ix, x := range index { 51 | v1, v2 := c.data[x], otherI.data[otherIndex[ix]] 52 | if v1 != v2 { 53 | // NaN != NaN but for our purposes they are the same 54 | if !(math.IsNaN(v1) && math.IsNaN(v2)) { 55 | return false 56 | } 57 | } 58 | } 59 | 60 | return true 61 | } 62 | 63 | func (c Comparable) Compare(i, j uint32) column.CompareResult { 64 | x, y := c.data[i], c.data[j] 65 | if x < y { 66 | return c.ltValue 67 | } 68 | 69 | if x > y { 70 | return c.gtValue 71 | } 72 | 73 | if math.IsNaN(x) || math.IsNaN(y) { 74 | if !math.IsNaN(x) { 75 | return c.nullGtValue 76 | } 77 | 78 | if !math.IsNaN(y) { 79 | return c.nullLtValue 80 | } 81 | 82 | return c.equalNullValue 83 | } 84 | 85 | return column.Equal 86 | } 87 | 88 | func (c Comparable) Hash(i uint32, seed uint64) uint64 { 89 | f := c.data[i] 90 | if math.IsNaN(f) && c.equalNullValue == column.NotEqual { 91 | // Use a random value here to avoid hash collisions when 92 | // we don't consider null to equal null. 93 | return rand.Uint64() 94 | } 95 | 96 | bits := math.Float64bits(c.data[i]) 97 | b := (*[8]byte)(unsafe.Pointer(&bits))[:] 98 | return hash.HashBytes(b, seed) 99 | } 100 | 101 | func (c Column) filterBuiltIn(index index.Int, comparator string, comparatee interface{}, bIndex index.Bool) error { 102 | switch t := comparatee.(type) { 103 | case float64: 104 | if math.IsNaN(t) { 105 | return qerrors.New("filter float", "NaN not allowed as filter argument") 106 | } 107 | 108 | compFunc, ok := filterFuncs1[comparator] 109 | if !ok { 110 | return qerrors.New("filter float", "invalid comparison operator to single argument filter, %v", comparator) 111 | } 112 | compFunc(index, c.data, t, bIndex) 113 | case Column: 114 | compFunc, ok := filterFuncs2[comparator] 115 | if !ok { 116 | return qerrors.New("filter float", "invalid comparison operator to column - column filter, %v", comparator) 117 | } 118 | compFunc(index, c.data, t.data, bIndex) 119 | case nil: 120 | compFunc, ok := filterFuncs0[comparator] 121 | if !ok { 122 | return qerrors.New("filter float", "invalid comparison operator to zero argument filter, %v", comparator) 123 | } 124 | compFunc(index, c.data, bIndex) 125 | default: 126 | return qerrors.New("filter float", "invalid comparison value type %v", reflect.TypeOf(comparatee)) 127 | } 128 | return nil 129 | } 130 | 131 | func (c Column) filterCustom1(index index.Int, fn func(float64) bool, bIndex index.Bool) { 132 | for i, x := range bIndex { 133 | if !x { 134 | bIndex[i] = fn(c.data[index[i]]) 135 | } 136 | } 137 | } 138 | 139 | func (c Column) filterCustom2(index index.Int, fn func(float64, float64) bool, comparatee interface{}, bIndex index.Bool) error { 140 | otherC, ok := comparatee.(Column) 141 | if !ok { 142 | return qerrors.New("filter float", "expected comparatee to be float column, was %v", reflect.TypeOf(comparatee)) 143 | } 144 | 145 | for i, x := range bIndex { 146 | if !x { 147 | bIndex[i] = fn(c.data[index[i]], otherC.data[index[i]]) 148 | } 149 | } 150 | 151 | return nil 152 | } 153 | 154 | func (c Column) Filter(index index.Int, comparator interface{}, comparatee interface{}, bIndex index.Bool) error { 155 | var err error 156 | switch t := comparator.(type) { 157 | case string: 158 | err = c.filterBuiltIn(index, t, comparatee, bIndex) 159 | case func(float64) bool: 160 | c.filterCustom1(index, t, bIndex) 161 | case func(float64, float64) bool: 162 | err = c.filterCustom2(index, t, comparatee, bIndex) 163 | default: 164 | err = qerrors.New("filter float", "invalid filter type %v", reflect.TypeOf(comparator)) 165 | } 166 | return err 167 | } 168 | 169 | func (c Column) FunctionType() types.FunctionType { 170 | return types.FunctionTypeFloat 171 | } 172 | 173 | func (c Column) Append(cols ...column.Column) (column.Column, error) { 174 | // TODO Append 175 | return nil, qerrors.New("Append", "Not implemented yet") 176 | } 177 | -------------------------------------------------------------------------------- /internal/fcolumn/column_gen.go: -------------------------------------------------------------------------------- 1 | // Code generated by genny. DO NOT EDIT. 2 | // This file was automatically generated by genny. 3 | // Any changes will be lost if this file is regenerated. 4 | // see https://github.com/mauricelam/genny 5 | 6 | package fcolumn 7 | 8 | // Code generated from template/column.go DO NOT EDIT 9 | 10 | import ( 11 | "fmt" 12 | 13 | "github.com/tobgu/qframe/config/rolling" 14 | 15 | "github.com/tobgu/qframe/internal/column" 16 | "github.com/tobgu/qframe/internal/index" 17 | "github.com/tobgu/qframe/qerrors" 18 | ) 19 | 20 | type Column struct { 21 | data []float64 22 | } 23 | 24 | func New(d []float64) Column { 25 | return Column{data: d} 26 | } 27 | 28 | func NewConst(val float64, count int) Column { 29 | var nullVal float64 30 | data := make([]float64, count) 31 | if val != nullVal { 32 | for i := range data { 33 | data[i] = val 34 | } 35 | } 36 | 37 | return Column{data: data} 38 | } 39 | 40 | func (c Column) fnName(name string) string { 41 | return fmt.Sprintf("%s.%s", c.DataType(), name) 42 | } 43 | 44 | // Apply single argument function. The result may be a column 45 | // of a different type than the current column. 46 | func (c Column) Apply1(fn interface{}, ix index.Int) (interface{}, error) { 47 | switch t := fn.(type) { 48 | case func(float64) int: 49 | result := make([]int, len(c.data)) 50 | for _, i := range ix { 51 | result[i] = t(c.data[i]) 52 | } 53 | return result, nil 54 | case func(float64) float64: 55 | result := make([]float64, len(c.data)) 56 | for _, i := range ix { 57 | result[i] = t(c.data[i]) 58 | } 59 | return result, nil 60 | case func(float64) bool: 61 | result := make([]bool, len(c.data)) 62 | for _, i := range ix { 63 | result[i] = t(c.data[i]) 64 | } 65 | return result, nil 66 | case func(float64) *string: 67 | result := make([]*string, len(c.data)) 68 | for _, i := range ix { 69 | result[i] = t(c.data[i]) 70 | } 71 | return result, nil 72 | default: 73 | return nil, qerrors.New(c.fnName("Apply1"), "cannot apply type %#v to column", fn) 74 | } 75 | } 76 | 77 | // Apply double argument function to two columns. Both columns must have the 78 | // same type. The resulting column will have the same type as this column. 79 | func (c Column) Apply2(fn interface{}, s2 column.Column, ix index.Int) (column.Column, error) { 80 | ss2, ok := s2.(Column) 81 | if !ok { 82 | return Column{}, qerrors.New(c.fnName("Apply2"), "invalid column type: %s", s2.DataType()) 83 | } 84 | 85 | t, ok := fn.(func(float64, float64) float64) 86 | if !ok { 87 | return Column{}, qerrors.New("Apply2", "invalid function type: %#v", fn) 88 | } 89 | 90 | result := make([]float64, len(c.data)) 91 | for _, i := range ix { 92 | result[i] = t(c.data[i], ss2.data[i]) 93 | } 94 | 95 | return New(result), nil 96 | } 97 | 98 | func (c Column) subset(index index.Int) Column { 99 | data := make([]float64, len(index)) 100 | for i, ix := range index { 101 | data[i] = c.data[ix] 102 | } 103 | 104 | return Column{data: data} 105 | } 106 | 107 | func (c Column) Subset(index index.Int) column.Column { 108 | return c.subset(index) 109 | } 110 | 111 | func (c Column) Comparable(reverse, equalNull, nullLast bool) column.Comparable { 112 | result := Comparable{data: c.data, ltValue: column.LessThan, gtValue: column.GreaterThan, nullLtValue: column.LessThan, nullGtValue: column.GreaterThan, equalNullValue: column.NotEqual} 113 | if reverse { 114 | result.ltValue, result.nullLtValue, result.gtValue, result.nullGtValue = 115 | result.gtValue, result.nullGtValue, result.ltValue, result.nullLtValue 116 | } 117 | 118 | if nullLast { 119 | result.nullLtValue, result.nullGtValue = result.nullGtValue, result.nullLtValue 120 | } 121 | 122 | if equalNull { 123 | result.equalNullValue = column.Equal 124 | } 125 | 126 | return result 127 | } 128 | 129 | func (c Column) String() string { 130 | return fmt.Sprintf("%v", c.data) 131 | } 132 | 133 | func (c Column) Len() int { 134 | return len(c.data) 135 | } 136 | 137 | func (c Column) Aggregate(indices []index.Int, fn interface{}) (column.Column, error) { 138 | var actualFn func([]float64) float64 139 | var ok bool 140 | 141 | switch t := fn.(type) { 142 | case string: 143 | actualFn, ok = aggregations[t] 144 | if !ok { 145 | return nil, qerrors.New(c.fnName("Aggregate"), "aggregation function %c is not defined for column", fn) 146 | } 147 | case func([]float64) float64: 148 | actualFn = t 149 | default: 150 | return nil, qerrors.New(c.fnName("Aggregate"), "invalid aggregation function type: %v", t) 151 | } 152 | 153 | data := make([]float64, 0, len(indices)) 154 | var buf []float64 155 | for _, ix := range indices { 156 | subS := c.subsetWithBuf(ix, &buf) 157 | data = append(data, actualFn(subS.data)) 158 | } 159 | 160 | return Column{data: data}, nil 161 | } 162 | 163 | func (c Column) subsetWithBuf(index index.Int, buf *[]float64) Column { 164 | if cap(*buf) < len(index) { 165 | *buf = make([]float64, 0, len(index)) 166 | } 167 | 168 | data := (*buf)[:0] 169 | for _, ix := range index { 170 | data = append(data, c.data[ix]) 171 | } 172 | 173 | return Column{data: data} 174 | } 175 | 176 | func (c Column) View(ix index.Int) View { 177 | return View{data: c.data, index: ix} 178 | } 179 | 180 | func (c Column) Rolling(fn interface{}, ix index.Int, config rolling.Config) (column.Column, error) { 181 | return c, nil 182 | } 183 | 184 | type Comparable struct { 185 | data []float64 186 | ltValue column.CompareResult 187 | nullLtValue column.CompareResult 188 | gtValue column.CompareResult 189 | nullGtValue column.CompareResult 190 | equalNullValue column.CompareResult 191 | } 192 | 193 | // View is a view into a column that allows access to individual elements by index. 194 | type View struct { 195 | data []float64 196 | index index.Int 197 | } 198 | 199 | // ItemAt returns the value at position i. 200 | func (v View) ItemAt(i int) float64 { 201 | return v.data[v.index[i]] 202 | } 203 | 204 | // Len returns the column length. 205 | func (v View) Len() int { 206 | return len(v.index) 207 | } 208 | 209 | // Slice returns a slice containing a copy of the column data. 210 | func (v View) Slice() []float64 { 211 | // TODO: This forces an alloc, as an alternative a slice could be taken 212 | // as input that can be (re)used by the client. Are there use cases 213 | // where this would actually make sense? 214 | result := make([]float64, v.Len()) 215 | for i, j := range v.index { 216 | result[i] = v.data[j] 217 | } 218 | return result 219 | } 220 | -------------------------------------------------------------------------------- /internal/fcolumn/doc_gen.go: -------------------------------------------------------------------------------- 1 | package fcolumn 2 | 3 | // Code generated from template/... DO NOT EDIT 4 | 5 | func Doc() string { 6 | return "\n Built in filters\n" + 7 | " !=\n" + 8 | " <\n" + 9 | " <=\n" + 10 | " =\n" + 11 | " >\n" + 12 | " >=\n" + 13 | " isnotnull\n" + 14 | " isnull\n" + 15 | 16 | "\n Built in aggregations\n" + 17 | " avg\n" + 18 | " max\n" + 19 | " min\n" + 20 | " sum\n" + 21 | "\n" 22 | } 23 | -------------------------------------------------------------------------------- /internal/fcolumn/filters.go: -------------------------------------------------------------------------------- 1 | package fcolumn 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/tobgu/qframe/filter" 7 | "github.com/tobgu/qframe/internal/index" 8 | ) 9 | 10 | var filterFuncs0 = map[string]func(index.Int, []float64, index.Bool){ 11 | filter.IsNull: isNull, 12 | filter.IsNotNull: isNotNull, 13 | } 14 | 15 | var filterFuncs1 = map[string]func(index.Int, []float64, float64, index.Bool){ 16 | filter.Gt: gt, 17 | filter.Gte: gte, 18 | filter.Lt: lt, 19 | filter.Lte: lte, 20 | filter.Eq: eq, 21 | filter.Neq: neq, 22 | } 23 | 24 | var filterFuncs2 = map[string]func(index.Int, []float64, []float64, index.Bool){ 25 | filter.Gt: gt2, 26 | filter.Gte: gte2, 27 | filter.Lt: lt2, 28 | filter.Lte: lte2, 29 | filter.Eq: eq2, 30 | filter.Neq: neq2, 31 | } 32 | 33 | func isNull(index index.Int, column []float64, bIndex index.Bool) { 34 | for i, x := range bIndex { 35 | if !x { 36 | bIndex[i] = math.IsNaN(column[index[i]]) 37 | } 38 | } 39 | } 40 | 41 | func isNotNull(index index.Int, column []float64, bIndex index.Bool) { 42 | for i, x := range bIndex { 43 | if !x { 44 | bIndex[i] = !math.IsNaN(column[index[i]]) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal/fcolumn/filters_gen.go: -------------------------------------------------------------------------------- 1 | package fcolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/internal/index" 5 | ) 6 | 7 | // Code generated from template/... DO NOT EDIT 8 | 9 | func lt(index index.Int, column []float64, comp float64, bIndex index.Bool) { 10 | for i, x := range bIndex { 11 | if !x { 12 | bIndex[i] = column[index[i]] < comp 13 | } 14 | } 15 | } 16 | 17 | func lte(index index.Int, column []float64, comp float64, bIndex index.Bool) { 18 | for i, x := range bIndex { 19 | if !x { 20 | bIndex[i] = column[index[i]] <= comp 21 | } 22 | } 23 | } 24 | 25 | func gt(index index.Int, column []float64, comp float64, bIndex index.Bool) { 26 | for i, x := range bIndex { 27 | if !x { 28 | bIndex[i] = column[index[i]] > comp 29 | } 30 | } 31 | } 32 | 33 | func gte(index index.Int, column []float64, comp float64, bIndex index.Bool) { 34 | for i, x := range bIndex { 35 | if !x { 36 | bIndex[i] = column[index[i]] >= comp 37 | } 38 | } 39 | } 40 | 41 | func eq(index index.Int, column []float64, comp float64, bIndex index.Bool) { 42 | for i, x := range bIndex { 43 | if !x { 44 | bIndex[i] = column[index[i]] == comp 45 | } 46 | } 47 | } 48 | 49 | func neq(index index.Int, column []float64, comp float64, bIndex index.Bool) { 50 | for i, x := range bIndex { 51 | if !x { 52 | bIndex[i] = column[index[i]] != comp 53 | } 54 | } 55 | } 56 | 57 | func lt2(index index.Int, column []float64, compCol []float64, bIndex index.Bool) { 58 | for i, x := range bIndex { 59 | if !x { 60 | pos := index[i] 61 | bIndex[i] = column[pos] < compCol[pos] 62 | } 63 | } 64 | } 65 | 66 | func lte2(index index.Int, column []float64, compCol []float64, bIndex index.Bool) { 67 | for i, x := range bIndex { 68 | if !x { 69 | pos := index[i] 70 | bIndex[i] = column[pos] <= compCol[pos] 71 | } 72 | } 73 | } 74 | 75 | func gt2(index index.Int, column []float64, compCol []float64, bIndex index.Bool) { 76 | for i, x := range bIndex { 77 | if !x { 78 | pos := index[i] 79 | bIndex[i] = column[pos] > compCol[pos] 80 | } 81 | } 82 | } 83 | 84 | func gte2(index index.Int, column []float64, compCol []float64, bIndex index.Bool) { 85 | for i, x := range bIndex { 86 | if !x { 87 | pos := index[i] 88 | bIndex[i] = column[pos] >= compCol[pos] 89 | } 90 | } 91 | } 92 | 93 | func eq2(index index.Int, column []float64, compCol []float64, bIndex index.Bool) { 94 | for i, x := range bIndex { 95 | if !x { 96 | pos := index[i] 97 | bIndex[i] = column[pos] == compCol[pos] 98 | } 99 | } 100 | } 101 | 102 | func neq2(index index.Int, column []float64, compCol []float64, bIndex index.Bool) { 103 | for i, x := range bIndex { 104 | if !x { 105 | pos := index[i] 106 | bIndex[i] = column[pos] != compCol[pos] 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /internal/fcolumn/generator.go: -------------------------------------------------------------------------------- 1 | package fcolumn 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/tobgu/qframe/filter" 7 | "github.com/tobgu/qframe/internal/maps" 8 | "github.com/tobgu/qframe/internal/template" 9 | ) 10 | 11 | //go:generate qfgenerate -source=ffilter -dst-file=filters_gen.go 12 | //go:generate qfgenerate -source=fdoc -dst-file=doc_gen.go 13 | 14 | func spec(name, operator, templateStr string) template.Spec { 15 | return template.Spec{ 16 | Name: name, 17 | Template: templateStr, 18 | Values: map[string]interface{}{"name": name, "dataType": "float64", "operator": operator}} 19 | } 20 | 21 | func colConstComparison(name, operator string) template.Spec { 22 | return spec(name, operator, template.BasicColConstComparison) 23 | } 24 | 25 | func colColComparison(name, operator string) template.Spec { 26 | return spec(name, operator, template.BasicColColComparison) 27 | } 28 | 29 | func GenerateFilters() (*bytes.Buffer, error) { 30 | // If adding more filters here make sure to also add a reference to them 31 | // in the corresponding filter map so that they can be looked up. 32 | return template.GenerateFilters("fcolumn", []template.Spec{ 33 | colConstComparison("lt", filter.Lt), 34 | colConstComparison("lte", filter.Lte), 35 | colConstComparison("gt", filter.Gt), 36 | colConstComparison("gte", filter.Gte), 37 | colConstComparison("eq", "=="), // Go eq ("==") differs from qframe eq ("=") 38 | colConstComparison("neq", filter.Neq), 39 | colColComparison("lt2", filter.Lt), 40 | colColComparison("lte2", filter.Lte), 41 | colColComparison("gt2", filter.Gt), 42 | colColComparison("gte2", filter.Gte), 43 | colColComparison("eq2", "=="), // Go eq ("==") differs from qframe eq ("=") 44 | colColComparison("neq2", filter.Neq), 45 | }) 46 | } 47 | 48 | func GenerateDoc() (*bytes.Buffer, error) { 49 | return template.GenerateDocs( 50 | "fcolumn", 51 | maps.StringKeys(filterFuncs0, filterFuncs1, filterFuncs2), 52 | maps.StringKeys(aggregations)) 53 | } 54 | -------------------------------------------------------------------------------- /internal/grouper/grouper.go: -------------------------------------------------------------------------------- 1 | package grouper 2 | 3 | import ( 4 | "math/bits" 5 | 6 | "github.com/tobgu/qframe/internal/column" 7 | "github.com/tobgu/qframe/internal/index" 8 | "github.com/tobgu/qframe/internal/math/integer" 9 | ) 10 | 11 | /* 12 | This package implements a basic hash table used for GroupBy and Distinct operations. 13 | 14 | Hashing is done using Go runtime memhash, collisions are handled using linear probing. 15 | 16 | When the table reaches a certain load factor it will be reallocated into a new, larger table. 17 | */ 18 | 19 | // An entry in the hash table. For group by operations a slice of all positions each group 20 | // are stored. For distinct operations only the first position is stored to avoid some overhead. 21 | type tableEntry struct { 22 | ix index.Int 23 | hash uint32 24 | firstPos uint32 25 | occupied bool 26 | } 27 | 28 | type table struct { 29 | entries []tableEntry 30 | comparables []column.Comparable 31 | stats GroupStats 32 | loadFactor float64 33 | groupCount uint32 34 | collectIx bool 35 | } 36 | 37 | const growthFactor = 2 38 | 39 | func (t *table) grow() { 40 | newLen := uint32(growthFactor * len(t.entries)) 41 | newEntries := make([]tableEntry, newLen) 42 | bitMask := newLen - 1 43 | for _, e := range t.entries { 44 | for pos := e.hash & bitMask; ; pos = (pos + 1) & bitMask { 45 | if !newEntries[pos].occupied { 46 | newEntries[pos] = e 47 | break 48 | } 49 | t.stats.RelocationCollisions++ 50 | } 51 | } 52 | 53 | t.stats.RelocationCount++ 54 | t.entries = newEntries 55 | t.loadFactor = t.loadFactor / growthFactor 56 | } 57 | 58 | func (t *table) hash(i uint32) uint32 { 59 | hashVal := uint64(0) 60 | for _, c := range t.comparables { 61 | hashVal = c.Hash(i, hashVal) 62 | } 63 | 64 | return uint32(hashVal) 65 | } 66 | 67 | const maxLoadFactor = 0.5 68 | 69 | func (t *table) insertEntry(i uint32) { 70 | if t.loadFactor > maxLoadFactor { 71 | t.grow() 72 | } 73 | 74 | hashSum := t.hash(i) 75 | bitMask := uint64(len(t.entries) - 1) 76 | startPos := uint64(hashSum) & bitMask 77 | var dstEntry *tableEntry 78 | for pos := startPos; dstEntry == nil; pos = (pos + 1) & bitMask { 79 | e := &t.entries[pos] 80 | if !e.occupied || e.hash == hashSum && equals(t.comparables, i, e.firstPos) { 81 | dstEntry = e 82 | } else { 83 | t.stats.InsertCollisions++ 84 | } 85 | } 86 | 87 | // Update entry 88 | if !dstEntry.occupied { 89 | // Eden entry 90 | dstEntry.hash = hashSum 91 | dstEntry.firstPos = i 92 | dstEntry.occupied = true 93 | t.groupCount++ 94 | t.loadFactor = float64(t.groupCount) / float64(len(t.entries)) 95 | } else { 96 | // Existing entry 97 | if t.collectIx { 98 | // Small hack to reduce number of allocations under some circumstances. Delay 99 | // creation of index slice until there are at least two entries in the group 100 | // since we store the first position in a separate variable on the entry anyway. 101 | if dstEntry.ix == nil { 102 | dstEntry.ix = index.Int{dstEntry.firstPos, i} 103 | } else { 104 | dstEntry.ix = append(dstEntry.ix, i) 105 | } 106 | } 107 | } 108 | } 109 | 110 | func newTable(sizeExp int, comparables []column.Comparable, collectIx bool) *table { 111 | return &table{ 112 | entries: make([]tableEntry, integer.Pow2(sizeExp)), 113 | comparables: comparables, 114 | collectIx: collectIx} 115 | } 116 | 117 | func equals(comparables []column.Comparable, i, j uint32) bool { 118 | for _, c := range comparables { 119 | if c.Compare(i, j) != column.Equal { 120 | return false 121 | } 122 | } 123 | return true 124 | } 125 | 126 | type GroupStats struct { 127 | RelocationCount int 128 | RelocationCollisions int 129 | InsertCollisions int 130 | GroupCount int 131 | LoadFactor float64 132 | } 133 | 134 | func calculateInitialSizeExp(ixLen int) int { 135 | // Size is expressed as 2^x to keep the size a multiple of two. 136 | // Initial size is picked fairly arbitrarily at the moment, we don't really know the distribution of 137 | // values within the index. Guarantee a minimum initial size of 8 (2³) for sanity. 138 | fitSize := uint64(ixLen) / 4 139 | return integer.Max(bits.Len64(fitSize), 3) 140 | } 141 | 142 | func groupIndex(ix index.Int, comparables []column.Comparable, collectIx bool) ([]tableEntry, GroupStats) { 143 | initialSizeExp := calculateInitialSizeExp(len(ix)) 144 | table := newTable(initialSizeExp, comparables, collectIx) 145 | for _, i := range ix { 146 | table.insertEntry(i) 147 | } 148 | 149 | stats := table.stats 150 | stats.LoadFactor = table.loadFactor 151 | stats.GroupCount = int(table.groupCount) 152 | return table.entries, stats 153 | } 154 | 155 | func GroupBy(ix index.Int, comparables []column.Comparable) ([]index.Int, GroupStats) { 156 | entries, stats := groupIndex(ix, comparables, true) 157 | result := make([]index.Int, 0, stats.GroupCount) 158 | for _, e := range entries { 159 | if e.occupied { 160 | if e.ix == nil { 161 | result = append(result, index.Int{e.firstPos}) 162 | } else { 163 | result = append(result, e.ix) 164 | } 165 | } 166 | } 167 | 168 | return result, stats 169 | } 170 | 171 | func Distinct(ix index.Int, comparables []column.Comparable) index.Int { 172 | entries, stats := groupIndex(ix, comparables, false) 173 | result := make(index.Int, 0, stats.GroupCount) 174 | for _, e := range entries { 175 | if e.occupied { 176 | result = append(result, e.firstPos) 177 | } 178 | } 179 | 180 | return result 181 | } 182 | -------------------------------------------------------------------------------- /internal/hash/memhash.go: -------------------------------------------------------------------------------- 1 | package hash 2 | 3 | import ( 4 | "unsafe" 5 | ) 6 | 7 | //go:noescape 8 | //go:linkname memhash runtime.memhash 9 | func memhash(p unsafe.Pointer, seed, s uintptr) uintptr 10 | 11 | type stringStruct struct { 12 | str unsafe.Pointer 13 | len int 14 | } 15 | 16 | func HashBytes(bb []byte, seed uint64) uint64 { 17 | ss := (*stringStruct)(unsafe.Pointer(&bb)) 18 | return uint64(memhash(ss.str, uintptr(seed), uintptr(ss.len))) 19 | } 20 | -------------------------------------------------------------------------------- /internal/hash/memhash_test.go: -------------------------------------------------------------------------------- 1 | package hash_test 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/tobgu/qframe/internal/hash" 9 | ) 10 | 11 | const noSeed int64 = 0 12 | const seed1 int64 = 1 13 | const seed2 int64 = 2 14 | const seed3 int64 = 3 15 | 16 | func genInts(seed int64, size int) []int { 17 | result := make([]int, size) 18 | r := rand.New(rand.NewSource(seed)) 19 | if seed == noSeed { 20 | // Sorted slice 21 | for ix := range result { 22 | result[ix] = ix 23 | } 24 | } else { 25 | // Random slice 26 | for ix := range result { 27 | result[ix] = r.Intn(size) 28 | } 29 | } 30 | 31 | return result 32 | } 33 | 34 | func genIntsWithCardinality(seed int64, size, cardinality int) []int { 35 | result := genInts(seed, size) 36 | for i, x := range result { 37 | result[i] = x % cardinality 38 | } 39 | 40 | return result 41 | } 42 | 43 | func genStringsWithCardinality(seed int64, size, cardinality, strLen int) []string { 44 | baseStr := "abcdefghijklmnopqrstuvxyz"[:strLen] 45 | result := make([]string, size) 46 | for i, x := range genIntsWithCardinality(seed, size, cardinality) { 47 | result[i] = fmt.Sprintf("%s%d", baseStr, x) 48 | } 49 | return result 50 | } 51 | 52 | func Test_StringDistribution(t *testing.T) { 53 | size := 100000 54 | strs1 := genStringsWithCardinality(seed1, 100000, 1000, 10) 55 | strs2 := genStringsWithCardinality(seed2, 100000, 10, 10) 56 | strs3 := genStringsWithCardinality(seed3, 100000, 2, 10) 57 | 58 | hashCounter := make(map[uint32]int) 59 | stringCounter := make(map[string]int) 60 | for i := 0; i < size; i++ { 61 | val := hash.HashBytes([]byte(strs1[i]), 0) 62 | val = hash.HashBytes([]byte(strs2[i]), val) 63 | val = hash.HashBytes([]byte(strs3[i]), val) 64 | h := uint32(val) 65 | hashCounter[h] += 1 66 | stringCounter[strs1[i]+strs2[i]+strs3[i]] += 1 67 | } 68 | 69 | // For this input the hash is perfect so we expect the same number 70 | // of different hashes as actual values. 71 | if len(hashCounter) < len(stringCounter) { 72 | t.Errorf("Unexpected hash count: %d, %d", len(hashCounter), len(stringCounter)) 73 | } 74 | } 75 | 76 | func Test_SmallIntDistribution(t *testing.T) { 77 | result := make(map[uint64]uint64) 78 | for i := 1; i < 177; i++ { 79 | val := hash.HashBytes([]byte{0, 0, 0, 0, 0, 0, 0, byte(i)}, 0) 80 | result[val] = result[val] + 1 81 | } 82 | 83 | if len(result) != 176 { 84 | t.Errorf("%d: %v", len(result), result) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /internal/icolumn/aggregations.go: -------------------------------------------------------------------------------- 1 | package icolumn 2 | 3 | import "github.com/tobgu/qframe/internal/math/integer" 4 | 5 | var aggregations = map[string]func([]int) int{ 6 | "sum": sum, 7 | "max": max, 8 | "min": min, 9 | } 10 | 11 | func sum(values []int) int { 12 | result := 0 13 | for _, v := range values { 14 | result += v 15 | } 16 | return result 17 | } 18 | 19 | func max(values []int) int { 20 | result := values[0] 21 | for _, v := range values[1:] { 22 | result = integer.Max(result, v) 23 | } 24 | return result 25 | } 26 | 27 | func min(values []int) int { 28 | result := values[0] 29 | for _, v := range values[1:] { 30 | result = integer.Min(result, v) 31 | } 32 | return result 33 | } 34 | -------------------------------------------------------------------------------- /internal/icolumn/column.go: -------------------------------------------------------------------------------- 1 | package icolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/internal/column" 5 | "github.com/tobgu/qframe/internal/hash" 6 | "github.com/tobgu/qframe/internal/index" 7 | "github.com/tobgu/qframe/qerrors" 8 | "github.com/tobgu/qframe/types" 9 | "reflect" 10 | "strconv" 11 | "unsafe" 12 | ) 13 | 14 | func (c Column) DataType() types.DataType { 15 | return types.Int 16 | } 17 | 18 | func (c Column) StringAt(i uint32, _ string) string { 19 | return strconv.FormatInt(int64(c.data[i]), 10) 20 | } 21 | 22 | func (c Column) AppendByteStringAt(buf []byte, i uint32) []byte { 23 | return strconv.AppendInt(buf, int64(c.data[i]), 10) 24 | } 25 | 26 | func (c Column) ByteSize() int { 27 | // Slice header + data 28 | return 2*8 + 8*cap(c.data) 29 | } 30 | 31 | func (c Column) Equals(index index.Int, other column.Column, otherIndex index.Int) bool { 32 | otherI, ok := other.(Column) 33 | if !ok { 34 | return false 35 | } 36 | 37 | for ix, x := range index { 38 | if c.data[x] != otherI.data[otherIndex[ix]] { 39 | return false 40 | } 41 | } 42 | 43 | return true 44 | } 45 | 46 | func (c Column) FloatSlice() []float64 { 47 | result := make([]float64, len(c.data)) 48 | for i, v := range c.data { 49 | result[i] = float64(v) 50 | } 51 | 52 | return result 53 | } 54 | 55 | func (c Comparable) Compare(i, j uint32) column.CompareResult { 56 | x, y := c.data[i], c.data[j] 57 | if x < y { 58 | return c.ltValue 59 | } 60 | 61 | if x > y { 62 | return c.gtValue 63 | } 64 | 65 | return column.Equal 66 | } 67 | 68 | func (c Comparable) Hash(i uint32, seed uint64) uint64 { 69 | x := &c.data[i] 70 | b := (*[8]byte)(unsafe.Pointer(x))[:] 71 | return hash.HashBytes(b, seed) 72 | } 73 | 74 | func intComp(comparatee interface{}) (int, bool) { 75 | comp, ok := comparatee.(int) 76 | if !ok { 77 | // Accept floats by truncating them 78 | compFloat, ok := comparatee.(float64) 79 | if !ok { 80 | return 0, false 81 | } 82 | comp = int(compFloat) 83 | } 84 | 85 | return comp, true 86 | } 87 | 88 | type intSet map[int]struct{} 89 | 90 | func interfaceSliceToIntSlice(ss []interface{}) ([]int, bool) { 91 | result := make([]int, len(ss)) 92 | for i, s := range ss { 93 | switch t := s.(type) { 94 | case int: 95 | result[i] = t 96 | case float64: 97 | result[i] = int(t) 98 | default: 99 | return nil, false 100 | } 101 | } 102 | return result, true 103 | } 104 | 105 | func newIntSet(input interface{}) (intSet, bool) { 106 | var result intSet 107 | var ok bool 108 | switch t := input.(type) { 109 | case []int: 110 | result, ok = make(intSet, len(t)), true 111 | for _, v := range t { 112 | result[v] = struct{}{} 113 | } 114 | case []float64: 115 | result, ok = make(intSet, len(t)), true 116 | for _, v := range t { 117 | result[int(v)] = struct{}{} 118 | } 119 | case []interface{}: 120 | if intSlice, innerOk := interfaceSliceToIntSlice(t); innerOk { 121 | result, ok = newIntSet(intSlice) 122 | } 123 | } 124 | 125 | return result, ok 126 | } 127 | 128 | func (is intSet) Contains(x int) bool { 129 | _, ok := is[x] 130 | return ok 131 | } 132 | 133 | func (c Column) filterBuiltIn(index index.Int, comparator string, comparatee interface{}, bIndex index.Bool) error { 134 | if intC, ok := intComp(comparatee); ok { 135 | filterFn, ok := filterFuncs[comparator] 136 | if !ok { 137 | return qerrors.New("filter int", "unknown filter operator %v", comparator) 138 | } 139 | filterFn(index, c.data, intC, bIndex) 140 | } else if set, ok := newIntSet(comparatee); ok { 141 | filterFn, ok := multiInputFilterFuncs[comparator] 142 | if !ok { 143 | return qerrors.New("filter int", "unknown filter operator %v", comparator) 144 | } 145 | filterFn(index, c.data, set, bIndex) 146 | } else if columnC, ok := comparatee.(Column); ok { 147 | filterFn, ok := filterFuncs2[comparator] 148 | if !ok { 149 | return qerrors.New("filter int", "unknown filter operator %v", comparator) 150 | } 151 | filterFn(index, c.data, columnC.data, bIndex) 152 | } else if comparatee == nil { 153 | compFunc, ok := filterFuncs0[comparator] 154 | if !ok { 155 | return qerrors.New("filter int", "invalid comparison operator to zero argument filter, %v", comparator) 156 | } 157 | compFunc(index, c.data, bIndex) 158 | } else { 159 | return qerrors.New("filter int", "invalid comparison value type %v", reflect.TypeOf(comparatee)) 160 | } 161 | 162 | return nil 163 | } 164 | 165 | func (c Column) filterCustom1(index index.Int, fn func(int) bool, bIndex index.Bool) { 166 | for i, x := range bIndex { 167 | if !x { 168 | bIndex[i] = fn(c.data[index[i]]) 169 | } 170 | } 171 | } 172 | 173 | func (c Column) filterCustom2(index index.Int, fn func(int, int) bool, comparatee interface{}, bIndex index.Bool) error { 174 | otherC, ok := comparatee.(Column) 175 | if !ok { 176 | return qerrors.New("filter int", "expected comparatee to be int column, was %v", reflect.TypeOf(comparatee)) 177 | } 178 | 179 | for i, x := range bIndex { 180 | if !x { 181 | bIndex[i] = fn(c.data[index[i]], otherC.data[index[i]]) 182 | } 183 | } 184 | 185 | return nil 186 | } 187 | 188 | func (c Column) Filter(index index.Int, comparator interface{}, comparatee interface{}, bIndex index.Bool) error { 189 | var err error 190 | switch t := comparator.(type) { 191 | case string: 192 | err = c.filterBuiltIn(index, t, comparatee, bIndex) 193 | case func(int) bool: 194 | c.filterCustom1(index, t, bIndex) 195 | case func(int, int) bool: 196 | err = c.filterCustom2(index, t, comparatee, bIndex) 197 | default: 198 | err = qerrors.New("filter int", "invalid filter type %v", reflect.TypeOf(comparator)) 199 | } 200 | return err 201 | } 202 | 203 | func (c Column) FunctionType() types.FunctionType { 204 | return types.FunctionTypeInt 205 | } 206 | 207 | func (c Column) Append(cols ...column.Column) (column.Column, error) { 208 | // TODO Improve, currently copies all data over to a new column, this may not be the best solution... 209 | newLen := c.Len() 210 | intCols := append(make([]Column, 0, len(cols)+1), c) 211 | for _, col := range cols { 212 | intCol, ok := col.(Column) 213 | if !ok { 214 | return nil, qerrors.New("append int", "can only append integer columns to integer column") 215 | } 216 | newLen += intCol.Len() 217 | intCols = append(intCols, intCol) 218 | } 219 | 220 | newData := make([]int, newLen) 221 | offset := 0 222 | for _, col := range intCols { 223 | offset += copy(newData[offset:], col.data) 224 | } 225 | 226 | return New(newData), nil 227 | } 228 | -------------------------------------------------------------------------------- /internal/icolumn/column_gen.go: -------------------------------------------------------------------------------- 1 | // Code generated by genny. DO NOT EDIT. 2 | // This file was automatically generated by genny. 3 | // Any changes will be lost if this file is regenerated. 4 | // see https://github.com/mauricelam/genny 5 | 6 | package icolumn 7 | 8 | // Code generated from template/column.go DO NOT EDIT 9 | 10 | import ( 11 | "fmt" 12 | 13 | "github.com/tobgu/qframe/config/rolling" 14 | 15 | "github.com/tobgu/qframe/internal/column" 16 | "github.com/tobgu/qframe/internal/index" 17 | "github.com/tobgu/qframe/qerrors" 18 | ) 19 | 20 | type Column struct { 21 | data []int 22 | } 23 | 24 | func New(d []int) Column { 25 | return Column{data: d} 26 | } 27 | 28 | func NewConst(val int, count int) Column { 29 | var nullVal int 30 | data := make([]int, count) 31 | if val != nullVal { 32 | for i := range data { 33 | data[i] = val 34 | } 35 | } 36 | 37 | return Column{data: data} 38 | } 39 | 40 | func (c Column) fnName(name string) string { 41 | return fmt.Sprintf("%s.%s", c.DataType(), name) 42 | } 43 | 44 | // Apply single argument function. The result may be a column 45 | // of a different type than the current column. 46 | func (c Column) Apply1(fn interface{}, ix index.Int) (interface{}, error) { 47 | switch t := fn.(type) { 48 | case func(int) int: 49 | result := make([]int, len(c.data)) 50 | for _, i := range ix { 51 | result[i] = t(c.data[i]) 52 | } 53 | return result, nil 54 | case func(int) float64: 55 | result := make([]float64, len(c.data)) 56 | for _, i := range ix { 57 | result[i] = t(c.data[i]) 58 | } 59 | return result, nil 60 | case func(int) bool: 61 | result := make([]bool, len(c.data)) 62 | for _, i := range ix { 63 | result[i] = t(c.data[i]) 64 | } 65 | return result, nil 66 | case func(int) *string: 67 | result := make([]*string, len(c.data)) 68 | for _, i := range ix { 69 | result[i] = t(c.data[i]) 70 | } 71 | return result, nil 72 | default: 73 | return nil, qerrors.New(c.fnName("Apply1"), "cannot apply type %#v to column", fn) 74 | } 75 | } 76 | 77 | // Apply double argument function to two columns. Both columns must have the 78 | // same type. The resulting column will have the same type as this column. 79 | func (c Column) Apply2(fn interface{}, s2 column.Column, ix index.Int) (column.Column, error) { 80 | ss2, ok := s2.(Column) 81 | if !ok { 82 | return Column{}, qerrors.New(c.fnName("Apply2"), "invalid column type: %s", s2.DataType()) 83 | } 84 | 85 | t, ok := fn.(func(int, int) int) 86 | if !ok { 87 | return Column{}, qerrors.New("Apply2", "invalid function type: %#v", fn) 88 | } 89 | 90 | result := make([]int, len(c.data)) 91 | for _, i := range ix { 92 | result[i] = t(c.data[i], ss2.data[i]) 93 | } 94 | 95 | return New(result), nil 96 | } 97 | 98 | func (c Column) subset(index index.Int) Column { 99 | data := make([]int, len(index)) 100 | for i, ix := range index { 101 | data[i] = c.data[ix] 102 | } 103 | 104 | return Column{data: data} 105 | } 106 | 107 | func (c Column) Subset(index index.Int) column.Column { 108 | return c.subset(index) 109 | } 110 | 111 | func (c Column) Comparable(reverse, equalNull, nullLast bool) column.Comparable { 112 | result := Comparable{data: c.data, ltValue: column.LessThan, gtValue: column.GreaterThan, nullLtValue: column.LessThan, nullGtValue: column.GreaterThan, equalNullValue: column.NotEqual} 113 | if reverse { 114 | result.ltValue, result.nullLtValue, result.gtValue, result.nullGtValue = 115 | result.gtValue, result.nullGtValue, result.ltValue, result.nullLtValue 116 | } 117 | 118 | if nullLast { 119 | result.nullLtValue, result.nullGtValue = result.nullGtValue, result.nullLtValue 120 | } 121 | 122 | if equalNull { 123 | result.equalNullValue = column.Equal 124 | } 125 | 126 | return result 127 | } 128 | 129 | func (c Column) String() string { 130 | return fmt.Sprintf("%v", c.data) 131 | } 132 | 133 | func (c Column) Len() int { 134 | return len(c.data) 135 | } 136 | 137 | func (c Column) Aggregate(indices []index.Int, fn interface{}) (column.Column, error) { 138 | var actualFn func([]int) int 139 | var ok bool 140 | 141 | switch t := fn.(type) { 142 | case string: 143 | actualFn, ok = aggregations[t] 144 | if !ok { 145 | return nil, qerrors.New(c.fnName("Aggregate"), "aggregation function %c is not defined for column", fn) 146 | } 147 | case func([]int) int: 148 | actualFn = t 149 | default: 150 | return nil, qerrors.New(c.fnName("Aggregate"), "invalid aggregation function type: %v", t) 151 | } 152 | 153 | data := make([]int, 0, len(indices)) 154 | var buf []int 155 | for _, ix := range indices { 156 | subS := c.subsetWithBuf(ix, &buf) 157 | data = append(data, actualFn(subS.data)) 158 | } 159 | 160 | return Column{data: data}, nil 161 | } 162 | 163 | func (c Column) subsetWithBuf(index index.Int, buf *[]int) Column { 164 | if cap(*buf) < len(index) { 165 | *buf = make([]int, 0, len(index)) 166 | } 167 | 168 | data := (*buf)[:0] 169 | for _, ix := range index { 170 | data = append(data, c.data[ix]) 171 | } 172 | 173 | return Column{data: data} 174 | } 175 | 176 | func (c Column) View(ix index.Int) View { 177 | return View{data: c.data, index: ix} 178 | } 179 | 180 | func (c Column) Rolling(fn interface{}, ix index.Int, config rolling.Config) (column.Column, error) { 181 | return c, nil 182 | } 183 | 184 | type Comparable struct { 185 | data []int 186 | ltValue column.CompareResult 187 | nullLtValue column.CompareResult 188 | gtValue column.CompareResult 189 | nullGtValue column.CompareResult 190 | equalNullValue column.CompareResult 191 | } 192 | 193 | // View is a view into a column that allows access to individual elements by index. 194 | type View struct { 195 | data []int 196 | index index.Int 197 | } 198 | 199 | // ItemAt returns the value at position i. 200 | func (v View) ItemAt(i int) int { 201 | return v.data[v.index[i]] 202 | } 203 | 204 | // Len returns the column length. 205 | func (v View) Len() int { 206 | return len(v.index) 207 | } 208 | 209 | // Slice returns a slice containing a copy of the column data. 210 | func (v View) Slice() []int { 211 | // TODO: This forces an alloc, as an alternative a slice could be taken 212 | // as input that can be (re)used by the client. Are there use cases 213 | // where this would actually make sense? 214 | result := make([]int, v.Len()) 215 | for i, j := range v.index { 216 | result[i] = v.data[j] 217 | } 218 | return result 219 | } 220 | -------------------------------------------------------------------------------- /internal/icolumn/doc_gen.go: -------------------------------------------------------------------------------- 1 | package icolumn 2 | 3 | // Code generated from template/... DO NOT EDIT 4 | 5 | func Doc() string { 6 | return "\n Built in filters\n" + 7 | " !=\n" + 8 | " <\n" + 9 | " <=\n" + 10 | " =\n" + 11 | " >\n" + 12 | " >=\n" + 13 | " all_bits\n" + 14 | " any_bits\n" + 15 | " in\n" + 16 | 17 | "\n Built in aggregations\n" + 18 | " max\n" + 19 | " min\n" + 20 | " sum\n" + 21 | "\n" 22 | } 23 | -------------------------------------------------------------------------------- /internal/icolumn/filters.go: -------------------------------------------------------------------------------- 1 | package icolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/filter" 5 | "github.com/tobgu/qframe/internal/index" 6 | ) 7 | 8 | // Column - constant 9 | var filterFuncs = map[string]func(index.Int, []int, int, index.Bool){ 10 | filter.Gt: gt, 11 | filter.Gte: gte, 12 | filter.Lt: lt, 13 | filter.Lte: lte, 14 | filter.Eq: eq, 15 | filter.Neq: neq, 16 | "any_bits": anyBits, 17 | "all_bits": allBits, 18 | } 19 | 20 | // Comparisons against multiple values 21 | var multiInputFilterFuncs = map[string]func(index.Int, []int, intSet, index.Bool){ 22 | filter.In: in, 23 | } 24 | 25 | // Column - Column 26 | var filterFuncs2 = map[string]func(index.Int, []int, []int, index.Bool){ 27 | filter.Gt: gt2, 28 | filter.Gte: gte2, 29 | filter.Lt: lt2, 30 | filter.Lte: lte2, 31 | filter.Eq: eq2, 32 | filter.Neq: neq2, 33 | } 34 | 35 | // Column only 36 | var filterFuncs0 = map[string]func(index.Int, []int, index.Bool){ 37 | filter.IsNull: isNull, 38 | filter.IsNotNull: isNotNull, 39 | } 40 | 41 | func isNull(_ index.Int, _ []int, bIndex index.Bool) { 42 | // Int columns are never null, this function is provided for convenience to avoid 43 | // clients from having to keep track of if a column is of type int or float for 44 | // common operations. 45 | for i := range bIndex { 46 | bIndex[i] = false 47 | } 48 | } 49 | 50 | func isNotNull(_ index.Int, _ []int, bIndex index.Bool) { 51 | // Int columns are never null, this function is provided for convenience to avoid 52 | // clients from having to keep track of if a column is of type int or float for 53 | // common operations. 54 | for i := range bIndex { 55 | bIndex[i] = true 56 | } 57 | } 58 | 59 | func in(index index.Int, column []int, comp intSet, bIndex index.Bool) { 60 | for i, x := range bIndex { 61 | if !x { 62 | bIndex[i] = comp.Contains(column[index[i]]) 63 | } 64 | } 65 | } 66 | 67 | func anyBits(index index.Int, column []int, comp int, bIndex index.Bool) { 68 | for i, x := range bIndex { 69 | if !x { 70 | bIndex[i] = column[index[i]]&comp > 0 71 | } 72 | } 73 | } 74 | 75 | func allBits(index index.Int, column []int, comp int, bIndex index.Bool) { 76 | for i, x := range bIndex { 77 | if !x { 78 | bIndex[i] = column[index[i]]&comp == comp 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /internal/icolumn/filters_gen.go: -------------------------------------------------------------------------------- 1 | package icolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/internal/index" 5 | ) 6 | 7 | // Code generated from template/... DO NOT EDIT 8 | 9 | func lt(index index.Int, column []int, comp int, bIndex index.Bool) { 10 | for i, x := range bIndex { 11 | if !x { 12 | bIndex[i] = column[index[i]] < comp 13 | } 14 | } 15 | } 16 | 17 | func lte(index index.Int, column []int, comp int, bIndex index.Bool) { 18 | for i, x := range bIndex { 19 | if !x { 20 | bIndex[i] = column[index[i]] <= comp 21 | } 22 | } 23 | } 24 | 25 | func gt(index index.Int, column []int, comp int, bIndex index.Bool) { 26 | for i, x := range bIndex { 27 | if !x { 28 | bIndex[i] = column[index[i]] > comp 29 | } 30 | } 31 | } 32 | 33 | func gte(index index.Int, column []int, comp int, bIndex index.Bool) { 34 | for i, x := range bIndex { 35 | if !x { 36 | bIndex[i] = column[index[i]] >= comp 37 | } 38 | } 39 | } 40 | 41 | func eq(index index.Int, column []int, comp int, bIndex index.Bool) { 42 | for i, x := range bIndex { 43 | if !x { 44 | bIndex[i] = column[index[i]] == comp 45 | } 46 | } 47 | } 48 | 49 | func neq(index index.Int, column []int, comp int, bIndex index.Bool) { 50 | for i, x := range bIndex { 51 | if !x { 52 | bIndex[i] = column[index[i]] != comp 53 | } 54 | } 55 | } 56 | 57 | func lt2(index index.Int, column []int, compCol []int, bIndex index.Bool) { 58 | for i, x := range bIndex { 59 | if !x { 60 | pos := index[i] 61 | bIndex[i] = column[pos] < compCol[pos] 62 | } 63 | } 64 | } 65 | 66 | func lte2(index index.Int, column []int, compCol []int, bIndex index.Bool) { 67 | for i, x := range bIndex { 68 | if !x { 69 | pos := index[i] 70 | bIndex[i] = column[pos] <= compCol[pos] 71 | } 72 | } 73 | } 74 | 75 | func gt2(index index.Int, column []int, compCol []int, bIndex index.Bool) { 76 | for i, x := range bIndex { 77 | if !x { 78 | pos := index[i] 79 | bIndex[i] = column[pos] > compCol[pos] 80 | } 81 | } 82 | } 83 | 84 | func gte2(index index.Int, column []int, compCol []int, bIndex index.Bool) { 85 | for i, x := range bIndex { 86 | if !x { 87 | pos := index[i] 88 | bIndex[i] = column[pos] >= compCol[pos] 89 | } 90 | } 91 | } 92 | 93 | func eq2(index index.Int, column []int, compCol []int, bIndex index.Bool) { 94 | for i, x := range bIndex { 95 | if !x { 96 | pos := index[i] 97 | bIndex[i] = column[pos] == compCol[pos] 98 | } 99 | } 100 | } 101 | 102 | func neq2(index index.Int, column []int, compCol []int, bIndex index.Bool) { 103 | for i, x := range bIndex { 104 | if !x { 105 | pos := index[i] 106 | bIndex[i] = column[pos] != compCol[pos] 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /internal/icolumn/generator.go: -------------------------------------------------------------------------------- 1 | package icolumn 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/tobgu/qframe/filter" 7 | "github.com/tobgu/qframe/internal/maps" 8 | "github.com/tobgu/qframe/internal/template" 9 | ) 10 | 11 | //go:generate qfgenerate -source=ifilter -dst-file=filters_gen.go 12 | //go:generate qfgenerate -source=idoc -dst-file=doc_gen.go 13 | 14 | func spec(name, operator, templateStr string) template.Spec { 15 | return template.Spec{ 16 | Name: name, 17 | Template: templateStr, 18 | Values: map[string]interface{}{"name": name, "dataType": "int", "operator": operator}} 19 | } 20 | 21 | func colConstComparison(name, operator string) template.Spec { 22 | return spec(name, operator, template.BasicColConstComparison) 23 | } 24 | 25 | func colColComparison(name, operator string) template.Spec { 26 | return spec(name, operator, template.BasicColColComparison) 27 | } 28 | 29 | func GenerateFilters() (*bytes.Buffer, error) { 30 | // If adding more filters here make sure to also add a reference to them 31 | // in the corresponding filter map so that they can be looked up. 32 | return template.GenerateFilters("icolumn", []template.Spec{ 33 | colConstComparison("lt", filter.Lt), 34 | colConstComparison("lte", filter.Lte), 35 | colConstComparison("gt", filter.Gt), 36 | colConstComparison("gte", filter.Gte), 37 | colConstComparison("eq", "=="), // Go eq ("==") differs from qframe eq ("=") 38 | colConstComparison("neq", filter.Neq), 39 | colColComparison("lt2", filter.Lt), 40 | colColComparison("lte2", filter.Lte), 41 | colColComparison("gt2", filter.Gt), 42 | colColComparison("gte2", filter.Gte), 43 | colColComparison("eq2", "=="), // Go eq ("==") differs from qframe eq ("=") 44 | colColComparison("neq2", filter.Neq), 45 | }) 46 | } 47 | 48 | func GenerateDoc() (*bytes.Buffer, error) { 49 | return template.GenerateDocs( 50 | "icolumn", 51 | maps.StringKeys(filterFuncs, filterFuncs2, multiInputFilterFuncs), 52 | maps.StringKeys(aggregations)) 53 | } 54 | -------------------------------------------------------------------------------- /internal/index/index.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | type Int []uint32 4 | 5 | type Bool []bool 6 | 7 | func NewBool(size int) Bool { 8 | return make(Bool, size) 9 | } 10 | 11 | func NewAscending(size uint32) Int { 12 | newIndex := make(Int, size) 13 | for i := range newIndex { 14 | newIndex[i] = uint32(i) 15 | } 16 | 17 | return newIndex 18 | } 19 | 20 | func (ix Int) Filter(bIx Bool) Int { 21 | count := 0 22 | for _, b := range bIx { 23 | if b { 24 | count++ 25 | } 26 | } 27 | 28 | result := make(Int, 0, count) 29 | for i, b := range bIx { 30 | if b { 31 | result = append(result, ix[i]) 32 | } 33 | } 34 | 35 | return result 36 | } 37 | 38 | func (ix Int) ByteSize() int { 39 | return 4 * cap(ix) 40 | } 41 | 42 | func (ix Int) Len() int { 43 | return len(ix) 44 | } 45 | 46 | func (ix Int) Copy() Int { 47 | newIndex := make(Int, len(ix)) 48 | copy(newIndex, ix) 49 | return newIndex 50 | } 51 | 52 | func (ix Bool) Len() int { 53 | return len(ix) 54 | } 55 | -------------------------------------------------------------------------------- /internal/io/json.go: -------------------------------------------------------------------------------- 1 | package io 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/tobgu/qframe/qerrors" 6 | "io" 7 | ) 8 | 9 | type JSONRecords []map[string]interface{} 10 | 11 | type JSONColumns map[string]json.RawMessage 12 | 13 | func fillInts(col []int, records JSONRecords, colName string) error { 14 | for i := range col { 15 | record := records[i] 16 | value, ok := record[colName] 17 | if !ok { 18 | return qerrors.New("fillInts", "missing value for column %s, row %d", colName, i) 19 | } 20 | 21 | intValue, ok := value.(int) 22 | if !ok { 23 | return qerrors.New("fillInts", "wrong type for column %s, row %d, expected int", colName, i) 24 | } 25 | col[i] = intValue 26 | } 27 | 28 | return nil 29 | } 30 | 31 | func fillFloats(col []float64, records JSONRecords, colName string) error { 32 | for i := range col { 33 | record := records[i] 34 | value, ok := record[colName] 35 | if !ok { 36 | return qerrors.New("fillFloats", "missing value for column %s, row %d", colName, i) 37 | } 38 | 39 | floatValue, ok := value.(float64) 40 | if !ok { 41 | return qerrors.New("fillFloats", "wrong type for column %s, row %d, expected float", colName, i) 42 | } 43 | col[i] = floatValue 44 | } 45 | 46 | return nil 47 | } 48 | 49 | func fillBools(col []bool, records JSONRecords, colName string) error { 50 | for i := range col { 51 | record := records[i] 52 | value, ok := record[colName] 53 | if !ok { 54 | return qerrors.New("fillBools", "wrong type for column %s, row %d", colName, i) 55 | } 56 | 57 | boolValue, ok := value.(bool) 58 | if !ok { 59 | return qerrors.New("fillBools", "wrong type for column %s, row %d, expected bool", colName, i) 60 | } 61 | col[i] = boolValue 62 | } 63 | 64 | return nil 65 | } 66 | 67 | func fillStrings(col []*string, records JSONRecords, colName string) error { 68 | for i := range col { 69 | record := records[i] 70 | value, ok := record[colName] 71 | if !ok { 72 | return qerrors.New("fillStrings", "wrong type for column %s, row %d", colName, i) 73 | } 74 | 75 | switch t := value.(type) { 76 | case string: 77 | col[i] = &t 78 | case nil: 79 | col[i] = nil 80 | default: 81 | return qerrors.New("fillStrings", "wrong type for column %s, row %d, expected int", colName, i) 82 | } 83 | } 84 | 85 | return nil 86 | } 87 | 88 | func jsonRecordsToData(records JSONRecords) (map[string]interface{}, error) { 89 | result := map[string]interface{}{} 90 | if len(records) == 0 { 91 | return result, nil 92 | } 93 | 94 | r0 := records[0] 95 | for colName, value := range r0 { 96 | switch t := value.(type) { 97 | case int: 98 | col := make([]int, len(records)) 99 | if err := fillInts(col, records, colName); err != nil { 100 | return nil, err 101 | } 102 | result[colName] = col 103 | case float64: 104 | col := make([]float64, len(records)) 105 | if err := fillFloats(col, records, colName); err != nil { 106 | return nil, err 107 | } 108 | result[colName] = col 109 | case bool: 110 | col := make([]bool, len(records)) 111 | if err := fillBools(col, records, colName); err != nil { 112 | return nil, err 113 | } 114 | result[colName] = col 115 | case nil, string: 116 | col := make([]*string, len(records)) 117 | if err := fillStrings(col, records, colName); err != nil { 118 | return nil, err 119 | } 120 | result[colName] = col 121 | default: 122 | return nil, qerrors.New("jsonRecordsToData", "unknown type of %s", t) 123 | } 124 | } 125 | return result, nil 126 | } 127 | 128 | // UnmarshalJSON transforms JSON containing data records or columns into a map of columns 129 | // that can be used to create a QFrame. 130 | func UnmarshalJSON(r io.Reader) (map[string]interface{}, error) { 131 | var records JSONRecords 132 | decoder := json.NewDecoder(r) 133 | err := decoder.Decode(&records) 134 | if err != nil { 135 | return nil, qerrors.Propagate("UnmarshalJSON", err) 136 | } 137 | 138 | return jsonRecordsToData(records) 139 | } 140 | -------------------------------------------------------------------------------- /internal/io/sql/coerce.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "reflect" 5 | "strconv" 6 | 7 | "github.com/tobgu/qframe/qerrors" 8 | ) 9 | 10 | // CoerceFunc returns a function that does an explicit 11 | // type cast from one input type and sets an internal 12 | // column type. 13 | type CoerceFunc func(c *Column) func(t interface{}) error 14 | 15 | // Int64ToBool casts an int64 type into a boolean. This 16 | // is useful for casting columns in SQLite which stores 17 | // BOOL as INT types natively. 18 | func Int64ToBool(c *Column) func(t interface{}) error { 19 | return func(t interface{}) error { 20 | v, ok := t.(int64) 21 | if !ok { 22 | return qerrors.New( 23 | "Coercion Int64ToBool", "type %s is not int64", reflect.TypeOf(t).Kind()) 24 | } 25 | c.Bool(v != 0) 26 | return nil 27 | } 28 | } 29 | 30 | func StringToFloat(c *Column) func(t interface{}) error { 31 | return func(t interface{}) error { 32 | v, ok := t.(string) 33 | if !ok { 34 | return qerrors.New( 35 | "Coercion StringToFloat", "type %s is not float", reflect.TypeOf(t).Kind()) 36 | } 37 | f, err := strconv.ParseFloat(v, 64) 38 | if err != nil { 39 | return qerrors.New( 40 | "Coercion StringToFloat", "Could not convert %s", v) 41 | } 42 | c.Float(f) 43 | return nil 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /internal/io/sql/column.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "math" 5 | "reflect" 6 | 7 | "github.com/tobgu/qframe/internal/math/float" 8 | 9 | "github.com/tobgu/qframe/qerrors" 10 | ) 11 | 12 | // Column implements the sql.Scanner interface 13 | // and allows arbitrary data types to be loaded from 14 | // any database/sql/driver into a QFrame. 15 | type Column struct { 16 | kind reflect.Kind 17 | nulls int 18 | // pointer to the data slice which 19 | // contains the inferred data type 20 | ptr interface{} 21 | data struct { 22 | Ints []int 23 | Floats []float64 24 | Bools []bool 25 | Strings []*string 26 | } 27 | coerce func(t interface{}) error 28 | precision int 29 | } 30 | 31 | // Null appends a new Null value to 32 | // the underlying column data. 33 | func (c *Column) Null() error { 34 | // If we haven't inferred the type of 35 | // data we are scanning simply count 36 | // the number of NULL values we receive. 37 | // The only scenario this will happen is 38 | // when the first returned values are NULL. 39 | if c.kind == reflect.Invalid { 40 | c.nulls++ 41 | return nil 42 | } 43 | switch c.kind { 44 | case reflect.Float64: 45 | c.data.Floats = append(c.data.Floats, math.NaN()) 46 | case reflect.String: 47 | c.data.Strings = append(c.data.Strings, nil) 48 | default: 49 | return qerrors.New("Column Null", "non-nullable type: %s", c.kind) 50 | } 51 | return nil 52 | } 53 | 54 | // Int adds a new int to the underlying data slice 55 | func (c *Column) Int(i int) { 56 | if c.ptr == nil { 57 | c.kind = reflect.Int 58 | c.ptr = &c.data.Ints 59 | } 60 | c.data.Ints = append(c.data.Ints, i) 61 | } 62 | 63 | // Float adds a new float to the underlying data slice 64 | func (c *Column) Float(f float64) { 65 | if c.ptr == nil { 66 | c.kind = reflect.Float64 67 | c.ptr = &c.data.Floats 68 | // add any NULL floats previously scanned 69 | if c.nulls > 0 { 70 | for i := 0; i < c.nulls; i++ { 71 | c.data.Floats = append(c.data.Floats, math.NaN()) 72 | } 73 | c.nulls = 0 74 | } 75 | } 76 | if c.precision > 0 { 77 | f = float.Fixed(f, c.precision) 78 | } 79 | c.data.Floats = append(c.data.Floats, f) 80 | } 81 | 82 | // String adds a new string to the underlying data slice 83 | func (c *Column) String(s string) { 84 | if c.ptr == nil { 85 | c.kind = reflect.String 86 | c.ptr = &c.data.Strings 87 | // add any NULL strings previously scanned 88 | if c.nulls > 0 { 89 | for i := 0; i < c.nulls; i++ { 90 | c.data.Strings = append(c.data.Strings, nil) 91 | } 92 | c.nulls = 0 93 | } 94 | } 95 | c.data.Strings = append(c.data.Strings, &s) 96 | } 97 | 98 | // Bool adds a new bool to the underlying data slice 99 | func (c *Column) Bool(b bool) { 100 | if c.ptr == nil { 101 | c.kind = reflect.Bool 102 | c.ptr = &c.data.Bools 103 | } 104 | c.data.Bools = append(c.data.Bools, b) 105 | } 106 | 107 | // Scan implements the sql.Scanner interface 108 | func (c *Column) Scan(t interface{}) error { 109 | if c.coerce != nil { 110 | return c.coerce(t) 111 | } 112 | switch v := t.(type) { 113 | case bool: 114 | c.Bool(v) 115 | case string: 116 | c.String(v) 117 | case int64: 118 | c.Int(int(v)) 119 | case []uint8: 120 | c.String(string(v)) 121 | case float64: 122 | c.Float(v) 123 | case nil: 124 | err := c.Null() 125 | if err != nil { 126 | return err 127 | } 128 | default: 129 | return qerrors.New( 130 | "Column Scan", "unsupported scan type: %s", reflect.ValueOf(t).Kind()) 131 | } 132 | return nil 133 | } 134 | 135 | // Data returns the underlying data slice 136 | func (c *Column) Data() interface{} { 137 | if c.ptr == nil { 138 | return nil 139 | } 140 | // *[] -> [] 141 | return reflect.ValueOf(c.ptr).Elem().Interface() 142 | } 143 | -------------------------------------------------------------------------------- /internal/io/sql/column_test.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "math" 5 | "testing" 6 | ) 7 | 8 | func assertEqual(t *testing.T, expected, actual interface{}) { 9 | t.Helper() 10 | if expected != actual { 11 | t.Errorf("%v != %v", expected, actual) 12 | } 13 | } 14 | 15 | func panicOnErr(err error) { 16 | if err != nil { 17 | panic(err) 18 | } 19 | } 20 | 21 | func TestColumn(t *testing.T) { 22 | // Column with two NULL values 23 | col := &Column{} 24 | panicOnErr(col.Scan(0.0)) 25 | panicOnErr(col.Scan(nil)) 26 | panicOnErr(col.Scan(2.0)) 27 | panicOnErr(col.Scan(nil)) 28 | data := col.Data().([]float64) 29 | assertEqual(t, 4, len(data)) 30 | assertEqual(t, data[0], 0.0) 31 | assertEqual(t, true, math.IsNaN(data[1])) 32 | assertEqual(t, data[2], 2.0) 33 | assertEqual(t, true, math.IsNaN(data[3])) 34 | 35 | // Column with NULL values at the head 36 | col = &Column{} 37 | panicOnErr(col.Scan(nil)) 38 | panicOnErr(col.Scan(nil)) 39 | panicOnErr(col.Scan(0.0)) 40 | panicOnErr(col.Scan(1.0)) 41 | data = col.Data().([]float64) 42 | assertEqual(t, 4, len(data)) 43 | 44 | // Column with all NULL values 45 | col = &Column{} 46 | panicOnErr(col.Scan(nil)) 47 | panicOnErr(col.Scan(nil)) 48 | panicOnErr(col.Scan(nil)) 49 | panicOnErr(col.Scan(nil)) 50 | assertEqual(t, nil, col.Data()) 51 | 52 | } 53 | 54 | func TestColumnCoercion(t *testing.T) { 55 | col := &Column{} 56 | col.coerce = Int64ToBool(col) 57 | panicOnErr(col.Scan(int64(1))) 58 | panicOnErr(col.Scan(int64(0))) 59 | panicOnErr(col.Scan(int64(1))) 60 | panicOnErr(col.Scan(int64(0))) 61 | data := col.Data().([]bool) 62 | assertEqual(t, 4, len(data)) 63 | assertEqual(t, true, data[0]) 64 | assertEqual(t, false, data[1]) 65 | assertEqual(t, true, data[2]) 66 | assertEqual(t, false, data[3]) 67 | } 68 | 69 | func BenchmarkColumn(b *testing.B) { 70 | col := &Column{} 71 | for n := 0; n < b.N; n++ { 72 | _ = col.Scan(1.0) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /internal/io/sql/reader.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/tobgu/qframe/qerrors" 7 | "github.com/tobgu/qframe/types" 8 | ) 9 | 10 | // ReadSQL returns a named map of types.DataSlice for consumption 11 | // by the qframe.New constructor. 12 | func ReadSQL(rows *sql.Rows, conf SQLConfig) (map[string]types.DataSlice, []string, error) { 13 | var ( 14 | columns []interface{} 15 | colNames []string 16 | ) 17 | for rows.Next() { 18 | // Allocate columns for the returning query 19 | if columns == nil { 20 | names, err := rows.Columns() 21 | if err != nil { 22 | return nil, colNames, qerrors.New("ReadSQL Columns", err.Error()) 23 | } 24 | for _, name := range names { 25 | col := &Column{precision: conf.Precision} 26 | if conf.CoerceMap != nil { 27 | fn, ok := conf.CoerceMap[name] 28 | if ok { 29 | col.coerce = fn(col) 30 | } 31 | } 32 | columns = append(columns, col) 33 | } 34 | // ensure any column in the coercion map 35 | // exists in the resulting columns or return 36 | // an error explicitly. 37 | if conf.CoerceMap != nil { 38 | checkMap: 39 | for name := range conf.CoerceMap { 40 | for _, colName := range colNames { 41 | if name == colName { 42 | continue checkMap 43 | } 44 | return nil, colNames, qerrors.New("ReadSQL Columns", "column %s does not exist to coerce", name) 45 | } 46 | } 47 | } 48 | colNames = names 49 | } 50 | // Scan the result into our columns 51 | err := rows.Scan(columns...) 52 | if err != nil { 53 | return nil, colNames, qerrors.New("ReadSQL Scan", err.Error()) 54 | } 55 | } 56 | result := map[string]types.DataSlice{} 57 | for i, column := range columns { 58 | result[colNames[i]] = column.(*Column).Data() 59 | } 60 | return result, colNames, nil 61 | } 62 | -------------------------------------------------------------------------------- /internal/io/sql/stmt.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | ) 7 | 8 | func escape(s string, char rune, buf *bytes.Buffer) { 9 | if char == 0 { 10 | buf.WriteString(s) 11 | return 12 | } 13 | buf.WriteRune(char) 14 | buf.WriteString(s) 15 | buf.WriteRune(char) 16 | } 17 | 18 | // Insert generates a SQL insert statement 19 | // for each colName. There are several variations 20 | // of SQL that need to be produced for each driver. 21 | // This has been tested with the following: 22 | // PostgreSQL - github.com/lib/pq 23 | // MySQL/MariaDB - github.com/go-sql-driver/mysql 24 | // SQLite - github.com/mattn/go-sqlite3 25 | // 26 | // "Parameter markers" are used to specify placeholders 27 | // for values scanned by the implementing driver: 28 | // PostgreSQL accepts "incrementing" markers e.g. $1..$2 29 | // While MySQL/MariaDB and SQLite accept ?..?. 30 | func Insert(colNames []string, conf SQLConfig) string { 31 | buf := bytes.NewBuffer(nil) 32 | buf.WriteString("INSERT INTO ") 33 | escape(conf.Table, conf.EscapeChar, buf) 34 | buf.WriteString(" (") 35 | for i, name := range colNames { 36 | escape(name, conf.EscapeChar, buf) 37 | if i+1 < len(colNames) { 38 | buf.WriteString(",") 39 | } 40 | } 41 | buf.WriteString(") VALUES (") 42 | for i := range colNames { 43 | if conf.Incrementing { 44 | buf.WriteString(fmt.Sprintf("$%d", i+1)) 45 | } else { 46 | buf.WriteString("?") 47 | } 48 | if i+1 < len(colNames) { 49 | buf.WriteString(",") 50 | } 51 | } 52 | buf.WriteString(");") 53 | return buf.String() 54 | } 55 | -------------------------------------------------------------------------------- /internal/io/sql/stmt_test.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestInsert(t *testing.T) { 8 | // Unescaped 9 | query := Insert([]string{"COL1", "COL2"}, SQLConfig{Table: "test"}) 10 | expected := `INSERT INTO test (COL1,COL2) VALUES (?,?);` 11 | assertEqual(t, expected, query) 12 | 13 | // Double quote escaped 14 | query = Insert([]string{"COL1", "COL2"}, SQLConfig{ 15 | Table: "test", EscapeChar: '"'}) 16 | expected = "INSERT INTO \"test\" (\"COL1\",\"COL2\") VALUES (?,?);" 17 | assertEqual(t, expected, query) 18 | 19 | // Backtick escaped 20 | query = Insert([]string{"COL1", "COL2"}, SQLConfig{ 21 | Table: "test", EscapeChar: '`'}) 22 | expected = "INSERT INTO `test` (`COL1`,`COL2`) VALUES (?,?);" 23 | assertEqual(t, expected, query) 24 | } 25 | -------------------------------------------------------------------------------- /internal/io/sql/types.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | "github.com/tobgu/qframe/internal/bcolumn" 8 | "github.com/tobgu/qframe/internal/column" 9 | "github.com/tobgu/qframe/internal/ecolumn" 10 | "github.com/tobgu/qframe/internal/fcolumn" 11 | "github.com/tobgu/qframe/internal/icolumn" 12 | "github.com/tobgu/qframe/internal/index" 13 | "github.com/tobgu/qframe/internal/scolumn" 14 | "github.com/tobgu/qframe/qerrors" 15 | ) 16 | 17 | type SQLConfig struct { 18 | // Query is a Raw SQL statement which must return 19 | // appropriate types which can be inferred 20 | // and loaded into a new QFrame. 21 | Query string 22 | // Incrementing indicates the PostgreSQL variant 23 | // of parameter markers will be used, e.g. $1..$2. 24 | // The default style is ?..?. 25 | Incrementing bool 26 | // Table is the name of the table to be used 27 | // for generating an INSERT statement. 28 | Table string 29 | // EscapeChar is a rune which column and table 30 | // names will be escaped with. PostgreSQL and SQLite 31 | // both accept double quotes "" while MariaDB/MySQL 32 | // only accept backticks. 33 | EscapeChar rune 34 | // CoerceMap is a map of columns to perform explicit 35 | // type coercion on. 36 | CoerceMap map[string]CoerceFunc 37 | // Precision specifies how much precision float values 38 | // should have. 0 has no effect. 39 | Precision int 40 | } 41 | 42 | type ArgBuilder func(ix index.Int, i int) interface{} 43 | 44 | func NewArgBuilder(col column.Column) (ArgBuilder, error) { 45 | switch c := col.(type) { 46 | case bcolumn.Column: 47 | return func(ix index.Int, i int) interface{} { 48 | return c.View(ix).ItemAt(i) 49 | }, nil 50 | case icolumn.Column: 51 | return func(ix index.Int, i int) interface{} { 52 | return c.View(ix).ItemAt(i) 53 | }, nil 54 | case fcolumn.Column: 55 | return func(ix index.Int, i int) interface{} { 56 | return c.View(ix).ItemAt(i) 57 | }, nil 58 | case scolumn.Column: 59 | return func(ix index.Int, i int) interface{} { 60 | return c.View(ix).ItemAt(i) 61 | }, nil 62 | case ecolumn.Column: 63 | return func(ix index.Int, i int) interface{} { 64 | return c.View(ix).ItemAt(i) 65 | }, nil 66 | } 67 | return nil, qerrors.New("NewArgBuilder", fmt.Sprintf("bad column type: %s", reflect.TypeOf(col).Name())) 68 | } 69 | -------------------------------------------------------------------------------- /internal/maps/maps.go: -------------------------------------------------------------------------------- 1 | package maps 2 | 3 | import ( 4 | "reflect" 5 | "sort" 6 | 7 | "github.com/tobgu/qframe/internal/strings" 8 | ) 9 | 10 | // StringKeys returns a sorted list of all unique keys present in mm. 11 | // This function will panic if mm contains non-maps or maps containing 12 | // other key types than string. 13 | func StringKeys(mm ...interface{}) []string { 14 | keySet := strings.NewStringSet(nil) 15 | for _, m := range mm { 16 | v := reflect.ValueOf(m) 17 | keys := v.MapKeys() 18 | for _, k := range keys { 19 | keySet.Add(k.String()) 20 | } 21 | } 22 | 23 | result := keySet.AsSlice() 24 | sort.Strings(result) 25 | return result 26 | } 27 | -------------------------------------------------------------------------------- /internal/math/float/float.go: -------------------------------------------------------------------------------- 1 | package float 2 | 3 | import ( 4 | "math" 5 | ) 6 | 7 | func Round(n float64) int { 8 | return int(n + math.Copysign(0.5, n)) 9 | } 10 | 11 | func Fixed(num float64, precision int) float64 { 12 | i := math.Pow(10, float64(precision)) 13 | return float64(Round(num*i)) / i 14 | } 15 | -------------------------------------------------------------------------------- /internal/math/integer/int.go: -------------------------------------------------------------------------------- 1 | package integer 2 | 3 | import "math" 4 | 5 | func Max(x, y int) int { 6 | if x > y { 7 | return x 8 | } 9 | return y 10 | } 11 | 12 | func Min(x, y int) int { 13 | if x < y { 14 | return x 15 | } 16 | return y 17 | } 18 | 19 | func Pow2(exp int) int { 20 | return int(math.Pow(2, float64(exp))) 21 | } 22 | -------------------------------------------------------------------------------- /internal/ncolumn/column.go: -------------------------------------------------------------------------------- 1 | package ncolumn 2 | 3 | /* 4 | Package ncolumn contains a "null implementation" of the Column interface. It is typeless and of size 0. 5 | 6 | It is for example used when reading zero row CSVs without type hints. 7 | */ 8 | 9 | import ( 10 | "github.com/tobgu/qframe/config/rolling" 11 | "github.com/tobgu/qframe/internal/column" 12 | "github.com/tobgu/qframe/internal/index" 13 | "github.com/tobgu/qframe/qerrors" 14 | "github.com/tobgu/qframe/types" 15 | ) 16 | 17 | type Column struct{} 18 | 19 | func (c Column) String() string { 20 | return "[]" 21 | } 22 | 23 | func (c Column) Filter(index index.Int, comparator interface{}, comparatee interface{}, bIndex index.Bool) error { 24 | return nil 25 | } 26 | 27 | func (c Column) Subset(index index.Int) column.Column { 28 | return c 29 | } 30 | 31 | func (c Column) Equals(index index.Int, other column.Column, otherIndex index.Int) bool { 32 | return false 33 | } 34 | 35 | func (c Column) Comparable(reverse, equalNull, nullLast bool) column.Comparable { 36 | return Comparable{} 37 | } 38 | 39 | func (c Column) Aggregate(indices []index.Int, fn interface{}) (column.Column, error) { 40 | return c, nil 41 | } 42 | 43 | func (c Column) StringAt(i uint32, naRep string) string { 44 | return naRep 45 | } 46 | 47 | func (c Column) AppendByteStringAt(buf []byte, i uint32) []byte { 48 | return buf 49 | } 50 | 51 | func (c Column) ByteSize() int { 52 | return 0 53 | } 54 | 55 | func (c Column) Len() int { 56 | return 0 57 | } 58 | 59 | func (c Column) Apply1(fn interface{}, ix index.Int) (interface{}, error) { 60 | return c, nil 61 | } 62 | 63 | func (c Column) Apply2(fn interface{}, s2 column.Column, ix index.Int) (column.Column, error) { 64 | return c, nil 65 | } 66 | 67 | func (c Column) Rolling(fn interface{}, ix index.Int, config rolling.Config) (column.Column, error) { 68 | return c, nil 69 | } 70 | 71 | func (c Column) FunctionType() types.FunctionType { 72 | return types.FunctionTypeUndefined 73 | } 74 | 75 | func (c Column) DataType() types.DataType { 76 | return types.Undefined 77 | } 78 | 79 | type Comparable struct{} 80 | 81 | func (c Comparable) Compare(i, j uint32) column.CompareResult { 82 | return column.NotEqual 83 | } 84 | 85 | func (c Comparable) Hash(i uint32, seed uint64) uint64 { 86 | return 0 87 | } 88 | 89 | func (c Column) Append(cols ...column.Column) (column.Column, error) { 90 | // TODO Append 91 | return nil, qerrors.New("Append", "Not implemented yet") 92 | } 93 | -------------------------------------------------------------------------------- /internal/qframe/generator/generator.go: -------------------------------------------------------------------------------- 1 | package generator 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | 7 | "github.com/tobgu/qframe/internal/template" 8 | ) 9 | 10 | //go:generate qfgenerate -source=qframe -dst-file=../../../qframe_gen.go 11 | 12 | const viewTemplate = ` 13 | // {{.type}}View provides a "view" into an {{.lowerType}} column and can be used for access to individual elements. 14 | type {{.type}}View struct { 15 | {{.package}}.View 16 | } 17 | 18 | // {{.type}}View returns a view into an {{.lowerType}} column identified by name. 19 | // 20 | // colName - Name of the column. 21 | // 22 | // Returns an error if the column is missing or of wrong type. 23 | // Time complexity O(1). 24 | func (qf QFrame) {{.type}}View(colName string) ({{.type}}View, error) { 25 | namedColumn, ok := qf.columnsByName[colName] 26 | if !ok { 27 | return {{.type}}View{}, qerrors.New("{{.type}}View", "unknown column: %s", colName) 28 | } 29 | 30 | col, ok := namedColumn.Column.({{.package}}.Column) 31 | if !ok { 32 | return {{.type}}View{}, qerrors.New( 33 | "{{.type}}View", 34 | "invalid column type, expected: %s, was: %s", "{{.lowerType}}", namedColumn.DataType()) 35 | } 36 | 37 | return {{.type}}View{View: col.View(qf.index)}, nil 38 | } 39 | 40 | // Must{{.type}}View returns a view into an {{.lowerType}} column identified by name. 41 | // 42 | // colName - Name of the column. 43 | // 44 | // Panics if the column is missing or of wrong type. 45 | // Time complexity O(1). 46 | func (qf QFrame) Must{{.type}}View(colName string) {{.type}}View { 47 | view, err := qf.{{.type}}View(colName) 48 | if err != nil { 49 | panic(qerrors.Propagate("Must{{.type}}View", err)) 50 | } 51 | return view 52 | } 53 | 54 | ` 55 | 56 | func spec(typeName, srcPackage string) template.Spec { 57 | return template.Spec{ 58 | Name: typeName, 59 | Template: viewTemplate, 60 | Values: map[string]interface{}{"type": typeName, "lowerType": strings.ToLower(typeName), "package": srcPackage}} 61 | } 62 | 63 | func view(typeName, srcPackage string) template.Spec { 64 | return spec(typeName, srcPackage) 65 | } 66 | 67 | func GenerateQFrame() (*bytes.Buffer, error) { 68 | return template.Generate("qframe", []template.Spec{ 69 | view("Int", "icolumn"), 70 | view("Float", "fcolumn"), 71 | view("Bool", "bcolumn"), 72 | view("String", "scolumn"), 73 | view("Enum", "ecolumn"), 74 | }, []string{ 75 | "github.com/tobgu/qframe/qerrors", 76 | "github.com/tobgu/qframe/internal/icolumn", 77 | "github.com/tobgu/qframe/internal/fcolumn", 78 | "github.com/tobgu/qframe/internal/bcolumn", 79 | "github.com/tobgu/qframe/internal/scolumn", 80 | "github.com/tobgu/qframe/internal/ecolumn", 81 | }) 82 | } 83 | -------------------------------------------------------------------------------- /internal/ryu/README.md: -------------------------------------------------------------------------------- 1 | This is a copy of selected parts from https://github.com/cespare/ryu/. -------------------------------------------------------------------------------- /internal/scolumn/doc_gen.go: -------------------------------------------------------------------------------- 1 | package scolumn 2 | 3 | // Code generated from template/... DO NOT EDIT 4 | 5 | func Doc() string { 6 | return "\n Built in filters\n" + 7 | " !=\n" + 8 | " <\n" + 9 | " <=\n" + 10 | " =\n" + 11 | " >\n" + 12 | " >=\n" + 13 | " ilike\n" + 14 | " in\n" + 15 | " isnotnull\n" + 16 | " isnull\n" + 17 | " like\n" + 18 | 19 | "\n Built in aggregations\n" + 20 | "\n" 21 | } 22 | -------------------------------------------------------------------------------- /internal/scolumn/filters.go: -------------------------------------------------------------------------------- 1 | package scolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/filter" 5 | "github.com/tobgu/qframe/internal/index" 6 | qfstrings "github.com/tobgu/qframe/internal/strings" 7 | "github.com/tobgu/qframe/qerrors" 8 | ) 9 | 10 | var filterFuncs0 = map[string]func(index.Int, Column, index.Bool) error{ 11 | filter.IsNull: isNull, 12 | filter.IsNotNull: isNotNull, 13 | } 14 | 15 | var filterFuncs1 = map[string]func(index.Int, Column, string, index.Bool) error{ 16 | filter.Gt: gt, 17 | filter.Gte: gte, 18 | filter.Lt: lt, 19 | filter.Lte: lte, 20 | filter.Eq: eq, 21 | filter.Neq: neq, 22 | "like": like, 23 | "ilike": ilike, 24 | } 25 | 26 | var multiInputFilterFuncs = map[string]func(index.Int, Column, qfstrings.StringSet, index.Bool) error{ 27 | filter.In: in, 28 | } 29 | 30 | var filterFuncs2 = map[string]func(index.Int, Column, Column, index.Bool) error{ 31 | filter.Gt: gt2, 32 | filter.Gte: gte2, 33 | filter.Lt: lt2, 34 | filter.Lte: lte2, 35 | filter.Eq: eq2, 36 | filter.Neq: neq2, 37 | } 38 | 39 | func neq(index index.Int, s Column, comparatee string, bIndex index.Bool) error { 40 | for i, x := range bIndex { 41 | if !x { 42 | s, isNull := s.stringAt(index[i]) 43 | bIndex[i] = isNull || s != comparatee 44 | } 45 | } 46 | 47 | return nil 48 | } 49 | 50 | func like(index index.Int, s Column, comparatee string, bIndex index.Bool) error { 51 | return regexFilter(index, s, comparatee, bIndex, true) 52 | } 53 | 54 | func ilike(index index.Int, s Column, comparatee string, bIndex index.Bool) error { 55 | return regexFilter(index, s, comparatee, bIndex, false) 56 | } 57 | 58 | func in(index index.Int, s Column, comparatee qfstrings.StringSet, bIndex index.Bool) error { 59 | for i, x := range bIndex { 60 | if !x { 61 | s, isNull := s.stringAt(index[i]) 62 | if !isNull { 63 | bIndex[i] = comparatee.Contains(s) 64 | } 65 | } 66 | } 67 | 68 | return nil 69 | } 70 | 71 | func regexFilter(index index.Int, s Column, comparatee string, bIndex index.Bool, caseSensitive bool) error { 72 | matcher, err := qfstrings.NewMatcher(comparatee, caseSensitive) 73 | if err != nil { 74 | return qerrors.Propagate("Regex filter", err) 75 | } 76 | 77 | for i, x := range bIndex { 78 | if !x { 79 | s, isNull := s.stringAt(index[i]) 80 | if !isNull { 81 | bIndex[i] = matcher.Matches(s) 82 | } 83 | } 84 | } 85 | 86 | return nil 87 | } 88 | 89 | func neq2(index index.Int, col, col2 Column, bIndex index.Bool) error { 90 | for i, x := range bIndex { 91 | if !x { 92 | s, isNull := col.stringAt(index[i]) 93 | s2, isNull2 := col2.stringAt(index[i]) 94 | bIndex[i] = isNull || isNull2 || s != s2 95 | } 96 | } 97 | return nil 98 | } 99 | 100 | func isNull(index index.Int, col Column, bIndex index.Bool) error { 101 | for i, x := range bIndex { 102 | if !x { 103 | _, isNull := col.stringAt(index[i]) 104 | bIndex[i] = isNull 105 | } 106 | } 107 | return nil 108 | } 109 | 110 | func isNotNull(index index.Int, col Column, bIndex index.Bool) error { 111 | for i, x := range bIndex { 112 | if !x { 113 | _, isNull := col.stringAt(index[i]) 114 | bIndex[i] = !isNull 115 | } 116 | } 117 | return nil 118 | } 119 | -------------------------------------------------------------------------------- /internal/scolumn/filters_gen.go: -------------------------------------------------------------------------------- 1 | package scolumn 2 | 3 | import ( 4 | "github.com/tobgu/qframe/internal/index" 5 | ) 6 | 7 | // Code generated from template/... DO NOT EDIT 8 | 9 | func lt(index index.Int, c Column, comparatee string, bIndex index.Bool) error { 10 | for i, x := range bIndex { 11 | if !x { 12 | s, isNull := c.stringAt(index[i]) 13 | bIndex[i] = !isNull && s < comparatee 14 | } 15 | } 16 | 17 | return nil 18 | } 19 | 20 | func lte(index index.Int, c Column, comparatee string, bIndex index.Bool) error { 21 | for i, x := range bIndex { 22 | if !x { 23 | s, isNull := c.stringAt(index[i]) 24 | bIndex[i] = !isNull && s <= comparatee 25 | } 26 | } 27 | 28 | return nil 29 | } 30 | 31 | func gt(index index.Int, c Column, comparatee string, bIndex index.Bool) error { 32 | for i, x := range bIndex { 33 | if !x { 34 | s, isNull := c.stringAt(index[i]) 35 | bIndex[i] = !isNull && s > comparatee 36 | } 37 | } 38 | 39 | return nil 40 | } 41 | 42 | func gte(index index.Int, c Column, comparatee string, bIndex index.Bool) error { 43 | for i, x := range bIndex { 44 | if !x { 45 | s, isNull := c.stringAt(index[i]) 46 | bIndex[i] = !isNull && s >= comparatee 47 | } 48 | } 49 | 50 | return nil 51 | } 52 | 53 | func eq(index index.Int, c Column, comparatee string, bIndex index.Bool) error { 54 | for i, x := range bIndex { 55 | if !x { 56 | s, isNull := c.stringAt(index[i]) 57 | bIndex[i] = !isNull && s == comparatee 58 | } 59 | } 60 | 61 | return nil 62 | } 63 | 64 | func lt2(index index.Int, col, col2 Column, bIndex index.Bool) error { 65 | for i, x := range bIndex { 66 | if !x { 67 | s, isNull := col.stringAt(index[i]) 68 | s2, isNull2 := col2.stringAt(index[i]) 69 | bIndex[i] = !isNull && !isNull2 && s < s2 70 | } 71 | } 72 | return nil 73 | } 74 | 75 | func lte2(index index.Int, col, col2 Column, bIndex index.Bool) error { 76 | for i, x := range bIndex { 77 | if !x { 78 | s, isNull := col.stringAt(index[i]) 79 | s2, isNull2 := col2.stringAt(index[i]) 80 | bIndex[i] = !isNull && !isNull2 && s <= s2 81 | } 82 | } 83 | return nil 84 | } 85 | 86 | func gt2(index index.Int, col, col2 Column, bIndex index.Bool) error { 87 | for i, x := range bIndex { 88 | if !x { 89 | s, isNull := col.stringAt(index[i]) 90 | s2, isNull2 := col2.stringAt(index[i]) 91 | bIndex[i] = !isNull && !isNull2 && s > s2 92 | } 93 | } 94 | return nil 95 | } 96 | 97 | func gte2(index index.Int, col, col2 Column, bIndex index.Bool) error { 98 | for i, x := range bIndex { 99 | if !x { 100 | s, isNull := col.stringAt(index[i]) 101 | s2, isNull2 := col2.stringAt(index[i]) 102 | bIndex[i] = !isNull && !isNull2 && s >= s2 103 | } 104 | } 105 | return nil 106 | } 107 | 108 | func eq2(index index.Int, col, col2 Column, bIndex index.Bool) error { 109 | for i, x := range bIndex { 110 | if !x { 111 | s, isNull := col.stringAt(index[i]) 112 | s2, isNull2 := col2.stringAt(index[i]) 113 | bIndex[i] = !isNull && !isNull2 && s == s2 114 | } 115 | } 116 | return nil 117 | } 118 | -------------------------------------------------------------------------------- /internal/scolumn/generator.go: -------------------------------------------------------------------------------- 1 | package scolumn 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/tobgu/qframe/filter" 7 | "github.com/tobgu/qframe/internal/maps" 8 | "github.com/tobgu/qframe/internal/template" 9 | ) 10 | 11 | //go:generate qfgenerate -source=sfilter -dst-file=filters_gen.go 12 | //go:generate qfgenerate -source=sdoc -dst-file=doc_gen.go 13 | 14 | const basicColConstComparison = ` 15 | func {{.name}}(index index.Int, c Column, comparatee string, bIndex index.Bool) error { 16 | for i, x := range bIndex { 17 | if !x { 18 | s, isNull := c.stringAt(index[i]) 19 | bIndex[i] = !isNull && s {{.operator}} comparatee 20 | } 21 | } 22 | 23 | return nil 24 | } 25 | ` 26 | 27 | const basicColColComparison = ` 28 | func {{.name}}(index index.Int, col, col2 Column, bIndex index.Bool) error { 29 | for i, x := range bIndex { 30 | if !x { 31 | s, isNull := col.stringAt(index[i]) 32 | s2, isNull2 := col2.stringAt(index[i]) 33 | bIndex[i] = !isNull && !isNull2 && s {{.operator}} s2 34 | } 35 | } 36 | return nil 37 | } 38 | ` 39 | 40 | func spec(name, operator, templateStr string) template.Spec { 41 | return template.Spec{ 42 | Name: name, 43 | Template: templateStr, 44 | Values: map[string]interface{}{"name": name, "operator": operator}} 45 | } 46 | 47 | func colConstComparison(name, operator string) template.Spec { 48 | return spec(name, operator, basicColConstComparison) 49 | } 50 | 51 | func colColComparison(name, operator string) template.Spec { 52 | return spec(name, operator, basicColColComparison) 53 | } 54 | 55 | func GenerateFilters() (*bytes.Buffer, error) { 56 | // If adding more filters here make sure to also add a reference to them 57 | // in the corresponding filter map so that they can be looked up. 58 | return template.GenerateFilters("scolumn", []template.Spec{ 59 | colConstComparison("lt", filter.Lt), 60 | colConstComparison("lte", filter.Lte), 61 | colConstComparison("gt", filter.Gt), 62 | colConstComparison("gte", filter.Gte), 63 | colConstComparison("eq", "=="), // Go eq ("==") differs from qframe eq ("=") 64 | colColComparison("lt2", filter.Lt), 65 | colColComparison("lte2", filter.Lte), 66 | colColComparison("gt2", filter.Gt), 67 | colColComparison("gte2", filter.Gte), 68 | colColComparison("eq2", "=="), // Go eq ("==") differs from qframe eq ("=") 69 | }) 70 | } 71 | 72 | func GenerateDoc() (*bytes.Buffer, error) { 73 | return template.GenerateDocs( 74 | "scolumn", 75 | maps.StringKeys(filterFuncs0, filterFuncs1, filterFuncs2, multiInputFilterFuncs), 76 | maps.StringKeys()) 77 | } 78 | -------------------------------------------------------------------------------- /internal/scolumn/view.go: -------------------------------------------------------------------------------- 1 | package scolumn 2 | 3 | import "github.com/tobgu/qframe/internal/index" 4 | 5 | type View struct { 6 | column Column 7 | index index.Int 8 | } 9 | 10 | func (v View) ItemAt(i int) *string { 11 | return stringToPtr(v.column.stringAt(v.index[i])) 12 | } 13 | 14 | func (v View) Len() int { 15 | return len(v.index) 16 | } 17 | 18 | func (v View) Slice() []*string { 19 | result := make([]*string, v.Len()) 20 | for i, j := range v.index { 21 | result[i] = stringToPtr(v.column.stringCopyAt(j)) 22 | } 23 | 24 | return result 25 | } 26 | -------------------------------------------------------------------------------- /internal/sort/GO-LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009 The Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /internal/sort/sorter.go: -------------------------------------------------------------------------------- 1 | // This is a straight copy of the sort functions found in the Go stdlib with 2 | // the interface type Interface replaced with a concrete type for performance reasons 3 | // the original licence text is available in the GO-LICENCE file. 4 | 5 | package sort 6 | 7 | import ( 8 | "github.com/tobgu/qframe/internal/column" 9 | "github.com/tobgu/qframe/internal/index" 10 | ) 11 | 12 | type Sorter struct { 13 | index index.Int 14 | columns []column.Comparable 15 | } 16 | 17 | func New(ix index.Int, columns []column.Comparable) Sorter { 18 | return Sorter{index: ix, columns: columns} 19 | } 20 | 21 | func (s Sorter) Sort() { 22 | n := s.Len() 23 | quickSort(s, 0, n, maxDepth(n)) 24 | } 25 | 26 | func (s Sorter) Len() int { 27 | return len(s.index) 28 | } 29 | 30 | func (s Sorter) Swap(i, j int) { 31 | s.index[i], s.index[j] = s.index[j], s.index[i] 32 | } 33 | 34 | func (s Sorter) Less(i, j int) bool { 35 | di, dj := s.index[i], s.index[j] 36 | for _, s := range s.columns { 37 | r := s.Compare(di, dj) 38 | if r == column.LessThan { 39 | return true 40 | } 41 | 42 | if r == column.GreaterThan { 43 | return false 44 | } 45 | } 46 | 47 | return false 48 | } 49 | 50 | // Copyright 2009 The Go Authors. All rights reserved. 51 | // Use of this source code is governed by a BSD-style 52 | // license that can be found in the LICENSE file. 53 | 54 | // Insertion sort 55 | func insertionSort(data Sorter, a, b int) { 56 | for i := a + 1; i < b; i++ { 57 | for j := i; j > a && data.Less(j, j-1); j-- { 58 | data.Swap(j, j-1) 59 | } 60 | } 61 | } 62 | 63 | // siftDown implements the heap property on data[lo, hi). 64 | // first is an offset into the array where the root of the heap lies. 65 | func siftDown(data Sorter, lo, hi, first int) { 66 | root := lo 67 | for { 68 | child := 2*root + 1 69 | if child >= hi { 70 | break 71 | } 72 | if child+1 < hi && data.Less(first+child, first+child+1) { 73 | child++ 74 | } 75 | if !data.Less(first+root, first+child) { 76 | return 77 | } 78 | data.Swap(first+root, first+child) 79 | root = child 80 | } 81 | } 82 | 83 | func heapSort(data Sorter, a, b int) { 84 | first := a 85 | lo := 0 86 | hi := b - a 87 | 88 | // Build heap with greatest element at top. 89 | for i := (hi - 1) / 2; i >= 0; i-- { 90 | siftDown(data, i, hi, first) 91 | } 92 | 93 | // Pop elements, largest first, into end of data. 94 | for i := hi - 1; i >= 0; i-- { 95 | data.Swap(first, first+i) 96 | siftDown(data, lo, i, first) 97 | } 98 | } 99 | 100 | // Quicksort, loosely following Bentley and McIlroy, 101 | // ``Engineering a Sort Function,'' SP&E November 1993. 102 | 103 | // medianOfThree moves the median of the three values data[m0], data[m1], data[m2] into data[m1]. 104 | func medianOfThree(data Sorter, m1, m0, m2 int) { 105 | // sort 3 elements 106 | if data.Less(m1, m0) { 107 | data.Swap(m1, m0) 108 | } 109 | // data[m0] <= data[m1] 110 | if data.Less(m2, m1) { 111 | data.Swap(m2, m1) 112 | // data[m0] <= data[m2] && data[m1] < data[m2] 113 | if data.Less(m1, m0) { 114 | data.Swap(m1, m0) 115 | } 116 | } 117 | // now data[m0] <= data[m1] <= data[m2] 118 | } 119 | 120 | func doPivot(data Sorter, lo, hi int) (midlo, midhi int) { 121 | m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow. 122 | if hi-lo > 40 { 123 | // Tukey's ``Ninther,'' median of three medians of three. 124 | s := (hi - lo) / 8 125 | medianOfThree(data, lo, lo+s, lo+2*s) 126 | medianOfThree(data, m, m-s, m+s) 127 | medianOfThree(data, hi-1, hi-1-s, hi-1-2*s) 128 | } 129 | medianOfThree(data, lo, m, hi-1) 130 | 131 | // Invariants are: 132 | // data[lo] = pivot (set up by ChoosePivot) 133 | // data[lo < i < a] < pivot 134 | // data[a <= i < b] <= pivot 135 | // data[b <= i < c] unexamined 136 | // data[c <= i < hi-1] > pivot 137 | // data[hi-1] >= pivot 138 | pivot := lo 139 | a, c := lo+1, hi-1 140 | 141 | for ; a < c && data.Less(a, pivot); a++ { 142 | } 143 | b := a 144 | for { 145 | for ; b < c && !data.Less(pivot, b); b++ { // data[b] <= pivot 146 | } 147 | for ; b < c && data.Less(pivot, c-1); c-- { // data[c-1] > pivot 148 | } 149 | if b >= c { 150 | break 151 | } 152 | // data[b] > pivot; data[c-1] <= pivot 153 | data.Swap(b, c-1) 154 | b++ 155 | c-- 156 | } 157 | // If hi-c<3 then there are duplicates (by property of median of nine). 158 | // Let be a bit more conservative, and set border to 5. 159 | protect := hi-c < 5 160 | if !protect && hi-c < (hi-lo)/4 { 161 | // Lets test some points for equality to pivot 162 | dups := 0 163 | if !data.Less(pivot, hi-1) { // data[hi-1] = pivot 164 | data.Swap(c, hi-1) 165 | c++ 166 | dups++ 167 | } 168 | if !data.Less(b-1, pivot) { // data[b-1] = pivot 169 | b-- 170 | dups++ 171 | } 172 | // m-lo = (hi-lo)/2 > 6 173 | // b-lo > (hi-lo)*3/4-1 > 8 174 | // ==> m < b ==> data[m] <= pivot 175 | if !data.Less(m, pivot) { // data[m] = pivot 176 | data.Swap(m, b-1) 177 | b-- 178 | dups++ 179 | } 180 | // if at least 2 points are equal to pivot, assume skewed distribution 181 | protect = dups > 1 182 | } 183 | if protect { 184 | // Protect against a lot of duplicates 185 | // Add invariant: 186 | // data[a <= i < b] unexamined 187 | // data[b <= i < c] = pivot 188 | for { 189 | for ; a < b && !data.Less(b-1, pivot); b-- { // data[b] == pivot 190 | } 191 | for ; a < b && data.Less(a, pivot); a++ { // data[a] < pivot 192 | } 193 | if a >= b { 194 | break 195 | } 196 | // data[a] == pivot; data[b-1] < pivot 197 | data.Swap(a, b-1) 198 | a++ 199 | b-- 200 | } 201 | } 202 | // Swap pivot into middle 203 | data.Swap(pivot, b-1) 204 | return b - 1, c 205 | } 206 | 207 | func quickSort(data Sorter, a, b, maxDepth int) { 208 | for b-a > 12 { // Use ShellSort for slices <= 12 elements 209 | if maxDepth == 0 { 210 | heapSort(data, a, b) 211 | return 212 | } 213 | maxDepth-- 214 | mlo, mhi := doPivot(data, a, b) 215 | // Avoiding recursion on the larger subproblem guarantees 216 | // a stack depth of at most lg(b-a). 217 | if mlo-a < b-mhi { 218 | quickSort(data, a, mlo, maxDepth) 219 | a = mhi // i.e., quickSort(data, mhi, b) 220 | } else { 221 | quickSort(data, mhi, b, maxDepth) 222 | b = mlo // i.e., quickSort(data, a, mlo) 223 | } 224 | } 225 | if b-a > 1 { 226 | // Do ShellSort pass with gap 6 227 | // It could be written in this simplified form cause b-a <= 12 228 | for i := a + 6; i < b; i++ { 229 | if data.Less(i, i-6) { 230 | data.Swap(i, i-6) 231 | } 232 | } 233 | insertionSort(data, a, b) 234 | } 235 | } 236 | 237 | // maxDepth returns a threshold at which quicksort should switch 238 | // to heapsort. It returns 2*ceil(lg(n+1)). 239 | func maxDepth(n int) int { 240 | var depth int 241 | for i := n; i > 0; i >>= 1 { 242 | depth++ 243 | } 244 | return depth * 2 245 | } 246 | -------------------------------------------------------------------------------- /internal/strings/convert.go: -------------------------------------------------------------------------------- 1 | package strings 2 | 3 | import ( 4 | "strconv" 5 | "unicode" 6 | "unicode/utf8" 7 | "unsafe" 8 | ) 9 | 10 | func ParseInt(b []byte) (i int, err error) { 11 | s := UnsafeBytesToString(b) 12 | return strconv.Atoi(s) 13 | } 14 | 15 | func ParseFloat(b []byte) (float64, error) { 16 | s := UnsafeBytesToString(b) 17 | return strconv.ParseFloat(s, 64) 18 | } 19 | 20 | func ParseBool(b []byte) (bool, error) { 21 | return strconv.ParseBool(UnsafeBytesToString(b)) 22 | } 23 | 24 | func UnsafeBytesToString(in []byte) string { 25 | return unsafe.String(unsafe.SliceData(in), len(in)) 26 | } 27 | 28 | func QuotedBytes(s string) []byte { 29 | result := make([]byte, 0, len(s)+2) 30 | result = append(result, byte('"')) 31 | result = append(result, []byte(s)...) 32 | return append(result, byte('"')) 33 | } 34 | 35 | // This is a modified, zero alloc, version of the stdlib function strings.ToUpper. 36 | // The passed in byte buffer is used to hold the converted string. The returned 37 | // string is not safe to use when bP goes out of scope and the content may 38 | // be overwritten upon next call to this function. 39 | func ToUpper(bP *[]byte, s string) string { 40 | // nbytes is the number of bytes encoded in b. 41 | var nbytes int 42 | 43 | var b []byte 44 | for i, c := range s { 45 | r := unicode.ToUpper(c) 46 | if r == c { 47 | continue 48 | } 49 | 50 | if len(*bP) >= len(s)+utf8.UTFMax { 51 | b = *bP 52 | } else { 53 | b = make([]byte, len(s)+utf8.UTFMax) 54 | } 55 | nbytes = copy(b, s[:i]) 56 | if r >= 0 { 57 | if r <= utf8.RuneSelf { 58 | b[nbytes] = byte(r) 59 | nbytes++ 60 | } else { 61 | nbytes += utf8.EncodeRune(b[nbytes:], r) 62 | } 63 | } 64 | 65 | if c == utf8.RuneError { 66 | // RuneError is the result of either decoding 67 | // an invalid sequence or '\uFFFD'. Determine 68 | // the correct number of bytes we need to advance. 69 | _, w := utf8.DecodeRuneInString(s[i:]) 70 | i += w 71 | } else { 72 | i += utf8.RuneLen(c) 73 | } 74 | 75 | s = s[i:] 76 | break 77 | } 78 | 79 | if b == nil { 80 | return s 81 | } 82 | 83 | for _, c := range s { 84 | r := unicode.ToUpper(c) 85 | 86 | // common case 87 | if (0 <= r && r <= utf8.RuneSelf) && nbytes < len(b) { 88 | b[nbytes] = byte(r) 89 | nbytes++ 90 | continue 91 | } 92 | 93 | // b is not big enough or r is not a ASCII rune. 94 | if r >= 0 { 95 | if nbytes+utf8.UTFMax >= len(b) { 96 | // Grow the buffer. 97 | nb := make([]byte, 2*len(b)) 98 | copy(nb, b[:nbytes]) 99 | b = nb 100 | } 101 | nbytes += utf8.EncodeRune(b[nbytes:], r) 102 | } 103 | } 104 | 105 | *bP = b 106 | return UnsafeBytesToString(b[:nbytes]) 107 | } 108 | 109 | // InterfaceSliceToStringSlice converts a slice of interface{} to a slice of strings. 110 | // If the input is not a slice of interface{} it is returned unmodified. If the input 111 | // slice does not consist of strings (only) the input is returned unmodified. 112 | func InterfaceSliceToStringSlice(input interface{}) interface{} { 113 | ifSlice, ok := input.([]interface{}) 114 | if !ok { 115 | return input 116 | } 117 | 118 | result := make([]string, len(ifSlice)) 119 | for i, intfc := range ifSlice { 120 | s, ok := intfc.(string) 121 | if !ok { 122 | return input 123 | } 124 | result[i] = s 125 | } 126 | 127 | return result 128 | } 129 | -------------------------------------------------------------------------------- /internal/strings/match.go: -------------------------------------------------------------------------------- 1 | package strings 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | 7 | "github.com/tobgu/qframe/qerrors" 8 | ) 9 | 10 | type Matcher interface { 11 | Matches(s string) bool 12 | } 13 | 14 | type CIStringMatcher struct { 15 | matchString string //nolint:structcheck 16 | buf []byte //nolint:structcheck 17 | } 18 | 19 | type CIPrefixMatcher CIStringMatcher 20 | 21 | func (m *CIPrefixMatcher) Matches(s string) bool { 22 | return strings.HasPrefix(ToUpper(&m.buf, s), m.matchString) 23 | } 24 | 25 | type CISuffixMatcher CIStringMatcher 26 | 27 | func (m *CISuffixMatcher) Matches(s string) bool { 28 | return strings.HasSuffix(ToUpper(&m.buf, s), m.matchString) 29 | } 30 | 31 | type CIContainsMatcher CIStringMatcher 32 | 33 | func (m *CIContainsMatcher) Matches(s string) bool { 34 | return strings.Contains(ToUpper(&m.buf, s), m.matchString) 35 | } 36 | 37 | type CIExactMatcher CIStringMatcher 38 | 39 | func (m *CIExactMatcher) Matches(s string) bool { 40 | return ToUpper(&m.buf, s) == m.matchString 41 | } 42 | 43 | type StringMatcher struct { 44 | matchString string //nolint:structcheck 45 | } 46 | 47 | type PrefixMatcher StringMatcher 48 | 49 | func (m *PrefixMatcher) Matches(s string) bool { 50 | return strings.HasPrefix(s, m.matchString) 51 | } 52 | 53 | type SuffixMatcher StringMatcher 54 | 55 | func (m *SuffixMatcher) Matches(s string) bool { 56 | return strings.HasSuffix(s, m.matchString) 57 | } 58 | 59 | type ContainsMatcher StringMatcher 60 | 61 | func (m *ContainsMatcher) Matches(s string) bool { 62 | return strings.Contains(s, m.matchString) 63 | } 64 | 65 | type ExactMatcher StringMatcher 66 | 67 | func (m *ExactMatcher) Matches(s string) bool { 68 | return s == m.matchString 69 | } 70 | 71 | type RegexpMatcher struct { 72 | r *regexp.Regexp 73 | } 74 | 75 | func (m *RegexpMatcher) Matches(s string) bool { 76 | return m.r.MatchString(s) 77 | } 78 | 79 | func trimPercent(s string) string { 80 | s = strings.TrimPrefix(s, "%") 81 | s = strings.TrimSuffix(s, "%") 82 | return s 83 | } 84 | 85 | func NewMatcher(comparatee string, caseSensitive bool) (Matcher, error) { 86 | fuzzyStart := strings.HasPrefix(comparatee, "%") 87 | fuzzyEnd := strings.HasSuffix(comparatee, "%") 88 | if regexp.QuoteMeta(comparatee) != comparatee { 89 | // There are regex characters in the match string 90 | if !fuzzyStart { 91 | comparatee = "^" + comparatee 92 | } else { 93 | comparatee = comparatee[1:] 94 | } 95 | 96 | if !fuzzyEnd { 97 | comparatee = comparatee + "$" 98 | } else { 99 | comparatee = comparatee[:len(comparatee)-1] 100 | } 101 | 102 | if !caseSensitive { 103 | comparatee = "(?i)" + comparatee 104 | } 105 | 106 | r, err := regexp.Compile(comparatee) 107 | if err != nil { 108 | return nil, qerrors.Propagate("string like", err) 109 | } 110 | 111 | return &RegexpMatcher{r: r}, nil 112 | } 113 | 114 | if !caseSensitive { 115 | comparatee = strings.ToUpper(comparatee) 116 | 117 | // Initial size, this will grow if needed 118 | buf := make([]byte, 10) 119 | if fuzzyStart && fuzzyEnd { 120 | return &CIContainsMatcher{matchString: trimPercent(comparatee), buf: buf}, nil 121 | } 122 | 123 | if fuzzyStart { 124 | return &CISuffixMatcher{matchString: trimPercent(comparatee), buf: buf}, nil 125 | } 126 | 127 | if fuzzyEnd { 128 | return &CIPrefixMatcher{matchString: trimPercent(comparatee), buf: buf}, nil 129 | } 130 | 131 | return &CIExactMatcher{matchString: comparatee, buf: buf}, nil 132 | } 133 | 134 | if fuzzyStart && fuzzyEnd { 135 | return &ContainsMatcher{matchString: trimPercent(comparatee)}, nil 136 | } 137 | 138 | if fuzzyStart { 139 | return &SuffixMatcher{matchString: trimPercent(comparatee)}, nil 140 | } 141 | 142 | if fuzzyEnd { 143 | return &PrefixMatcher{matchString: trimPercent(comparatee)}, nil 144 | } 145 | 146 | return &ExactMatcher{matchString: comparatee}, nil 147 | } 148 | -------------------------------------------------------------------------------- /internal/strings/name.go: -------------------------------------------------------------------------------- 1 | package strings 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/tobgu/qframe/qerrors" 7 | ) 8 | 9 | func isQuoted(s string) bool { 10 | return len(s) > 2 && 11 | ((strings.HasPrefix(s, "'") && strings.HasSuffix(s, "'")) || 12 | (strings.HasPrefix(s, `"`) && strings.HasSuffix(s, `"`))) 13 | } 14 | 15 | func CheckName(name string) error { 16 | if len(name) == 0 { 17 | return qerrors.New("CheckName", "column name must not be empty") 18 | } 19 | 20 | if isQuoted(name) { 21 | // Reserved for future use 22 | return qerrors.New("CheckName", "column name must not be quoted: %s", name) 23 | } 24 | 25 | // Reserved for future use of variables in Eval 26 | if strings.HasPrefix(name, "$") { 27 | return qerrors.New("CheckName", "column name must not start with $: %s", name) 28 | } 29 | 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /internal/strings/pointer.go: -------------------------------------------------------------------------------- 1 | package strings 2 | 3 | import "fmt" 4 | 5 | // Pointer identifies a string within a StringBlob. 6 | // Max individual string size 2^28 byte ~ 268 Mb 7 | // Max total size 2^35 byte ~ 34 Gb 8 | type Pointer uint64 9 | 10 | // StringBlob represents a set of strings. 11 | // The underlying data is stored in a byte blob which can be interpreted through 12 | // the pointers which identifies the start and end of individual strings in the blob. 13 | // 14 | // This structure is used instead of a slice of strings or a slice of 15 | // string pointers is to avoid that the GC has to scan all pointers which 16 | // takes quite some time with large/many live frames. 17 | type StringBlob struct { 18 | Pointers []Pointer 19 | Data []byte 20 | } 21 | 22 | const nullBit = 0x8000000000000000 23 | 24 | func NewPointer(offset, length int, isNull bool) Pointer { 25 | result := Pointer(offset<<28 | length) 26 | if isNull { 27 | result |= nullBit 28 | } 29 | return result 30 | } 31 | 32 | func (p Pointer) Offset() int { 33 | return int(p>>28) & 0x7FFFFFFFF 34 | } 35 | 36 | func (p Pointer) Len() int { 37 | return int(p) & 0xFFFFFFF 38 | } 39 | 40 | func (p Pointer) IsNull() bool { 41 | return p&nullBit > 0 42 | } 43 | 44 | func (p Pointer) String() string { 45 | return fmt.Sprintf("{offset: %d, len: %d, isNull: %v}", 46 | p.Offset(), p.Len(), p.IsNull()) 47 | } 48 | -------------------------------------------------------------------------------- /internal/strings/serialize.go: -------------------------------------------------------------------------------- 1 | package strings 2 | 3 | import ( 4 | "unicode/utf8" 5 | ) 6 | 7 | const chars = "0123456789abcdef" 8 | 9 | func AppendQuotedString(buf []byte, str string) []byte { 10 | // String escape code is highly inspired by the escape code in easyjson. 11 | buf = append(buf, '"') 12 | p := 0 13 | // last non-escape symbol 14 | for i := 0; i < len(str); { 15 | c := str[i] 16 | 17 | if c != '\\' && c != '"' && c >= 0x20 && c < utf8.RuneSelf { 18 | // single-width character, no escaping is required 19 | i++ 20 | continue 21 | } 22 | 23 | if c < utf8.RuneSelf { 24 | // single-with character, need to escape 25 | buf = append(buf, str[p:i]...) 26 | 27 | switch c { 28 | case '\t': 29 | buf = append(buf, `\t`...) 30 | case '\r': 31 | buf = append(buf, `\r`...) 32 | case '\n': 33 | buf = append(buf, `\n`...) 34 | case '\\': 35 | buf = append(buf, `\\`...) 36 | case '"': 37 | buf = append(buf, `\"`...) 38 | default: 39 | buf = append(buf, `\u00`...) 40 | buf = append(buf, chars[c>>4]) 41 | buf = append(buf, chars[c&0xf]) 42 | } 43 | 44 | i++ 45 | p = i 46 | continue 47 | } 48 | 49 | // broken utf 50 | runeValue, runeWidth := utf8.DecodeRuneInString(str[i:]) 51 | if runeValue == utf8.RuneError && runeWidth == 1 { 52 | buf = append(buf, str[p:i]...) 53 | buf = append(buf, `\ufffd`...) 54 | i++ 55 | p = i 56 | continue 57 | } 58 | 59 | // jsonp stuff - tab separator and line separator 60 | if runeValue == '\u2028' || runeValue == '\u2029' { 61 | buf = append(buf, str[p:i]...) 62 | buf = append(buf, `\u202`...) 63 | buf = append(buf, chars[runeValue&0xf]) 64 | i += runeWidth 65 | p = i 66 | continue 67 | } 68 | i += runeWidth 69 | } 70 | 71 | buf = append(buf, str[p:]...) 72 | buf = append(buf, '"') 73 | return buf 74 | } 75 | -------------------------------------------------------------------------------- /internal/strings/set.go: -------------------------------------------------------------------------------- 1 | package strings 2 | 3 | type StringSet map[string]struct{} 4 | 5 | func NewEmptyStringSet() StringSet { 6 | return make(StringSet) 7 | } 8 | 9 | func NewStringSet(input []string) StringSet { 10 | result := make(StringSet, len(input)) 11 | for _, s := range input { 12 | result.Add(s) 13 | } 14 | 15 | return result 16 | } 17 | 18 | func (ss StringSet) Contains(s string) bool { 19 | _, ok := ss[s] 20 | return ok 21 | } 22 | 23 | func (ss StringSet) Add(s string) { 24 | ss[s] = struct{}{} 25 | } 26 | 27 | func (ss StringSet) AsSlice() []string { 28 | result := make([]string, 0, len(ss)) 29 | for k := range ss { 30 | result = append(result, k) 31 | } 32 | 33 | return result 34 | } 35 | -------------------------------------------------------------------------------- /internal/template/docs.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import "bytes" 4 | 5 | const DocTemplate = ` 6 | func Doc() string { 7 | return "\n Built in filters\n" + 8 | {{ range $name := .filters }}" {{$name}}\n" + 9 | {{ end }} 10 | "\n Built in aggregations\n" + 11 | {{ range $name := .aggregations }}" {{$name}}\n" + 12 | {{ end }}"\n" 13 | } 14 | ` 15 | 16 | func GenerateDocs(pkgName string, filters, aggregations []string) (*bytes.Buffer, error) { 17 | values := map[string]interface{}{ 18 | "filters": filters, 19 | "aggregations": aggregations} 20 | 21 | return Generate(pkgName, []Spec{{Name: "filterdocs", Template: DocTemplate, Values: values}}, []string{}) 22 | } 23 | -------------------------------------------------------------------------------- /internal/template/filters.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import ( 4 | "bytes" 5 | ) 6 | 7 | const BasicColConstComparison = ` 8 | func {{.name}}(index index.Int, column []{{.dataType}}, comp {{.dataType}}, bIndex index.Bool) { 9 | for i, x := range bIndex { 10 | if !x { 11 | bIndex[i] = column[index[i]] {{.operator}} comp 12 | } 13 | } 14 | } 15 | ` 16 | 17 | const BasicColColComparison = ` 18 | func {{.name}}(index index.Int, column []{{.dataType}}, compCol []{{.dataType}}, bIndex index.Bool) { 19 | for i, x := range bIndex { 20 | if !x { 21 | pos := index[i] 22 | bIndex[i] = column[pos] {{.operator}} compCol[pos] 23 | } 24 | } 25 | } 26 | ` 27 | 28 | func GenerateFilters(pkgName string, specs []Spec) (*bytes.Buffer, error) { 29 | return Generate(pkgName, specs, []string{"github.com/tobgu/qframe/internal/index"}) 30 | } 31 | -------------------------------------------------------------------------------- /internal/template/generate.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "text/template" 7 | ) 8 | 9 | const HeaderTemplate = ` 10 | package {{.pkgName}} 11 | 12 | {{if .imports}} 13 | import ( 14 | {{ range $_, $imp := .imports }} 15 | "{{$imp}}"{{ end }} 16 | ) 17 | {{end}} 18 | 19 | // Code generated from template/... DO NOT EDIT 20 | ` 21 | 22 | type Spec struct { 23 | Name string 24 | Template string 25 | Values map[string]interface{} 26 | } 27 | 28 | func render(name, templateStr string, templateData interface{}, dst io.Writer) error { 29 | t := template.New(name) 30 | t, err := t.Parse(templateStr) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | err = t.Execute(dst, templateData) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | return nil 41 | } 42 | 43 | func Generate(pkgName string, specs []Spec, imports []string) (*bytes.Buffer, error) { 44 | var buf bytes.Buffer 45 | values := map[string]interface{}{"pkgName": pkgName, "imports": imports} 46 | renderValues := append([]Spec{{Name: "header", Template: HeaderTemplate, Values: values}}, specs...) 47 | for _, v := range renderValues { 48 | if err := render(v.Name, v.Template, v.Values, &buf); err != nil { 49 | return nil, err 50 | } 51 | } 52 | 53 | return &buf, nil 54 | } 55 | -------------------------------------------------------------------------------- /internal/template/placeholders.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/tobgu/qframe/qerrors" 6 | 7 | "github.com/tobgu/qframe/internal/column" 8 | "github.com/tobgu/qframe/internal/index" 9 | "github.com/tobgu/qframe/types" 10 | ) 11 | 12 | // This file contains definitions for data and functions that need to be added 13 | // manually for each data type. 14 | 15 | // TODO: Probably need a more general aggregation pattern, int -> float (average for example) 16 | var aggregations = map[string]func([]genericDataType) genericDataType{} 17 | 18 | func (c Column) DataType() types.DataType { 19 | return types.None 20 | } 21 | 22 | // Functions not generated but needed to fulfill interface 23 | func (c Column) AppendByteStringAt(buf []byte, i uint32) []byte { 24 | return nil 25 | } 26 | 27 | func (c Column) ByteSize() int { 28 | return 0 29 | } 30 | 31 | func (c Column) Equals(index index.Int, other column.Column, otherIndex index.Int) bool { 32 | return false 33 | } 34 | 35 | func (c Column) Filter(index index.Int, comparator interface{}, comparatee interface{}, bIndex index.Bool) error { 36 | return nil 37 | } 38 | 39 | func (c Column) FunctionType() types.FunctionType { 40 | return types.FunctionTypeBool 41 | } 42 | 43 | func (c Column) Marshaler(index index.Int) json.Marshaler { 44 | return nil 45 | } 46 | 47 | func (c Column) StringAt(i uint32, naRep string) string { 48 | return "" 49 | } 50 | 51 | func (c Column) Append(cols ...column.Column) (column.Column, error) { 52 | return nil, qerrors.New("Append", "Not implemented") 53 | } 54 | 55 | func (c Comparable) Compare(i, j uint32) column.CompareResult { 56 | return column.Equal 57 | } 58 | 59 | func (c Comparable) Hash(i uint32, seed uint64) uint64 { 60 | return 0 61 | } 62 | -------------------------------------------------------------------------------- /qerrors/error.go: -------------------------------------------------------------------------------- 1 | package qerrors 2 | 3 | import "fmt" 4 | 5 | // Error holds data identifying an error that occurred 6 | // while executing a qframe operation. 7 | type Error struct { 8 | source error 9 | operation string 10 | reason string 11 | } 12 | 13 | // Error returns a string representation of the error. 14 | func (e Error) Error() string { 15 | result := e.operation 16 | if e.reason != "" { 17 | result += ": " + e.reason 18 | } 19 | 20 | if e.source != nil { 21 | result += fmt.Sprintf(" (%s)", e.source) 22 | } 23 | 24 | return result 25 | } 26 | 27 | // New creates a new error instance. 28 | func New(operation, reason string, params ...interface{}) Error { 29 | return Error{operation: operation, reason: fmt.Sprintf(reason, params...)} 30 | } 31 | 32 | // Propagate propagates an existing error with added context. 33 | func Propagate(operation string, err error) Error { 34 | return Error{operation: operation, source: err} 35 | } 36 | 37 | // Error types: 38 | // - Type error 39 | // - Input error (which would basically always be the case...) 40 | -------------------------------------------------------------------------------- /types/aliases.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | /* 4 | This slightly unconventional use of type aliasing is meant to provide a hook for documentation 5 | of the different uses of interface{} that exists in QFrame. Since there is nothing like a union 6 | or a sum type in Go, QFrame settles for the use of interface{} for some input. 7 | 8 | Hopefully this construct says a bit more than nothing about the empty interfaces used. 9 | */ 10 | 11 | /* 12 | DataSlice can be a slice of any of the supported data types. 13 | 14 | The following types are currently supported: 15 | []bool 16 | []float64 17 | []int 18 | []string 19 | []*string 20 | */ 21 | type DataSlice = interface{} 22 | 23 | /* 24 | SliceFuncOrBuiltInId can be a function taking a slice of type T and returning a value of type T. 25 | 26 | For example: 27 | func(x []float64) float64 28 | func(x []int) int 29 | func(x []*string) *string 30 | func(x []bool) bool 31 | 32 | Or it can be a string identifying a built in function. 33 | 34 | For example: 35 | "sum" 36 | 37 | IMPORTANT: Reference arguments (eg. slices) must never be assumed to be valid after that the passed function returns. 38 | Under the hood reuse and other performance enhancements may trigger unexpected behaviour if this is ever done. 39 | If, for some reason, you want to retain the data a copy must be made. 40 | */ 41 | type SliceFuncOrBuiltInId = interface{} 42 | 43 | /* 44 | DataFuncOrBuiltInId can be a function taking one argument of type T and returning a value of type U. 45 | 46 | For example: 47 | func(x float64) float64 48 | func(x float64) int 49 | 50 | Or it can be a function taking zero arguments returning a value of type T. 51 | 52 | For example: 53 | func() float64 54 | func() int 55 | 56 | Or it can be a function taking two arguments of type T and returning a value of type T. Note that arguments 57 | and return values must all have the same type in this case. 58 | 59 | For example: 60 | func(x, y float64) float64 61 | func(x, y int) int 62 | 63 | Or it can be a string identifying a built in function. 64 | 65 | For example: 66 | "abs" 67 | 68 | IMPORTANT: Pointer arguments (eg. *string) must never be assumed to be valid after that the passed function returns. 69 | Under the hood reuse and other performance enhancements may trigger unexpected behaviour if this is ever done. 70 | If, for some reason, you want to retain the data a copy must be made. 71 | */ 72 | type DataFuncOrBuiltInId = interface{} 73 | -------------------------------------------------------------------------------- /types/markers.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // ColumnName is used to separate a column identifier from a string constant. 4 | // It is used when filtering and evaluating expressions where ambiguities 5 | // would otherwise arise. 6 | type ColumnName string 7 | -------------------------------------------------------------------------------- /types/types.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // DataType represents any of the data types valid in a QFrame. 4 | type DataType string 5 | 6 | const ( 7 | // None represents an unknown data type. 8 | // This is mainly used to indicate that the type of a column should be auto detected. 9 | None DataType = "" 10 | 11 | // Int translates into the Go int type. Missing values cannot be represented explicitly. 12 | Int = "int" 13 | 14 | // String translates into the Go *string type. nil represents a missing value. 15 | // Internally a string currently has an overhead of eight bytes (64 bits) in 16 | // addition to the bytes actually used to hold the string. 17 | String = "string" 18 | 19 | // Float translates into the Go float64 type. NaN represents a missing value. 20 | Float = "float" 21 | 22 | // Bool translates into the Go bool type. Missing values cannot be represented explicitly. 23 | Bool = "bool" 24 | 25 | // Enum translates into the Go *string type. nil represents a missing value. 26 | // An enum column can, at most, have 254 distinct values. 27 | Enum = "enum" 28 | 29 | // Undefined represents an unspecified data type. 30 | // This is used for zero length columns where the datatype could not be identified. 31 | Undefined DataType = "Undefined" 32 | ) 33 | 34 | // FunctionType represents the different types of input that functions operating on columns can take. 35 | type FunctionType byte 36 | 37 | const ( 38 | FunctionTypeUndefined FunctionType = iota 39 | FunctionTypeInt 40 | FunctionTypeFloat 41 | FunctionTypeBool 42 | FunctionTypeString 43 | ) 44 | 45 | func (t FunctionType) String() string { 46 | switch t { 47 | case FunctionTypeInt: 48 | return "Int function" 49 | case FunctionTypeBool: 50 | return "Bool function" 51 | case FunctionTypeString: 52 | return "String function" 53 | case FunctionTypeFloat: 54 | return "Float function" 55 | case FunctionTypeUndefined: 56 | return "Undefined type function" 57 | default: 58 | return "Unknown function" 59 | } 60 | } 61 | --------------------------------------------------------------------------------