├── .github └── main.workflow ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── backends ├── aggregator.go ├── backends.go ├── bleve-indexer.go ├── caching-backend.go ├── dynamo-indexer.go ├── dynamo.go ├── dynamo_test.go ├── elasticsearch-aggregator.go ├── elasticsearch-indexer.go ├── elasticsearch.go ├── embedded-record-backend.go ├── embedded_records_test.go ├── file_backend.go ├── fs-indexer.go ├── fs.go ├── indexer-multi.go ├── indexer-null.go ├── indexers.go ├── mapper.go ├── metaindex.go ├── mongo-aggregator.go ├── mongo-indexer.go ├── mongo.go ├── monitoring-backend.go.disabled ├── options.go ├── redis-indexer.go ├── redis.go ├── redis_test.go ├── relationships.go ├── sql-aggregator.go ├── sql-backend-mysql.go ├── sql-backend-postgres.go ├── sql-backend-sqlite.go ├── sql-indexer.go ├── sql.go └── sql_test.go ├── client └── client.go ├── cmd └── pivot │ └── main.go ├── config.go ├── dal ├── collection.go ├── collection_test.go ├── connection.go ├── constraint.go ├── errors.go ├── field.go ├── field_test.go ├── formatters.go ├── record.go ├── record_loader.go ├── record_loader_test.go ├── record_test.go ├── recordset.go ├── recordset_test.go ├── relationship.go ├── types.go └── validators.go ├── database.go ├── db_test.go ├── docs ├── - │ ├── about.html │ ├── bootstrap.min.css │ ├── bootstrap.min.js │ ├── jquery-2.2.4.min.js │ ├── module.html │ ├── site.css │ └── site.js ├── index.html ├── module.json └── pkg │ ├── backends.html │ ├── backends.json │ ├── client.html │ ├── client.json │ ├── dal.html │ ├── dal.json │ ├── filter.html │ ├── filter.json │ ├── filter │ ├── generators.html │ └── generators.json │ ├── mapper.html │ ├── mapper.json │ ├── pivot.html │ ├── pivot.json │ ├── util.html │ └── util.json ├── examples ├── basic-crud │ └── main.go └── embedded-collections │ ├── fixtures │ └── contacts.json │ └── main.go ├── filter ├── README.md ├── filter.go ├── filter_matches_record_test.go ├── filter_test.go ├── generator.go └── generators │ ├── elasticsearch.go │ ├── elasticsearch_util.go │ ├── mongodb-util.go │ ├── mongodb.go │ ├── mongodb_test.go │ ├── sql.go │ └── sql_test.go ├── go.mod ├── go.sum ├── lib └── python │ ├── Makefile │ ├── README.md │ ├── pivot │ ├── __init__.py │ ├── client.py │ ├── collection.py │ ├── exceptions.py │ ├── results.py │ └── utils.py │ ├── requirements.txt │ ├── setup.py │ └── shell.py ├── mapper └── model.go ├── pivot.go ├── server.go ├── static.go ├── test ├── elasticsearch.yml ├── fixtures │ ├── README.md │ └── testing.json ├── schema │ ├── README.md │ ├── groups.json │ └── users.json ├── test.yml └── us-fips.csv ├── ui ├── _includes │ └── paginator.html ├── _layouts │ └── default.html ├── _query.html ├── css │ ├── app.css │ ├── bootstrap.min.css │ ├── bootstrap.min.css.map │ ├── codemirror-theme.css │ ├── codemirror.css │ ├── font-awesome.min.css │ └── jquery.json-viewer.css ├── editor.html ├── fonts │ ├── FontAwesome.otf │ ├── fontawesome-webfont.eot │ ├── fontawesome-webfont.svg │ ├── fontawesome-webfont.ttf │ ├── fontawesome-webfont.woff │ ├── fontawesome-webfont.woff2 │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.svg │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ └── glyphicons-halflings-regular.woff2 ├── index.html ├── js │ ├── URI.min.js │ ├── app.js │ ├── bootstrap.bundle.min.js │ ├── bootstrap.bundle.min.js.map │ ├── bootstrap.min.js │ ├── codemirror.js │ ├── codemirror │ │ └── addons │ │ │ ├── active-line.js │ │ │ ├── closebrackets.js │ │ │ ├── matchbrackets.js │ │ │ └── show-hint.js │ ├── director.min.js │ ├── jquery-2.2.4.min.js │ ├── jquery.json-viewer.js │ ├── stapes.min.js │ └── tether.min.js └── views │ ├── backend.html │ └── index.html ├── util ├── features.go ├── http.go ├── types.go └── version.go ├── v3 └── version.go /.github/main.workflow: -------------------------------------------------------------------------------- 1 | # .github/main.workflow 2 | 3 | workflow "Build" { 4 | on = "release" 5 | resolves = [ 6 | "release darwin/amd64", 7 | "release freebsd/amd64", 8 | "release linux/amd64", 9 | ] 10 | } 11 | 12 | action "release darwin/amd64" { 13 | uses = "ghetzel/go-release.action@d2a93b3e6c7c606f23c1d6dc07797b357e455fa5" 14 | env = { 15 | GOOS = "darwin" 16 | GOARCH = "amd64" 17 | } 18 | secrets = ["GITHUB_TOKEN"] 19 | } 20 | 21 | action "release freebsd/amd64" { 22 | uses = "ghetzel/go-release.action@d2a93b3e6c7c606f23c1d6dc07797b357e455fa5" 23 | env = { 24 | GOOS = "freebsd" 25 | GOARCH = "amd64" 26 | } 27 | secrets = ["GITHUB_TOKEN"] 28 | } 29 | 30 | action "release linux/amd64" { 31 | uses = "ghetzel/go-release.action@d2a93b3e6c7c606f23c1d6dc07797b357e455fa5" 32 | env = { 33 | GOOS = "linux" 34 | GOARCH = "amd64" 35 | } 36 | secrets = ["GITHUB_TOKEN"] 37 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | /vendor 3 | /public 4 | /test.yml 5 | /*.db 6 | /*.db-* 7 | /tmp 8 | /*.json 9 | *.pyc 10 | *.egg-info/ 11 | /lib/python/env 12 | .DS_Store 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.11" 5 | - "1.12" 6 | - master 7 | 8 | dist: xenial 9 | 10 | # sudo is required for docker 11 | sudo: required 12 | 13 | # Enable docker 14 | services: 15 | - docker 16 | 17 | before_install: 18 | - go get github.com/ory/dockertest 19 | - go get github.com/stretchr/testify 20 | - go get github.com/mjibson/esc/... 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.16.5-alpine3.13 2 | 3 | ENV GO111MODULE on 4 | ENV LOGLEVEL debug 5 | 6 | RUN apk update && apk add --no-cache bash gcc g++ ca-certificates curl wget make socat git jq libsass-dev libsass 7 | RUN go get github.com/ghetzel/pivot/v3/cmd/pivot@v3.4.9 8 | RUN rm -rf /go/pkg /go/src 9 | RUN mv /go/bin/pivot /usr/bin/pivot 10 | RUN rm -rf /usr/local/go /usr/libexec/gcc 11 | RUN mkdir /config 12 | RUN echo '---' > /config/pivot.yml 13 | 14 | RUN test -x /usr/bin/pivot 15 | 16 | EXPOSE 29029 17 | ENTRYPOINT ["/usr/bin/pivot", "--config", "/config/pivot.yml", "web", "--address", ":29029"] 18 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .EXPORT_ALL_VARIABLES: 2 | EXAMPLES := $(wildcard examples/*) 3 | GO111MODULE ?= on 4 | CGO_CFLAGS = -I/opt/homebrew/include 5 | CGO_LDFLAGS = -L/opt/homebrew/lib 6 | DOCKER_VERSION = $(shell grep -m1 'go get github.com/ghetzel/pivot' Dockerfile | cut -d@ -f2 | tr -d v) 7 | 8 | all: deps fmt test build docs 9 | 10 | deps: 11 | @go list github.com/mjibson/esc > /dev/null || go get github.com/mjibson/esc/... 12 | @go get ./... 13 | 14 | fmt: 15 | @go generate -x ./... 16 | @gofmt -w $(shell find . -type f -name '*.go') 17 | @go vet ./... 18 | @go mod tidy 19 | 20 | docs: 21 | owndoc render --property rootpath=/pivot/ 22 | 23 | test: 24 | go test -count=1 --tags json1 ./... 25 | 26 | $(EXAMPLES): 27 | go build --tags json1 -o bin/example-$(notdir $(@)) $(@)/*.go 28 | 29 | build: $(EXAMPLES) 30 | go build --tags json1 -o bin/pivot cmd/pivot/*.go 31 | which pivot && cp -v bin/pivot `which pivot` || true 32 | 33 | docker-build: 34 | @docker build -t ghetzel/pivot:$(DOCKER_VERSION) . 35 | 36 | docker-push: 37 | @docker tag ghetzel/pivot:$(DOCKER_VERSION) ghetzel/pivot:latest 38 | @docker push ghetzel/pivot:$(DOCKER_VERSION) 39 | @docker push ghetzel/pivot:latest 40 | 41 | docker: docker-build docker-push 42 | 43 | .PHONY: test deps docs $(EXAMPLES) build docker docker-build docker-push 44 | -------------------------------------------------------------------------------- /backends/aggregator.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "github.com/ghetzel/pivot/v3/dal" 5 | "github.com/ghetzel/pivot/v3/filter" 6 | ) 7 | 8 | type Aggregator interface { 9 | AggregatorConnectionString() *dal.ConnectionString 10 | AggregatorInitialize(Backend) error 11 | Sum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) 12 | Count(collection *dal.Collection, f ...*filter.Filter) (uint64, error) 13 | Minimum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) 14 | Maximum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) 15 | Average(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) 16 | GroupBy(collection *dal.Collection, fields []string, aggregates []filter.Aggregate, f ...*filter.Filter) (*dal.RecordSet, error) 17 | } 18 | -------------------------------------------------------------------------------- /backends/backends.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/alexcesaro/statsd" 8 | "github.com/ghetzel/go-stockutil/log" 9 | "github.com/ghetzel/pivot/v3/dal" 10 | "github.com/ghetzel/pivot/v3/filter" 11 | "github.com/ghetzel/pivot/v3/util" 12 | ) 13 | 14 | const ClientUserAgent = `pivot/` + util.Version 15 | 16 | var querylog = log.Logger() 17 | var stats, _ = statsd.New() 18 | var DefaultAutoregister = false 19 | var AutopingTimeout = 5 * time.Second 20 | 21 | type BackendFeature int 22 | 23 | const ( 24 | PartialSearch BackendFeature = iota 25 | CompositeKeys 26 | Constraints 27 | ) 28 | 29 | type Backend interface { 30 | Initialize() error 31 | SetIndexer(dal.ConnectionString) error 32 | RegisterCollection(*dal.Collection) 33 | GetConnectionString() *dal.ConnectionString 34 | Exists(collection string, id interface{}) bool 35 | Retrieve(collection string, id interface{}, fields ...string) (*dal.Record, error) 36 | Insert(collection string, records *dal.RecordSet) error 37 | Update(collection string, records *dal.RecordSet, target ...string) error 38 | Delete(collection string, ids ...interface{}) error 39 | CreateCollection(definition *dal.Collection) error 40 | DeleteCollection(collection string) error 41 | ListCollections() ([]string, error) 42 | GetCollection(collection string) (*dal.Collection, error) 43 | WithSearch(collection *dal.Collection, filters ...*filter.Filter) Indexer 44 | WithAggregator(collection *dal.Collection) Aggregator 45 | Flush() error 46 | Ping(time.Duration) error 47 | String() string 48 | Supports(feature ...BackendFeature) bool 49 | } 50 | 51 | var NotImplementedError = fmt.Errorf("Not Implemented") 52 | 53 | type BackendFunc func(dal.ConnectionString) Backend 54 | 55 | var backendMap = map[string]BackendFunc{ 56 | `dynamodb`: NewDynamoBackend, 57 | `file`: NewFileBackend, 58 | `fs`: NewFilesystemBackend, 59 | `mongodb`: NewMongoBackend, 60 | `mongo`: NewMongoBackend, 61 | `mysql`: NewSqlBackend, 62 | `postgres`: NewSqlBackend, 63 | `postgresql`: NewSqlBackend, 64 | `psql`: NewSqlBackend, 65 | `sqlite`: NewSqlBackend, 66 | `redis`: NewRedisBackend, 67 | `elasticsearch`: NewElasticsearchBackend, 68 | `es`: NewElasticsearchBackend, 69 | } 70 | 71 | // Register a new or replacement backend for the given connection string scheme. 72 | // For example, registering backend "foo" will allow Pivot to handle "foo://" 73 | // connection strings. 74 | func RegisterBackend(name string, fn BackendFunc) { 75 | backendMap[name] = fn 76 | } 77 | 78 | func startPeriodicPinger(interval time.Duration, backend Backend) { 79 | for { 80 | if err := backend.Ping(AutopingTimeout); err != nil { 81 | log.Warningf("%v: ping failed with error: %v", backend, err) 82 | } 83 | 84 | time.Sleep(interval) 85 | } 86 | } 87 | 88 | // Instantiate the appropriate Backend for the given connection string. 89 | func MakeBackend(connection dal.ConnectionString) (Backend, error) { 90 | var autopingInterval time.Duration 91 | 92 | backendName := connection.Backend() 93 | log.Infof("Creating backend: %v", connection.String()) 94 | 95 | if fn, ok := backendMap[backendName]; ok { 96 | if i := connection.OptDuration(`ping`, 0); i > 0 { 97 | autopingInterval = i 98 | } 99 | 100 | connection.ClearOpt(`ping`) 101 | 102 | if backend := fn(connection); backend != nil { 103 | if autopingInterval > 0 { 104 | go startPeriodicPinger(autopingInterval, backend) 105 | } 106 | 107 | return backend, nil 108 | } else { 109 | return nil, fmt.Errorf("Error occurred instantiating backend %q", backendName) 110 | } 111 | } else { 112 | return nil, fmt.Errorf("Unknown backend type %q", backendName) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /backends/caching-backend.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/ghetzel/go-stockutil/typeutil" 8 | "github.com/ghetzel/pivot/v3/dal" 9 | "github.com/ghetzel/pivot/v3/filter" 10 | ) 11 | 12 | type CachingBackend struct { 13 | backend Backend 14 | cache sync.Map 15 | } 16 | 17 | func NewCachingBackend(parent Backend) *CachingBackend { 18 | return &CachingBackend{ 19 | backend: parent, 20 | } 21 | } 22 | 23 | func (self *CachingBackend) ResetCache() { 24 | self.cache = sync.Map{} 25 | } 26 | 27 | func (self *CachingBackend) Retrieve(collection string, id interface{}, fields ...string) (*dal.Record, error) { 28 | cacheset := make(map[interface{}]interface{}) 29 | 30 | if c, ok := self.cache.LoadOrStore(collection, cacheset); ok { 31 | cacheset = c.(map[interface{}]interface{}) 32 | } 33 | 34 | if typeutil.IsScalar(id) { 35 | if recordI, ok := cacheset[id]; ok { 36 | return recordI.(*dal.Record), nil 37 | } 38 | } 39 | 40 | if record, err := self.backend.Retrieve(collection, id, fields...); err == nil { 41 | cacheset[id] = record 42 | return record, nil 43 | } else { 44 | return nil, err 45 | } 46 | } 47 | 48 | // passthrough the remaining functions to fulfill the Backend interface 49 | // ------------------------------------------------------------------------------------------------- 50 | func (self *CachingBackend) Exists(collection string, id interface{}) bool { 51 | return self.backend.Exists(collection, id) 52 | } 53 | 54 | func (self *CachingBackend) Initialize() error { 55 | return self.backend.Initialize() 56 | } 57 | 58 | func (self *CachingBackend) SetIndexer(cs dal.ConnectionString) error { 59 | return self.backend.SetIndexer(cs) 60 | } 61 | 62 | func (self *CachingBackend) RegisterCollection(c *dal.Collection) { 63 | self.backend.RegisterCollection(c) 64 | } 65 | 66 | func (self *CachingBackend) GetConnectionString() *dal.ConnectionString { 67 | return self.backend.GetConnectionString() 68 | } 69 | 70 | func (self *CachingBackend) Insert(collection string, records *dal.RecordSet) error { 71 | return self.backend.Insert(collection, records) 72 | } 73 | 74 | func (self *CachingBackend) Update(collection string, records *dal.RecordSet, target ...string) error { 75 | return self.backend.Update(collection, records, target...) 76 | } 77 | 78 | func (self *CachingBackend) Delete(collection string, ids ...interface{}) error { 79 | return self.backend.Delete(collection, ids...) 80 | } 81 | 82 | func (self *CachingBackend) CreateCollection(definition *dal.Collection) error { 83 | return self.backend.CreateCollection(definition) 84 | } 85 | 86 | func (self *CachingBackend) DeleteCollection(collection string) error { 87 | return self.backend.DeleteCollection(collection) 88 | } 89 | 90 | func (self *CachingBackend) ListCollections() ([]string, error) { 91 | return self.backend.ListCollections() 92 | } 93 | 94 | func (self *CachingBackend) GetCollection(collection string) (*dal.Collection, error) { 95 | return self.backend.GetCollection(collection) 96 | } 97 | 98 | func (self *CachingBackend) WithSearch(collection *dal.Collection, filters ...*filter.Filter) Indexer { 99 | return self.backend.WithSearch(collection, filters...) 100 | } 101 | 102 | func (self *CachingBackend) WithAggregator(collection *dal.Collection) Aggregator { 103 | return self.backend.WithAggregator(collection) 104 | } 105 | 106 | func (self *CachingBackend) Flush() error { 107 | return self.backend.Flush() 108 | } 109 | 110 | func (self *CachingBackend) Ping(d time.Duration) error { 111 | return self.backend.Ping(d) 112 | } 113 | 114 | func (self *CachingBackend) String() string { 115 | return self.backend.String() 116 | } 117 | 118 | func (self *CachingBackend) Supports(feature ...BackendFeature) bool { 119 | return self.backend.Supports(feature...) 120 | } 121 | -------------------------------------------------------------------------------- /backends/dynamo_test.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/dynamodb" 8 | "github.com/ghetzel/pivot/v3/dal" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | var dynamoTestCollection = &dal.Collection{ 13 | Name: `TestDynamoRecordToItem`, 14 | Fields: []dal.Field{ 15 | { 16 | Name: `name`, 17 | Type: dal.StringType, 18 | }, { 19 | Name: `enabled`, 20 | Type: dal.BooleanType, 21 | }, { 22 | Name: `age`, 23 | Type: dal.IntType, 24 | }, 25 | }, 26 | } 27 | 28 | func TestDynamoRecordToItem(t *testing.T) { 29 | assert := require.New(t) 30 | attr, err := dynamoRecordToItem( 31 | dynamoTestCollection, 32 | dal.NewRecord(123).Set(`name`, `tester`).Set(`age`, 42), 33 | ) 34 | 35 | assert.NoError(err) 36 | assert.Equal(map[string]*dynamodb.AttributeValue{ 37 | `id`: { 38 | N: aws.String(`123`), 39 | }, 40 | `name`: { 41 | S: aws.String(`tester`), 42 | }, 43 | `age`: { 44 | N: aws.String(`42`), 45 | }, 46 | }, attr) 47 | } 48 | 49 | func TestDynamoRecordFromItem(t *testing.T) { 50 | assert := require.New(t) 51 | record, err := dynamoRecordFromItem( 52 | dynamoTestCollection, 53 | 123, 54 | map[string]*dynamodb.AttributeValue{ 55 | `id`: { 56 | N: aws.String(`123`), 57 | }, 58 | `name`: { 59 | S: aws.String(`tester`), 60 | }, 61 | `age`: { 62 | N: aws.String(`42`), 63 | }, 64 | }, 65 | ) 66 | 67 | assert.NoError(err) 68 | assert.EqualValues(123, record.ID) 69 | assert.EqualValues(`tester`, record.Get(`name`)) 70 | assert.EqualValues(42, record.Get(`age`)) 71 | } 72 | -------------------------------------------------------------------------------- /backends/elasticsearch-aggregator.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | // this file satifies the Aggregator interface for ElasticsearchIndexer 4 | 5 | import ( 6 | "encoding/json" 7 | "fmt" 8 | 9 | "github.com/ghetzel/go-stockutil/httputil" 10 | "github.com/ghetzel/go-stockutil/maputil" 11 | "github.com/ghetzel/go-stockutil/typeutil" 12 | "github.com/ghetzel/pivot/v3/dal" 13 | "github.com/ghetzel/pivot/v3/filter" 14 | "github.com/ghetzel/pivot/v3/filter/generators" 15 | ) 16 | 17 | type esAggregationQuery struct { 18 | Aggregations map[string]esAggregation `json:"aggs"` 19 | Query map[string]interface{} `json:"query,omitempty"` 20 | Size int `json:"size"` 21 | From int `json:"from"` 22 | Sort []string `json:"sort,omitempty"` 23 | } 24 | 25 | type esAggregation map[string]interface{} 26 | 27 | func (self *ElasticsearchIndexer) Sum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 28 | return self.aggregateFloat(collection, filter.Sum, field, f) 29 | } 30 | 31 | func (self *ElasticsearchIndexer) Count(collection *dal.Collection, flt ...*filter.Filter) (uint64, error) { 32 | var f *filter.Filter 33 | 34 | if len(flt) > 0 { 35 | f = flt[0] 36 | } 37 | 38 | if query, err := filter.Render( 39 | generators.NewElasticsearchGenerator(), 40 | collection.GetAggregatorName(), 41 | f, 42 | ); err == nil { 43 | var q = maputil.M(query) 44 | 45 | q.Delete(`size`) 46 | q.Delete(`from`) 47 | q.Delete(`sort`) 48 | 49 | if res, err := self.client.GetWithBody( 50 | fmt.Sprintf("/%s/_doc/_count", collection.GetAggregatorName()), 51 | httputil.Literal(q.JSON()), 52 | nil, 53 | nil, 54 | ); err == nil { 55 | var rv map[string]interface{} 56 | 57 | if err := self.client.Decode(res.Body, &rv); err == nil { 58 | return uint64(typeutil.Int(rv[`count`])), nil 59 | } else { 60 | return 0, err 61 | } 62 | } else { 63 | return 0, err 64 | } 65 | } else { 66 | return 0, err 67 | } 68 | } 69 | 70 | func (self *ElasticsearchIndexer) Minimum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 71 | return self.aggregateFloat(collection, filter.Minimum, field, f) 72 | } 73 | 74 | func (self *ElasticsearchIndexer) Maximum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 75 | return self.aggregateFloat(collection, filter.Maximum, field, f) 76 | } 77 | 78 | func (self *ElasticsearchIndexer) Average(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 79 | return self.aggregateFloat(collection, filter.Average, field, f) 80 | } 81 | 82 | func (self *ElasticsearchIndexer) GroupBy(collection *dal.Collection, groupBy []string, aggregates []filter.Aggregate, flt ...*filter.Filter) (*dal.RecordSet, error) { 83 | if result, err := self.aggregate(collection, groupBy, aggregates, flt, false); err == nil { 84 | return result.(*dal.RecordSet), nil 85 | } else { 86 | return nil, err 87 | } 88 | } 89 | 90 | func (self *ElasticsearchIndexer) aggregateFloat(collection *dal.Collection, aggregation filter.Aggregation, field string, flt []*filter.Filter) (float64, error) { 91 | if result, err := self.aggregate(collection, nil, []filter.Aggregate{ 92 | { 93 | Aggregation: aggregation, 94 | Field: field, 95 | }, 96 | }, flt, true); err == nil { 97 | var aggkey string 98 | 99 | switch aggregation { 100 | case filter.Minimum: 101 | aggkey = `min` 102 | case filter.Maximum: 103 | aggkey = `max` 104 | case filter.Sum: 105 | aggkey = `sum` 106 | case filter.Average: 107 | aggkey = `avg` 108 | } 109 | 110 | if aggkey != `` { 111 | return maputil.M(result).Float(`aggregations.` + field + `.` + aggkey), nil 112 | } else { 113 | return 0, fmt.Errorf("unknown aggregation") 114 | } 115 | } else { 116 | return 0, err 117 | } 118 | } 119 | 120 | func (self *ElasticsearchIndexer) aggregate(collection *dal.Collection, groupBy []string, aggregates []filter.Aggregate, flt []*filter.Filter, single bool) (interface{}, error) { 121 | var f *filter.Filter 122 | 123 | if len(flt) > 0 { 124 | f = flt[0] 125 | } 126 | 127 | if query, err := filter.Render( 128 | generators.NewElasticsearchGenerator(), 129 | collection.GetAggregatorName(), 130 | f, 131 | ); err == nil { 132 | var esFilter map[string]interface{} 133 | 134 | if err := json.Unmarshal(query, &esFilter); err == nil { 135 | var aggs = esAggregationQuery{ 136 | Aggregations: make(map[string]esAggregation), 137 | } 138 | 139 | for _, aggregate := range aggregates { 140 | var statsKey = aggregate.Field 141 | var statsField esAggregation 142 | 143 | if s, ok := aggs.Aggregations[statsKey]; ok { 144 | statsField = s 145 | } else { 146 | statsField = make(esAggregation) 147 | } 148 | 149 | statsField[`stats`] = map[string]interface{}{ 150 | `field`: aggregate.Field, 151 | } 152 | 153 | aggs.Aggregations[statsKey] = statsField 154 | } 155 | 156 | if len(esFilter) > 0 { 157 | aggs.Query = maputil.M(esFilter).Get(`query`).MapNative() 158 | aggs.Size = 0 159 | } 160 | 161 | if response, err := self.client.GetWithBody( 162 | fmt.Sprintf("/%s/_search", collection.GetAggregatorName()), 163 | &aggs, 164 | nil, 165 | nil, 166 | ); err == nil { 167 | var output = make(map[string]interface{}) 168 | 169 | if err := self.client.Decode(response.Body, &output); err == nil { 170 | return output, nil 171 | } else { 172 | return nil, fmt.Errorf("response decode error: %v", err) 173 | } 174 | } else { 175 | return nil, err 176 | } 177 | } else { 178 | return nil, fmt.Errorf("filter encode error: %v", err) 179 | } 180 | } else { 181 | return nil, fmt.Errorf("filter error: %v", err) 182 | } 183 | } 184 | 185 | func (self *ElasticsearchIndexer) AggregatorConnectionString() *dal.ConnectionString { 186 | return self.conn 187 | } 188 | 189 | func (self *ElasticsearchIndexer) AggregatorInitialize(parent Backend) error { 190 | return nil 191 | } 192 | -------------------------------------------------------------------------------- /backends/embedded_records_test.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestInflateEmbeddedRecords(t *testing.T) { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /backends/fs-indexer.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/ghetzel/go-stockutil/sliceutil" 7 | "github.com/ghetzel/pivot/v3/dal" 8 | "github.com/ghetzel/pivot/v3/filter" 9 | ) 10 | 11 | func (self *FilesystemBackend) IndexConnectionString() *dal.ConnectionString { 12 | return &dal.ConnectionString{} 13 | } 14 | 15 | func (self *FilesystemBackend) IndexInitialize(_ Backend) error { 16 | return nil 17 | } 18 | 19 | func (self *FilesystemBackend) GetBackend() Backend { 20 | return self 21 | } 22 | 23 | func (self *FilesystemBackend) IndexExists(collection *dal.Collection, id interface{}) bool { 24 | return self.Exists(collection.GetIndexName(), id) 25 | } 26 | 27 | func (self *FilesystemBackend) IndexRetrieve(collection *dal.Collection, id interface{}) (*dal.Record, error) { 28 | defer stats.NewTiming().Send(`pivot.indexers.filesystem.retrieve_time`) 29 | return self.Retrieve(collection.GetIndexName(), id) 30 | } 31 | 32 | func (self *FilesystemBackend) IndexRemove(collection *dal.Collection, ids []interface{}) error { 33 | return nil 34 | } 35 | 36 | func (self *FilesystemBackend) Index(collection *dal.Collection, records *dal.RecordSet) error { 37 | return nil 38 | } 39 | 40 | func (self *FilesystemBackend) QueryFunc(collection *dal.Collection, filter *filter.Filter, resultFn IndexResultFunc) error { 41 | defer stats.NewTiming().Send(`pivot.indexers.filesystem.query_time`) 42 | querylog.Debugf("[%T] Query using filter %q", self, filter.String()) 43 | 44 | if filter.IdOnly() { 45 | if id, ok := filter.GetFirstValue(); ok { 46 | if record, err := self.Retrieve(collection.GetIndexName(), id); err == nil { 47 | querylog.Debugf("[%T] Record %v matches filter %q", self, id, filter.String()) 48 | 49 | if err := resultFn(record, err, IndexPage{ 50 | Page: 1, 51 | TotalPages: 1, 52 | Limit: filter.Limit, 53 | Offset: 0, 54 | TotalResults: 1, 55 | }); err != nil { 56 | return err 57 | } 58 | } else { 59 | return err 60 | } 61 | } 62 | } else { 63 | if ids, err := self.listObjectIdsInCollection(collection); err == nil { 64 | var page = 1 65 | var processed = 0 66 | var offset = filter.Offset 67 | 68 | for _, id := range ids { 69 | var parts = strings.Split(id, FilesystemKeyJoiner) 70 | var record *dal.Record 71 | var err error 72 | 73 | if len(parts) == 1 { 74 | record, err = self.Retrieve(collection.Name, parts[0]) 75 | } else { 76 | record, err = self.Retrieve(collection.Name, parts) 77 | } 78 | 79 | // retrieve the record by id 80 | if err == nil { 81 | // if matching all records OR the found record matches the filter 82 | if filter.MatchesRecord(record) { 83 | if processed >= offset { 84 | querylog.Debugf("[%T] Record %v matches filter %q", self, record.ID, filter.String()) 85 | 86 | if err := resultFn(record, err, IndexPage{ 87 | Page: page, 88 | TotalPages: 1, 89 | Limit: filter.Limit, 90 | Offset: offset, 91 | TotalResults: -1, 92 | }); err != nil { 93 | return err 94 | } 95 | } 96 | } 97 | } else { 98 | if err := resultFn(dal.NewRecord(nil), err, IndexPage{ 99 | Page: page, 100 | TotalPages: 1, 101 | Limit: filter.Limit, 102 | Offset: offset, 103 | TotalResults: -1, 104 | }); err != nil { 105 | return err 106 | } 107 | } 108 | 109 | processed += 1 110 | page = int(float64(processed) / float64(filter.Limit)) 111 | 112 | if filter.Limit > 0 && processed >= (offset+filter.Limit) { 113 | querylog.Debugf("[%T] %d at or beyond limit %d, returning results", self, processed, filter.Limit) 114 | break 115 | } 116 | } 117 | } else { 118 | return err 119 | } 120 | } 121 | 122 | return nil 123 | } 124 | 125 | func (self *FilesystemBackend) Query(collection *dal.Collection, f *filter.Filter, resultFns ...IndexResultFunc) (*dal.RecordSet, error) { 126 | return DefaultQueryImplementation(self, collection, f, resultFns...) 127 | } 128 | 129 | func (self *FilesystemBackend) ListValues(collection *dal.Collection, fields []string, f *filter.Filter) (map[string][]interface{}, error) { 130 | values := make(map[string][]interface{}) 131 | 132 | if err := self.QueryFunc(collection, f, func(record *dal.Record, err error, page IndexPage) error { 133 | if err == nil { 134 | for _, field := range fields { 135 | var v []interface{} 136 | 137 | switch field { 138 | case collection.IdentityField: 139 | field = collection.IdentityField 140 | 141 | if current, ok := values[field]; ok { 142 | v = current 143 | } else { 144 | v = make([]interface{}, 0) 145 | } 146 | 147 | if record.ID != nil { 148 | v = sliceutil.Unique(append(v, record.ID)) 149 | } 150 | default: 151 | if current, ok := values[field]; ok { 152 | v = current 153 | } else { 154 | v = make([]interface{}, 0) 155 | } 156 | 157 | if newV := record.Get(field); newV != nil { 158 | v = sliceutil.Unique(append(v, newV)) 159 | } 160 | } 161 | 162 | values[field] = v 163 | } 164 | } 165 | 166 | return nil 167 | }); err == nil { 168 | return values, nil 169 | } else { 170 | return values, err 171 | } 172 | } 173 | 174 | func (self *FilesystemBackend) DeleteQuery(collection *dal.Collection, f *filter.Filter) error { 175 | idsToRemove := make([]interface{}, 0) 176 | 177 | if err := self.QueryFunc(collection, f, func(record *dal.Record, err error, page IndexPage) error { 178 | if err == nil { 179 | idsToRemove = append(idsToRemove, record.ID) 180 | } 181 | 182 | return nil 183 | }); err == nil { 184 | return self.Delete(collection.Name, idsToRemove...) 185 | } else { 186 | return err 187 | } 188 | } 189 | 190 | func (self *FilesystemBackend) FlushIndex() error { 191 | return nil 192 | } 193 | -------------------------------------------------------------------------------- /backends/indexer-null.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "github.com/ghetzel/pivot/v3/dal" 5 | "github.com/ghetzel/pivot/v3/filter" 6 | ) 7 | 8 | type NullIndexer struct { 9 | } 10 | 11 | func (self *NullIndexer) IndexConnectionString() *dal.ConnectionString { 12 | return nil 13 | } 14 | 15 | func (self *NullIndexer) IndexInitialize(Backend) error { 16 | return NotImplementedError 17 | } 18 | 19 | func (self *NullIndexer) GetBackend() Backend { 20 | return nil 21 | } 22 | 23 | func (self *NullIndexer) IndexExists(collection *dal.Collection, id interface{}) bool { 24 | return false 25 | } 26 | 27 | func (self *NullIndexer) IndexRetrieve(collection *dal.Collection, id interface{}) (*dal.Record, error) { 28 | return nil, NotImplementedError 29 | } 30 | 31 | func (self *NullIndexer) IndexRemove(collection *dal.Collection, ids []interface{}) error { 32 | return NotImplementedError 33 | } 34 | 35 | func (self *NullIndexer) Index(collection *dal.Collection, records *dal.RecordSet) error { 36 | return NotImplementedError 37 | } 38 | 39 | func (self *NullIndexer) QueryFunc(collection *dal.Collection, filter filter.Filter, resultFn IndexResultFunc) error { 40 | return NotImplementedError 41 | } 42 | 43 | func (self *NullIndexer) Query(collection *dal.Collection, filter filter.Filter, resultFns ...IndexResultFunc) (*dal.RecordSet, error) { 44 | return nil, NotImplementedError 45 | } 46 | 47 | func (self *NullIndexer) ListValues(collection *dal.Collection, fields []string, filter filter.Filter) (map[string][]interface{}, error) { 48 | return nil, NotImplementedError 49 | } 50 | 51 | func (self *NullIndexer) DeleteQuery(collection *dal.Collection, f filter.Filter) error { 52 | return NotImplementedError 53 | } 54 | 55 | func (self *NullIndexer) FlushIndex() error { 56 | return NotImplementedError 57 | } 58 | -------------------------------------------------------------------------------- /backends/indexers.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "strings" 7 | 8 | "github.com/ghetzel/go-stockutil/log" 9 | "github.com/ghetzel/go-stockutil/sliceutil" 10 | "github.com/ghetzel/pivot/v3/dal" 11 | "github.com/ghetzel/pivot/v3/filter" 12 | ) 13 | 14 | var IndexerPageSize int = 100 15 | var MaxFacetCardinality int = 10000 16 | var DefaultCompoundJoiner = `:` 17 | 18 | type IndexPage struct { 19 | Page int 20 | TotalPages int 21 | Limit int 22 | Offset int 23 | TotalResults int64 24 | } 25 | 26 | type IndexResultFunc func(record *dal.Record, err error, page IndexPage) error // {} 27 | 28 | type Indexer interface { 29 | IndexConnectionString() *dal.ConnectionString 30 | IndexInitialize(Backend) error 31 | IndexExists(collection *dal.Collection, id interface{}) bool 32 | IndexRetrieve(collection *dal.Collection, id interface{}) (*dal.Record, error) 33 | IndexRemove(collection *dal.Collection, ids []interface{}) error 34 | Index(collection *dal.Collection, records *dal.RecordSet) error 35 | QueryFunc(collection *dal.Collection, filter *filter.Filter, resultFn IndexResultFunc) error 36 | Query(collection *dal.Collection, filter *filter.Filter, resultFns ...IndexResultFunc) (*dal.RecordSet, error) 37 | ListValues(collection *dal.Collection, fields []string, filter *filter.Filter) (map[string][]interface{}, error) 38 | DeleteQuery(collection *dal.Collection, f *filter.Filter) error 39 | FlushIndex() error 40 | GetBackend() Backend 41 | } 42 | 43 | func MakeIndexer(connection dal.ConnectionString) (Indexer, error) { 44 | log.Infof("Creating indexer: %v", connection.String()) 45 | 46 | switch connection.Backend() { 47 | case `bleve`: 48 | return NewBleveIndexer(connection), nil 49 | case `elasticsearch`: 50 | return NewElasticsearchIndexer(connection), nil 51 | default: 52 | return nil, fmt.Errorf("Unknown indexer type %q", connection.Backend()) 53 | } 54 | } 55 | 56 | func PopulateRecordSetPageDetails(recordset *dal.RecordSet, f *filter.Filter, page IndexPage) { 57 | // result count is whatever we were told it was for this query 58 | if page.TotalResults >= 0 { 59 | recordset.KnownSize = true 60 | recordset.ResultCount = page.TotalResults 61 | } else { 62 | recordset.ResultCount = int64(len(recordset.Records)) 63 | } 64 | 65 | if page.TotalPages > 0 { 66 | recordset.TotalPages = page.TotalPages 67 | } else if recordset.ResultCount >= 0 && f.Limit > 0 { 68 | // total pages = ceil(result count / page size) 69 | recordset.TotalPages = int(math.Ceil(float64(recordset.ResultCount) / float64(f.Limit))) 70 | } else { 71 | recordset.TotalPages = 1 72 | } 73 | 74 | if recordset.RecordsPerPage == 0 { 75 | recordset.RecordsPerPage = page.Limit 76 | } 77 | 78 | // page is the last page number set 79 | if page.Limit > 0 { 80 | recordset.Page = int(math.Ceil(float64(f.Offset+1) / float64(page.Limit))) 81 | } 82 | } 83 | 84 | func DefaultQueryImplementation(indexer Indexer, collection *dal.Collection, f *filter.Filter, resultFns ...IndexResultFunc) (*dal.RecordSet, error) { 85 | var recordset = dal.NewRecordSet() 86 | 87 | if err := indexer.QueryFunc(collection, f, func(indexRecord *dal.Record, err error, page IndexPage) error { 88 | defer PopulateRecordSetPageDetails(recordset, f, page) 89 | 90 | var parent = indexer.GetBackend() 91 | var forceIndexRecord bool 92 | 93 | // look for a filter option that specifies that we should explicitly NOT attempt to retrieve the 94 | // record from the parent by ID, but rather always use the index record as-is. 95 | if f != nil { 96 | if v, ok := f.Options[`ForceIndexRecord`].(bool); ok { 97 | forceIndexRecord = v 98 | } 99 | } 100 | 101 | // index compound field processing 102 | if parent != nil { 103 | if len(collection.IndexCompoundFields) > 1 { 104 | var joiner = collection.IndexCompoundFieldJoiner 105 | 106 | if joiner == `` { 107 | joiner = DefaultCompoundJoiner 108 | } 109 | 110 | var values = sliceutil.Sliceify( 111 | strings.Split(fmt.Sprintf("%v", indexRecord.ID), joiner), 112 | ) 113 | 114 | if len(values) != len(collection.IndexCompoundFields) { 115 | // if the index record's ID isn't holding _all_ of the field components, 116 | // attempt to retrieve the rest of the values from the record itself. 117 | if diff := len(collection.IndexCompoundFields) - len(values); diff >= 1 { 118 | for _, cf := range collection.IndexCompoundFields[diff:] { 119 | if v := indexRecord.Get(cf); v != nil { 120 | values = append(values, v) 121 | } 122 | } 123 | } 124 | 125 | if len(values) != len(collection.IndexCompoundFields) { 126 | return fmt.Errorf( 127 | "Index item %v: expected %d compound field components, got %d", 128 | indexRecord.ID, 129 | len(collection.IndexCompoundFields), 130 | len(values), 131 | ) 132 | } 133 | } 134 | 135 | for i, field := range collection.IndexCompoundFields { 136 | if i == 0 { 137 | indexRecord.ID = values 138 | } else { 139 | indexRecord.Set(field, values[i]) 140 | } 141 | } 142 | } 143 | } 144 | 145 | var emptyRecord = dal.NewRecord(indexRecord.ID) 146 | emptyRecord.Error = err 147 | 148 | if len(resultFns) > 0 { 149 | var resultFn = resultFns[0] 150 | 151 | if f.IdOnly() { 152 | return resultFn(emptyRecord, err, page) 153 | } else if parent != nil && !forceIndexRecord { 154 | if record, err := parent.Retrieve(collection.Name, indexRecord.ID, f.Fields...); err == nil { 155 | return resultFn(record, err, page) 156 | } else { 157 | return resultFn(emptyRecord, err, page) 158 | } 159 | } else { 160 | return resultFn(indexRecord, err, page) 161 | } 162 | } else { 163 | if f.IdOnly() { 164 | recordset.Records = append(recordset.Records, dal.NewRecord(indexRecord.ID)) 165 | 166 | } else if parent != nil && !forceIndexRecord { 167 | if record, err := parent.Retrieve(collection.Name, indexRecord.ID, f.Fields...); err == nil { 168 | recordset.Records = append(recordset.Records, record) 169 | 170 | } else { 171 | recordset.Records = append(recordset.Records, dal.NewRecordErr(indexRecord.ID, err)) 172 | } 173 | } else { 174 | recordset.Records = append(recordset.Records, indexRecord) 175 | } 176 | 177 | return nil 178 | } 179 | }); err != nil { 180 | return nil, err 181 | } 182 | 183 | return recordset, nil 184 | } 185 | -------------------------------------------------------------------------------- /backends/mapper.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "github.com/ghetzel/pivot/v3/dal" 5 | "github.com/ghetzel/pivot/v3/filter" 6 | ) 7 | 8 | type ResultFunc func(ptrToInstance interface{}, err error) // {} 9 | 10 | type Mapper interface { 11 | GetBackend() Backend 12 | GetCollection() *dal.Collection 13 | Migrate() error 14 | Drop() error 15 | Exists(id interface{}) bool 16 | Create(from interface{}) error 17 | Get(id interface{}, into interface{}) error 18 | Update(from interface{}) error 19 | CreateOrUpdate(id interface{}, from interface{}) error 20 | Delete(ids ...interface{}) error 21 | DeleteQuery(flt interface{}) error 22 | Find(flt interface{}, into interface{}) error 23 | FindFunc(flt interface{}, destZeroValue interface{}, resultFn ResultFunc) error 24 | All(into interface{}) error 25 | Each(destZeroValue interface{}, resultFn ResultFunc) error 26 | List(fields []string) (map[string][]interface{}, error) 27 | ListWithFilter(fields []string, flt interface{}) (map[string][]interface{}, error) 28 | Sum(field string, flt interface{}) (float64, error) 29 | Count(flt interface{}) (uint64, error) 30 | Minimum(field string, flt interface{}) (float64, error) 31 | Maximum(field string, flt interface{}) (float64, error) 32 | Average(field string, flt interface{}) (float64, error) 33 | GroupBy(fields []string, aggregates []filter.Aggregate, flt interface{}) (*dal.RecordSet, error) 34 | } 35 | -------------------------------------------------------------------------------- /backends/mongo-aggregator.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | // this file satifies the Aggregator interface for MongoBackend 4 | 5 | import ( 6 | "fmt" 7 | "strings" 8 | "time" 9 | 10 | "github.com/ghetzel/go-stockutil/stringutil" 11 | "github.com/ghetzel/pivot/v3/dal" 12 | "github.com/ghetzel/pivot/v3/filter" 13 | "github.com/globalsign/mgo/bson" 14 | ) 15 | 16 | func (self *MongoBackend) Sum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 17 | return self.aggregateFloat(collection, filter.Sum, field, f) 18 | } 19 | 20 | func (self *MongoBackend) Count(collection *dal.Collection, flt ...*filter.Filter) (uint64, error) { 21 | var f *filter.Filter 22 | 23 | if len(flt) > 0 { 24 | f = flt[0] 25 | } 26 | 27 | if query, err := self.filterToNative(collection, f); err == nil { 28 | q := self.db.C(collection.Name).Find(query) 29 | 30 | if totalResults, err := q.Count(); err == nil { 31 | return uint64(totalResults), nil 32 | } else { 33 | return 0, err 34 | } 35 | } else { 36 | return 0, err 37 | } 38 | } 39 | 40 | func (self *MongoBackend) Minimum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 41 | return self.aggregateFloat(collection, filter.Minimum, field, f) 42 | } 43 | 44 | func (self *MongoBackend) Maximum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 45 | return self.aggregateFloat(collection, filter.Maximum, field, f) 46 | } 47 | 48 | func (self *MongoBackend) Average(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 49 | return self.aggregateFloat(collection, filter.Average, field, f) 50 | } 51 | 52 | func (self *MongoBackend) GroupBy(collection *dal.Collection, groupBy []string, aggregates []filter.Aggregate, flt ...*filter.Filter) (*dal.RecordSet, error) { 53 | if result, err := self.aggregate(collection, groupBy, aggregates, flt, false); err == nil { 54 | return result.(*dal.RecordSet), nil 55 | } else { 56 | return nil, err 57 | } 58 | } 59 | 60 | func (self *MongoBackend) aggregateFloat(collection *dal.Collection, aggregation filter.Aggregation, field string, flt []*filter.Filter) (float64, error) { 61 | if result, err := self.aggregate(collection, nil, []filter.Aggregate{ 62 | { 63 | Aggregation: aggregation, 64 | Field: field, 65 | }, 66 | }, flt, true); err == nil { 67 | if vF, ok := result.(float64); ok { 68 | return vF, nil 69 | } else { 70 | return 0, err 71 | } 72 | } else { 73 | return 0, err 74 | } 75 | } 76 | 77 | func (self *MongoBackend) aggregate(collection *dal.Collection, groupBy []string, aggregates []filter.Aggregate, flt []*filter.Filter, single bool) (interface{}, error) { 78 | var f *filter.Filter 79 | 80 | if len(flt) > 0 { 81 | f = flt[0] 82 | } 83 | 84 | if query, err := self.filterToNative(collection, f); err == nil { 85 | var aggGroups []bson.M 86 | var firstKey string 87 | 88 | for _, aggregate := range aggregates { 89 | var mongoFn string 90 | 91 | switch aggregate.Aggregation { 92 | case filter.Sum: 93 | mongoFn = `$sum` 94 | case filter.First: 95 | mongoFn = `$first` 96 | case filter.Last: 97 | mongoFn = `$last` 98 | case filter.Minimum: 99 | mongoFn = `$min` 100 | case filter.Maximum: 101 | mongoFn = `$max` 102 | case filter.Average: 103 | mongoFn = `$avg` 104 | } 105 | 106 | aggGroups = append(aggGroups, bson.M{ 107 | `$group`: bson.M{ 108 | `_id`: aggregate.Field, 109 | strings.TrimPrefix(mongoFn, `$`): bson.M{ 110 | mongoFn: fmt.Sprintf("$%s", aggregate.Field), 111 | }, 112 | }, 113 | }) 114 | 115 | if firstKey == `` { 116 | firstKey = strings.TrimPrefix(mongoFn, `$`) 117 | } 118 | } 119 | 120 | var finalQuery []bson.M 121 | 122 | if len(query) > 0 { 123 | finalQuery = append(finalQuery, query) 124 | } 125 | 126 | finalQuery = append(finalQuery, aggGroups...) 127 | 128 | q := self.db.C(collection.Name).Pipe(finalQuery) 129 | iter := q.Iter() 130 | 131 | var result map[string]interface{} 132 | 133 | for iter.Next(&result) { 134 | if err := iter.Err(); err != nil { 135 | return nil, err 136 | } else if single { 137 | _id, _ := result[`_id`] 138 | 139 | if v, ok := result[firstKey]; ok { 140 | if vF, err := stringutil.ConvertToFloat(v); err == nil { 141 | return vF, nil 142 | } else if vT, err := stringutil.ConvertToTime(v); err == nil { 143 | return float64(vT.UnixNano()) / float64(time.Second), nil 144 | } else { 145 | return 0, fmt.Errorf("'%s' aggregation not supported for field %v", firstKey, _id) 146 | } 147 | } else { 148 | return 0, fmt.Errorf("missing aggregation value '%s'", firstKey) 149 | } 150 | } else { 151 | return nil, fmt.Errorf("Not implemented") 152 | } 153 | } 154 | 155 | return nil, nil 156 | } else { 157 | return nil, fmt.Errorf("filter error: %v", err) 158 | } 159 | } 160 | 161 | func (self *MongoBackend) AggregatorConnectionString() *dal.ConnectionString { 162 | return self.GetConnectionString() 163 | } 164 | 165 | func (self *MongoBackend) AggregatorInitialize(parent Backend) error { 166 | return nil 167 | } 168 | -------------------------------------------------------------------------------- /backends/mongo-indexer.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/ghetzel/go-stockutil/maputil" 8 | "github.com/ghetzel/go-stockutil/sliceutil" 9 | "github.com/ghetzel/go-stockutil/stringutil" 10 | "github.com/ghetzel/go-stockutil/typeutil" 11 | "github.com/ghetzel/pivot/v3/dal" 12 | "github.com/ghetzel/pivot/v3/filter" 13 | "github.com/ghetzel/pivot/v3/filter/generators" 14 | "github.com/globalsign/mgo/bson" 15 | ) 16 | 17 | func (self *MongoBackend) IndexConnectionString() *dal.ConnectionString { 18 | return self.conn 19 | } 20 | 21 | func (self *MongoBackend) IndexInitialize(Backend) error { 22 | return nil 23 | } 24 | 25 | func (self *MongoBackend) GetBackend() Backend { 26 | return self 27 | } 28 | 29 | func (self *MongoBackend) IndexExists(collection *dal.Collection, id interface{}) bool { 30 | return self.Exists(collection.GetIndexName(), id) 31 | } 32 | 33 | func (self *MongoBackend) IndexRetrieve(collection *dal.Collection, id interface{}) (*dal.Record, error) { 34 | return self.Retrieve(collection.GetIndexName(), id) 35 | } 36 | 37 | func (self *MongoBackend) IndexRemove(collection *dal.Collection, ids []interface{}) error { 38 | return nil 39 | } 40 | 41 | func (self *MongoBackend) Index(collection *dal.Collection, records *dal.RecordSet) error { 42 | return nil 43 | } 44 | 45 | func (self *MongoBackend) QueryFunc(collection *dal.Collection, flt *filter.Filter, resultFn IndexResultFunc) error { 46 | var result map[string]interface{} 47 | 48 | if query, err := self.filterToNative(collection, flt); err == nil { 49 | q := self.db.C(collection.Name).Find(query) 50 | 51 | if totalResults, err := q.Count(); err == nil { 52 | if flt.Limit > 0 { 53 | q = q.Limit(flt.Limit) 54 | } 55 | 56 | if flt.Offset > 0 { 57 | q = q.Skip(flt.Offset) 58 | } 59 | 60 | if len(flt.Sort) > 0 { 61 | q = q.Sort(flt.Sort...) 62 | } 63 | 64 | q = self.prepMongoQuery(q, flt.Fields) 65 | 66 | iter := q.Iter() 67 | 68 | for iter.Next(&result) { 69 | if err := iter.Err(); err != nil { 70 | return err 71 | } else { 72 | if record, err := self.recordFromResult(collection, result, flt.Fields...); err == nil { 73 | if err := resultFn(record, nil, IndexPage{ 74 | Limit: flt.Limit, 75 | Offset: flt.Offset, 76 | TotalResults: int64(totalResults), 77 | }); err != nil { 78 | return err 79 | } 80 | 81 | result = nil 82 | } else { 83 | return err 84 | } 85 | } 86 | } 87 | 88 | return iter.Close() 89 | } else { 90 | return err 91 | } 92 | } else { 93 | return err 94 | } 95 | } 96 | 97 | func (self *MongoBackend) Query(collection *dal.Collection, f *filter.Filter, resultFns ...IndexResultFunc) (*dal.RecordSet, error) { 98 | if f != nil { 99 | if f.IdentityField == `` { 100 | f.IdentityField = MongoIdentityField 101 | } 102 | 103 | f.Options[`ForceIndexRecord`] = true 104 | } 105 | 106 | return DefaultQueryImplementation(self, collection, f, resultFns...) 107 | } 108 | 109 | func (self *MongoBackend) ListValues(collection *dal.Collection, fields []string, flt *filter.Filter) (map[string][]interface{}, error) { 110 | if query, err := self.filterToNative(collection, flt); err == nil { 111 | rv := make(map[string][]interface{}) 112 | 113 | for _, field := range fields { 114 | qfield := field 115 | 116 | if qfield == `id` { 117 | qfield = MongoIdentityField 118 | } 119 | 120 | var results []interface{} 121 | 122 | if err := self.db.C(collection.Name).Find(&query).Distinct(qfield, &results); err == nil { 123 | rv[field] = sliceutil.Autotype(results) 124 | } else { 125 | return nil, err 126 | } 127 | } 128 | 129 | return rv, nil 130 | } else { 131 | return nil, err 132 | } 133 | } 134 | 135 | func (self *MongoBackend) DeleteQuery(collection *dal.Collection, flt *filter.Filter) error { 136 | if query, err := self.filterToNative(collection, flt); err == nil { 137 | if _, err := self.db.C(collection.Name).RemoveAll(&query); err == nil { 138 | return nil 139 | } else { 140 | return err 141 | } 142 | } else { 143 | return err 144 | } 145 | } 146 | 147 | func (self *MongoBackend) FlushIndex() error { 148 | return nil 149 | } 150 | 151 | func (self *MongoBackend) filterToNative(collection *dal.Collection, flt *filter.Filter) (bson.M, error) { 152 | if data, err := filter.Render( 153 | generators.NewMongoDBGenerator(), 154 | collection.GetIndexName(), 155 | flt, 156 | ); err == nil { 157 | var query bson.M 158 | 159 | if err := json.Unmarshal(data, &query); err != nil { 160 | return nil, err 161 | } 162 | 163 | // handle type-specific processing of values; nuances that get lost in the JSON-to-Map serialization 164 | // process. I *wanted* to just serialize to BSON directly from the query generator interface, but 165 | // that turned into a messy time-wasting boondoggle #neat. 166 | // 167 | query = bson.M(maputil.Apply(query, func(key []string, value interface{}) (interface{}, bool) { 168 | vS := fmt.Sprintf("%v", value) 169 | 170 | // I'm half-a-tick from just forking the mgo library for how absolutely maddening this 171 | // Stringer output is. 172 | vS = stringutil.Unwrap(vS, `ObjectIdHex("`, `")`) 173 | 174 | if bson.IsObjectIdHex(vS) { 175 | return bson.ObjectIdHex(vS), true 176 | } else if stringutil.IsTime(value) { 177 | if vT, err := stringutil.ConvertToTime(value); err == nil { 178 | return vT, true 179 | } 180 | } 181 | 182 | return nil, false 183 | })) 184 | 185 | querylog.Debugf("[%T] query %v: %v", self, collection.Name, typeutil.Dump(query)) 186 | return query, nil 187 | } else { 188 | return nil, err 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /backends/monitoring-backend.go.disabled: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ghetzel/pivot/v3/dal" 7 | "github.com/ghetzel/pivot/v3/filter" 8 | ) 9 | 10 | type MonitoringBackend struct { 11 | backend Backend 12 | } 13 | 14 | func NewMonitoringBackend(parent Backend) *MonitoringBackend { 15 | return &MonitoringBackend{ 16 | backend: parent, 17 | } 18 | } 19 | 20 | func (self *MonitoringBackend) Retrieve(collection string, id interface{}, fields ...string) (*dal.Record, error) { 21 | defer stats.NewTiming().Send(`pivot.operations.retrieve.timing`) 22 | stats.Increment(`pivot.operations.retrieve.calls`) 23 | 24 | record, err := self.backend.Retrieve(collection, id) 25 | 26 | if err != nil { 27 | stats.Increment(`pivot.operations.retrieve.errors`) 28 | } 29 | 30 | return record, err 31 | } 32 | 33 | func (self *MonitoringBackend) Exists(collection string, id interface{}) bool { 34 | } 35 | 36 | func (self *MonitoringBackend) Insert(collection string, records *dal.RecordSet) error { 37 | } 38 | 39 | func (self *MonitoringBackend) Update(collection string, records *dal.RecordSet, target ...string) error { 40 | } 41 | 42 | func (self *MonitoringBackend) Delete(collection string, ids ...interface{}) error { 43 | } 44 | 45 | func (self *MonitoringBackend) CreateCollection(definition *dal.Collection) error { 46 | } 47 | 48 | func (self *MonitoringBackend) DeleteCollection(collection string) error { 49 | } 50 | 51 | func (self *MonitoringBackend) ListCollections() ([]string, error) { 52 | } 53 | 54 | func (self *MonitoringBackend) GetCollection(collection string) (*dal.Collection, error) { 55 | } 56 | 57 | func (self *MonitoringBackend) Ping(d time.Duration) error { 58 | } 59 | 60 | 61 | // passthrough the remaining functions to fulfill the Backend interface 62 | // ------------------------------------------------------------------------------------------------- 63 | func (self *MonitoringBackend) Initialize() error { 64 | return self.backend.Initialize() 65 | } 66 | 67 | func (self *MonitoringBackend) SetIndexer(cs dal.ConnectionString) error { 68 | return self.backend.SetIndexer(cs) 69 | } 70 | 71 | func (self *MonitoringBackend) RegisterCollection(c *dal.Collection) { 72 | self.backend.RegisterCollection(c) 73 | } 74 | 75 | func (self *MonitoringBackend) GetConnectionString() *dal.ConnectionString { 76 | return self.backend.GetConnectionString() 77 | } 78 | 79 | func (self *MonitoringBackend) WithSearch(collection *dal.Collection, filters ...*filter.Filter) Indexer { 80 | return self.backend.WithSearch(collection, filters...) 81 | } 82 | 83 | func (self *MonitoringBackend) WithAggregator(collection *dal.Collection) Aggregator { 84 | return self.backend.WithAggregator(collection) 85 | } 86 | 87 | func (self *MonitoringBackend) Flush() error { 88 | return self.backend.Flush() 89 | } 90 | 91 | func (self *MonitoringBackend) String() string { 92 | return self.backend.String() 93 | } 94 | 95 | func (self *MonitoringBackend) Supports(feature ...BackendFeature) bool { 96 | return self.backend.Supports(feature...) 97 | } 98 | -------------------------------------------------------------------------------- /backends/options.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | type ConnectOptions struct { 4 | Indexer string `json:"indexer"` 5 | AdditionalIndexers []string `json:"additional_indexers"` 6 | SkipInitialize bool `json:"skip_initialize"` 7 | AutocreateCollections bool `json:"autocreate_collections"` 8 | } 9 | -------------------------------------------------------------------------------- /backends/redis-indexer.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | 7 | "github.com/ghetzel/pivot/v3/dal" 8 | "github.com/ghetzel/pivot/v3/filter" 9 | "github.com/gomodule/redigo/redis" 10 | ) 11 | 12 | func (self *RedisBackend) IndexConnectionString() *dal.ConnectionString { 13 | return &self.cs 14 | } 15 | 16 | func (self *RedisBackend) IndexInitialize(Backend) error { 17 | return nil 18 | } 19 | 20 | func (self *RedisBackend) GetBackend() Backend { 21 | return self 22 | } 23 | 24 | func (self *RedisBackend) IndexExists(collection *dal.Collection, id interface{}) bool { 25 | return self.Exists(collection.Name, id) 26 | } 27 | 28 | func (self *RedisBackend) IndexRetrieve(collection *dal.Collection, id interface{}) (*dal.Record, error) { 29 | return self.Retrieve(collection.Name, id) 30 | } 31 | 32 | func (self *RedisBackend) IndexRemove(collection *dal.Collection, ids []interface{}) error { 33 | return nil 34 | } 35 | 36 | func (self *RedisBackend) Index(collection *dal.Collection, records *dal.RecordSet) error { 37 | return nil 38 | } 39 | 40 | func (self *RedisBackend) QueryFunc(collection *dal.Collection, flt *filter.Filter, resultFn IndexResultFunc) error { 41 | if err := self.validateFilter(collection, flt); err != nil { 42 | return err 43 | } 44 | 45 | var keyPattern string 46 | 47 | // full keyscan 48 | if flt == nil || flt.IsMatchAll() { 49 | keyPattern = self.key(collection.Name, `*`) 50 | } else { 51 | placeholders := make([]interface{}, 0) 52 | 53 | for i := 0; i < collection.KeyCount(); i++ { 54 | placeholders = append(placeholders, `*`) 55 | } 56 | 57 | for i, criterion := range flt.Criteria { 58 | if !criterion.IsExactMatch() || len(criterion.Values) != 1 { 59 | return fmt.Errorf( 60 | "%v: filters can only contain exact match criteria (%q is invalid on %d values)", 61 | self, 62 | criterion.Operator, 63 | len(criterion.Values), 64 | ) 65 | } else if i < len(placeholders) { 66 | placeholders[i] = fmt.Sprintf("%v", criterion.Values[0]) 67 | } else { 68 | return fmt.Errorf("%v: too many criteria", self) 69 | } 70 | } 71 | 72 | keyPattern = self.key(collection.Name, placeholders...) 73 | } 74 | 75 | if keys, err := redis.Strings(self.run(`KEYS`, keyPattern)); err == nil { 76 | limit := len(keys) 77 | total := int64(len(keys)) 78 | 79 | if flt.Limit <= 0 { 80 | flt.Limit = IndexerPageSize 81 | } 82 | 83 | if flt.Limit < limit { 84 | limit = flt.Limit 85 | } 86 | 87 | for i := flt.Offset; i < limit; i++ { 88 | if _, values := redisSplitKey(keys[i]); len(values) > 0 { 89 | if values[0] == `__schema__` { 90 | continue 91 | } 92 | 93 | record, err := self.Retrieve(collection.Name, values, flt.Fields...) 94 | 95 | // fire off the result handler 96 | if err := resultFn(record, err, IndexPage{ 97 | Page: 1, 98 | TotalPages: int(math.Ceil(float64(total) / float64(flt.Limit))), 99 | Limit: flt.Limit, 100 | Offset: (1 - 1) * flt.Limit, 101 | TotalResults: total, 102 | }); err != nil { 103 | return err 104 | } 105 | } 106 | } 107 | 108 | return nil 109 | } else { 110 | return err 111 | } 112 | } 113 | 114 | func (self *RedisBackend) Query(collection *dal.Collection, f *filter.Filter, resultFns ...IndexResultFunc) (*dal.RecordSet, error) { 115 | if f != nil { 116 | f.Options[`ForceIndexRecord`] = true 117 | } 118 | 119 | return DefaultQueryImplementation(self, collection, f, resultFns...) 120 | } 121 | 122 | func (self *RedisBackend) ListValues(collection *dal.Collection, fields []string, flt *filter.Filter) (map[string][]interface{}, error) { 123 | return nil, fmt.Errorf("%T.ListValues: Not Implemented", self) 124 | } 125 | 126 | func (self *RedisBackend) DeleteQuery(collection *dal.Collection, flt *filter.Filter) error { 127 | return fmt.Errorf("%T.DeleteQuery: Not Implemented", self) 128 | } 129 | 130 | func (self *RedisBackend) FlushIndex() error { 131 | return nil 132 | } 133 | 134 | func (self *RedisBackend) validateFilter(collection *dal.Collection, flt *filter.Filter) error { 135 | if flt != nil { 136 | for _, field := range flt.CriteriaFields() { 137 | if collection.IsIdentityField(field) { 138 | continue 139 | } 140 | 141 | if collection.IsKeyField(field) { 142 | continue 143 | } 144 | 145 | return fmt.Errorf("Filter field '%v' cannot be used: not a key field", field) 146 | } 147 | } 148 | 149 | return nil 150 | } 151 | 152 | // func (self *RedisBackend) iterResult(collection *dal.Collection, flt *filter.Filter, items []map[string]*dynamodb.AttributeValue, processed int, totalResults int64, pageNumber int, lastPage bool, resultFn IndexResultFunc) bool { 153 | // if len(items) > 0 { 154 | // for _, item := range items { 155 | // record, err := dynamoRecordFromItem(collection, item) 156 | 157 | // // fire off the result handler 158 | // if err := resultFn(record, err, IndexPage{ 159 | // Page: pageNumber, 160 | // TotalPages: int(math.Ceil(float64(totalResults) / float64(25))), 161 | // Limit: flt.Limit, 162 | // Offset: (pageNumber - 1) * 25, 163 | // TotalResults: totalResults, 164 | // }); err != nil { 165 | // return false 166 | // } 167 | 168 | // // perform bounds checking 169 | // if processed += 1; flt.Limit > 0 && processed >= flt.Limit { 170 | // return false 171 | // } 172 | // } 173 | 174 | // return !lastPage 175 | // } else { 176 | // return false 177 | // } 178 | // } 179 | -------------------------------------------------------------------------------- /backends/redis_test.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestRedisSplitKey(t *testing.T) { 10 | assert := require.New(t) 11 | 12 | collection, keys := redisSplitKey(``) 13 | assert.Zero(collection) 14 | assert.Empty(keys) 15 | 16 | collection, keys = redisSplitKey(`testing:123`) 17 | assert.Equal(`testing`, collection) 18 | assert.Equal([]string{`123`}, keys) 19 | 20 | collection, keys = redisSplitKey(`pivot.testing:123`) 21 | assert.Equal(`testing`, collection) 22 | assert.Equal([]string{`123`}, keys) 23 | 24 | collection, keys = redisSplitKey(`pivot.testing:123:456`) 25 | assert.Equal(`testing`, collection) 26 | assert.Equal([]string{`123`, `456`}, keys) 27 | 28 | collection, keys = redisSplitKey(`pivot.deeply.nested.whatsawhat.testing:123:456`) 29 | assert.Equal(`testing`, collection) 30 | assert.Equal([]string{`123`, `456`}, keys) 31 | } 32 | -------------------------------------------------------------------------------- /backends/sql-aggregator.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | // this file satifies the Aggregator interface for SqlBackend 4 | 5 | import ( 6 | "database/sql" 7 | "reflect" 8 | 9 | "github.com/ghetzel/go-stockutil/typeutil" 10 | "github.com/ghetzel/pivot/v3/dal" 11 | "github.com/ghetzel/pivot/v3/filter" 12 | "github.com/ghetzel/pivot/v3/filter/generators" 13 | ) 14 | 15 | type sqlAggResultFunc func(*sql.Rows, *generators.Sql, *dal.Collection, *filter.Filter) (interface{}, error) 16 | 17 | func (self *SqlBackend) Sum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 18 | return self.aggregateFloat(collection, filter.Sum, field, f) 19 | } 20 | 21 | func (self *SqlBackend) Count(collection *dal.Collection, f ...*filter.Filter) (uint64, error) { 22 | whatToCount := collection.IdentityField 23 | 24 | if typeutil.IsZero(whatToCount) { 25 | whatToCount = `1` 26 | } 27 | 28 | v, err := self.aggregateFloat(collection, filter.Count, whatToCount, f) 29 | return uint64(v), err 30 | } 31 | 32 | func (self *SqlBackend) Minimum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 33 | return self.aggregateFloat(collection, filter.Minimum, field, f) 34 | } 35 | 36 | func (self *SqlBackend) Maximum(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 37 | return self.aggregateFloat(collection, filter.Maximum, field, f) 38 | } 39 | 40 | func (self *SqlBackend) Average(collection *dal.Collection, field string, f ...*filter.Filter) (float64, error) { 41 | return self.aggregateFloat(collection, filter.Average, field, f) 42 | } 43 | 44 | func (self *SqlBackend) GroupBy(collection *dal.Collection, groupBy []string, aggregates []filter.Aggregate, f ...*filter.Filter) (*dal.RecordSet, error) { 45 | if result, err := self.aggregate(collection, groupBy, aggregates, f, self.extractRecordSet); err == nil { 46 | return result.(*dal.RecordSet), nil 47 | } else { 48 | return nil, err 49 | } 50 | } 51 | 52 | func (self *SqlBackend) aggregateFloat(collection *dal.Collection, aggregation filter.Aggregation, field string, f []*filter.Filter) (float64, error) { 53 | if result, err := self.aggregate(collection, nil, []filter.Aggregate{ 54 | { 55 | Aggregation: aggregation, 56 | Field: field, 57 | }, 58 | }, f, self.extractSingleFloat64); err == nil { 59 | return result.(float64), nil 60 | } else { 61 | return 0, err 62 | } 63 | } 64 | 65 | func (self *SqlBackend) aggregate(collection *dal.Collection, groupBy []string, aggregates []filter.Aggregate, f []*filter.Filter, resultFn sqlAggResultFunc) (interface{}, error) { 66 | queryGen := self.makeQueryGen(collection) 67 | var flt *filter.Filter 68 | 69 | if len(f) == 0 { 70 | flt = filter.New() 71 | } else { 72 | flt = f[0] 73 | } 74 | 75 | for _, g := range groupBy { 76 | queryGen.GroupByField(g) 77 | } 78 | 79 | for _, agg := range aggregates { 80 | queryGen.AggregateByField(agg.Aggregation, agg.Field) 81 | } 82 | 83 | if err := queryGen.Initialize(collection.Name); err == nil { 84 | if stmt, err := filter.Render(queryGen, collection.Name, flt); err == nil { 85 | querylog.Debugf("[%T] %s %v", self, string(stmt[:]), queryGen.GetValues()) 86 | 87 | // perform query 88 | if rows, err := self.db.Query(string(stmt[:]), queryGen.GetValues()...); err == nil { 89 | defer rows.Close() 90 | return resultFn(rows, queryGen, collection, flt) 91 | } else { 92 | return nil, err 93 | } 94 | } else { 95 | return nil, err 96 | } 97 | } else { 98 | return nil, err 99 | } 100 | } 101 | 102 | func (self *SqlBackend) AggregatorConnectionString() *dal.ConnectionString { 103 | return self.GetConnectionString() 104 | } 105 | 106 | func (self *SqlBackend) AggregatorInitialize(parent Backend) error { 107 | return nil 108 | } 109 | 110 | func (self *SqlBackend) extractSingleFloat64(rows *sql.Rows, _ *generators.Sql, _ *dal.Collection, _ *filter.Filter) (interface{}, error) { 111 | if rows.Next() { 112 | var rv sql.NullFloat64 113 | 114 | if err := rows.Scan(&rv); err == nil { 115 | return rv.Float64, nil 116 | } else { 117 | return float64(0), err 118 | } 119 | } else { 120 | return float64(0), nil 121 | } 122 | } 123 | 124 | func (self *SqlBackend) extractRecordSet(rows *sql.Rows, queryGen *generators.Sql, collection *dal.Collection, flt *filter.Filter) (interface{}, error) { 125 | recordset := dal.NewRecordSet() 126 | 127 | if columns, err := rows.Columns(); err == nil { 128 | for rows.Next() { 129 | if record, err := self.scanFnValueToRecord(queryGen, collection, columns, reflect.ValueOf(rows.Scan), flt.Fields); err == nil { 130 | recordset.Push(record) 131 | } else { 132 | return nil, err 133 | } 134 | } 135 | } else { 136 | return nil, err 137 | } 138 | 139 | return recordset, nil 140 | } 141 | -------------------------------------------------------------------------------- /backends/sql-backend-mysql.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/ghetzel/go-stockutil/stringutil" 9 | "github.com/ghetzel/pivot/v3/dal" 10 | "github.com/ghetzel/pivot/v3/filter" 11 | "github.com/ghetzel/pivot/v3/filter/generators" 12 | _ "github.com/go-sql-driver/mysql" 13 | ) 14 | 15 | func preinitializeMysql(self *SqlBackend) { 16 | // tell the backend cool details about generating compatible SQL 17 | self.queryGenTypeMapping = generators.MysqlTypeMapping 18 | self.queryGenNormalizerFormat = "LOWER(REPLACE(REPLACE(REPLACE(REPLACE(%v, ':', ' '), '[', ' '), ']', ' '), '*', ' '))" 19 | self.listAllTablesQuery = `SHOW TABLES` 20 | self.createPrimaryKeyIntFormat = `%s INT AUTO_INCREMENT NOT NULL` 21 | self.createPrimaryKeyStrFormat = `%s VARCHAR(255) NOT NULL` 22 | self.foreignKeyConstraintFormat = `FOREIGN KEY(%s) REFERENCES %s (%s) %s` 23 | self.defaultCurrentTimeString = `CURRENT_TIMESTAMP` 24 | } 25 | 26 | func initializeMysql(self *SqlBackend) (string, string, error) { 27 | // the bespoke method for determining table information for sqlite3 28 | self.refreshCollectionFunc = func(datasetName string, collectionName string) (*dal.Collection, error) { 29 | if f, err := filter.FromMap(map[string]interface{}{ 30 | `TABLE_SCHEMA`: datasetName, 31 | `TABLE_NAME`: collectionName, 32 | }); err == nil { 33 | f.Fields = []string{ 34 | `ORDINAL_POSITION`, 35 | `COLUMN_NAME`, 36 | `COLUMN_TYPE`, 37 | `IS_NULLABLE`, 38 | `COLUMN_DEFAULT`, 39 | `COLUMN_KEY`, 40 | } 41 | 42 | queryGen := self.makeQueryGen(nil) 43 | 44 | // make this instance of the query generator use the table name as given because 45 | // we need to reference another database (information_schema) 46 | queryGen.TypeMapping.TableNameFormat = "%s" 47 | 48 | if stmt, err := filter.Render(queryGen, "`information_schema`.`COLUMNS`", f); err == nil { 49 | querylog.Debugf("[%T] %s %v", self, string(stmt[:]), queryGen.GetValues()) 50 | 51 | if rows, err := self.db.Query(string(stmt[:]), queryGen.GetValues()...); err == nil { 52 | defer rows.Close() 53 | 54 | collection := dal.NewCollection(collectionName) 55 | 56 | // for each field in the schema description for this table... 57 | for rows.Next() { 58 | var i int 59 | var column, columnType, nullable string 60 | var defaultValue, keyType sql.NullString 61 | 62 | // populate variables from column values 63 | if err := rows.Scan(&i, &column, &columnType, &nullable, &defaultValue, &keyType); err == nil { 64 | // start building the dal.Field 65 | field := dal.Field{ 66 | Name: column, 67 | NativeType: columnType, 68 | Required: (nullable != `YES`), 69 | } 70 | 71 | // set default value if it's not NULL 72 | if defaultValue.Valid { 73 | field.DefaultValue = stringutil.Autotype(defaultValue.String) 74 | } 75 | 76 | // tease out type, length, and precision from the native type 77 | // e.g: DOULBE(8,12) -> "DOUBLE", 8, 12 78 | columnType, field.Length, field.Precision = queryGen.SplitTypeLength(columnType) 79 | 80 | // map native types to DAL types 81 | if strings.HasSuffix(columnType, `CHAR`) || strings.HasSuffix(columnType, `TEXT`) { 82 | field.Type = dal.StringType 83 | 84 | } else if strings.HasPrefix(columnType, `BOOL`) || columnType == `BIT` { 85 | field.Type = dal.BooleanType 86 | 87 | } else if strings.HasSuffix(columnType, `INT`) { 88 | if field.Length == 1 { 89 | field.Type = dal.BooleanType 90 | } else { 91 | field.Type = dal.IntType 92 | } 93 | 94 | } else if columnType == `FLOAT` || columnType == `DOUBLE` || columnType == `DECIMAL` { 95 | field.Type = dal.FloatType 96 | 97 | } else if strings.HasPrefix(columnType, `DATE`) || strings.Contains(columnType, `TIME`) { 98 | field.Type = dal.TimeType 99 | 100 | } else { 101 | switch field.Length { 102 | case SqlObjectFieldHintLength: 103 | field.Type = dal.ObjectType 104 | case SqlArrayFieldHintLength: 105 | field.Type = dal.ArrayType 106 | default: 107 | field.Type = dal.RawType 108 | } 109 | } 110 | 111 | // figure out keying 112 | switch keyType.String { 113 | case `PRI`: 114 | field.Identity = true 115 | collection.IdentityField = column 116 | collection.IdentityFieldType = field.Type 117 | case `UNI`: 118 | field.Unique = true 119 | case `MUL`: 120 | field.Key = true 121 | } 122 | 123 | // add field to the collection we're building 124 | collection.Fields = append(collection.Fields, field) 125 | } else { 126 | return nil, err 127 | } 128 | } 129 | 130 | return collection, rows.Err() 131 | } else { 132 | return nil, err 133 | } 134 | } else { 135 | return nil, err 136 | } 137 | } else { 138 | return nil, err 139 | } 140 | } 141 | 142 | var dsn, protocol, host string 143 | 144 | // set or autodetect protocol 145 | if v := self.conn.Protocol(); v != `` { 146 | protocol = v 147 | } else if strings.HasPrefix(self.conn.Host(), `/`) { 148 | protocol = `unix` 149 | } else { 150 | protocol = `tcp` 151 | } 152 | 153 | // prepend port to host if not present 154 | if strings.Contains(self.conn.Host(), `:`) { 155 | host = self.conn.Host() 156 | } else { 157 | host = fmt.Sprintf("%s:3306", self.conn.Host()) 158 | } 159 | 160 | if u, p, ok := self.conn.Credentials(); ok { 161 | dsn += fmt.Sprintf("%s:%s@", u, p) 162 | } 163 | 164 | dsn += fmt.Sprintf( 165 | "%s(%s)/%s", 166 | protocol, 167 | host, 168 | self.conn.Dataset(), 169 | ) 170 | 171 | return `mysql`, dsn, nil 172 | } 173 | -------------------------------------------------------------------------------- /backends/sql_test.go: -------------------------------------------------------------------------------- 1 | package backends 2 | 3 | // func TestSqlAlterStatements(t *testing.T) { 4 | // assert := require.New(t) 5 | // b := NewSqlBackend(dal.MustParseConnectionString(`sqlite://temporary`)).(*SqlBackend) 6 | // assert.NoError(b.Initialize()) 7 | 8 | // have := &dal.Collection{ 9 | // Name: `TestSqlAlterStatements`, 10 | // IdentityField: `id`, 11 | // Fields: []dal.Field{ 12 | // { 13 | // Name: `name`, 14 | // Type: dal.StringType, 15 | // Required: true, 16 | // }, { 17 | // Name: `created_at`, 18 | // Type: dal.IntType, 19 | // Required: true, 20 | // }, 21 | // }, 22 | // } 23 | 24 | // want := &dal.Collection{ 25 | // Name: `TestSqlAlterStatements`, 26 | // IdentityField: `id`, 27 | // Fields: []dal.Field{ 28 | // { 29 | // Name: `name`, 30 | // Type: dal.StringType, 31 | // Required: true, 32 | // }, { 33 | // Name: `age`, 34 | // Type: dal.IntType, 35 | // Required: true, 36 | // DefaultValue: 1, 37 | // }, { 38 | // Name: `created_at`, 39 | // Type: dal.TimeType, 40 | // Required: true, 41 | // DefaultValue: `now`, 42 | // }, 43 | // }, 44 | // } 45 | 46 | // b.RegisterCollection(have) 47 | // assert.NoError(b.Migrate()) 48 | 49 | // for _, delta := range want.Diff(have) { 50 | // stmt, _, err := b.generateAlterStatement(delta) 51 | // assert.NoError(err) 52 | 53 | // // TODO: this is the wrong order, need to work out whats going on 54 | // switch delta.Name { 55 | // case `age`: 56 | // assert.Equal(`ALTER TABLE "TestSqlAlterStatements" ADD "created_at" INTEGER NOT NULL DEFAULT CURRENT_TIMESTAMP`, stmt) 57 | // case `created_at`: 58 | // assert.Equal(`ALTER TABLE "TestSqlAlterStatements" ADD "age" BIGINT NOT NULL`, stmt) 59 | // default: 60 | // assert.NoError(fmt.Errorf("extra diff: %v", delta)) 61 | // } 62 | // } 63 | // } 64 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strings" 7 | 8 | "github.com/ghetzel/go-stockutil/httputil" 9 | "github.com/ghetzel/go-stockutil/sliceutil" 10 | "github.com/ghetzel/go-stockutil/typeutil" 11 | "github.com/ghetzel/pivot/v3/dal" 12 | "github.com/ghetzel/pivot/v3/util" 13 | ) 14 | 15 | const DefaultPivotUrl = `http://localhost:29029` 16 | 17 | type Status = util.Status 18 | 19 | type QueryOptions struct { 20 | Limit int `json:"limit"` 21 | Offset int `json:"offset"` 22 | Sort []string `json:"sort,omitempty"` 23 | Fields []string `json:"fields,omitempty"` 24 | Conjunction string `json:"conjunction,omitempty"` 25 | } 26 | 27 | type Pivot struct { 28 | *httputil.Client 29 | } 30 | 31 | func New(url string) (*Pivot, error) { 32 | if url == `` { 33 | url = DefaultPivotUrl 34 | } 35 | 36 | if client, err := httputil.NewClient(url); err == nil { 37 | return &Pivot{ 38 | Client: client, 39 | }, nil 40 | } else { 41 | return nil, err 42 | } 43 | } 44 | 45 | func (self *Pivot) Status() (*Status, error) { 46 | if response, err := self.Get(`/api/status`, nil, nil); err == nil { 47 | status := Status{} 48 | 49 | if err := self.Decode(response.Body, &status); err == nil { 50 | return &status, nil 51 | } else { 52 | return nil, err 53 | } 54 | } else { 55 | return nil, err 56 | } 57 | } 58 | 59 | func (self *Pivot) Collections() ([]string, error) { 60 | if response, err := self.Get(`/api/schema`, nil, nil); err == nil { 61 | var names []string 62 | 63 | if err := self.Decode(response.Body, &names); err == nil { 64 | return names, nil 65 | } else { 66 | return nil, err 67 | } 68 | } else { 69 | return nil, err 70 | } 71 | } 72 | 73 | func (self *Pivot) CreateCollection(def *dal.Collection) error { 74 | return fmt.Errorf("Not Implemented") 75 | } 76 | 77 | func (self *Pivot) DeleteCollection(name string) error { 78 | _, err := self.Delete(fmt.Sprintf("/api/schema/%s", name), nil, nil) 79 | return err 80 | } 81 | 82 | func (self *Pivot) Collection(name string) (*dal.Collection, error) { 83 | if response, err := self.Get(fmt.Sprintf("/api/schema/%s", name), nil, nil); err == nil { 84 | var collection dal.Collection 85 | 86 | if err := self.Decode(response.Body, &collection); err == nil { 87 | return &collection, nil 88 | } else { 89 | return nil, err 90 | } 91 | } else { 92 | return nil, err 93 | } 94 | } 95 | 96 | func (self *Pivot) Query(collection string, query interface{}, options *QueryOptions) (*dal.RecordSet, error) { 97 | var response *http.Response 98 | var err error 99 | 100 | opts := make(map[string]interface{}) 101 | 102 | if options != nil { 103 | opts[`limit`] = options.Limit 104 | opts[`offset`] = options.Offset 105 | 106 | if len(options.Sort) > 0 { 107 | opts[`sort`] = strings.Join(options.Sort, `,`) 108 | } 109 | 110 | if len(options.Fields) > 0 { 111 | opts[`fields`] = strings.Join(options.Fields, `,`) 112 | } 113 | } 114 | 115 | if typeutil.IsMap(query) { 116 | response, err = self.Post(fmt.Sprintf("/api/collections/%s/query/", collection), query, opts, nil) 117 | } else if typeutil.IsArray(query) { 118 | qS := sliceutil.Stringify(query) 119 | 120 | if len(qS) == 0 { 121 | qS = []string{`all`} 122 | } 123 | 124 | response, err = self.Get(fmt.Sprintf("/api/collections/%s/where/%s", collection, strings.Join(qS, `/`)), opts, nil) 125 | } else { 126 | q := typeutil.String(query) 127 | 128 | if q == `` { 129 | q = `all` 130 | } 131 | 132 | response, err = self.Get(fmt.Sprintf("/api/collections/%s/where/%s", collection, q), opts, nil) 133 | } 134 | 135 | if err == nil { 136 | var recordset dal.RecordSet 137 | 138 | if err := self.Decode(response.Body, &recordset); err == nil { 139 | return &recordset, nil 140 | } else { 141 | return nil, err 142 | } 143 | } else { 144 | return nil, err 145 | } 146 | } 147 | 148 | func (self *Pivot) Aggregate(collection string, query interface{}) (*dal.RecordSet, error) { 149 | return nil, fmt.Errorf("Not Implemented") 150 | } 151 | 152 | func (self *Pivot) GetRecord(collection string, id interface{}) (*dal.Record, error) { 153 | if response, err := self.Get(fmt.Sprintf("/api/collections/%s/records/%v", collection, id), nil, nil); err == nil { 154 | var record dal.Record 155 | 156 | if err := self.Decode(response.Body, &record); err == nil { 157 | return &record, nil 158 | } else { 159 | return nil, err 160 | } 161 | } else { 162 | return nil, err 163 | } 164 | } 165 | 166 | func (self *Pivot) CreateRecord(collection string, records ...*dal.Record) (*dal.RecordSet, error) { 167 | return self.upsertRecord(true, collection, records...) 168 | } 169 | 170 | func (self *Pivot) UpdateRecord(collection string, records ...*dal.Record) (*dal.RecordSet, error) { 171 | return self.upsertRecord(false, collection, records...) 172 | } 173 | 174 | func (self *Pivot) upsertRecord(create bool, collection string, records ...*dal.Record) (*dal.RecordSet, error) { 175 | var response *http.Response 176 | var err error 177 | 178 | recordset := dal.NewRecordSet(records...) 179 | 180 | if create { 181 | response, err = self.Post(fmt.Sprintf("/api/collections/%s", collection), recordset, nil, nil) 182 | } else { 183 | response, err = self.Put(fmt.Sprintf("/api/collections/%s", collection), recordset, nil, nil) 184 | } 185 | 186 | if err == nil { 187 | if err := self.Decode(response.Body, recordset); err == nil { 188 | return recordset, nil 189 | } else { 190 | return recordset, err 191 | } 192 | } else { 193 | return nil, err 194 | } 195 | } 196 | 197 | func (self *Pivot) DeleteRecords(collection string, ids ...interface{}) error { 198 | _, err := self.Delete(fmt.Sprintf( 199 | "/api/collections/%s/records/%s", 200 | collection, 201 | strings.Join(sliceutil.Stringify(ids), `/`), 202 | ), nil, nil) 203 | return err 204 | } 205 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package pivot 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | "github.com/fatih/structs" 7 | "github.com/ghodss/yaml" 8 | ) 9 | 10 | type Configuration struct { 11 | Backend string `json:"backend"` 12 | Indexer string `json:"indexer"` 13 | Autoexpand bool `json:"autoexpand"` 14 | AutocreateCollections bool `json:"autocreate"` 15 | Environments map[string]Configuration `json:"environments"` 16 | } 17 | 18 | func LoadConfigFile(path string) (Configuration, error) { 19 | config := Configuration{} 20 | 21 | if data, err := ioutil.ReadFile(path); err == nil { 22 | if err := yaml.Unmarshal(data, &config); err != nil { 23 | return config, err 24 | } 25 | } else { 26 | return config, err 27 | } 28 | 29 | return config, nil 30 | } 31 | 32 | func (self *Configuration) ForEnv(env string) Configuration { 33 | config := *self 34 | base := structs.New(&config) 35 | 36 | if envConfig, ok := self.Environments[env]; ok { 37 | specific := structs.New(envConfig) 38 | 39 | // merge the environment-specific values over top of the general ones 40 | for _, field := range base.Fields() { 41 | if field.IsExported() { 42 | switch field.Name() { 43 | case `Environment`: 44 | continue 45 | default: 46 | if specificField, ok := specific.FieldOk(field.Name()); ok { 47 | if !specificField.IsZero() { 48 | field.Set(specificField.Value()) 49 | } 50 | } 51 | } 52 | } 53 | } 54 | } 55 | 56 | return config 57 | } 58 | -------------------------------------------------------------------------------- /dal/constraint.go: -------------------------------------------------------------------------------- 1 | package dal 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | ) 7 | 8 | type Constraint struct { 9 | // Represents the name (or array of names) of the local field the constraint is being applied to. 10 | On interface{} `json:"on"` 11 | 12 | // The remote collection the constraint applies to. 13 | Collection string `json:"collection"` 14 | 15 | // The remote field (or fields) in the remote collection the constraint applies to. 16 | Field interface{} `json:"field"` 17 | 18 | // Provides backend-specific additional options for the constraint. 19 | Options string `json:"options,omitempty"` 20 | 21 | // Specifies the local field that related records will be put into. Defaults to the field specified in On. 22 | Into string `json:"into,omitempty"` 23 | 24 | // Whether to omit this constraint when determining embedded collections. 25 | NoEmbed bool `json:"noembed,omitempty"` 26 | } 27 | 28 | func (self Constraint) Validate() error { 29 | if self.On == `` { 30 | return fmt.Errorf("invalid constraint missing local field") 31 | } 32 | 33 | if self.Collection == `` { 34 | return fmt.Errorf("invalid constraint missing remote collection name") 35 | } 36 | 37 | if self.Field == `` { 38 | return fmt.Errorf("invalid constraint missing remote field") 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func (self Constraint) Equal(other *Constraint) bool { 45 | if self.On == other.On { 46 | if self.Collection == other.Collection { 47 | if reflect.DeepEqual(self.Field, other.Field) { 48 | return true 49 | } 50 | } 51 | } 52 | 53 | return false 54 | } 55 | -------------------------------------------------------------------------------- /dal/errors.go: -------------------------------------------------------------------------------- 1 | package dal 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | ) 7 | 8 | var CollectionNotFound = errors.New(`Collection not found`) 9 | var FieldNotFound = errors.New(`Field not found`) 10 | 11 | func IsCollectionNotFoundErr(err error) bool { 12 | return (err == CollectionNotFound) 13 | } 14 | 15 | func IsNotExistError(err error) bool { 16 | if err == nil { 17 | return false 18 | } 19 | 20 | return strings.HasSuffix(err.Error(), ` does not exist`) 21 | } 22 | 23 | func IsExistError(err error) bool { 24 | if err == nil { 25 | return false 26 | } 27 | 28 | return strings.HasSuffix(err.Error(), ` already exists`) 29 | } 30 | 31 | func IsFieldNotFoundErr(err error) bool { 32 | return (err == FieldNotFound) 33 | } 34 | -------------------------------------------------------------------------------- /dal/record_loader.go: -------------------------------------------------------------------------------- 1 | package dal 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | 8 | "github.com/ghetzel/go-stockutil/sliceutil" 9 | "github.com/ghetzel/go-stockutil/stringutil" 10 | "github.com/ghetzel/go-stockutil/structutil" 11 | "github.com/ghetzel/go-stockutil/typeutil" 12 | "github.com/ghetzel/pivot/v3/util" 13 | ) 14 | 15 | var RecordStructTag = util.RecordStructTag 16 | var DefaultStructIdentityFieldName = `ID` 17 | 18 | type fieldDescription struct { 19 | OriginalName string 20 | RecordKey string 21 | Identity bool 22 | OmitEmpty bool 23 | FieldValue reflect.Value 24 | FieldType reflect.Type 25 | DataValue interface{} 26 | } 27 | 28 | func (self *fieldDescription) Set(value interface{}) error { 29 | if self.FieldValue.IsValid() { 30 | if self.FieldValue.CanSet() { 31 | return typeutil.SetValue(self.FieldValue, value) 32 | } else { 33 | return fmt.Errorf("cannot set field %q: field is unsettable", self.OriginalName) 34 | } 35 | } else { 36 | return fmt.Errorf("cannot set field %q: no value available", self.OriginalName) 37 | } 38 | } 39 | 40 | type Model interface{} 41 | 42 | func structFieldToDesc(field *reflect.StructField) *fieldDescription { 43 | desc := new(fieldDescription) 44 | desc.OriginalName = field.Name 45 | desc.RecordKey = field.Name 46 | 47 | if tag := field.Tag.Get(RecordStructTag); tag != `` { 48 | tag = strings.TrimSpace(tag) 49 | key, rest := stringutil.SplitPair(tag, `,`) 50 | options := strings.Split(rest, `,`) 51 | 52 | if key != `` { 53 | desc.RecordKey = key 54 | } 55 | 56 | for _, opt := range options { 57 | switch opt { 58 | case `identity`: 59 | desc.Identity = true 60 | case `omitempty`: 61 | desc.OmitEmpty = true 62 | } 63 | } 64 | } 65 | 66 | return desc 67 | } 68 | 69 | func getIdentityFieldName(in interface{}, collection *Collection) string { 70 | candidates := make([]string, 0) 71 | 72 | if err := structutil.FieldsFunc(in, func(field *reflect.StructField, value reflect.Value) error { 73 | desc := structFieldToDesc(field) 74 | 75 | if desc.Identity { 76 | candidates = append(candidates, field.Name) 77 | } else if collection != nil && field.Name == collection.GetIdentityFieldName() { 78 | candidates = append(candidates, field.Name) 79 | } else { 80 | switch field.Name { 81 | case `id`, `ID`, `Id`: 82 | candidates = append(candidates, field.Name) 83 | } 84 | } 85 | 86 | return nil 87 | }); err == nil { 88 | if len(candidates) > 0 { 89 | return candidates[0] 90 | } 91 | } 92 | 93 | return `` 94 | } 95 | 96 | // Retrieves the struct field name and key name that represents the identity field for a given struct. 97 | func getIdentityFieldNameFromStruct(instance interface{}, fallbackIdentityFieldName string) (string, string, error) { 98 | if err := validatePtrToStructType(instance); err != nil { 99 | return ``, ``, err 100 | } 101 | 102 | var structFieldName string 103 | var dbFieldName string 104 | var fieldNames = make(map[string]bool) 105 | 106 | // find a field with an ",identity" tag and get its value 107 | var fn structutil.StructFieldFunc 108 | 109 | fn = func(field *reflect.StructField, v reflect.Value) error { 110 | fieldNames[field.Name] = true 111 | 112 | if tag := field.Tag.Get(RecordStructTag); tag != `` { 113 | if v := strings.Split(tag, `,`); sliceutil.ContainsString(v[1:], `identity`) { 114 | structFieldName = field.Name 115 | 116 | if v[0] != `` { 117 | dbFieldName = v[0] 118 | } 119 | 120 | return structutil.StopIterating 121 | } 122 | } 123 | 124 | return nil 125 | } 126 | 127 | if err := structutil.FieldsFunc(instance, fn); err != nil { 128 | return ``, ``, err 129 | } 130 | 131 | if structFieldName != `` { 132 | if dbFieldName != `` { 133 | return structFieldName, dbFieldName, nil 134 | } else { 135 | return structFieldName, structFieldName, nil 136 | } 137 | } 138 | 139 | if fallbackIdentityFieldName == `` { 140 | fallbackIdentityFieldName = DefaultStructIdentityFieldName 141 | } 142 | 143 | if _, ok := fieldNames[fallbackIdentityFieldName]; ok { 144 | return fallbackIdentityFieldName, fallbackIdentityFieldName, nil 145 | } else if _, ok := fieldNames[DefaultStructIdentityFieldName]; ok { 146 | return DefaultStructIdentityFieldName, DefaultStructIdentityFieldName, nil 147 | } 148 | 149 | return ``, ``, fmt.Errorf("No identity field could be found for type %T", instance) 150 | } 151 | 152 | func validatePtrToStructType(instance interface{}) error { 153 | vInstance := reflect.ValueOf(instance) 154 | 155 | if vInstance.IsValid() { 156 | if vInstance.Kind() == reflect.Ptr { 157 | vInstance = vInstance.Elem() 158 | } else { 159 | return fmt.Errorf("Can only operate on pointer to struct, got %T", instance) 160 | } 161 | 162 | if vInstance.Kind() == reflect.Struct { 163 | return nil 164 | } else { 165 | return fmt.Errorf("Can only operate on pointer to struct, got %T", instance) 166 | } 167 | } else { 168 | return fmt.Errorf("invalid value %T", instance) 169 | } 170 | } 171 | 172 | // Retrieve details about a specific field in the given struct. This function parses the `pivot` 173 | // tag details into discrete values, extracts the concrete value of the field, and returns the 174 | // reflected Type and Value of the field. 175 | func getFieldForStruct(instance interface{}, key string) (*fieldDescription, error) { 176 | var desc *fieldDescription 177 | 178 | // iterate over all exported struct fields 179 | if err := structutil.FieldsFunc(instance, func(field *reflect.StructField, value reflect.Value) error { 180 | desc = structFieldToDesc(field) 181 | 182 | // either the field name OR the name specified in the "pivot" tag will match 183 | if field.Name == key || desc.RecordKey == key { 184 | desc.FieldValue = value 185 | desc.FieldType = value.Type() 186 | 187 | if value.CanInterface() { 188 | desc.DataValue = value.Interface() 189 | } 190 | 191 | return structutil.StopIterating 192 | } else { 193 | return nil 194 | } 195 | }); err == nil { 196 | if desc == nil { 197 | return nil, fmt.Errorf("No such field %q", key) 198 | } else { 199 | return desc, nil 200 | } 201 | } else { 202 | return nil, err 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /dal/record_loader_test.go: -------------------------------------------------------------------------------- 1 | package dal 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | type TestRecord struct { 10 | ID int 11 | Name string `pivot:"name,omitempty"` 12 | Size int `pivot:"size"` 13 | } 14 | 15 | type TestRecordTwo struct { 16 | UUID string `pivot:"uuid,identity"` 17 | } 18 | 19 | type TestRecordThree struct { 20 | UUID string 21 | } 22 | 23 | type TestRecordEmbedded struct { 24 | TestRecordTwo 25 | TestRecord 26 | Local bool 27 | } 28 | 29 | func TestGetIdentityFieldNameFromStruct(t *testing.T) { 30 | assert := require.New(t) 31 | 32 | f := TestRecord{ 33 | ID: 1234, 34 | } 35 | 36 | field, key, err := getIdentityFieldNameFromStruct(&f, ``) 37 | assert.NoError(err) 38 | assert.Equal(`ID`, key) 39 | assert.Equal(`ID`, key) 40 | 41 | f = TestRecord{} 42 | field, key, err = getIdentityFieldNameFromStruct(&f, `Size`) 43 | assert.NoError(err) 44 | assert.Equal(`Size`, field) 45 | assert.Equal(`Size`, key) 46 | 47 | f2 := TestRecordTwo{`42`} 48 | field, key, err = getIdentityFieldNameFromStruct(&f2, ``) 49 | assert.Equal(`UUID`, field) 50 | assert.Equal(`uuid`, key) 51 | 52 | f3 := TestRecordThree{} 53 | field, key, err = getIdentityFieldNameFromStruct(&f3, ``) 54 | assert.Error(err) 55 | 56 | f4 := TestRecordThree{} 57 | field, key, err = getIdentityFieldNameFromStruct(&f4, `UUID`) 58 | assert.Equal(`UUID`, field) 59 | assert.Equal(`UUID`, key) 60 | 61 | f5 := TestRecordEmbedded{} 62 | f5.UUID = `42` 63 | f5.UUID = `42` 64 | f5.ID = 42 65 | f5.Name = `Fourty Two` 66 | f5.Size = 42 67 | f5.Local = true 68 | 69 | field, key, err = getIdentityFieldNameFromStruct(&f5, ``) 70 | assert.NoError(err) 71 | assert.Equal(`UUID`, field) 72 | assert.Equal(`uuid`, key) 73 | } 74 | -------------------------------------------------------------------------------- /dal/recordset.go: -------------------------------------------------------------------------------- 1 | package dal 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | "github.com/ghetzel/go-stockutil/typeutil" 8 | ) 9 | 10 | type RecordSet struct { 11 | ResultCount int64 `json:"result_count"` 12 | Page int `json:"page,omitempty"` 13 | TotalPages int `json:"total_pages,omitempty"` 14 | RecordsPerPage int `json:"records_per_page,omitempty"` 15 | Records []*Record `json:"records"` 16 | Options map[string]interface{} `json:"options"` 17 | KnownSize bool `json:"known_size"` 18 | } 19 | 20 | func NewRecordSet(records ...*Record) *RecordSet { 21 | return &RecordSet{ 22 | Records: records, 23 | Options: make(map[string]interface{}), 24 | } 25 | } 26 | 27 | func (self *RecordSet) IDs() (ids []interface{}) { 28 | for _, record := range self.Records { 29 | ids = append(ids, record.ID) 30 | } 31 | 32 | return 33 | } 34 | 35 | func (self *RecordSet) Push(record *Record) *RecordSet { 36 | self.Records = append(self.Records, record) 37 | self.ResultCount = self.ResultCount + 1 38 | return self 39 | } 40 | 41 | func (self *RecordSet) Append(other *RecordSet) *RecordSet { 42 | for _, record := range other.Records { 43 | self.Push(record) 44 | } 45 | 46 | return self 47 | } 48 | 49 | func (self *RecordSet) GetRecord(index int) (*Record, bool) { 50 | if index < len(self.Records) { 51 | return self.Records[index], true 52 | } 53 | 54 | return nil, false 55 | } 56 | 57 | func (self *RecordSet) GetRecordByID(id interface{}) (*Record, bool) { 58 | for _, record := range self.Records { 59 | if typeutil.String(record.ID) == typeutil.String(id) { 60 | return record, true 61 | } 62 | } 63 | 64 | return nil, false 65 | } 66 | 67 | func (self *RecordSet) Pluck(field string, fallback ...interface{}) []interface{} { 68 | rv := make([]interface{}, 0) 69 | 70 | for _, record := range self.Records { 71 | rv = append(rv, record.Get(field, fallback...)) 72 | } 73 | 74 | return rv 75 | } 76 | 77 | func (self *RecordSet) IsEmpty() bool { 78 | if self.ResultCount == 0 { 79 | return true 80 | } else { 81 | return false 82 | } 83 | } 84 | 85 | // Takes a slice of structs or maps and fills it with instances populated by the records in this RecordSet 86 | // in accordance with the types specified in the given collection definition, as well as which 87 | // fields are available in the given struct. 88 | func (self *RecordSet) PopulateFromRecords(into interface{}, schema *Collection) error { 89 | vInto := reflect.ValueOf(into) 90 | 91 | // get value pointed to if we were given a pointer 92 | if vInto.Kind() == reflect.Ptr { 93 | vInto = vInto.Elem() 94 | } else { 95 | return fmt.Errorf("Output argument must be a pointer") 96 | } 97 | 98 | // we're going to fill arrays or slices 99 | switch vInto.Type().Kind() { 100 | case reflect.Array, reflect.Slice: 101 | indirectResult := true 102 | 103 | // get the type of the underlying slice element 104 | sliceType := vInto.Type().Elem() 105 | 106 | // get the type pointed to 107 | if sliceType.Kind() == reflect.Ptr { 108 | sliceType = sliceType.Elem() 109 | indirectResult = false 110 | } 111 | 112 | // for each resulting record... 113 | for _, record := range self.Records { 114 | // make a new zero-valued instance of the slice type 115 | elem := reflect.New(sliceType) 116 | 117 | // populate that type with data from this record 118 | if err := record.Populate(elem.Interface(), schema); err == nil { 119 | // if the slice elements are pointers, we can append the pointer we just created as-is 120 | // otherwise, we need to indirect the value and append a copy 121 | 122 | if indirectResult { 123 | vInto.Set(reflect.Append(vInto, reflect.Indirect(elem))) 124 | } else { 125 | vInto.Set(reflect.Append(vInto, elem)) 126 | } 127 | } else { 128 | return fmt.Errorf("Cannot populate record: %v", err) 129 | } 130 | } 131 | 132 | return nil 133 | case reflect.Struct: 134 | if rs, ok := into.(*RecordSet); ok { 135 | *rs = *self 136 | return nil 137 | } 138 | } 139 | 140 | return fmt.Errorf("RecordSet can only populate records into slice or array, got %T", into) 141 | } 142 | -------------------------------------------------------------------------------- /dal/recordset_test.go: -------------------------------------------------------------------------------- 1 | package dal 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | var deftime = time.Date(2006, 1, 2, 15, 4, 5, 999999, time.FixedZone(`MST`, -420)) 11 | var othtime = time.Date(2015, 1, 2, 5, 4, 3, 0, time.UTC) 12 | 13 | type testRecordSetRecordDest struct { 14 | ID int `pivot:"id,identity"` 15 | Name string `pivot:"name"` 16 | Factor float64 `pivot:"factor"` 17 | CreatedAt time.Time `pivot:"created_at"` 18 | } 19 | 20 | func TestRecordSet(t *testing.T) { 21 | assert := require.New(t) 22 | 23 | demo := &Collection{ 24 | Fields: []Field{ 25 | { 26 | Name: `name`, 27 | Type: StringType, 28 | }, { 29 | Name: `factor`, 30 | Type: FloatType, 31 | Required: true, 32 | DefaultValue: 0.2, 33 | }, { 34 | Name: `created_at`, 35 | Type: TimeType, 36 | Required: true, 37 | DefaultValue: func() interface{} { 38 | return deftime 39 | }, 40 | }, 41 | }, 42 | } 43 | 44 | recordset := NewRecordSet( 45 | NewRecord(1).SetFields(map[string]interface{}{ 46 | `name`: `First`, 47 | `factor`: 0.1, 48 | `created_at`: nil, 49 | }), 50 | NewRecord(3).SetFields(map[string]interface{}{ 51 | `name`: `Second`, 52 | `factor`: nil, 53 | `created_at`: othtime, 54 | }), 55 | NewRecord(5).SetFields(map[string]interface{}{ 56 | `name`: `Third`, 57 | `factor`: 0.3, 58 | `created_at`: nil, 59 | }), 60 | ) 61 | 62 | dest := make([]*testRecordSetRecordDest, 0) 63 | assert.NoError(recordset.PopulateFromRecords(&dest, demo)) 64 | assert.Len(dest, 3) 65 | 66 | assert.Equal(1, dest[0].ID) 67 | assert.Equal(`First`, dest[0].Name) 68 | assert.Equal(0.1, dest[0].Factor) 69 | assert.Equal(deftime, dest[0].CreatedAt) 70 | 71 | assert.Equal(3, dest[1].ID) 72 | assert.Equal(`Second`, dest[1].Name) 73 | assert.Equal(0.2, dest[1].Factor) 74 | assert.Equal(othtime, dest[1].CreatedAt) 75 | 76 | assert.Equal(5, dest[2].ID) 77 | assert.Equal(`Third`, dest[2].Name) 78 | assert.Equal(0.3, dest[2].Factor) 79 | assert.Equal(deftime, dest[2].CreatedAt) 80 | } 81 | -------------------------------------------------------------------------------- /dal/relationship.go: -------------------------------------------------------------------------------- 1 | package dal 2 | 3 | type Relationship struct { 4 | Keys interface{} `json:"key"` 5 | Collection *Collection `json:"-"` 6 | CollectionName string `json:"collection,omitempty"` 7 | Fields []string `json:"fields,omitempty"` 8 | Force bool `json:"force,omitempty"` 9 | } 10 | 11 | func (self *Relationship) RelatedCollectionName() string { 12 | if self.Collection != nil { 13 | return self.Collection.Name 14 | } else { 15 | return self.CollectionName 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dal/types.go: -------------------------------------------------------------------------------- 1 | package dal 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/ghetzel/go-stockutil/log" 8 | "github.com/ghetzel/go-stockutil/maputil" 9 | ) 10 | 11 | type Type string 12 | 13 | const ( 14 | StringType Type = `str` 15 | AutoType = `auto` 16 | BooleanType = `bool` 17 | IntType = `int` 18 | FloatType = `float` 19 | TimeType = `time` 20 | ObjectType = `object` 21 | RawType = `raw` 22 | ArrayType = `array` 23 | ) 24 | 25 | func (self Type) String() string { 26 | return string(self) 27 | } 28 | 29 | func ParseFieldType(in string) Type { 30 | switch in { 31 | case `str`: 32 | return StringType 33 | case `bool`: 34 | return BooleanType 35 | case `int`: 36 | return IntType 37 | case `float`: 38 | return FloatType 39 | case `time`: 40 | return TimeType 41 | case `object`: 42 | return ObjectType 43 | case `raw`: 44 | return RawType 45 | case `array`: 46 | return ArrayType 47 | default: 48 | return `` 49 | } 50 | } 51 | 52 | type FieldOperation int 53 | 54 | const ( 55 | PersistOperation FieldOperation = iota 56 | RetrieveOperation 57 | ) 58 | 59 | type FieldValidatorFunc func(interface{}) error 60 | type FieldFormatterFunc func(interface{}, FieldOperation) (interface{}, error) 61 | type CollectionValidatorFunc func(*Record) error 62 | 63 | type DeltaType string 64 | 65 | const ( 66 | CollectionDelta DeltaType = `collection` 67 | FieldDelta = `field` 68 | ) 69 | 70 | type DeltaIssue int 71 | 72 | const ( 73 | UnknownIssue DeltaIssue = iota 74 | CollectionNameIssue 75 | CollectionKeyNameIssue 76 | CollectionKeyTypeIssue 77 | FieldMissingIssue 78 | FieldNameIssue 79 | FieldLengthIssue 80 | FieldTypeIssue 81 | FieldPropertyIssue 82 | ) 83 | 84 | type SchemaDelta struct { 85 | Type DeltaType 86 | Issue DeltaIssue 87 | Message string 88 | Collection string 89 | Name string 90 | Parameter string 91 | Desired interface{} 92 | Actual interface{} 93 | ReferenceField *Field 94 | } 95 | 96 | func (self SchemaDelta) DesiredField(from Field) *Field { 97 | field := &from 98 | maputil.M(field).Set(self.Parameter, self.Desired) 99 | 100 | log.Noticef("DESIRED: %+v", self.Actual) 101 | 102 | return field 103 | } 104 | 105 | func (self SchemaDelta) String() string { 106 | msg := fmt.Sprintf("%s '%s'", strings.Title(string(self.Type)), self.Name) 107 | 108 | if self.Parameter != `` { 109 | msg += fmt.Sprintf(", parameter '%s'", self.Parameter) 110 | } 111 | 112 | msg += fmt.Sprintf(": %s", self.Message) 113 | 114 | dV := fmt.Sprintf("%v", self.Desired) 115 | aV := fmt.Sprintf("%v", self.Actual) 116 | 117 | if len(dV) <= 12 && len(aV) <= 12 { 118 | msg += fmt.Sprintf(" (desired: %v, actual: %v)", dV, aV) 119 | } 120 | 121 | return msg 122 | } 123 | 124 | type Migratable interface { 125 | Migrate() error 126 | } 127 | -------------------------------------------------------------------------------- /dal/validators.go: -------------------------------------------------------------------------------- 1 | package dal 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "regexp" 7 | 8 | "github.com/ghetzel/go-stockutil/maputil" 9 | "github.com/ghetzel/go-stockutil/sliceutil" 10 | "github.com/ghetzel/go-stockutil/stringutil" 11 | "github.com/ghetzel/go-stockutil/typeutil" 12 | ) 13 | 14 | // Retrieve a validator by name. Used by the ValidatorConfig configuration on Field. 15 | func ValidatorFromMap(in map[string]interface{}) (FieldValidatorFunc, error) { 16 | validators := make([]FieldValidatorFunc, 0) 17 | 18 | for name, defn := range in { 19 | if validator, err := GetValidator(name, defn); err == nil { 20 | validators = append(validators, validator) 21 | } else { 22 | return nil, fmt.Errorf("Invalid validator configuration %v: %v", name, err) 23 | } 24 | } 25 | 26 | return ValidateAll(validators...), nil 27 | } 28 | 29 | // Retrieve a validator by name. Used by the ValidatorConfig configuration on Field. 30 | func GetValidator(name string, args interface{}) (FieldValidatorFunc, error) { 31 | switch name { 32 | case `one-of`: 33 | if typeutil.IsArray(args) { 34 | var values []interface{} = sliceutil.Sliceify(args) 35 | 36 | for i, v := range values { 37 | if vAtKey := maputil.M(v).Get(`value`); !vAtKey.IsNil() { 38 | values[i] = vAtKey.Value 39 | } 40 | } 41 | 42 | return ValidateIsOneOf(values...), nil 43 | } else if typeutil.IsMap(args) { 44 | return ValidateIsOneOf(maputil.MapValues(args)...), nil 45 | } else { 46 | return nil, fmt.Errorf("Must specify an array of values for validator 'one-of'") 47 | } 48 | 49 | case `not-zero`: 50 | return ValidateNonZero, nil 51 | 52 | case `not-empty`: 53 | return ValidateNotEmpty, nil 54 | 55 | case `positive-integer`: 56 | return ValidatePositiveInteger, nil 57 | 58 | case `positive-or-zero-integer`: 59 | return ValidatePositiveOrZeroInteger, nil 60 | 61 | case `url`: 62 | return ValidateIsURL, nil 63 | 64 | case `match`, `match-all`: 65 | if typeutil.IsArray(args) { 66 | return ValidateMatchAll(sliceutil.Stringify(args)...), nil 67 | } else { 68 | return nil, fmt.Errorf("Must specify an array of values for validator 'match'") 69 | } 70 | 71 | case `match-any`: 72 | if typeutil.IsArray(args) { 73 | return ValidateMatchAny(sliceutil.Stringify(args)...), nil 74 | } else { 75 | return nil, fmt.Errorf("Must specify an array of values for validator 'match-any'") 76 | } 77 | 78 | default: 79 | return nil, fmt.Errorf("Unknown validator %q", name) 80 | } 81 | } 82 | 83 | // Validate that the given value matches all of the given regular expressions. 84 | func ValidateMatchAll(patterns ...string) FieldValidatorFunc { 85 | prx := make([]*regexp.Regexp, len(patterns)) 86 | 87 | for i, rxs := range patterns { 88 | prx[i] = regexp.MustCompile(rxs) 89 | } 90 | 91 | return func(value interface{}) error { 92 | for _, rx := range prx { 93 | if !rx.MatchString(typeutil.String(value)) { 94 | return fmt.Errorf("Value does not match pattern %q", rx.String()) 95 | } 96 | } 97 | 98 | return nil 99 | } 100 | } 101 | 102 | // Validate that the given value matches at least one of the given regular expressions. 103 | func ValidateMatchAny(patterns ...string) FieldValidatorFunc { 104 | prx := make([]*regexp.Regexp, len(patterns)) 105 | 106 | for i, rxs := range patterns { 107 | prx[i] = regexp.MustCompile(rxs) 108 | } 109 | 110 | return func(value interface{}) error { 111 | for _, rx := range prx { 112 | if rx.MatchString(typeutil.String(value)) { 113 | return nil 114 | } 115 | } 116 | 117 | return fmt.Errorf("Value does not match any valid pattern") 118 | } 119 | } 120 | 121 | // Validate that all of the given validator functions pass. 122 | func ValidateAll(validators ...FieldValidatorFunc) FieldValidatorFunc { 123 | return func(value interface{}) error { 124 | for _, validator := range validators { 125 | if err := validator(value); err != nil { 126 | return err 127 | } 128 | } 129 | 130 | return nil 131 | } 132 | } 133 | 134 | // Validate that the given value is among the given choices. 135 | func ValidateIsOneOf(choices ...interface{}) FieldValidatorFunc { 136 | return func(value interface{}) error { 137 | for _, choice := range choices { 138 | if ok, err := stringutil.RelaxedEqual(choice, value); err == nil && ok { 139 | return nil 140 | } 141 | } 142 | 143 | return fmt.Errorf("value must be one of: %+v", choices) 144 | } 145 | } 146 | 147 | // Validate that the given value is not a zero value (false, 0, 0.0, "", null). 148 | func ValidateNonZero(value interface{}) error { 149 | if typeutil.IsZero(value) { 150 | return fmt.Errorf("expected non-zero value, got: %v", value) 151 | } 152 | 153 | return nil 154 | } 155 | 156 | // Validate that the given value is not a zero value, and if it's a string, that the string 157 | // does not contain only whitespace. 158 | func ValidateNotEmpty(value interface{}) error { 159 | if typeutil.IsEmpty(value) { 160 | return fmt.Errorf("expected non-empty value, got: %v", value) 161 | } 162 | 163 | return nil 164 | } 165 | 166 | // Validate that the given value is an integer > 0. 167 | func ValidatePositiveInteger(value interface{}) error { 168 | if v, err := stringutil.ConvertToInteger(value); err == nil { 169 | if v <= 0 { 170 | return fmt.Errorf("expected value > 0, got: %v", v) 171 | } 172 | } else { 173 | return err 174 | } 175 | 176 | return nil 177 | } 178 | 179 | // Validate that the given value is an integer >= 0. 180 | func ValidatePositiveOrZeroInteger(value interface{}) error { 181 | if v, err := stringutil.ConvertToInteger(value); err == nil { 182 | if v < 0 { 183 | return fmt.Errorf("expected value >= 0, got: %v", v) 184 | } 185 | } else { 186 | return err 187 | } 188 | 189 | return nil 190 | } 191 | 192 | // Validate that the value is a URL with a non-empty scheme and host component. 193 | func ValidateIsURL(value interface{}) error { 194 | if u, err := url.Parse(typeutil.String(value)); err == nil { 195 | if u.Scheme == `` || u.Host == `` || u.Path == `` { 196 | return fmt.Errorf("Invalid URL") 197 | } 198 | } else { 199 | return err 200 | } 201 | 202 | return nil 203 | } 204 | -------------------------------------------------------------------------------- /database.go: -------------------------------------------------------------------------------- 1 | package pivot 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/ghetzel/pivot/v3/backends" 7 | "github.com/ghetzel/pivot/v3/dal" 8 | "github.com/ghetzel/pivot/v3/mapper" 9 | ) 10 | 11 | type DB interface { 12 | backends.Backend 13 | AttachCollection(*Collection) Model 14 | C(string) *Collection 15 | Migrate() error 16 | Models() []Model 17 | ApplySchemata(fileOrDirPath string) error 18 | LoadFixtures(fileOrDirPath string) error 19 | GetBackend() Backend 20 | SetBackend(Backend) 21 | } 22 | 23 | type schemaModel struct { 24 | Collection *dal.Collection 25 | Model Model 26 | } 27 | 28 | func (self *schemaModel) String() string { 29 | if self.Collection != nil { 30 | return self.Collection.Name 31 | } else { 32 | return `` 33 | } 34 | } 35 | 36 | type db struct { 37 | backends.Backend 38 | models []*schemaModel 39 | } 40 | 41 | func newdb(backend backends.Backend) *db { 42 | return &db{ 43 | Backend: backend, 44 | models: make([]*schemaModel, 0), 45 | } 46 | } 47 | 48 | func (self *db) GetBackend() Backend { 49 | return self.Backend 50 | } 51 | 52 | func (self *db) SetBackend(backend Backend) { 53 | self.Backend = backend 54 | } 55 | 56 | // A version of GetCollection that panics if the collection does not exist. 57 | func (self *db) C(name string) *Collection { 58 | if collection, err := self.GetCollection(name); err == nil { 59 | return collection 60 | } else { 61 | panic("C(" + name + "): " + err.Error()) 62 | } 63 | } 64 | 65 | func (self *db) AttachCollection(collection *Collection) Model { 66 | if collection == nil { 67 | panic("cannot attach nil Collection") 68 | } 69 | 70 | for _, sm := range self.models { 71 | if sm.String() == collection.Name { 72 | panic(fmt.Sprintf("Collection %q is already registered", collection.Name)) 73 | } 74 | } 75 | 76 | sm := &schemaModel{ 77 | Collection: collection, 78 | Model: mapper.NewModel(self, collection), 79 | } 80 | 81 | self.models = append(self.models, sm) 82 | return sm.Model 83 | } 84 | 85 | func (self *db) Migrate() error { 86 | for _, sm := range self.models { 87 | if err := sm.Model.Migrate(); err != nil { 88 | return fmt.Errorf("failed to migrate %v: %v", sm, err) 89 | } 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func (self *db) Models() (models []Model) { 96 | for _, sm := range self.models { 97 | models = append(models, sm.Model) 98 | } 99 | 100 | return 101 | } 102 | 103 | func (self *db) ApplySchemata(fileOrDirPath string) error { 104 | return ApplySchemata(fileOrDirPath, self) 105 | } 106 | 107 | func (self *db) LoadFixtures(fileOrDirPath string) error { 108 | return LoadFixtures(fileOrDirPath, self) 109 | } 110 | -------------------------------------------------------------------------------- /docs/-/about.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | pivot - Golang Package Documentation 13 | 14 | 15 | 88 | 89 |
90 |

About go-owndoc

91 | 92 |
93 |

94 | go-owndoc is a project that aims to provide a clean, flexible, and customizable 95 | mechanism for generating and self-hosting documentation and API reference for Golang projects. 96 |

97 | 98 |

99 | It uses the same underlying mechanisms as the godoc 100 | command, but offers much greater ability to customize the output of the documentation. The aim is to allow projects, 101 | especially private repositories, to generate and host their own high-quality updated code documentation. 102 |

103 | 104 |
105 |

106 | This project is built and maintained by Gary Hetzel, and the source code may 107 | be viewed, copied, and modified on GitHub. 108 |

109 |
110 |
111 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /docs/-/site.css: -------------------------------------------------------------------------------- 1 | html { 2 | background-color: whitesmoke; 3 | } 4 | 5 | body { 6 | background-color: white; 7 | } 8 | 9 | h4 { 10 | margin-top: 20px; 11 | } 12 | 13 | .container { 14 | max-width: 728px; 15 | } 16 | 17 | #x-projnav { 18 | min-height: 20px; 19 | margin-bottom: 20px; 20 | background-color: #eee; 21 | padding: 9px; 22 | border-radius: 3px; 23 | } 24 | 25 | #x-footer { 26 | padding-top: 14px; 27 | padding-bottom: 15px; 28 | margin-top: 5px; 29 | border-top: 1px solid #ccc; 30 | background-color: #e1e7f4; 31 | } 32 | 33 | .highlighted { 34 | background-color: #fdff9e; 35 | } 36 | 37 | #x-pkginfo { 38 | margin-top: 25px; 39 | border-top: 1px solid #ccc; 40 | padding-top: 20px; 41 | margin-bottom: 15px; 42 | } 43 | 44 | code { 45 | background-color: inherit; 46 | border: none; 47 | color: #222; 48 | padding: 0; 49 | } 50 | 51 | pre { 52 | color: #222; 53 | overflow: auto; 54 | white-space: pre; 55 | word-break: normal; 56 | word-wrap: normal; 57 | } 58 | 59 | .funcdecl>pre { 60 | white-space: pre-wrap; 61 | word-break: break-all; 62 | word-wrap: break-word; 63 | } 64 | 65 | pre .com { 66 | color: #006600; 67 | } 68 | 69 | .decl { 70 | position: relative; 71 | } 72 | 73 | .decl>a { 74 | position: absolute; 75 | top: 0px; 76 | right: 0px; 77 | display: none; 78 | border: 1px solid #ccc; 79 | border-top-right-radius: 4px; 80 | border-bottom-left-radius: 4px; 81 | padding-left: 4px; 82 | padding-right: 4px; 83 | } 84 | 85 | .decl>a:hover { 86 | background-color: white; 87 | text-decoration: none; 88 | } 89 | 90 | .decl:hover>a { 91 | display: block; 92 | } 93 | 94 | .navbar { 95 | border-radius: 0; 96 | margin-bottom: 0; 97 | } 98 | 99 | a { 100 | color: #375eab; 101 | } 102 | 103 | .navbar-default .navbar-brand { 104 | color: white; 105 | } 106 | 107 | .navbar-default { 108 | background-color: #375eab; 109 | border: 0; 110 | color: white; 111 | } 112 | 113 | .navbar-nav .text-muted, 114 | .navbar-nav .navbar-text { 115 | color: #ccc; 116 | } 117 | 118 | .navbar-default .navbar-nav>li>a { 119 | color: white; 120 | } 121 | 122 | .navbar-default .navbar-nav>.active>a, 123 | .navbar-default .navbar-nav>.active>a:hover, 124 | .navbar-default .navbar-nav>.active>a:focus { 125 | background-color: #6281bd; 126 | color: white; 127 | } 128 | 129 | .navbar-default .navbar-nav a:hover, 130 | .navbar-default .navbar-nav a:focus { 131 | background-color: #42567f !important; 132 | color: #ddd !important; 133 | } 134 | 135 | .navbar-default .navbar-brand:hover, 136 | .navbar-default .navbar-brand:focus { 137 | color: #ddd !important; 138 | } 139 | 140 | .banner { 141 | align-items: center; 142 | background-color: #fffbe0; 143 | display: flex; 144 | justify-content: space-between; 145 | margin-bottom: 20px; 146 | min-height: 40px; 147 | padding: 8px 16px; 148 | } 149 | 150 | .banner-message { 151 | margin-right: 90px; 152 | } 153 | 154 | .banner-action-container { 155 | text-align: right; 156 | } 157 | 158 | .banner-action { 159 | white-space: nowrap; 160 | } 161 | 162 | @media (max-width: 768px) { 163 | .banner { 164 | flex-direction: column; 165 | } 166 | 167 | .banner-message { 168 | align-items: flex-start; 169 | margin-bottom: 10px; 170 | margin-right: 0; 171 | } 172 | 173 | .banner-action-container { 174 | width: 100%; 175 | } 176 | } 177 | 178 | .panel-default>.panel-heading { 179 | color: #333; 180 | background-color: transparent; 181 | } 182 | 183 | a.permalink { 184 | display: none; 185 | } 186 | 187 | a.uses { 188 | display: none; 189 | color: #666; 190 | font-size: 0.8em; 191 | } 192 | 193 | h1:hover .permalink, 194 | h2:hover .permalink, 195 | h3:hover .permalink, 196 | h4:hover .permalink, 197 | h5:hover .permalink, 198 | h6:hover .permalink, 199 | h1:hover .uses, 200 | h2:hover .uses, 201 | h3:hover .uses, 202 | h4:hover .uses, 203 | h5:hover .uses, 204 | h6:hover .uses { 205 | display: inline; 206 | } 207 | 208 | @media (max-width: 768px) { 209 | .form-control { 210 | font-size: 16px; 211 | } 212 | } 213 | 214 | .synopsis { 215 | opacity: 0.87; 216 | } 217 | 218 | .additional-info { 219 | display: block; 220 | opacity: 0.54; 221 | text-transform: uppercase; 222 | font-size: 0.75em; 223 | } -------------------------------------------------------------------------------- /examples/basic-crud/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/mail" 7 | "os" 8 | "time" 9 | 10 | "github.com/ghetzel/pivot/v3" 11 | "github.com/ghetzel/pivot/v3/dal" 12 | "github.com/ghetzel/pivot/v3/mapper" 13 | ) 14 | 15 | type User struct { 16 | Username string `pivot:"id,identity"` 17 | Email string `pivot:"email"` 18 | Enabled bool `pivot:"enabled"` 19 | CreatedAt time.Time `pivot:"created_at"` 20 | UpdatedAt time.Time `pivot:"updated_at"` 21 | } 22 | 23 | var UsersTable = &dal.Collection{ 24 | Name: `users`, 25 | 26 | // primary key is the username 27 | IdentityField: `username`, 28 | IdentityFieldType: dal.StringType, 29 | 30 | // enforce that usernames be snake_cased and whitespace-trimmed 31 | IdentityFieldFormatter: dal.FormatAll( 32 | dal.TrimSpace, 33 | dal.ChangeCase(`underscore`), 34 | ), 35 | 36 | Fields: []dal.Field{ 37 | { 38 | Name: `email`, 39 | Type: dal.StringType, 40 | Required: true, 41 | 42 | // enforces RFC 5322 formatting on email addresses 43 | Formatter: func(email interface{}, inOrOut dal.FieldOperation) (interface{}, error) { 44 | if addr, err := mail.ParseAddress(fmt.Sprintf("%v", email)); err == nil { 45 | return addr.String(), nil 46 | } else { 47 | return nil, err 48 | } 49 | }, 50 | }, { 51 | Name: `enabled`, 52 | Type: dal.BooleanType, 53 | DefaultValue: true, 54 | Required: true, 55 | }, { 56 | Name: `created_at`, 57 | Type: dal.TimeType, 58 | Required: true, 59 | 60 | // set created_at only if it's currently nil 61 | Formatter: dal.CurrentTimeIfUnset, 62 | }, { 63 | Name: `updated_at`, 64 | Type: dal.TimeType, 65 | Required: true, 66 | 67 | // set updated_at to the current time, every time 68 | Formatter: dal.CurrentTime, 69 | }, 70 | }, 71 | } 72 | 73 | func main() { 74 | var connectionString string 75 | 76 | if len(os.Args) < 2 { 77 | connectionString = `sqlite:///tmp/.pivot-basic-crud.db` 78 | } else { 79 | connectionString = os.Args[1] 80 | } 81 | 82 | // setup a connection to the database 83 | if db, err := pivot.NewDatabase(connectionString); err == nil { 84 | // make sure we can actually talk to the database 85 | if err := db.Ping(10 * time.Second); err != nil { 86 | log.Fatalf("Database connection test failed: %v", err) 87 | } 88 | 89 | Users := mapper.NewModel(db, UsersTable) 90 | 91 | // creates the table, and fails if the existing schema does not match the 92 | // UsersTable collection definition above 93 | if err := Users.Migrate(); err != nil { 94 | log.Fatalf("migrating users table failed: %v", err) 95 | } 96 | 97 | // CREATE 98 | // ----------------------------------------------------------------------------------------- 99 | 100 | // make a user object resembling user input 101 | user := User{ 102 | Username: " Test\nUser ", 103 | Email: "test.user+testing@example.com", 104 | } 105 | 106 | // create the user 107 | if err := Users.Create(&user); err != nil { 108 | log.Fatalf("failed to create user: %v", err) 109 | } 110 | 111 | // print out what we've done 112 | log.Printf("User %q created email=%q", user.Username, user.Email) 113 | 114 | // UPDATES 115 | // ----------------------------------------------------------------------------------------- 116 | 117 | // update the user's email 118 | user.Email = "other.user@example.com" 119 | 120 | if err := Users.Update(&user); err != nil { 121 | log.Fatalf("failed to update user: %v", err) 122 | } 123 | 124 | // RETRIEVAL 125 | // ----------------------------------------------------------------------------------------- 126 | 127 | // read back the user 128 | if err := Users.Get(user.Username, &user); err != nil { 129 | log.Fatalf("failed to retrieve user: %v", err) 130 | } 131 | 132 | log.Printf("User %q now has email=%q", user.Username, user.Email) 133 | 134 | // DELETE 135 | // ----------------------------------------------------------------------------------------- 136 | 137 | // delete the user 138 | if err := Users.Delete(user.Username); err != nil { 139 | log.Fatalf("failed to delete user: %v", err) 140 | } 141 | 142 | // EXISTS 143 | // ----------------------------------------------------------------------------------------- 144 | 145 | // make SURE we deleted the user 146 | if Users.Exists(user.Username) { 147 | log.Fatalf("user %q still exists", user.Username) 148 | } else { 149 | log.Println("OK") 150 | } 151 | 152 | } else { 153 | log.Fatalf("failed to instantiate database: %v", err) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /examples/embedded-collections/fixtures/contacts.json: -------------------------------------------------------------------------------- 1 | // names and other made-up things pulled from https://jsonplaceholder.typicode.com 2 | [{ 3 | "id": 1000, 4 | "fields": { 5 | "first_name": "Leanne", 6 | "last_name": "Graham", 7 | "address": "556 Kulas St.", 8 | "city": "Gwenborough", 9 | "state": "ZT", 10 | "country": "us" 11 | } 12 | }] -------------------------------------------------------------------------------- /examples/embedded-collections/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | "github.com/ghetzel/go-stockutil/log" 8 | "github.com/ghetzel/pivot/v3" 9 | "github.com/ghetzel/pivot/v3/backends" 10 | "github.com/ghetzel/pivot/v3/dal" 11 | ) 12 | 13 | type Contact struct { 14 | ID int64 `pivot:"id,identity"` 15 | FirstName string `pivot:"first_name"` 16 | LastName string `pivot:"last_name"` 17 | Address string `pivot:"address"` 18 | City string `pivot:"city"` 19 | State string `pivot:"state"` 20 | Country string `pivot:"country"` 21 | } 22 | 23 | type Item struct { 24 | ID int `pivot:"id,identity"` 25 | Name string `pivot:"name"` 26 | Description string `pivot:"description"` 27 | Cost float64 `pivot:"cost"` 28 | Currency string `pivot:"currency"` 29 | } 30 | 31 | type Order struct { 32 | ID string `pivot:"id,identity"` 33 | Status string `pivot:"status"` 34 | Items []Item `pivot:"-"` 35 | ShippingAddress Contact `pivot:"shipping_address"` 36 | BillingAddress Contact `pivot:"billing_address"` 37 | CreatedAt time.Time `pivot:"created_at"` 38 | UpdatedAt time.Time `pivot:"updated_at"` 39 | } 40 | 41 | var ItemsTable = &dal.Collection{ 42 | Name: `items`, 43 | Fields: []dal.Field{ 44 | { 45 | Name: `name`, 46 | Type: dal.StringType, 47 | Required: true, 48 | }, { 49 | Name: `description`, 50 | Type: dal.StringType, 51 | Required: true, 52 | }, { 53 | Name: `cost`, 54 | Type: dal.FloatType, 55 | Required: true, 56 | }, { 57 | Name: `currency`, 58 | Type: dal.StringType, 59 | Required: true, 60 | }, 61 | }, 62 | } 63 | 64 | var OrdersTable = &dal.Collection{ 65 | Name: `items`, 66 | Fields: []dal.Field{ 67 | { 68 | Name: `status`, 69 | Type: dal.StringType, 70 | Required: true, 71 | DefaultValue: `pending`, 72 | Validator: dal.ValidateIsOneOf( 73 | `pending`, 74 | `received`, 75 | `processing`, 76 | `shipped`, 77 | `delivered`, 78 | `canceled`, 79 | ), 80 | }, { 81 | Name: `shipping_address`, 82 | Type: dal.IntType, 83 | Required: true, 84 | }, { 85 | Name: `billing_address`, 86 | Type: dal.IntType, 87 | Required: true, 88 | }, { 89 | Name: `currency`, 90 | Type: dal.StringType, 91 | Required: true, 92 | }, 93 | }, 94 | } 95 | 96 | var OrdersItemsTable = &dal.Collection{ 97 | Fields: []dal.Field{ 98 | { 99 | Name: `order_id`, 100 | Type: dal.StringType, 101 | Required: true, 102 | BelongsTo: OrdersTable, 103 | }, { 104 | Name: `item_id`, 105 | Type: dal.IntType, 106 | Required: true, 107 | BelongsTo: ItemsTable, 108 | }, 109 | }, 110 | } 111 | 112 | var ContactsTable = &dal.Collection{ 113 | Name: `contacts`, 114 | Fields: []dal.Field{ 115 | { 116 | Name: `first_name`, 117 | Type: dal.StringType, 118 | Required: true, 119 | }, { 120 | Name: `last_name`, 121 | Type: dal.StringType, 122 | }, { 123 | Name: `address`, 124 | Type: dal.StringType, 125 | }, { 126 | Name: `city`, 127 | Type: dal.StringType, 128 | Required: true, 129 | }, { 130 | Name: `state`, 131 | Type: dal.StringType, 132 | Required: true, 133 | }, { 134 | Name: `country`, 135 | Type: dal.StringType, 136 | Required: true, 137 | }, 138 | }, 139 | } 140 | 141 | var Contacts pivot.Model 142 | var Items pivot.Model 143 | var Orders pivot.Model 144 | var OrdersItems pivot.Model 145 | 146 | func main() { 147 | var connectionString string 148 | 149 | if len(os.Args) < 2 { 150 | connectionString = `sqlite:///tmp/.pivot-embedded-collections.db` 151 | } else { 152 | connectionString = os.Args[1] 153 | } 154 | 155 | // setup a connection to the database 156 | if db, err := pivot.NewDatabase(connectionString); err == nil { 157 | db.SetBackend(backends.NewEmbeddedRecordBackend(db.GetBackend())) 158 | 159 | // make sure we can actually talk to the database 160 | if err := db.Ping(10 * time.Second); err != nil { 161 | log.Fatalf("Database connection test failed: %v", err) 162 | } 163 | 164 | Contacts = db.AttachCollection(ContactsTable) 165 | Items = db.AttachCollection(ItemsTable) 166 | Orders = db.AttachCollection(OrdersTable) 167 | OrdersItems = db.AttachCollection(OrdersItemsTable) 168 | 169 | // create tables as necessary 170 | log.FatalfIf("migrate failed: %v", db.Migrate()) 171 | 172 | // load data into tables 173 | log.FatalfIf("load fixtures failed: %v", db.LoadFixtures(`./examples/embedded-collections/fixtures/*.json`)) 174 | 175 | // TODO: need to actually demonstrate how embedding works 176 | } else { 177 | log.Fatalf("failed to instantiate database: %v", err) 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /filter/README.md: -------------------------------------------------------------------------------- 1 | # Filter Syntax 2 | 3 | Pivot uses a simplified filter syntax that is URL-friendly and fairly straightforward to use. It consists of a sequence of `field/value` pairs that are themselves separated by a forward slash. Generally, the best way to learn this syntax is through examples: 4 | 5 | ## Examples 6 | 7 | ``` 8 | # Where field "id" is exactly equal to 123 9 | id/123 10 | 11 | # Where "id" is exactly 3, 14, OR 27 12 | id/3|14|27 13 | 14 | # Where "id" is exactly 3, 14, OR 27 AND "enabled" is true 15 | id/3|14|27/enabled/true 16 | 17 | # Where "name" is "Bob" AND "age" is 42. 18 | name/Bob/age/42 19 | 20 | # Where "product" contains the string "usb" and "price" is less than 5.00 21 | product/contains:usb/price/lt:5.00 22 | 23 | # Where "product" contains the string "usb" and "price" is between 10.00 (inclusive) and 20.01 (exclusive) 24 | product/contains:usb/price/range:10|20.01 25 | ``` 26 | 27 | ## General Form 28 | 29 | The general form of this filter syntax is: 30 | 31 | `[[type:]field/[operator:]value[|orvalue ..] ..]` 32 | 33 | ## Operators 34 | 35 | Supported operators are as follows: 36 | 37 | | Operator | Description | 38 | | ---------- | ----------- | 39 | | `is` | Values must exactly match (this is the default if no operator is provided) | 40 | | `not` | Values may be anything _except_ an exact match | 41 | | `contains` | String value must contain the given substring (case sensitive) | 42 | | `like` | String value must contain the given substring (case insensitive) | 43 | | `unlike` | String value may not contain the given substring (case insensitive) | 44 | | `prefix` | String value must start with the given string | 45 | | `suffix` | String value must end with the given string | 46 | | `gt` | Numeric or date value must be strictly greater than | 47 | | `gte` | Numeric or date value must be strictly greater than or equal to | 48 | | `lt` | Numeric or date value must be strictly less than | 49 | | `lte` | Numeric or date value must be strictly less than or equal to | 50 | | `range` | Numeric or date value must be between two values (separated by `|`; first value is inclusive, second value exclusive | 51 | 52 | 53 | -------------------------------------------------------------------------------- /filter/filter_matches_record_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ghetzel/pivot/v3/dal" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestFilterMatchesRecord(t *testing.T) { 11 | assert := require.New(t) 12 | 13 | assert.True(MustParse(`id/1`).MatchesRecord(dal.NewRecord(1))) 14 | assert.True(MustParse(`id/1`).MatchesRecord(dal.NewRecord(`1`))) 15 | assert.True(MustParse(`id/is:1`).MatchesRecord(dal.NewRecord(1))) 16 | assert.True(MustParse(`id/is:1`).MatchesRecord(dal.NewRecord(`1`))) 17 | assert.True(MustParse(`int:id/1`).MatchesRecord(dal.NewRecord(1))) 18 | assert.True(MustParse(`str:id/1`).MatchesRecord(dal.NewRecord(`1`))) 19 | assert.False(MustParse(`str:id/is:1`).MatchesRecord(dal.NewRecord(1))) 20 | assert.True(MustParse(`id/not:1`).MatchesRecord(dal.NewRecord(2))) 21 | assert.True(MustParse(`id/not:1`).MatchesRecord(dal.NewRecord(`2`))) 22 | 23 | assert.True(MustParse(`id/1/test/true`).MatchesRecord(dal.NewRecord(1).Set(`test`, true))) 24 | assert.True(MustParse(`id/1/test/true`).MatchesRecord(dal.NewRecord(1).Set(`test`, `true`))) 25 | assert.True(MustParse(`id/1/bool:test/true`).MatchesRecord(dal.NewRecord(1).Set(`test`, true))) 26 | assert.True(MustParse(`id/1/str:test/true`).MatchesRecord(dal.NewRecord(1).Set(`test`, `true`))) 27 | 28 | assert.False(MustParse(`id/1/test/true`).MatchesRecord(dal.NewRecord(1).Set(`test`, false))) 29 | assert.False(MustParse(`id/1/test/true`).MatchesRecord(dal.NewRecord(1).Set(`test`, `false`))) 30 | assert.False(MustParse(`id/1/test/true`).MatchesRecord(dal.NewRecord(1).Set(`test`, 1))) 31 | assert.False(MustParse(`id/1/test/true`).MatchesRecord(dal.NewRecord(1).Set(`test`, `1`))) 32 | assert.False(MustParse(`id/1/test/false`).MatchesRecord(dal.NewRecord(1).Set(`test`, 0))) 33 | assert.False(MustParse(`id/1/test/false`).MatchesRecord(dal.NewRecord(1).Set(`test`, `0`))) 34 | assert.False(MustParse(`id/1/str:test/true`).MatchesRecord(dal.NewRecord(1).Set(`test`, true))) 35 | 36 | assert.False(MustParse(`id/gt:1`).MatchesRecord(dal.NewRecord(0))) 37 | assert.False(MustParse(`id/gt:1`).MatchesRecord(dal.NewRecord(1))) 38 | assert.True(MustParse(`id/gt:1`).MatchesRecord(dal.NewRecord(2))) 39 | assert.False(MustParse(`id/gte:1`).MatchesRecord(dal.NewRecord(0))) 40 | assert.True(MustParse(`id/gte:1`).MatchesRecord(dal.NewRecord(1))) 41 | assert.True(MustParse(`id/gte:1`).MatchesRecord(dal.NewRecord(2))) 42 | 43 | assert.False(MustParse(`id/lt:1`).MatchesRecord(dal.NewRecord(2))) 44 | assert.False(MustParse(`id/lt:1`).MatchesRecord(dal.NewRecord(1))) 45 | assert.True(MustParse(`id/lt:1`).MatchesRecord(dal.NewRecord(0))) 46 | assert.False(MustParse(`id/lte:1`).MatchesRecord(dal.NewRecord(2))) 47 | assert.True(MustParse(`id/lte:1`).MatchesRecord(dal.NewRecord(1))) 48 | assert.True(MustParse(`id/lte:1`).MatchesRecord(dal.NewRecord(0))) 49 | 50 | assert.True(MustParse(`name/contains:old`).MatchesRecord(dal.NewRecord(1).Set(`name`, `Goldenrod`))) 51 | assert.True(MustParse(`name/prefix:gold`).MatchesRecord(dal.NewRecord(1).Set(`name`, `Gold`))) 52 | assert.True(MustParse(`name/prefix:Gold`).MatchesRecord(dal.NewRecord(1).Set(`name`, `Gold`))) 53 | assert.True(MustParse(`name/suffix:rod`).MatchesRecord(dal.NewRecord(1).Set(`name`, `Goldenrod`))) 54 | assert.True(MustParse(`name/suffix:apple|orange|rod`).MatchesRecord(dal.NewRecord(1).Set(`name`, `Goldenrod`))) 55 | 56 | assert.True(MustParse(`name/contains:olden rod`).MatchesRecord(dal.NewRecord(1).Set(`name`, `Golden rod`))) 57 | assert.True(MustParse(`name/Golden rod`).MatchesRecord(dal.NewRecord(1).Set(`name`, `Golden rod`))) 58 | assert.True(MustParse(`name/like:golden rod`).MatchesRecord(dal.NewRecord(1).Set(`name`, `Golden rod`))) 59 | } 60 | -------------------------------------------------------------------------------- /filter/generator.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | type IGenerator interface { 4 | Initialize(string) error 5 | Finalize(*Filter) error 6 | Push([]byte) 7 | Set([]byte) 8 | Payload() []byte 9 | WithCriterion(Criterion) error 10 | OrCriterion(Criterion) error 11 | WithField(string) error 12 | GroupByField(string) error 13 | AggregateByField(Aggregation, string) error 14 | SetOption(string, interface{}) error 15 | GetValues() []interface{} 16 | Reset() 17 | } 18 | 19 | type Generator struct { 20 | IGenerator 21 | payload []byte 22 | } 23 | 24 | func Render(generator IGenerator, collectionName string, filter *Filter) ([]byte, error) { 25 | if err := generator.Initialize(collectionName); err != nil { 26 | return nil, err 27 | } 28 | 29 | // add options 30 | for key, value := range filter.Options { 31 | if err := generator.SetOption(key, value); err != nil { 32 | return nil, err 33 | } 34 | } 35 | 36 | // add fields 37 | for _, fieldName := range filter.Fields { 38 | if filter.IdentityField != `` && fieldName == `id` { 39 | fieldName = filter.IdentityField 40 | } 41 | 42 | if err := generator.WithField(fieldName); err != nil { 43 | return nil, err 44 | } 45 | } 46 | 47 | // add criteria 48 | for _, criterion := range filter.Criteria { 49 | if filter.IdentityField != `` && criterion.Field == `id` { 50 | criterion.Field = filter.IdentityField 51 | } 52 | 53 | if err := generator.WithCriterion(criterion); err != nil { 54 | return nil, err 55 | } 56 | } 57 | 58 | // finalize the payload 59 | if err := generator.Finalize(filter); err != nil { 60 | return nil, err 61 | } 62 | 63 | // return the payload 64 | return generator.Payload(), nil 65 | } 66 | 67 | func (self *Generator) Set(data []byte) { 68 | self.payload = data 69 | } 70 | 71 | func (self *Generator) Push(data []byte) { 72 | if self.payload == nil { 73 | self.payload = make([]byte, 0) 74 | } 75 | 76 | self.payload = append(self.payload, data...) 77 | } 78 | 79 | func (self *Generator) Reset() { 80 | self.payload = nil 81 | } 82 | 83 | func (self *Generator) Payload() []byte { 84 | return self.payload 85 | } 86 | 87 | func (self *Generator) Finalize(_ *Filter) error { 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /filter/generators/elasticsearch.go: -------------------------------------------------------------------------------- 1 | package generators 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/ghetzel/pivot/v3/filter" 8 | ) 9 | 10 | // Elasticsearch Generator 11 | var DefaultMinVersionCompat = float64(6) 12 | 13 | type Elasticsearch struct { 14 | filter.Generator 15 | collection string 16 | fields []string 17 | criteria []map[string]interface{} 18 | options map[string]interface{} 19 | values []interface{} 20 | facetFields []string 21 | aggregateBy []filter.Aggregate 22 | compat float64 23 | } 24 | 25 | func NewElasticsearchGenerator() *Elasticsearch { 26 | return &Elasticsearch{ 27 | Generator: filter.Generator{}, 28 | compat: DefaultMinVersionCompat, 29 | } 30 | } 31 | 32 | func (self *Elasticsearch) SetMinimumVersion(v float64) { 33 | self.compat = v 34 | } 35 | 36 | func (self *Elasticsearch) Initialize(collectionName string) error { 37 | self.Reset() 38 | self.collection = collectionName 39 | self.fields = make([]string, 0) 40 | self.criteria = make([]map[string]interface{}, 0) 41 | self.options = make(map[string]interface{}) 42 | self.values = make([]interface{}, 0) 43 | 44 | return nil 45 | } 46 | 47 | func (self *Elasticsearch) Finalize(flt *filter.Filter) error { 48 | var conjunction = `must` 49 | 50 | if flt.Conjunction == filter.OrConjunction { 51 | conjunction = `should` 52 | } 53 | 54 | var query map[string]interface{} 55 | 56 | if flt.Spec == `all` { 57 | query = map[string]interface{}{ 58 | `match_all`: map[string]interface{}{}, 59 | } 60 | } else { 61 | query = map[string]interface{}{ 62 | `bool`: map[string]interface{}{ 63 | conjunction: self.criteria, 64 | }, 65 | } 66 | } 67 | 68 | var payload = map[string]interface{}{ 69 | `query`: query, 70 | `size`: flt.Limit, 71 | `from`: flt.Offset, 72 | } 73 | 74 | if len(flt.Fields) > 0 { 75 | if self.compat >= 5 { 76 | payload[`_source`] = map[string]interface{}{ 77 | `include`: flt.Fields, 78 | } 79 | } else { 80 | payload[`fields`] = flt.Fields 81 | } 82 | } 83 | 84 | if len(flt.Sort) > 0 { 85 | var sorts = make([]interface{}, 0) 86 | 87 | for _, sort := range flt.Sort { 88 | if len(sort) > 1 && sort[0] == '-' { 89 | sorts = append(sorts, map[string]interface{}{ 90 | sort[1:]: `desc`, 91 | }) 92 | } else { 93 | sorts = append(sorts, map[string]interface{}{ 94 | sort: `asc`, 95 | }) 96 | } 97 | } 98 | 99 | payload[`sort`] = sorts 100 | } else { 101 | payload[`sort`] = []string{`_doc`} 102 | } 103 | 104 | for k, v := range self.options { 105 | payload[k] = v 106 | } 107 | 108 | if data, err := json.MarshalIndent(payload, ``, ` `); err == nil { 109 | self.Push(data) 110 | } else { 111 | return err 112 | } 113 | 114 | return nil 115 | } 116 | 117 | func (self *Elasticsearch) WithField(field string) error { 118 | self.fields = append(self.fields, field) 119 | return nil 120 | } 121 | 122 | func (self *Elasticsearch) SetOption(key string, value interface{}) error { 123 | self.options[key] = value 124 | return nil 125 | } 126 | 127 | func (self *Elasticsearch) GroupByField(field string) error { 128 | self.facetFields = append(self.facetFields, field) 129 | return nil 130 | } 131 | 132 | func (self *Elasticsearch) AggregateByField(agg filter.Aggregation, field string) error { 133 | self.aggregateBy = append(self.aggregateBy, filter.Aggregate{ 134 | Aggregation: agg, 135 | Field: field, 136 | }) 137 | 138 | return nil 139 | } 140 | 141 | func (self *Elasticsearch) GetValues() []interface{} { 142 | return self.values 143 | } 144 | 145 | func (self *Elasticsearch) WithCriterion(criterion filter.Criterion) error { 146 | var c map[string]interface{} 147 | var err error 148 | 149 | switch criterion.Operator { 150 | case `is`, ``: 151 | c, err = esCriterionOperatorIs(self, criterion) 152 | case `not`: 153 | c, err = esCriterionOperatorNot(self, criterion) 154 | case `like`: 155 | c, err = esCriterionOperatorLike(self, criterion) 156 | case `unlike`: 157 | c, err = esCriterionOperatorUnlike(self, criterion) 158 | case `contains`, `prefix`, `suffix`: 159 | c, err = esCriterionOperatorPattern(self, criterion.Operator, criterion) 160 | case `gt`, `gte`, `lt`, `lte`: 161 | c, err = esCriterionOperatorRange(self, criterion, criterion.Operator) 162 | case `fulltext`: 163 | c, err = esCriterionOperatorFulltext(self, criterion) 164 | default: 165 | return fmt.Errorf("Unimplemented operator '%s'", criterion.Operator) 166 | } 167 | 168 | if err != nil { 169 | return err 170 | } else { 171 | self.criteria = append(self.criteria, c) 172 | } 173 | 174 | return nil 175 | } 176 | -------------------------------------------------------------------------------- /filter/generators/mongodb-util.go: -------------------------------------------------------------------------------- 1 | package generators 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | 7 | "github.com/ghetzel/go-stockutil/stringutil" 8 | "github.com/ghetzel/pivot/v3/filter" 9 | ) 10 | 11 | var rxCharFilter = regexp.MustCompile(`[\W\s]`) 12 | 13 | func mongoCriterionOperatorIs(gen *MongoDB, criterion filter.Criterion) (map[string]interface{}, error) { 14 | c := make(map[string]interface{}) 15 | 16 | if len(criterion.Values) == 1 && criterion.Values[0] == nil { 17 | gen.values = append(gen.values, nil) 18 | 19 | c[`$or`] = []map[string]interface{}{ 20 | { 21 | criterion.Field: map[string]interface{}{ 22 | `$exists`: false, 23 | }, 24 | }, { 25 | criterion.Field: nil, 26 | }, 27 | } 28 | } else { 29 | for _, value := range criterion.Values { 30 | gen.values = append(gen.values, value) 31 | } 32 | 33 | if len(criterion.Values) == 1 { 34 | if criterion.Field == `_id` { 35 | c[criterion.Field] = fmt.Sprintf("%v", criterion.Values[0]) 36 | } else { 37 | c[criterion.Field] = criterion.Values[0] 38 | } 39 | } else { 40 | c[criterion.Field] = map[string]interface{}{ 41 | `$in`: criterion.Values, 42 | } 43 | } 44 | } 45 | 46 | return c, nil 47 | } 48 | 49 | func mongoCriterionOperatorNot(gen *MongoDB, criterion filter.Criterion) (map[string]interface{}, error) { 50 | c := make(map[string]interface{}) 51 | 52 | if len(criterion.Values) == 0 { 53 | return c, fmt.Errorf("The not criterion must have at least one value") 54 | 55 | } else if len(criterion.Values) == 1 && criterion.Values[0] == nil { 56 | gen.values = append(gen.values, nil) 57 | 58 | c[`$and`] = []map[string]interface{}{ 59 | { 60 | criterion.Field: map[string]interface{}{ 61 | `$exists`: true, 62 | }, 63 | }, { 64 | criterion.Field: map[string]interface{}{ 65 | `$not`: nil, 66 | }, 67 | }, 68 | } 69 | } else { 70 | for _, value := range criterion.Values { 71 | gen.values = append(gen.values, value) 72 | } 73 | 74 | if len(criterion.Values) == 1 { 75 | c[criterion.Field] = map[string]interface{}{ 76 | `$ne`: criterion.Values[0], 77 | } 78 | } else { 79 | c[criterion.Field] = map[string]interface{}{ 80 | `$nin`: criterion.Values, 81 | } 82 | } 83 | } 84 | 85 | return c, nil 86 | } 87 | 88 | func mongoCriterionOperatorPattern(gen *MongoDB, opname string, criterion filter.Criterion) (map[string]interface{}, error) { 89 | c := make(map[string]interface{}) 90 | 91 | if len(criterion.Values) == 0 { 92 | return nil, fmt.Errorf("The not criterion must have at least one value") 93 | } else { 94 | or_regexp := make([]map[string]interface{}, 0) 95 | 96 | for _, value := range criterion.Values { 97 | gen.values = append(gen.values, value) 98 | var valueClause string 99 | 100 | switch opname { 101 | case `contains`: 102 | valueClause = fmt.Sprintf(".*%v.*", value) 103 | case `prefix`: 104 | valueClause = fmt.Sprintf("^%v.*", value) 105 | case `suffix`: 106 | valueClause = fmt.Sprintf(".*%v$", value) 107 | case `like`, `unlike`: 108 | valueClause = rxCharFilter.ReplaceAllString(fmt.Sprintf("%v", value), `.`) 109 | default: 110 | return nil, fmt.Errorf("Unsupported pattern operator %q", opname) 111 | } 112 | 113 | if opname == `unlike` { 114 | or_regexp = append(or_regexp, map[string]interface{}{ 115 | `$not`: map[string]interface{}{ 116 | criterion.Field: map[string]interface{}{ 117 | `$regex`: valueClause, 118 | `$options`: `si`, 119 | }, 120 | }, 121 | }) 122 | } else { 123 | or_regexp = append(or_regexp, map[string]interface{}{ 124 | criterion.Field: map[string]interface{}{ 125 | `$regex`: valueClause, 126 | `$options`: `si`, 127 | }, 128 | }) 129 | } 130 | } 131 | 132 | if len(or_regexp) == 1 { 133 | c = or_regexp[0] 134 | } else { 135 | c[`$or`] = or_regexp 136 | } 137 | } 138 | 139 | return c, nil 140 | } 141 | 142 | func mongoCriterionOperatorRange(gen *MongoDB, criterion filter.Criterion, operator string) (map[string]interface{}, error) { 143 | c := make(map[string]interface{}) 144 | 145 | switch operator { 146 | case `range`: 147 | if l := len(criterion.Values); l > 0 && (l%2 == 0) { 148 | or_clauses := make([]map[string]interface{}, 0) 149 | 150 | for i := 0; i < l; i += 2 { 151 | value1 := stringutil.Autotype(criterion.Values[i]) 152 | value2 := stringutil.Autotype(criterion.Values[i+1]) 153 | 154 | gen.values = append(gen.values, value1, value2) 155 | 156 | c[criterion.Field] = map[string]interface{}{ 157 | `$gte`: value1, 158 | `$lt`: value2, 159 | } 160 | 161 | or_clauses = append(or_clauses, c) 162 | c = nil 163 | } 164 | 165 | if len(or_clauses) == 1 { 166 | return or_clauses[0], nil 167 | } else { 168 | return map[string]interface{}{ 169 | `$or`: or_clauses, 170 | }, nil 171 | } 172 | } else { 173 | return c, fmt.Errorf("Ranging criteria can only accept pairs of values, %d given", l) 174 | } 175 | 176 | default: 177 | switch l := len(criterion.Values); l { 178 | case 0: 179 | return c, fmt.Errorf("No values given for criterion %v", criterion.Field) 180 | case 1: 181 | value := stringutil.Autotype(criterion.Values[0]) 182 | gen.values = append(gen.values, value) 183 | 184 | c[criterion.Field] = map[string]interface{}{ 185 | `$` + operator: value, 186 | } 187 | default: 188 | return c, fmt.Errorf("Numeric comparators can only accept one value, %d given", l) 189 | } 190 | } 191 | 192 | return c, nil 193 | } 194 | -------------------------------------------------------------------------------- /filter/generators/mongodb.go: -------------------------------------------------------------------------------- 1 | package generators 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/ghetzel/go-stockutil/stringutil" 8 | "github.com/ghetzel/pivot/v3/filter" 9 | ) 10 | 11 | // MongoDB Query Generator 12 | 13 | type MongoDB struct { 14 | filter.Generator 15 | collection string 16 | fields []string 17 | criteria []map[string]interface{} 18 | options map[string]interface{} 19 | values []interface{} 20 | facetFields []string 21 | aggregateBy []filter.Aggregate 22 | } 23 | 24 | func NewMongoDBGenerator() *MongoDB { 25 | return &MongoDB{ 26 | Generator: filter.Generator{}, 27 | } 28 | } 29 | 30 | func (self *MongoDB) Initialize(collectionName string) error { 31 | self.Reset() 32 | self.collection = collectionName 33 | self.fields = make([]string, 0) 34 | self.criteria = make([]map[string]interface{}, 0) 35 | self.options = make(map[string]interface{}) 36 | self.values = make([]interface{}, 0) 37 | 38 | return nil 39 | } 40 | 41 | func (self *MongoDB) Finalize(f *filter.Filter) error { 42 | conjunction := `$and` 43 | 44 | var query map[string]interface{} 45 | 46 | if f.Conjunction == filter.OrConjunction { 47 | conjunction = `$or` 48 | } 49 | 50 | if f.Spec == `all` { 51 | query = map[string]interface{}{} 52 | } else if len(self.criteria) == 1 { 53 | query = self.criteria[0] 54 | } else { 55 | query = map[string]interface{}{ 56 | conjunction: self.criteria, 57 | } 58 | } 59 | 60 | if data, err := json.MarshalIndent(query, ``, ` `); err == nil { 61 | self.Push(data) 62 | } else { 63 | return err 64 | } 65 | 66 | return nil 67 | } 68 | 69 | func (self *MongoDB) WithField(field string) error { 70 | if field == `id` { 71 | field = `_id` 72 | } 73 | 74 | self.fields = append(self.fields, field) 75 | return nil 76 | } 77 | 78 | func (self *MongoDB) SetOption(key string, value interface{}) error { 79 | self.options[key] = value 80 | return nil 81 | } 82 | 83 | func (self *MongoDB) GroupByField(field string) error { 84 | if field == `id` { 85 | field = `_id` 86 | } 87 | 88 | self.facetFields = append(self.facetFields, field) 89 | return nil 90 | } 91 | 92 | func (self *MongoDB) AggregateByField(agg filter.Aggregation, field string) error { 93 | if field == `id` { 94 | field = `_id` 95 | } 96 | 97 | self.aggregateBy = append(self.aggregateBy, filter.Aggregate{ 98 | Aggregation: agg, 99 | Field: field, 100 | }) 101 | 102 | return nil 103 | } 104 | 105 | func (self *MongoDB) GetValues() []interface{} { 106 | return self.values 107 | } 108 | 109 | func (self *MongoDB) WithCriterion(criterion filter.Criterion) error { 110 | var c map[string]interface{} 111 | var err error 112 | 113 | if criterion.Field == `id` { 114 | criterion.Field = `_id` 115 | } 116 | 117 | for i, value := range criterion.Values { 118 | switch value.(type) { 119 | case string: 120 | criterion.Values[i] = stringutil.Autotype(value) 121 | } 122 | } 123 | 124 | switch criterion.Operator { 125 | case `is`, ``: 126 | c, err = mongoCriterionOperatorIs(self, criterion) 127 | case `not`: 128 | c, err = mongoCriterionOperatorNot(self, criterion) 129 | case `contains`, `prefix`, `suffix`, `like`, `unlike`: 130 | c, err = mongoCriterionOperatorPattern(self, criterion.Operator, criterion) 131 | case `gt`, `gte`, `lt`, `lte`, `range`: 132 | c, err = mongoCriterionOperatorRange(self, criterion, criterion.Operator) 133 | default: 134 | return fmt.Errorf("Unimplemented operator '%s'", criterion.Operator) 135 | } 136 | 137 | if err != nil { 138 | return err 139 | } else { 140 | self.criteria = append(self.criteria, c) 141 | } 142 | 143 | return nil 144 | } 145 | -------------------------------------------------------------------------------- /filter/generators/mongodb_test.go: -------------------------------------------------------------------------------- 1 | package generators 2 | 3 | import ( 4 | "testing" 5 | 6 | "encoding/json" 7 | 8 | "github.com/ghetzel/pivot/v3/filter" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | type mongoQv struct { 13 | query map[string]interface{} 14 | values []interface{} 15 | input map[string]interface{} 16 | } 17 | 18 | func TestMongodb(t *testing.T) { 19 | assert := require.New(t) 20 | 21 | tests := map[string]mongoQv{ 22 | `all`: { 23 | query: map[string]interface{}{}, 24 | values: []interface{}{}, 25 | }, 26 | `id/1`: { 27 | query: map[string]interface{}{ 28 | `_id`: `1`, 29 | }, 30 | values: []interface{}{int64(1)}, 31 | }, 32 | `id/not:1`: { 33 | query: map[string]interface{}{ 34 | `_id`: map[string]interface{}{ 35 | `$ne`: float64(1), 36 | }, 37 | }, 38 | values: []interface{}{int64(1)}, 39 | }, 40 | `name/Bob Johnson`: { 41 | query: map[string]interface{}{ 42 | `name`: `Bob Johnson`, 43 | }, 44 | values: []interface{}{`Bob Johnson`}, 45 | }, 46 | `age/21`: { 47 | query: map[string]interface{}{ 48 | `age`: float64(21), 49 | }, 50 | values: []interface{}{int64(21)}, 51 | }, 52 | `enabled/true`: { 53 | query: map[string]interface{}{ 54 | `enabled`: true, 55 | }, 56 | values: []interface{}{true}, 57 | }, 58 | `enabled/false`: { 59 | query: map[string]interface{}{ 60 | `enabled`: false, 61 | }, 62 | values: []interface{}{false}, 63 | }, 64 | `enabled/null`: { 65 | query: map[string]interface{}{ 66 | `$or`: []interface{}{ 67 | map[string]interface{}{ 68 | `enabled`: map[string]interface{}{ 69 | `$exists`: false, 70 | }, 71 | }, 72 | map[string]interface{}{ 73 | `enabled`: nil, 74 | }, 75 | }, 76 | }, 77 | values: []interface{}{nil}, 78 | }, 79 | `enabled/not:null`: { 80 | query: map[string]interface{}{ 81 | `$and`: []interface{}{ 82 | map[string]interface{}{ 83 | `enabled`: map[string]interface{}{ 84 | `$exists`: true, 85 | }, 86 | }, 87 | map[string]interface{}{ 88 | `enabled`: map[string]interface{}{ 89 | `$not`: nil, 90 | }, 91 | }, 92 | }, 93 | }, 94 | values: []interface{}{nil}, 95 | }, 96 | `age/lt:21`: { 97 | query: map[string]interface{}{ 98 | `age`: map[string]interface{}{ 99 | `$lt`: float64(21), 100 | }, 101 | }, 102 | values: []interface{}{int64(21)}, 103 | }, 104 | `age/lte:21`: { 105 | query: map[string]interface{}{ 106 | `age`: map[string]interface{}{ 107 | `$lte`: float64(21), 108 | }, 109 | }, 110 | values: []interface{}{int64(21)}, 111 | }, 112 | `age/gt:21`: { 113 | query: map[string]interface{}{ 114 | `age`: map[string]interface{}{ 115 | `$gt`: float64(21), 116 | }, 117 | }, 118 | values: []interface{}{int64(21)}, 119 | }, 120 | `age/gte:21`: { 121 | query: map[string]interface{}{ 122 | `age`: map[string]interface{}{ 123 | `$gte`: float64(21), 124 | }, 125 | }, 126 | values: []interface{}{int64(21)}, 127 | }, 128 | `factor/lt:3.141597`: { 129 | query: map[string]interface{}{ 130 | `factor`: map[string]interface{}{ 131 | `$lt`: float64(3.141597), 132 | }, 133 | }, 134 | values: []interface{}{float64(3.141597)}, 135 | }, 136 | `factor/lte:3.141597`: { 137 | query: map[string]interface{}{ 138 | `factor`: map[string]interface{}{ 139 | `$lte`: float64(3.141597), 140 | }, 141 | }, 142 | values: []interface{}{float64(3.141597)}, 143 | }, 144 | `factor/gt:3.141597`: { 145 | query: map[string]interface{}{ 146 | `factor`: map[string]interface{}{ 147 | `$gt`: float64(3.141597), 148 | }, 149 | }, 150 | values: []interface{}{float64(3.141597)}, 151 | }, 152 | `factor/gte:3.141597`: { 153 | query: map[string]interface{}{ 154 | `factor`: map[string]interface{}{ 155 | `$gte`: float64(3.141597), 156 | }, 157 | }, 158 | values: []interface{}{float64(3.141597)}, 159 | }, 160 | `name/contains:ob`: { 161 | query: map[string]interface{}{ 162 | `name`: map[string]interface{}{ 163 | `$regex`: `.*ob.*`, 164 | `$options`: `si`, 165 | }, 166 | }, 167 | values: []interface{}{`ob`}, 168 | }, 169 | `name/prefix:ob`: { 170 | query: map[string]interface{}{ 171 | `name`: map[string]interface{}{ 172 | `$regex`: `^ob.*`, 173 | `$options`: `si`, 174 | }, 175 | }, 176 | values: []interface{}{`ob`}, 177 | }, 178 | `name/suffix:ob`: { 179 | query: map[string]interface{}{ 180 | `name`: map[string]interface{}{ 181 | `$regex`: `.*ob$`, 182 | `$options`: `si`, 183 | }, 184 | }, 185 | values: []interface{}{`ob`}, 186 | }, 187 | `age/7/name/ted`: { 188 | query: map[string]interface{}{ 189 | `$and`: []interface{}{ 190 | map[string]interface{}{ 191 | `age`: float64(7), 192 | }, 193 | map[string]interface{}{ 194 | `name`: `ted`, 195 | }, 196 | }, 197 | }, 198 | values: []interface{}{int64(7), `ted`}, 199 | }, 200 | } 201 | 202 | for spec, expected := range tests { 203 | f, err := filter.Parse(spec) 204 | assert.NoError(err) 205 | 206 | gen := NewMongoDBGenerator() 207 | actual, err := filter.Render(gen, `foo`, f) 208 | assert.NoError(err) 209 | 210 | var query map[string]interface{} 211 | assert.NoError(json.Unmarshal(actual, &query)) 212 | 213 | assert.Equal(expected.query, query, "filter: %v", spec) 214 | assert.Equal(expected.values, gen.GetValues(), "filter: %v", spec) 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ghetzel/pivot/v3 2 | 3 | require ( 4 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect 5 | github.com/Microsoft/go-winio v0.4.11 // indirect 6 | github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect 7 | github.com/RoaringBitmap/roaring v0.4.4 // indirect 8 | github.com/Smerity/govarint v0.0.0-20150407073650-7265e41f48f1 // indirect 9 | github.com/alexcesaro/statsd v2.0.0+incompatible 10 | github.com/aws/aws-sdk-go v1.34.13 11 | github.com/blevesearch/bleve v0.7.0 12 | github.com/blevesearch/blevex v0.0.0-20180227211930-4b158bb555a3 // indirect 13 | github.com/blevesearch/go-porterstemmer v1.0.2 // indirect 14 | github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f // indirect 15 | github.com/boltdb/bolt v0.0.0-20171120010307-9da317453632 // indirect 16 | github.com/containerd/continuity v0.0.0-20180919190352-508d86ade3c2 // indirect 17 | github.com/couchbase/vellum v0.0.0-20180314210611-5083a469fcef // indirect 18 | github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07 // indirect 19 | github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7 // indirect 20 | github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186 // indirect 21 | github.com/deckarep/golang-set v0.0.0-20171013212420-1d4478f51bed 22 | github.com/docker/go-connections v0.3.0 // indirect 23 | github.com/docker/go-units v0.3.3 // indirect 24 | github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect 25 | github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect 26 | github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect 27 | github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect 28 | github.com/fatih/structs v1.0.0 29 | github.com/ghetzel/cli v1.17.0 30 | github.com/ghetzel/diecast v1.20.0 31 | github.com/ghetzel/go-stockutil v1.9.5 32 | github.com/ghodss/yaml v1.0.0 33 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 34 | github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd // indirect 35 | github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 // indirect 36 | github.com/go-sql-driver/mysql v1.5.0 37 | github.com/golang/snappy v0.0.0-20160407051505-cef980a12b31 // indirect 38 | github.com/gomodule/redigo v2.0.0+incompatible 39 | github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect 40 | github.com/gotestyourself/gotestyourself v2.1.0+incompatible // indirect 41 | github.com/hashicorp/golang-lru v0.5.1 42 | github.com/husobee/vestigo v1.1.0 43 | github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 44 | github.com/jdxcode/netrc v0.0.0-20201119100258-050cafb6dbe6 45 | github.com/jmhodges/levigo v0.0.0-20161115193449-c42d9e0ca023 // indirect 46 | github.com/jtolds/gls v4.2.1+incompatible // indirect 47 | github.com/lib/pq v1.1.0 48 | github.com/mattn/go-runewidth v0.0.9 // indirect 49 | github.com/mattn/go-shellwords v1.0.10 // indirect 50 | github.com/mattn/go-sqlite3 v1.14.4 51 | github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae // indirect 52 | github.com/opencontainers/go-digest v1.0.0-rc1 // indirect 53 | github.com/opencontainers/image-spec v1.0.1 // indirect 54 | github.com/opencontainers/runc v1.0.0-rc5 // indirect 55 | github.com/orcaman/concurrent-map v0.0.0-20180319144342-a05df785d2dc 56 | github.com/ory/dockertest v3.3.2+incompatible 57 | github.com/philhofer/fwd v1.0.0 // indirect 58 | github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 // indirect 59 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect 60 | github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect 61 | github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect 62 | github.com/stretchr/testify v1.6.1 63 | github.com/syndtr/goleveldb v0.0.0-20181105012736-f9080354173f // indirect 64 | github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect 65 | github.com/tinylib/msgp v1.0.2 // indirect 66 | github.com/urfave/negroni v1.0.1-0.20191011213438-f4316798d5d3 67 | github.com/willf/bitset v0.0.0-20161202170036-5c3c0fce4884 // indirect 68 | golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect 69 | golang.org/x/tools v0.1.3 // indirect 70 | gopkg.in/alexcesaro/statsd.v2 v2.0.0 // indirect 71 | gotest.tools v2.1.0+incompatible // indirect 72 | ) 73 | 74 | go 1.13 75 | 76 | replace github.com/marten-seemann/qtls-go1-15 v0.1.0 => github.com/marten-seemann/qtls-go1-16 v0.1.0-beta.1.1 77 | -------------------------------------------------------------------------------- /lib/python/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: pivot 2 | 3 | all: env deps test 4 | 5 | env: 6 | virtualenv env 7 | ./env/bin/pip install pytest ipython 8 | 9 | deps: 10 | ./env/bin/pip install -r requirements.txt 11 | 12 | test: 13 | ./env/bin/py.test -x ./pivot/ 14 | 15 | shell: 16 | ./env/bin/ipython --config=shell.py 17 | -------------------------------------------------------------------------------- /lib/python/README.md: -------------------------------------------------------------------------------- 1 | # Pivot Python Client Library 2 | 3 | ## Overview 4 | 5 | This is a Python module that is used as a client for interacting with the Pivot database 6 | abstraction service. Import this into your Python projects to interact with databases 7 | using Pivot. 8 | 9 | 10 | ## Installation 11 | 12 | To use this module, install it using the Python `pip` utility (directly, or via a _requirements.txt_ 13 | dependency file) like so: 14 | 15 | ``` 16 | pip install git+https://github.com/ghetzel/pivot.git#subdirectory=lib/python 17 | ``` 18 | 19 | ## Examples 20 | 21 | Here are some examples for working with data using this module: 22 | 23 | ### Query data from an existing collection 24 | 25 | ```python 26 | import pivot 27 | 28 | # Connect to a local Pivot instance running at http://localhost:29029 29 | client = pivot.Client() 30 | 31 | # get details about the "users" Collection 32 | users = client.collection('users') 33 | 34 | # query all users, print out their records 35 | for user in users.all(limit=False, sort): 36 | print('User {} (id={})'.format(user.name, user.id)) 37 | print(' email: {}'.format(user.email)) 38 | print(' roles: {}'.format( ','.join(user.roles or []) ) 39 | ``` 40 | 41 | 42 | ### Create a new record in the "orders" collection 43 | 44 | ```python 45 | import pivot 46 | 47 | # Connect to a local Pivot instance running at http://localhost:29029 48 | client = pivot.Client() 49 | orders = client.collection('orders') 50 | 51 | # Create a new record in the orders collection. Depending on the collection's 52 | # schema definition, additional fields may be added with default values that 53 | # aren't explicitly specified here. If any required fields are missing, this 54 | # call will raise a `pivot.exceptions.HttpError` exception containing the error 55 | # that Pivot returned. 56 | # 57 | orders.create({ 58 | 'user_id': 123, 59 | 'item_ids': [4, 8, 12], 60 | 'shipping_address': '123 Fake St., Anytown, MO, 64141', 61 | }) 62 | ``` 63 | -------------------------------------------------------------------------------- /lib/python/pivot/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pivot client library 3 | """ 4 | from .client import Client 5 | from .collection import Collection, Field 6 | -------------------------------------------------------------------------------- /lib/python/pivot/client.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from __future__ import absolute_import 3 | import requests 4 | import requests.exceptions 5 | from . import exceptions 6 | from .collection import Collection, Field 7 | from .utils import compact 8 | import json 9 | 10 | 11 | DEFAULT_URL = 'http://localhost:29029' 12 | 13 | 14 | class Client(object): 15 | def __init__(self, url=DEFAULT_URL): 16 | # disables the warnings requests emits, which ARE for our own good, but if we make the 17 | # decision to do something stupid, we'll own that and don't need to pollute the logs. 18 | requests.packages.urllib3.disable_warnings() 19 | self.url = url 20 | self.session = requests.Session() 21 | 22 | def request(self, method, path, data=None, params={}, headers={}, **kwargs): 23 | headers['Content-Type'] = 'application/json' 24 | 25 | response = getattr(self.session, method.lower())( 26 | self.make_url(path), 27 | json=data, 28 | params=params, 29 | headers=headers, 30 | **kwargs 31 | ) 32 | 33 | if response.status_code < 400: 34 | return response 35 | else: 36 | try: 37 | body = response.json() 38 | 39 | if response.status_code == 403: 40 | raise exceptions.AuthenticationFailed(response, body) 41 | elif response.status_code == 404: 42 | raise exceptions.NotFound(response, body) 43 | elif response.status_code >= 500: 44 | raise exceptions.ServiceUnavailable(response, body) 45 | else: 46 | raise exceptions.HttpError(response, body) 47 | except json.decoder.JSONDecodeError: 48 | raise exceptions.HttpError(response, None) 49 | 50 | def make_url(self, path): 51 | return '{}/{}'.format(self.url, path.lstrip('/')) 52 | 53 | @property 54 | def collections(self): 55 | return [ 56 | Collection(c, client=self) for c in self.request( 57 | 'get', 58 | '/api/schema' 59 | ).json() 60 | ] 61 | 62 | def collection(self, name): 63 | c = Collection(name, client=self) 64 | c.load() 65 | return c 66 | 67 | def create_collection( 68 | self, 69 | name, 70 | identity_field=None, 71 | identity_field_type=None, 72 | fields=[] 73 | ): 74 | for i, field in enumerate(fields): 75 | if isinstance(field, Field): 76 | fields[i] = field.as_dict() 77 | elif isinstance(field, dict): 78 | continue 79 | else: 80 | raise TypeError("Field specification must be a dict or a Field") 81 | 82 | body = compact({ 83 | 'name': name, 84 | 'identity_field': identity_field, 85 | 'identity_field_type': identity_field_type, 86 | 'fields': fields, 87 | }) 88 | 89 | self.request('post', '/api/schema', body).json() 90 | 91 | return Collection(name, client=self) 92 | 93 | def delete_collection(self, name): 94 | self.request('delete', '/api/schema/' + name) 95 | return True 96 | -------------------------------------------------------------------------------- /lib/python/pivot/exceptions.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from __future__ import absolute_import 3 | 4 | 5 | class HttpError(Exception): 6 | description = None 7 | 8 | def __init__(self, response, body): 9 | if isinstance(body, dict) and body.get('error'): 10 | super(Exception, self).__init__('HTTP {}: {}'.format( 11 | response.status_code, 12 | body['error'] 13 | )) 14 | 15 | elif self.description: 16 | super(Exception, self).__init__('HTTP {}: {}'.format( 17 | response.status_code, 18 | self.description 19 | )) 20 | 21 | else: 22 | super(Exception, self).__init__('HTTP {}: {}'.format( 23 | response.status_code, 24 | body 25 | )) 26 | 27 | 28 | class AuthenticationFailed(HttpError): 29 | description = 'Authentication Failed' 30 | 31 | 32 | class NotFound(HttpError): 33 | description = 'Not Found' 34 | 35 | 36 | class ServiceUnavailable(HttpError): 37 | description = 'Service Unavailable' 38 | 39 | 40 | class CollectionNotFound(Exception): 41 | pass 42 | 43 | 44 | class RecordNotFound(Exception): 45 | pass 46 | -------------------------------------------------------------------------------- /lib/python/pivot/results.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from __future__ import absolute_import 3 | from .utils import dotdict, mutate_dict, compact, uu 4 | 5 | 6 | class Record(dotdict): 7 | def __init__(self, record): 8 | if isinstance(record, Record): 9 | raise ValueError("Record cannot be given another Record") 10 | 11 | _record = record.get('fields', {}) 12 | _record['id'] = record['id'] 13 | 14 | super(dotdict, self).__init__( 15 | compact( 16 | mutate_dict( 17 | _record, 18 | keyFn=self.keyfn, 19 | valueFn=self.valuefn 20 | ) 21 | ) 22 | ) 23 | 24 | def keyfn(self, key, **kwargs): 25 | return uu(key) 26 | 27 | def valuefn(self, value): 28 | if isinstance(value, dict) and not isinstance(value, dotdict): 29 | return dotdict(value) 30 | return value 31 | 32 | 33 | class RecordSet(object): 34 | max_repr_preview = 10 35 | 36 | def __init__(self, response, client=None): 37 | if not isinstance(response, dict): 38 | raise ValueError('RecordSet must be populated with a dict, got {}'.format(response.__class__)) 39 | 40 | self._client = client 41 | self.response = response 42 | self._result_count = response.get('result_count', 0) 43 | self._results_iter = iter(self.records) 44 | 45 | @property 46 | def result_count(self): 47 | return self._result_count 48 | 49 | @property 50 | def records(self): 51 | _results = self.response.get('records') or [] 52 | out = [] 53 | 54 | for result in _results: 55 | out.append(Record(result)) 56 | 57 | return out 58 | 59 | def __getitem__(self, key): 60 | return self.records.__getitem__(key) 61 | 62 | def __len__(self): 63 | return len(self.records) 64 | 65 | def __iter__(self): 66 | self._results_iter = iter(self.records) 67 | return self 68 | 69 | def __next__(self): 70 | if '__next__' in dir(self._results_iter): 71 | return self._results_iter.__next__() 72 | else: 73 | return self._results_iter.next() 74 | 75 | def next(self): 76 | return self.__next__() 77 | 78 | def __repr__(self): 79 | out = ' self.max_repr_preview: 87 | out += '\n truncated [{} more, {} total]...'.format( 88 | len(records) - self.max_repr_preview, 89 | self.result_count 90 | ) 91 | 92 | if out.endswith('['): 93 | return out + ']>' 94 | else: 95 | return out + '\n]>' 96 | 97 | def __str__(self): 98 | return self.__repr__() 99 | -------------------------------------------------------------------------------- /lib/python/pivot/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from __future__ import absolute_import 3 | 4 | 5 | class dotdict(dict): 6 | """ 7 | Provides dot.notation access to dictionary attributes 8 | """ 9 | __getattr__ = dict.get 10 | __setattr__ = dict.__setitem__ 11 | __delattr__ = dict.__delitem__ 12 | 13 | 14 | def uu(input): 15 | try: 16 | if type(input) != unicode: 17 | input = input.decode('utf-8') 18 | except NameError: 19 | pass 20 | 21 | return input 22 | 23 | 24 | def mutate_dict(inValue, 25 | keyFn=lambda k: k, 26 | valueFn=lambda v: v, 27 | keyTypes=None, 28 | valueTypes=None, 29 | **kwargs): 30 | """ 31 | Takes an input dict or list-of-dicts and applies ``keyfn`` function to all of the keys in 32 | both the top-level and any nested dicts or lists, and ``valuefn`` to all 33 | If the input value is not of type `dict` or `list`, the value will be returned as-is. 34 | Args: 35 | inValue (any): The dict to mutate. 36 | keyFn (lambda): The function to apply to keys. 37 | valueFn (lambda): The function to apply to values. 38 | keyTypes (tuple, optional): If set, only keys of these types will be mutated 39 | with ``keyFn``. 40 | valueTypes (tuple, optional): If set, only values of these types will be mutated 41 | with ``valueFn``. 42 | Returns: 43 | A recursively mutated dict, list of dicts, or the value as-is (described above). 44 | """ 45 | 46 | # this is here as a way of making sure that the various places where recursion is done always 47 | # performs the same call, preserving all arguments except for value (which is what changes 48 | # between nested calls). 49 | def recurse(value): 50 | return mutate_dict(value, 51 | keyFn=keyFn, 52 | valueFn=valueFn, 53 | keyTypes=keyTypes, 54 | valueTypes=valueTypes, 55 | **kwargs) 56 | 57 | # handle dicts 58 | if isinstance(inValue, dict): 59 | # create the output dict 60 | outputDict = dict() 61 | 62 | # for each dict item... 63 | for k, v in inValue.items(): 64 | # apply the keyFn to some or all of the keys we encounter 65 | if keyTypes is None or (isinstance(keyTypes, tuple) and isinstance(k, keyTypes)): 66 | # prepare the new key 67 | k = keyFn(k, **kwargs) 68 | 69 | # apply the valueFn to some or all of the values we encounter 70 | if valueTypes is None or (isinstance(valueTypes, tuple) and isinstance(v, valueTypes)): 71 | v = valueFn(v) 72 | 73 | # recurse depending on the value's type 74 | # 75 | if isinstance(v, dict): 76 | # recursively call mutate_dict() for nested dicts 77 | outputDict[k] = recurse(v) 78 | elif isinstance(v, list): 79 | # recursively call mutate_dict() for each element in a list 80 | outputDict[k] = [recurse(i) for i in v] 81 | else: 82 | # set the value straight up 83 | outputDict[k] = v 84 | 85 | # return the now-populated output dict 86 | return outputDict 87 | 88 | # handle lists-of-dicts 89 | elif isinstance(inValue, list) and len(inValue) > 0: 90 | return [recurse(i) for i in inValue] 91 | 92 | else: 93 | # passthrough non-dict value as-is 94 | return inValue 95 | 96 | 97 | def compact(inDict, keep_if=lambda k, v: v is not None): 98 | """ 99 | Takes a dictionary and returns a copy with elements matching a given lambda removed. The 100 | default behavior will remove any values that are `None`. 101 | Args: 102 | inDict (dict): The dictionary to operate on. 103 | keep_if (lambda(k,v), optional): A function or lambda that will be called for each 104 | (key, value) pair. If the function returns truthy, the element will be left alone, 105 | otherwise it will be removed. 106 | """ 107 | if isinstance(inDict, dict): 108 | return { 109 | k: v for k, v in inDict.items() if keep_if(k, v) 110 | } 111 | 112 | raise ValueError("Expected: dict, got: {0}".format(type(inDict))) 113 | -------------------------------------------------------------------------------- /lib/python/requirements.txt: -------------------------------------------------------------------------------- 1 | --index-url https://pypi.python.org/simple/ 2 | -e . 3 | -------------------------------------------------------------------------------- /lib/python/setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from setuptools import setup, find_packages 3 | 4 | 5 | setup( 6 | name='pivot-client', 7 | description='Client library for integrating with the Pivot database abstraction service.', 8 | version='0.1.2', 9 | author='Gary Hetzel', 10 | author_email='garyhetzel+pivot@gmail.com', 11 | url='https://github.com/ghetzel/pivot', 12 | install_requires=[ 13 | 'requests', 14 | 'six', 15 | ], 16 | packages=find_packages(exclude=['*.tests']), 17 | classifiers=[], 18 | ) 19 | -------------------------------------------------------------------------------- /lib/python/shell.py: -------------------------------------------------------------------------------- 1 | c = get_config() 2 | 3 | c.TerminalIPythonApp.display_banner = False 4 | c.InteractiveShell.confirm_exit = False 5 | c.InteractiveShellApp.log_level = 20 6 | c.InteractiveShellApp.exec_lines = [ 7 | 'from pivot import *', 8 | ] 9 | -------------------------------------------------------------------------------- /test/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | backends: 3 | testing: 4 | type: elasticsearch 5 | addresses: 6 | - 'http://172.17.0.2:9200' 7 | -------------------------------------------------------------------------------- /test/fixtures/README.md: -------------------------------------------------------------------------------- 1 | # Fixtures Tests 2 | 3 | This folder contains files with data that should be automatically loaded by `db_test.go`. -------------------------------------------------------------------------------- /test/fixtures/testing.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "id": "users", 3 | "collection": "groups", 4 | "fields": { 5 | "Name": "Users", 6 | "Description": "A group containing all users." 7 | } 8 | }, { 9 | "id": "test01", 10 | "collection": "users", 11 | "fields": { 12 | "FirstName": "Test", 13 | "LastName": "User", 14 | "Email": " test01@example.com ", 15 | "PasswordHash": "invalid", 16 | "Salt": "invalid", 17 | "PrimaryGroupID": "users" 18 | } 19 | }, { 20 | "id": "test02", 21 | "collection": "users", 22 | "fields": { 23 | "FirstName": "Test", 24 | "LastName": "User 02", 25 | "Email": "test02@example.com", 26 | "PasswordHash": "invalid", 27 | "Salt": "invalid", 28 | "PrimaryGroupID": "users" 29 | } 30 | }, { 31 | "id": "test01", 32 | "collection": "users", 33 | "fields": { 34 | "LastName": "User 01" 35 | } 36 | }] -------------------------------------------------------------------------------- /test/schema/README.md: -------------------------------------------------------------------------------- 1 | # Schema Tests 2 | 3 | This folder contains files with schemata that should be automatically loaded by `db_test.go`. -------------------------------------------------------------------------------- /test/schema/groups.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "name": "groups", 3 | "identity_field": "ID", 4 | "identity_field_type": "str", 5 | "fields": [{ 6 | "name": "Name", 7 | "type": "str", 8 | "required": true, 9 | "formatters": { 10 | "trim-space": true 11 | } 12 | }, { 13 | "name": "Description", 14 | "type": "str" 15 | }, { 16 | "name": "UpdatedAt", 17 | "type": "time", 18 | "required": true, 19 | "default": "now", 20 | "formatters": { 21 | "current-time": true 22 | } 23 | }, { 24 | "name": "CreatedAt", 25 | "type": "time", 26 | "required": true, 27 | "default": "now" 28 | }] 29 | }] -------------------------------------------------------------------------------- /test/schema/users.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "name": "users", 3 | "identity_field": "ID", 4 | "identity_field_type": "str", 5 | "export": [ 6 | "FirstName", 7 | "LastName", 8 | "Email", 9 | "PrimaryGroupID" 10 | ], 11 | "constraints": [{ 12 | "on": "PrimaryGroupID", 13 | "collection": "groups", 14 | "field": "ID" 15 | }], 16 | "fields": [{ 17 | "name": "FirstName", 18 | "type": "str", 19 | "required": true, 20 | "formatters": { 21 | "trim-space": true 22 | } 23 | }, { 24 | "name": "LastName", 25 | "type": "str", 26 | "formatters": { 27 | "trim-space": true 28 | } 29 | }, { 30 | "name": "Email", 31 | "type": "str", 32 | "formatters": { 33 | "trim-space": true 34 | } 35 | }, { 36 | "name": "PrimaryGroupID", 37 | "type": "str", 38 | "required": true 39 | }, { 40 | "name": "PasswordHash", 41 | "type": "str", 42 | "required": true 43 | }, { 44 | "name": "Salt", 45 | "type": "str", 46 | "required": true 47 | }, { 48 | "name": "UpdatedAt", 49 | "type": "time", 50 | "required": true, 51 | "default": "now", 52 | "formatters": { 53 | "current-time": true 54 | } 55 | }, { 56 | "name": "CreatedAt", 57 | "type": "time", 58 | "required": true, 59 | "default": "now" 60 | }] 61 | }] -------------------------------------------------------------------------------- /test/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | backends: 3 | testing: 4 | type: dummy 5 | dataset: test-one 6 | addresses: 7 | - 'http://127.0.0.1:0' 8 | - 'http://127.0.0.1:1' 9 | 10 | options: 11 | first: 1 12 | second: true 13 | third: 'three' 14 | -------------------------------------------------------------------------------- /ui/_includes/paginator.html: -------------------------------------------------------------------------------- 1 | {{ $startWindow := 5 }} 2 | {{ $endWindow := 3 }} 3 | 4 | {{ if gtx $.bindings.results.total_pages 1 }} 5 | 55 | {{ end }} -------------------------------------------------------------------------------- /ui/_layouts/default.html: -------------------------------------------------------------------------------- 1 | --- 2 | bindings: 3 | - name: status 4 | resource: /api/status 5 | --- 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | Pivot 30 | 31 | 32 | 33 | 66 | 67 |
{{ template "content" . }}
68 | 69 | 70 | -------------------------------------------------------------------------------- /ui/css/app.css: -------------------------------------------------------------------------------- 1 | html, body { 2 | height: 100vh; 3 | overflow: hidden; 4 | } 5 | 6 | #content { 7 | margin-top: 56px; 8 | height: calc(100vh - 56px); 9 | } 10 | 11 | .navicon { 12 | width: 16px; 13 | height: 16px; 14 | margin-right: 4px; 15 | filter: grayscale(100%) brightness(0.75) contrast(200%); 16 | -webkit-filter: grayscale(100%) brightness(0.75) contrast(200%); 17 | } 18 | 19 | #content .backend-icon { 20 | width: 96px; 21 | height: 96px; 22 | } 23 | 24 | #content .panel-title { 25 | font-weight: bold; 26 | font-size: 10pt; 27 | } 28 | 29 | .table-plain, .table-plain tbody, .table-plain tr { 30 | margin-bottom: 0; 31 | border-width: 0; 32 | } 33 | 34 | .elide { 35 | overflow: hidden; 36 | text-overflow: ellipsis; 37 | white-space: nowrap; 38 | } 39 | 40 | .collections-list { 41 | background: #222; 42 | padding: 0; 43 | color: #DDD; 44 | overflow: auto; 45 | } 46 | 47 | .collections-list > .collections-list-header { 48 | font-size: 18px; 49 | background: black; 50 | color: white; 51 | padding: 1rem; 52 | margin: 0; 53 | } 54 | 55 | #collections { 56 | border-radius: 0; 57 | } 58 | 59 | #collections .list-group-item { 60 | background: transparent; 61 | padding: 0.25rem 1rem; 62 | border-radius: 0; 63 | border: 0; 64 | color: #DDD; 65 | overflow: hidden; 66 | text-overflow: ellipsis; 67 | } 68 | 69 | #collections .list-group-item:hover { 70 | background: #111; 71 | color: white; 72 | } 73 | 74 | #browser { 75 | padding: 0; 76 | overflow: auto; 77 | } 78 | 79 | #browser > .container-fluid { 80 | padding: 0; 81 | } 82 | 83 | #browser .browser-view { 84 | margin: 0.5rem; 85 | } 86 | 87 | #browser .browser-view th { 88 | cursor: pointer; 89 | } 90 | 91 | #browser .browser-header { 92 | font-size: 18px; 93 | background: #444; 94 | color: white; 95 | padding: 0.4rem 1rem; 96 | margin: 0; 97 | } 98 | 99 | .filter-criteria { 100 | height: calc(2.25rem + 2px); 101 | } 102 | 103 | .filter-criteria > .criterion { 104 | user-select: none; 105 | padding: 0.125em 0.25em; 106 | } 107 | 108 | .filter-criteria > .criterion:after { 109 | content: 'AND'; 110 | margin-left: 0.5em; 111 | } 112 | 113 | .filter-criteria > .criterion:last-child:after { 114 | content: ''; 115 | } 116 | 117 | .filter-criteria > .criterion > * { 118 | background-color: dodgerblue; 119 | border-radius: 4px; 120 | padding: 0.125em 0.2em; 121 | color: white; 122 | cursor: pointer; 123 | } 124 | 125 | .filter-criteria > .criterion .criterion-field { 126 | border-top-right-radius: 0; 127 | border-bottom-right-radius: 0; 128 | } 129 | 130 | .filter-criteria > .criterion .criterion-field:after { 131 | content: ' /'; 132 | } 133 | 134 | .filter-criteria > .criterion .criterion-operator { 135 | border-radius: 0; 136 | } 137 | 138 | .filter-criteria > .criterion .criterion-operator:after { 139 | content: ' :'; 140 | } 141 | 142 | .filter-criteria > .criterion .criterion-value { 143 | border-top-left-radius: 0; 144 | border-bottom-left-radius: 0; 145 | } -------------------------------------------------------------------------------- /ui/css/codemirror-theme.css: -------------------------------------------------------------------------------- 1 | /* Based on Sublime Text's Monokai theme */ 2 | 3 | .cm-s-monokai.CodeMirror { background: #272822; color: #f8f8f2; } 4 | .cm-s-monokai div.CodeMirror-selected { background: #49483E; } 5 | .cm-s-monokai .CodeMirror-line::selection, .cm-s-monokai .CodeMirror-line > span::selection, .cm-s-monokai .CodeMirror-line > span > span::selection { background: rgba(73, 72, 62, .99); } 6 | .cm-s-monokai .CodeMirror-line::-moz-selection, .cm-s-monokai .CodeMirror-line > span::-moz-selection, .cm-s-monokai .CodeMirror-line > span > span::-moz-selection { background: rgba(73, 72, 62, .99); } 7 | .cm-s-monokai .CodeMirror-gutters { background: #272822; border-right: 0px; } 8 | .cm-s-monokai .CodeMirror-guttermarker { color: white; } 9 | .cm-s-monokai .CodeMirror-guttermarker-subtle { color: #d0d0d0; } 10 | .cm-s-monokai .CodeMirror-linenumber { color: #d0d0d0; } 11 | .cm-s-monokai .CodeMirror-cursor { border-left: 1px solid #f8f8f0; } 12 | 13 | .cm-s-monokai span.cm-comment { color: #75715e; } 14 | .cm-s-monokai span.cm-atom { color: #ae81ff; } 15 | .cm-s-monokai span.cm-number { color: #ae81ff; } 16 | 17 | .cm-s-monokai span.cm-comment.cm-attribute { color: #97b757; } 18 | .cm-s-monokai span.cm-comment.cm-def { color: #bc9262; } 19 | .cm-s-monokai span.cm-comment.cm-tag { color: #bc6283; } 20 | .cm-s-monokai span.cm-comment.cm-type { color: #5998a6; } 21 | 22 | .cm-s-monokai span.cm-property, .cm-s-monokai span.cm-attribute { color: #a6e22e; } 23 | .cm-s-monokai span.cm-keyword { color: #f92672; } 24 | .cm-s-monokai span.cm-builtin { color: #66d9ef; } 25 | .cm-s-monokai span.cm-string { color: #e6db74; } 26 | 27 | .cm-s-monokai span.cm-variable { color: #f8f8f2; } 28 | .cm-s-monokai span.cm-variable-2 { color: #9effff; } 29 | .cm-s-monokai span.cm-variable-3, .cm-s-monokai span.cm-type { color: #66d9ef; } 30 | .cm-s-monokai span.cm-def { color: #fd971f; } 31 | .cm-s-monokai span.cm-bracket { color: #f8f8f2; } 32 | .cm-s-monokai span.cm-tag { color: #f92672; } 33 | .cm-s-monokai span.cm-header { color: #ae81ff; } 34 | .cm-s-monokai span.cm-link { color: #ae81ff; } 35 | .cm-s-monokai span.cm-error { background: #f92672; color: #f8f8f0; } 36 | 37 | .cm-s-monokai .CodeMirror-activeline-background { background: #373831; } 38 | .cm-s-monokai .CodeMirror-matchingbracket { 39 | text-decoration: underline; 40 | color: white !important; 41 | } 42 | -------------------------------------------------------------------------------- /ui/css/jquery.json-viewer.css: -------------------------------------------------------------------------------- 1 | /* Syntax highlighting for JSON objects */ 2 | ul.json-dict, ol.json-array { 3 | list-style-type: none; 4 | margin: 0 0 0 1px; 5 | border-left: 1px dotted #ccc; 6 | padding-left: 2em; 7 | } 8 | .json-string { 9 | color: #0B7500; 10 | } 11 | .json-literal { 12 | color: #1A01CC; 13 | font-weight: bold; 14 | } 15 | 16 | /* Toggle button */ 17 | a.json-toggle { 18 | position: relative; 19 | color: inherit; 20 | text-decoration: none; 21 | } 22 | a.json-toggle:focus { 23 | outline: none; 24 | } 25 | a.json-toggle:before { 26 | color: #aaa; 27 | content: "\25BC"; /* down arrow */ 28 | position: absolute; 29 | display: inline-block; 30 | width: 1em; 31 | left: -1em; 32 | } 33 | a.json-toggle.collapsed:before { 34 | transform: rotate(-90deg); /* Use rotated down arrow, prevents right arrow appearing smaller than down arrow in some browsers */ 35 | -ms-transform: rotate(-90deg); 36 | -webkit-transform: rotate(-90deg); 37 | } 38 | 39 | /* Collapsable placeholder links */ 40 | a.json-placeholder { 41 | color: #aaa; 42 | padding: 0 1em; 43 | text-decoration: none; 44 | } 45 | a.json-placeholder:hover { 46 | text-decoration: underline; 47 | } 48 | -------------------------------------------------------------------------------- /ui/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghetzel/pivot/b5f1c4afd22dffab2b02e5f5ef5cda6261c2f736/ui/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /ui/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghetzel/pivot/b5f1c4afd22dffab2b02e5f5ef5cda6261c2f736/ui/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /ui/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghetzel/pivot/b5f1c4afd22dffab2b02e5f5ef5cda6261c2f736/ui/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /ui/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghetzel/pivot/b5f1c4afd22dffab2b02e5f5ef5cda6261c2f736/ui/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /ui/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghetzel/pivot/b5f1c4afd22dffab2b02e5f5ef5cda6261c2f736/ui/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /ui/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghetzel/pivot/b5f1c4afd22dffab2b02e5f5ef5cda6261c2f736/ui/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /ui/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghetzel/pivot/b5f1c4afd22dffab2b02e5f5ef5cda6261c2f736/ui/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /ui/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghetzel/pivot/b5f1c4afd22dffab2b02e5f5ef5cda6261c2f736/ui/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /ui/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghetzel/pivot/b5f1c4afd22dffab2b02e5f5ef5cda6261c2f736/ui/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /ui/index.html: -------------------------------------------------------------------------------- 1 | --- 2 | page: 3 | class: 'd-flex flex-row justify-content-stretch' 4 | 5 | bindings: 6 | - name: collections 7 | resource: /api/schema 8 | --- 9 | 10 |
11 |

Collections

12 |
13 | {{ range $.bindings.collections }} 14 | 15 | {{ . }} 16 | 17 | {{ end }} 18 |
19 |
20 | 21 |
22 |
23 |

(no collection selected)

24 |
25 |
26 | 27 | 53 | -------------------------------------------------------------------------------- /ui/js/codemirror/addons/active-line.js: -------------------------------------------------------------------------------- 1 | // CodeMirror, copyright (c) by Marijn Haverbeke and others 2 | // Distributed under an MIT license: http://codemirror.net/LICENSE 3 | 4 | (function (mod) { 5 | if (typeof exports == "object" && typeof module == "object") // CommonJS 6 | mod(require("../../lib/codemirror")); 7 | else if (typeof define == "function" && define.amd) // AMD 8 | define(["../../lib/codemirror"], mod); 9 | else // Plain browser env 10 | mod(CodeMirror); 11 | })(function (CodeMirror) { 12 | "use strict"; 13 | var WRAP_CLASS = "CodeMirror-activeline"; 14 | var BACK_CLASS = "CodeMirror-activeline-background"; 15 | var GUTT_CLASS = "CodeMirror-activeline-gutter"; 16 | 17 | CodeMirror.defineOption("styleActiveLine", false, function (cm, val, old) { 18 | var prev = old == CodeMirror.Init ? false : old; 19 | if (val == prev) return 20 | if (prev) { 21 | cm.off("beforeSelectionChange", selectionChange); 22 | clearActiveLines(cm); 23 | delete cm.state.activeLines; 24 | } 25 | if (val) { 26 | cm.state.activeLines = []; 27 | updateActiveLines(cm, cm.listSelections()); 28 | cm.on("beforeSelectionChange", selectionChange); 29 | } 30 | }); 31 | 32 | function clearActiveLines(cm) { 33 | for (var i = 0; i < cm.state.activeLines.length; i++) { 34 | cm.removeLineClass(cm.state.activeLines[i], "wrap", WRAP_CLASS); 35 | cm.removeLineClass(cm.state.activeLines[i], "background", BACK_CLASS); 36 | cm.removeLineClass(cm.state.activeLines[i], "gutter", GUTT_CLASS); 37 | } 38 | } 39 | 40 | function sameArray(a, b) { 41 | if (a.length != b.length) return false; 42 | for (var i = 0; i < a.length; i++) 43 | if (a[i] != b[i]) return false; 44 | return true; 45 | } 46 | 47 | function updateActiveLines(cm, ranges) { 48 | var active = []; 49 | for (var i = 0; i < ranges.length; i++) { 50 | var range = ranges[i]; 51 | var option = cm.getOption("styleActiveLine"); 52 | if (typeof option == "object" && option.nonEmpty ? range.anchor.line != range.head.line : !range.empty()) 53 | continue 54 | var line = cm.getLineHandleVisualStart(range.head.line); 55 | if (active[active.length - 1] != line) active.push(line); 56 | } 57 | if (sameArray(cm.state.activeLines, active)) return; 58 | cm.operation(function () { 59 | clearActiveLines(cm); 60 | for (var i = 0; i < active.length; i++) { 61 | cm.addLineClass(active[i], "wrap", WRAP_CLASS); 62 | cm.addLineClass(active[i], "background", BACK_CLASS); 63 | cm.addLineClass(active[i], "gutter", GUTT_CLASS); 64 | } 65 | cm.state.activeLines = active; 66 | }); 67 | } 68 | 69 | function selectionChange(cm, sel) { 70 | updateActiveLines(cm, sel.ranges); 71 | } 72 | }); -------------------------------------------------------------------------------- /ui/js/codemirror/addons/closebrackets.js: -------------------------------------------------------------------------------- 1 | // CodeMirror 4.1.1, copyright (c) by Marijn Haverbeke and others 2 | // Distributed under an MIT license: http://codemirror.net/LICENSE 3 | 4 | (function(mod) { 5 | if (typeof exports == "object" && typeof module == "object") // CommonJS 6 | mod(require("../../lib/codemirror")); 7 | else if (typeof define == "function" && define.amd) // AMD 8 | define(["../../lib/codemirror"], mod); 9 | else // Plain browser env 10 | mod(CodeMirror); 11 | })(function(CodeMirror) { 12 | var DEFAULT_BRACKETS = "()[]{}''\"\""; 13 | var DEFAULT_EXPLODE_ON_ENTER = "[]{}"; 14 | var SPACE_CHAR_REGEX = /\s/; 15 | 16 | var Pos = CodeMirror.Pos; 17 | 18 | CodeMirror.defineOption("autoCloseBrackets", false, function(cm, val, old) { 19 | if (old != CodeMirror.Init && old) 20 | cm.removeKeyMap("autoCloseBrackets"); 21 | if (!val) return; 22 | var pairs = DEFAULT_BRACKETS, explode = DEFAULT_EXPLODE_ON_ENTER; 23 | if (typeof val == "string") pairs = val; 24 | else if (typeof val == "object") { 25 | if (val.pairs != null) pairs = val.pairs; 26 | if (val.explode != null) explode = val.explode; 27 | } 28 | var map = buildKeymap(pairs); 29 | if (explode) map.Enter = buildExplodeHandler(explode); 30 | cm.addKeyMap(map); 31 | }); 32 | 33 | function charsAround(cm, pos) { 34 | var str = cm.getRange(Pos(pos.line, pos.ch - 1), 35 | Pos(pos.line, pos.ch + 1)); 36 | return str.length == 2 ? str : null; 37 | } 38 | 39 | function buildKeymap(pairs) { 40 | var map = { 41 | name : "autoCloseBrackets", 42 | Backspace: function(cm) { 43 | if (cm.getOption("disableInput")) return CodeMirror.Pass; 44 | var ranges = cm.listSelections(); 45 | for (var i = 0; i < ranges.length; i++) { 46 | if (!ranges[i].empty()) return CodeMirror.Pass; 47 | var around = charsAround(cm, ranges[i].head); 48 | if (!around || pairs.indexOf(around) % 2 != 0) return CodeMirror.Pass; 49 | } 50 | for (var i = ranges.length - 1; i >= 0; i--) { 51 | var cur = ranges[i].head; 52 | cm.replaceRange("", Pos(cur.line, cur.ch - 1), Pos(cur.line, cur.ch + 1)); 53 | } 54 | } 55 | }; 56 | var closingBrackets = ""; 57 | for (var i = 0; i < pairs.length; i += 2) (function(left, right) { 58 | if (left != right) closingBrackets += right; 59 | map["'" + left + "'"] = function(cm) { 60 | if (cm.getOption("disableInput")) return CodeMirror.Pass; 61 | var ranges = cm.listSelections(), type, next; 62 | for (var i = 0; i < ranges.length; i++) { 63 | var range = ranges[i], cur = range.head, curType; 64 | if (left == "'" && cm.getTokenTypeAt(cur) == "comment") 65 | return CodeMirror.Pass; 66 | var next = cm.getRange(cur, Pos(cur.line, cur.ch + 1)); 67 | if (!range.empty()) 68 | curType = "surround"; 69 | else if (left == right && next == right) { 70 | if (cm.getRange(cur, Pos(cur.line, cur.ch + 3)) == left + left + left) 71 | curType = "skipThree"; 72 | else 73 | curType = "skip"; 74 | } else if (left == right && cur.ch > 1 && 75 | cm.getRange(Pos(cur.line, cur.ch - 2), cur) == left + left && 76 | (cur.ch <= 2 || cm.getRange(Pos(cur.line, cur.ch - 3), Pos(cur.line, cur.ch - 2)) != left)) 77 | curType = "addFour"; 78 | else if (left == right && CodeMirror.isWordChar(next)) 79 | return CodeMirror.Pass; 80 | else if (cm.getLine(cur.line).length == cur.ch || closingBrackets.indexOf(next) >= 0 || SPACE_CHAR_REGEX.test(next)) 81 | curType = "both"; 82 | else 83 | return CodeMirror.Pass; 84 | if (!type) type = curType; 85 | else if (type != curType) return CodeMirror.Pass; 86 | } 87 | 88 | cm.operation(function() { 89 | if (type == "skip") { 90 | cm.execCommand("goCharRight"); 91 | } else if (type == "skipThree") { 92 | for (var i = 0; i < 3; i++) 93 | cm.execCommand("goCharRight"); 94 | } else if (type == "surround") { 95 | var sels = cm.getSelections(); 96 | for (var i = 0; i < sels.length; i++) 97 | sels[i] = left + sels[i] + right; 98 | cm.replaceSelections(sels, "around"); 99 | } else if (type == "both") { 100 | cm.replaceSelection(left + right, null); 101 | cm.execCommand("goCharLeft"); 102 | } else if (type == "addFour") { 103 | cm.replaceSelection(left + left + left + left, "before"); 104 | cm.execCommand("goCharRight"); 105 | } 106 | }); 107 | }; 108 | if (left != right) map["'" + right + "'"] = function(cm) { 109 | var ranges = cm.listSelections(); 110 | for (var i = 0; i < ranges.length; i++) { 111 | var range = ranges[i]; 112 | if (!range.empty() || 113 | cm.getRange(range.head, Pos(range.head.line, range.head.ch + 1)) != right) 114 | return CodeMirror.Pass; 115 | } 116 | cm.execCommand("goCharRight"); 117 | }; 118 | })(pairs.charAt(i), pairs.charAt(i + 1)); 119 | return map; 120 | } 121 | 122 | function buildExplodeHandler(pairs) { 123 | return function(cm) { 124 | if (cm.getOption("disableInput")) return CodeMirror.Pass; 125 | var ranges = cm.listSelections(); 126 | for (var i = 0; i < ranges.length; i++) { 127 | if (!ranges[i].empty()) return CodeMirror.Pass; 128 | var around = charsAround(cm, ranges[i].head); 129 | if (!around || pairs.indexOf(around) % 2 != 0) return CodeMirror.Pass; 130 | } 131 | cm.operation(function() { 132 | cm.replaceSelection("\n\n", null); 133 | cm.execCommand("goCharLeft"); 134 | ranges = cm.listSelections(); 135 | for (var i = 0; i < ranges.length; i++) { 136 | var line = ranges[i].head.line; 137 | cm.indentLine(line, null, true); 138 | cm.indentLine(line + 1, null, true); 139 | } 140 | }); 141 | }; 142 | } 143 | }); 144 | -------------------------------------------------------------------------------- /ui/js/jquery.json-viewer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * jQuery json-viewer 3 | * @author: Alexandre Bodelot 4 | */ 5 | (function($){ 6 | 7 | /** 8 | * Check if arg is either an array with at least 1 element, or a dict with at least 1 key 9 | * @return boolean 10 | */ 11 | function isCollapsable(arg) { 12 | return arg instanceof Object && Object.keys(arg).length > 0; 13 | } 14 | 15 | /** 16 | * Check if a string represents a valid url 17 | * @return boolean 18 | */ 19 | function isUrl(string) { 20 | var regexp = /^(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/; 21 | return regexp.test(string); 22 | } 23 | 24 | /** 25 | * Transform a json object into html representation 26 | * @return string 27 | */ 28 | function json2html(json, options) { 29 | var html = ''; 30 | if (typeof json === 'string') { 31 | /* Escape tags */ 32 | json = json.replace(/&/g, '&').replace(//g, '>'); 33 | if (isUrl(json)) 34 | html += '' + json + ''; 35 | else 36 | html += '"' + json + '"'; 37 | } 38 | else if (typeof json === 'number') { 39 | html += '' + json + ''; 40 | } 41 | else if (typeof json === 'boolean') { 42 | html += '' + json + ''; 43 | } 44 | else if (json === null) { 45 | html += 'null'; 46 | } 47 | else if (json instanceof Array) { 48 | if (json.length > 0) { 49 | html += '[
    '; 50 | for (var i = 0; i < json.length; ++i) { 51 | html += '
  1. '; 52 | /* Add toggle button if item is collapsable */ 53 | if (isCollapsable(json[i])) { 54 | html += ''; 55 | } 56 | html += json2html(json[i], options); 57 | /* Add comma if item is not last */ 58 | if (i < json.length - 1) { 59 | html += ','; 60 | } 61 | html += '
    2. '; 62 | } 63 | html += '
]'; 64 | } 65 | else { 66 | html += '[]'; 67 | } 68 | } 69 | else if (typeof json === 'object') { 70 | var key_count = Object.keys(json).length; 71 | if (key_count > 0) { 72 | html += '{}'; 93 | } 94 | else { 95 | html += '{}'; 96 | } 97 | } 98 | return html; 99 | } 100 | 101 | /** 102 | * jQuery plugin method 103 | * @param json: a javascript object 104 | * @param options: an optional options hash 105 | */ 106 | $.fn.jsonViewer = function(json, options) { 107 | options = options || {}; 108 | 109 | /* jQuery chaining */ 110 | return this.each(function() { 111 | 112 | /* Transform to HTML */ 113 | var html = json2html(json, options); 114 | if (isCollapsable(json)) 115 | html = '' + html; 116 | 117 | /* Insert HTML in target DOM element */ 118 | $(this).html(html); 119 | 120 | /* Bind click on toggle buttons */ 121 | $(this).off('click'); 122 | $(this).on('click', 'a.json-toggle', function() { 123 | var target = $(this).toggleClass('collapsed').siblings('ul.json-dict, ol.json-array'); 124 | target.toggle(); 125 | if (target.is(':visible')) { 126 | target.siblings('.json-placeholder').remove(); 127 | } 128 | else { 129 | var count = target.children('li').length; 130 | var placeholder = count + (count > 1 ? ' items' : ' item'); 131 | target.after('' + placeholder + ''); 132 | } 133 | return false; 134 | }); 135 | 136 | /* Simulate click on toggle button when placeholder is clicked */ 137 | $(this).on('click', 'a.json-placeholder', function() { 138 | $(this).siblings('a.json-toggle').click(); 139 | return false; 140 | }); 141 | 142 | if (options.collapsed == true) { 143 | /* Trigger click to collapse all nodes */ 144 | $(this).find('a.json-toggle').click(); 145 | } 146 | }); 147 | }; 148 | })(jQuery); 149 | -------------------------------------------------------------------------------- /ui/js/stapes.min.js: -------------------------------------------------------------------------------- 1 | /*! Stapes.js < http://hay.github.com/stapes > */ 2 | ;(function(){"use strict";var a="1.0.0",b=1;if(!Object.create)var c=function(){};var d=Array.prototype.slice,e={attributes:{},eventHandlers:{"-1":{}},guid:-1,addEvent:function(a){e.eventHandlers[a.guid][a.type]||(e.eventHandlers[a.guid][a.type]=[]),e.eventHandlers[a.guid][a.type].push({guid:a.guid,handler:a.handler,scope:a.scope,type:a.type})},addEventHandler:function(a,b,c){var d={},f;typeof a=="string"?(f=c||!1,d[a]=b):(f=b||!1,d=a);for(var g in d){var h=d[g],i=g.split(" ");for(var j=0,k=i.length;j1){var b={};for(var c=0,d=arguments.length;c 2 |
3 |
4 | 5 |
6 | 7 |
8 |

{{ name }}

9 |

{{ configuration.type | titleize }} Backend

10 |
11 |
12 | 13 |
14 |
15 |
16 |
17 |

Status

18 |
19 |
20 | 21 | 22 | 23 | 24 | 28 | 32 | 33 | 34 | 35 | 36 | 40 | 44 | 45 | 46 | 47 | 48 | 51 | 54 | 55 | 56 |
Available 25 | 26 | 27 | 29 | This backend is available to serve requests. 30 | This backend is unavailable. 31 |
Connected 37 | 38 | 39 | 41 | Pivot is connected to this backend. 42 | Pivot is not connected to this backend. 43 |
Refresh 49 | 50 | 52 | Every {{ schema_refresh_interval | autotime }} / Timeout {{ schema_refresh_timeout | autotime }} / Max Failures: {{ schema_refresh_max_failures }} 53 |
57 |
58 |
59 |
60 | 61 |
62 |
63 |
64 |

Dataset Details

65 |
66 |
67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 80 | 81 | 82 | 83 | 84 | 85 | 86 |
Dataset Name{{ configuration.dataset }}
Addresses 76 |
    77 |
  • {{ address }}
    • 78 |
79 |
Collections{{ configuration.collections | length }}
87 |
88 |
89 |
90 |
91 | 92 |
93 |
94 |
95 |
96 |

Collections

97 |
98 |
99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 |
NameFields
{{ collection.name }}{{ collection.fields | length }}
113 |
114 |
115 |
116 |
117 | 118 |
119 |
120 |
121 |
122 |

Metadata

123 |
124 |
125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 |
{{ metadata.key }}{{ metadata.value }}
133 |
134 |
135 |
136 |
137 | 138 | -------------------------------------------------------------------------------- /ui/views/index.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |
6 |
7 | 8 |

{{ backend.configuration.type | titleize }} Backend – {{ backend.name }}

9 |
10 |
11 |
12 |
13 |
14 | 15 | 16 |

Available

17 | 18 | 19 | 20 | 21 |

Unavailable

22 | 23 |
24 | 25 |
26 | 27 | 28 |

Connected

29 | 30 | 31 | 32 | 33 |

Disconnected

34 | 35 |
36 |
37 | 38 |
39 |
40 | Disable 41 | Enable 42 | 43 | Manage 44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 | -------------------------------------------------------------------------------- /util/features.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | var features = make(map[string]bool) 4 | 5 | func EnableFeature(name string) { 6 | features[name] = true 7 | } 8 | 9 | func DisableFeature(name string) { 10 | features[name] = false 11 | } 12 | 13 | func Features(names ...string) bool { 14 | for _, name := range names { 15 | if v, ok := features[name]; !ok || !v { 16 | return false 17 | } 18 | } 19 | 20 | return true 21 | } 22 | 23 | func AnyFeatures(names ...string) bool { 24 | for _, name := range names { 25 | if v, ok := features[name]; ok && v { 26 | return true 27 | } 28 | } 29 | 30 | return false 31 | } 32 | -------------------------------------------------------------------------------- /util/http.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | type EndpointResponseFunc func(*http.Request, map[string]string) (int, interface{}, error) 8 | 9 | type Endpoint struct { 10 | BackendName string 11 | Method string 12 | Path string 13 | Handler EndpointResponseFunc 14 | } 15 | -------------------------------------------------------------------------------- /util/types.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | var RecordStructTag = `pivot` 4 | 5 | type Status struct { 6 | OK bool `json:"ok"` 7 | Application string `json:"application"` 8 | Version string `json:"version"` 9 | Backend string `json:"backend,omitempty"` 10 | Indexer string `json:"indexer,omitempty"` 11 | } 12 | -------------------------------------------------------------------------------- /util/version.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | const Version = `3.4.10` 4 | -------------------------------------------------------------------------------- /v3: -------------------------------------------------------------------------------- 1 | . -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package pivot 2 | 3 | import "github.com/ghetzel/pivot/v3/util" 4 | 5 | const ApplicationName = `pivot` 6 | const ApplicationSummary = `an extensible database abstraction service` 7 | const ApplicationVersion = util.Version 8 | --------------------------------------------------------------------------------