├── .github ├── FUNDING.yml └── workflows │ └── go.yml ├── LICENSE ├── README.md ├── cursor.go ├── cursor_test.go ├── doc.go ├── go.mod ├── go.sum ├── minquery.go └── minquery_test.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: icza 2 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: [ "master" ] 9 | pull_request: 10 | branches: [ "master" ] 11 | 12 | jobs: 13 | 14 | build: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | mongodb-version: ['4.2', '4.4', '5.0'] 19 | steps: 20 | - uses: actions/checkout@v3 21 | 22 | - name: Set up Go 23 | uses: actions/setup-go@v3 24 | with: 25 | go-version: 1.19 26 | 27 | - name: Build 28 | run: go build -v ./... 29 | 30 | - name: Start MongoDB 31 | uses: supercharge/mongodb-github-action@1.8.0 32 | with: 33 | mongodb-version: ${{ matrix.mongodb-version }} 34 | 35 | - name: Test 36 | run: go test -v ./... 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2016 Andras Belicza 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # minquery 2 | 3 | ![Build Status](https://github.com/icza/minquery/actions/workflows/go.yml/badge.svg) 4 | [![Go Reference](https://pkg.go.dev/badge/github.com/icza/minquery.svg)](https://pkg.go.dev/github.com/icza/minquery) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/icza/minquery)](https://goreportcard.com/report/github.com/icza/minquery) 6 | [![codecov](https://codecov.io/gh/icza/minquery/branch/master/graph/badge.svg)](https://codecov.io/gh/icza/minquery) 7 | 8 | MongoDB / `mgo` query that supports _efficient_ pagination (cursors to continue listing documents where we left off). 9 | 10 | **Note:** Only MongoDB 3.2 and newer versions support the feature used by this package. 11 | 12 | **Note #2:** minquery [v1.0.0](https://github.com/icza/minquery/releases/tag/v1.0.0) 13 | uses the `gopkg.in/mgo.v2` mgo driver which has gone unmaintained 14 | for a long time now. minquery [v2.0.0](https://github.com/icza/minquery/releases/tag/v2.0.0) 15 | (tip of master) uses the new, community supported fork `github.com/globalsign/mgo`. 16 | It is highly recommended to switch over to `globalsign/mgo`. If you can't or don't 17 | want to, you may continue to use the v1.0.0 release with `gopkg.in/mgo.v2`. 18 | 19 | ## Introduction 20 | 21 | Let's say we have a `users` collection in MongoDB modeled with this Go `struct`: 22 | 23 | type User struct { 24 | ID bson.ObjectId `bson:"_id"` 25 | Name string `bson:"name"` 26 | Country string `bson:"country"` 27 | } 28 | 29 | To achieve paging of the results of some query, MongoDB and the [`mgo`](https://godoc.org/github.com/globalsign/mgo) 30 | driver package has built-in support in the form of [`Query.Skip()`](https://godoc.org/github.com/globalsign/mgo#Query.Skip) and [`Query.Limit()`](https://godoc.org/github.com/globalsign/mgo#Query.Limit), e.g.: 31 | 32 | session, err := mgo.Dial(url) // Acquire Mongo session, handle error! 33 | 34 | c := session.DB("").C("users") 35 | q := c.Find(bson.M{"country" : "USA"}).Sort("name", "_id").Limit(10) 36 | 37 | // To get the nth page: 38 | q = q.Skip((n-1)*10) 39 | 40 | var users []*User 41 | err = q.All(&users) 42 | 43 | This however becomes slow if the page number increases, as MongoDB can't just "magically" jump to the xth document in the result, it has to iterate over all the result documents and omit (not return) the first `x` that need to be skipped. 44 | 45 | MongoDB provides the right solution: If the query operates on an index (it has to work on an index), [`cursor.min()`](https://docs.mongodb.com/manual/reference/method/cursor.min/) can be used to specify the first _index entry_ to start listing results from. 46 | 47 | This Stack Overflow answer shows how it can be done using a mongo client: [How to do pagination using range queries in MongoDB?](http://stackoverflow.com/questions/5525304/how-to-do-pagination-using-range-queries-in-mongodb/5526907#5526907) 48 | 49 | Note: the required index for the above query would be: 50 | 51 | db.users.createIndex( 52 | { 53 | country: 1, 54 | name: 1, 55 | _id: 1 56 | } 57 | ) 58 | 59 | There is one problem though: the `mgo` package has no support specifying this `min()`. 60 | 61 | ## Introducing `minquery` 62 | 63 | Unfortunately the [`mgo`](https://godoc.org/github.com/globalsign/mgo) driver does not provide API calls to specify [`cursor.min()`](https://docs.mongodb.com/manual/reference/method/cursor.min/). 64 | 65 | But there is a solution. The [`mgo.Database`](https://godoc.org/github.com/globalsign/mgo#Database) type provides a [`Database.Run()`](https://godoc.org/github.com/globalsign/mgo#Database.Run) method to run any MongoDB commands. The available commands and their documentation can be found here: [Database commands](https://docs.mongodb.com/manual/reference/command/) 66 | 67 | Starting with MongoDB 3.2, a new [`find`](https://docs.mongodb.com/manual/reference/command/find/) command is available which can be used to execute queries, and it supports specifying the `min` argument that denotes the first index entry to start listing results from. 68 | 69 | Good. What we need to do is after each batch (documents of a page) generate the `min` document from the last document of the query result, which must contain the values of the index entry that was used to execute the query, and then the next batch (the documents of the next page) can be acquired by setting this min index entry prior to executing the query. 70 | 71 | This index entry –let's call it _cursor_ from now on– may be encoded to a `string` and sent to the client along with the results, and when the client wants the next page, he sends back the _cursor_ saying he wants results starting after this cursor. 72 | 73 | And this is where `minquery` comes into the picture. It provides a wrapper to configure and execute a MongoDB `find` command, allowing you to specify a cursor, and after executing the query, it gives you back the new cursor to be used to query the next batch of results. The wrapper is the [`MinQuery`](https://godoc.org/github.com/icza/minquery#MinQuery) type which is very similar to [`mgo.Query`](https://godoc.org/github.com/globalsign/mgo#Query) but it supports specifying MongoDB's `min` via the `MinQuery.Cursor()` method. 74 | 75 | The above solution using `minquery` looks like this: 76 | 77 | q := minquery.New(session.DB(""), "users", bson.M{"country" : "USA"}). 78 | Sort("name", "_id").Limit(10) 79 | // If this is not the first page, set cursor: 80 | // getLastCursor() represents your logic how you acquire the last cursor. 81 | if cursor := getLastCursor(); cursor != "" { 82 | q = q.Cursor(cursor) 83 | } 84 | 85 | var users []*User 86 | newCursor, err := q.All(&users, "country", "name", "_id") 87 | 88 | And that's all. `newCursor` is the cursor to be used to fetch the next batch. 89 | 90 | **Note #1:** When calling `MinQuery.All()`, you have to provide the names of the cursor fields, this will be used to build the cursor data (and ultimately the cursor string) from. 91 | 92 | **Note #2:** If you're retrieving partial results (by using `MinQuery.Select()`), you have to include all the fields that are part of the cursor (the index entry) even if you don't intend to use them directly, else `MinQuery.All()` will not have all the values of the cursor fields, and so it will not be able to create the proper cursor value. 93 | -------------------------------------------------------------------------------- /cursor.go: -------------------------------------------------------------------------------- 1 | // This file contains the CursorCodec interface and a default implementation. 2 | 3 | package minquery 4 | 5 | import ( 6 | "encoding/base64" 7 | 8 | "github.com/globalsign/mgo/bson" 9 | ) 10 | 11 | // CursorCodec represents a symmetric pair of functions that can be used to 12 | // convert cursor data of type bson.D to a string and vice versa. 13 | type CursorCodec interface { 14 | // CreateCursor returns a cursor string from the specified fields. 15 | CreateCursor(cursorData bson.D) (string, error) 16 | 17 | // ParseCursor parses the cursor string and returns the cursor data. 18 | ParseCursor(c string) (cursorData bson.D, err error) 19 | } 20 | 21 | // cursorCodec is a default implementation of CursorCodec which produces 22 | // web-safe cursor strings by first marshaling the cursor data using 23 | // bson.Marshal(), then using base64.RawURLEncoding. 24 | type cursorCodec struct{} 25 | 26 | // CreateCursor implements CursorCodec.CreateCursor(). 27 | // The returned cursor string is web-safe, and so it's safe to include 28 | // in URL queries without escaping. 29 | func (cursorCodec) CreateCursor(cursorData bson.D) (string, error) { 30 | // bson.Marshal() never returns error, so I skip a check and early return 31 | // (but I do return the error if it would ever happen) 32 | data, err := bson.Marshal(cursorData) 33 | return base64.RawURLEncoding.EncodeToString(data), err 34 | } 35 | 36 | // ParseCursor implements CursorCodec.ParseCursor(). 37 | func (cursorCodec) ParseCursor(c string) (cursorData bson.D, err error) { 38 | var data []byte 39 | if data, err = base64.RawURLEncoding.DecodeString(c); err != nil { 40 | return 41 | } 42 | 43 | err = bson.Unmarshal(data, &cursorData) 44 | return 45 | } 46 | -------------------------------------------------------------------------------- /cursor_test.go: -------------------------------------------------------------------------------- 1 | package minquery 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/globalsign/mgo/bson" 8 | "github.com/icza/mighty" 9 | ) 10 | 11 | func TestDefaultCodec(t *testing.T) { 12 | eq, neq, expDeq := mighty.Eq(t), mighty.Neq(t), mighty.ExpDeq(t) 13 | 14 | cc := cursorCodec{} 15 | 16 | cd := bson.D{ 17 | {Name: "a", Value: 1}, 18 | {Name: "b", Value: "2"}, 19 | {Name: "c", Value: time.Date(3, 0, 0, 0, 0, 0, 0, time.UTC)}, 20 | } 21 | cursor, err := cc.CreateCursor(cd) 22 | eq(nil, err) 23 | 24 | expDeq(cd)(cc.ParseCursor(cursor)) 25 | 26 | _, err = cc.ParseCursor("%^&") 27 | neq(nil, err) 28 | _, err = cc.ParseCursor("ValidBas64ButInvalidCursor") 29 | neq(nil, err) 30 | } 31 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Package minquery provides a mgo-like Query type called MinQuery, which supports 4 | efficient query pagination (cursors to continue listing documents where 5 | we left off). 6 | 7 | Example using MinQuery 8 | 9 | Let's say we have a users collection in MongoDB modeled with this Go struct: 10 | 11 | type User struct { 12 | ID bson.ObjectId `bson:"_id"` 13 | Name string `bson:"name"` 14 | Country string `bson:"country"` 15 | } 16 | 17 | To query users having country=USA, sorted by Name and ID: 18 | 19 | q := minquery.New(session.DB(""), "users", bson.M{"country" : "USA"}). 20 | Sort("name", "_id").Limit(10) 21 | // If this is not the first page, set cursor: 22 | // getLastCursor() represents your logic how you acquire the last cursor. 23 | if cursor := getLastCursor(); cursor != "" { 24 | q = q.Cursor(cursor) 25 | } 26 | 27 | var users []*User 28 | newCursor, err := q.All(&users, "country", "name", "_id") 29 | 30 | And that's all. newCursor is the cursor to be used to fetch the next batch. 31 | 32 | Note #1: When calling MinQuery.All(), you have to provide the names 33 | of the cursor fields, this will be used to build the cursor data 34 | (and ultimately the cursor string) from. 35 | 36 | Note #2: If you're retrieving partial results (by using MinQuery.Select()), 37 | you have to include all the fields that are part of the cursor (the index entry) 38 | even if you don't intend to use them directly, else MinQuery.All() will not 39 | have all the values of the cursor fields, and so it will not be able to create 40 | the proper cursor value. 41 | 42 | */ 43 | package minquery 44 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/icza/minquery 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 7 | github.com/icza/mighty v0.0.0-20200205104645-c377cb773678 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= 2 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= 3 | github.com/icza/mighty v0.0.0-20200205104645-c377cb773678 h1:qDS615z2ayqE3jdG6SosEQknlqg5dhp3SSQ/ZPMSBZ0= 4 | github.com/icza/mighty v0.0.0-20200205104645-c377cb773678/go.mod h1:klfNufgs1IcVNz2fWjXufNHkhl2cqIUbFoia2580Iv4= 5 | -------------------------------------------------------------------------------- /minquery.go: -------------------------------------------------------------------------------- 1 | // This file contains the MinQuery interface and its implementation. 2 | 3 | package minquery 4 | 5 | import ( 6 | "errors" 7 | 8 | "github.com/globalsign/mgo" 9 | "github.com/globalsign/mgo/bson" 10 | ) 11 | 12 | // DefaultCursorCodec is the default CursorCodec value that is used if none 13 | // is specified. The default implementation produces web-safe cursor strings. 14 | var DefaultCursorCodec cursorCodec 15 | 16 | // MinQuery is an mgo-like Query that supports cursors to continue listing documents 17 | // where we left off. If a cursor is set, it specifies the last index entry 18 | // that was already returned, and result documents will be listed after this. 19 | type MinQuery interface { 20 | // Sort asks the database to order returned documents according to 21 | // the provided field names. 22 | Sort(fields ...string) MinQuery 23 | 24 | // Select enables selecting which fields should be retrieved for 25 | // the results found. 26 | Select(selector interface{}) MinQuery 27 | 28 | // Limit restricts the maximum number of documents retrieved to n, 29 | // and also changes the batch size to the same value. 30 | Limit(n int) MinQuery 31 | 32 | // Cursor sets the cursor, which specifies the last index entry 33 | // that was already returned, and result documents will be listed after this. 34 | // Parsing a cursor may fail which is not returned. If an invalid cursor 35 | // is specified, All() will fail and return the error. 36 | Cursor(c string) MinQuery 37 | 38 | // CursorCoded sets the CursorCodec to be used to parse and to create cursors. 39 | // This gives you the possibility to implement your own logic to create cursors, 40 | // including encryption should you need it. 41 | CursorCodec(cc CursorCodec) MinQuery 42 | 43 | // All retrieves all documents from the result set into the provided slice. 44 | // cursorFields lists the fields (in order) to be used to generate 45 | // the returned cursor. 46 | All(result interface{}, cursorFields ...string) (cursor string, err error) 47 | } 48 | 49 | // errTestValue is the error value returned for testing purposes. 50 | var errTestValue = errors.New("Intentional testing error") 51 | 52 | // minQuery is the MinQuery implementation. 53 | type minQuery struct { 54 | // db is the mgo Database to use 55 | db *mgo.Database 56 | hint map[string]int 57 | // Name of the collection 58 | coll string 59 | 60 | // filter document (query) 61 | filter interface{} 62 | 63 | // sort document 64 | sort bson.D 65 | 66 | // projection document (to retrieve only selected fields) 67 | projection interface{} 68 | 69 | // limit is the max number of results 70 | limit int 71 | 72 | // Cursor, need to store and supply it if query returns no results 73 | cursor string 74 | 75 | // cursorCodec to be used to parse and to create cursors 76 | cursorCodec CursorCodec 77 | 78 | // cursorErr contains an error if an invalid cursor is supplied 79 | cursorErr error 80 | 81 | // min specifies the last index entry 82 | min bson.D 83 | 84 | // testError is a helper field to aid testing errors to reach 100% coverage. 85 | // May only be changed from tests! Zero value means normal operation. 86 | testError bool 87 | } 88 | 89 | // New returns a new MinQuery. 90 | func New(db *mgo.Database, coll string, query interface{}) MinQuery { 91 | return NewWithHint(db, coll, query, nil) 92 | } 93 | 94 | // NewWithHint returns a new MinQuery with index hint. 95 | // MongoDB 4.2 requires an index hint. 96 | func NewWithHint(db *mgo.Database, coll string, query interface{}, hint map[string]int) MinQuery { 97 | return &minQuery{ 98 | db: db, 99 | coll: coll, 100 | filter: query, 101 | hint: hint, 102 | cursorCodec: DefaultCursorCodec, 103 | } 104 | } 105 | 106 | // Sort implements MinQuery.Sort(). 107 | func (mq *minQuery) Sort(fields ...string) MinQuery { 108 | mq.sort = make(bson.D, 0, len(fields)) 109 | for _, field := range fields { 110 | if field == "" { 111 | continue 112 | } 113 | n := 1 114 | if field[0] == '+' { 115 | field = field[1:] 116 | } else if field[0] == '-' { 117 | n, field = -1, field[1:] 118 | } 119 | mq.sort = append(mq.sort, bson.DocElem{Name: field, Value: n}) 120 | } 121 | return mq 122 | } 123 | 124 | // Select implements MinQuery.Select(). 125 | func (mq *minQuery) Select(selector interface{}) MinQuery { 126 | mq.projection = selector 127 | return mq 128 | } 129 | 130 | // Limit implements MinQuery.Limit(). 131 | func (mq *minQuery) Limit(n int) MinQuery { 132 | mq.limit = n 133 | return mq 134 | } 135 | 136 | // Cursor implements MinQuery.Cursor(). 137 | func (mq *minQuery) Cursor(c string) MinQuery { 138 | mq.cursor = c 139 | if c != "" { 140 | mq.min, mq.cursorErr = mq.cursorCodec.ParseCursor(c) 141 | } else { 142 | mq.min, mq.cursorErr = nil, nil 143 | } 144 | return mq 145 | } 146 | 147 | // CursorCodec implements MinQuery.CursorCodec(). 148 | func (mq *minQuery) CursorCodec(cc CursorCodec) MinQuery { 149 | mq.cursorCodec = cc 150 | return mq 151 | } 152 | 153 | // All implements MinQuery.All(). 154 | func (mq *minQuery) All(result interface{}, cursorFields ...string) (cursor string, err error) { 155 | if mq.cursorErr != nil { 156 | return "", mq.cursorErr 157 | } 158 | 159 | // Mongodb "find" reference: 160 | // https://docs.mongodb.com/manual/reference/command/find/ 161 | 162 | cmd := bson.D{ 163 | {Name: "find", Value: mq.coll}, 164 | {Name: "limit", Value: mq.limit}, 165 | {Name: "batchSize", Value: mq.limit}, 166 | {Name: "singleBatch", Value: true}, 167 | } 168 | if mq.filter != nil { 169 | cmd = append(cmd, bson.DocElem{Name: "filter", Value: mq.filter}) 170 | } 171 | if mq.sort != nil { 172 | cmd = append(cmd, bson.DocElem{Name: "sort", Value: mq.sort}) 173 | } 174 | if mq.projection != nil { 175 | cmd = append(cmd, bson.DocElem{Name: "projection", Value: mq.projection}) 176 | } 177 | if mq.min != nil { 178 | // min is inclusive, skip the first (which is the previous last) 179 | cmd = append(cmd, 180 | bson.DocElem{Name: "skip", Value: 1}, 181 | bson.DocElem{Name: "min", Value: mq.min}, 182 | ) 183 | if mq.hint != nil { 184 | cmd = append(cmd, bson.DocElem{Name: "hint", Value: mq.hint}) 185 | } 186 | } 187 | 188 | var res struct { 189 | OK int `bson:"ok"` 190 | WaitedMS int `bson:"waitedMS"` 191 | Cursor struct { 192 | ID interface{} `bson:"id"` 193 | NS string `bson:"ns"` 194 | FirstBatch []bson.Raw `bson:"firstBatch"` 195 | } `bson:"cursor"` 196 | } 197 | 198 | if err = mq.db.Run(cmd, &res); err != nil { 199 | return 200 | } 201 | 202 | firstBatch := res.Cursor.FirstBatch 203 | if len(firstBatch) > 0 { 204 | if len(cursorFields) > 0 { 205 | // create cursor from the last document 206 | var doc bson.M 207 | err = firstBatch[len(firstBatch)-1].Unmarshal(&doc) 208 | if mq.testError { 209 | err = errTestValue 210 | } 211 | if err != nil { 212 | return 213 | } 214 | cursorData := make(bson.D, len(cursorFields)) 215 | for i, cf := range cursorFields { 216 | cursorData[i] = bson.DocElem{Name: cf, Value: doc[cf]} 217 | } 218 | cursor, err = mq.cursorCodec.CreateCursor(cursorData) 219 | if err != nil { 220 | return 221 | } 222 | } 223 | } else { 224 | // No more results. Use the same cursor that was used for the query. 225 | // It's possible that the last doc was returned previously, and there 226 | // are no more. 227 | cursor = mq.cursor 228 | } 229 | 230 | // Unmarshal results (FirstBatch) into the user-provided value: 231 | err = mq.db.C(mq.coll).NewIter(nil, firstBatch, 0, nil).All(result) 232 | 233 | return 234 | } 235 | -------------------------------------------------------------------------------- /minquery_test.go: -------------------------------------------------------------------------------- 1 | package minquery 2 | 3 | import ( 4 | "encoding/hex" 5 | "log" 6 | "testing" 7 | 8 | "github.com/globalsign/mgo" 9 | "github.com/globalsign/mgo/bson" 10 | "github.com/icza/mighty" 11 | ) 12 | 13 | var sess *mgo.Session 14 | 15 | func init() { 16 | var err error 17 | sess, err = mgo.Dial("mongodb://localhost/minquery") 18 | if err != nil { 19 | panic(err) 20 | } 21 | 22 | // Check required min version (3.2) 23 | bi, err := sess.BuildInfo() 24 | if err != nil { 25 | panic(err) 26 | } 27 | if !bi.VersionAtLeast(3, 2) { 28 | log.Panicf("This test requires at least MongoDB 3.2, got only %v", bi.Version) 29 | } 30 | 31 | c := sess.DB("").C("users") 32 | if err := c.EnsureIndex(mgo.Index{Key: []string{"name", "_id"}}); err != nil { 33 | panic(err) 34 | } 35 | if err := c.EnsureIndex(mgo.Index{Key: []string{"name", "-_id"}}); err != nil { 36 | panic(err) 37 | } 38 | if _, err := c.RemoveAll(nil); err != nil { 39 | panic(err) 40 | } 41 | } 42 | 43 | type User struct { 44 | ID bson.ObjectId `bson:"_id"` 45 | Name string `bson:"name"` 46 | Country string `bson:"country"` 47 | } 48 | 49 | func TestMinQuery(t *testing.T) { 50 | eq, neq, deq := mighty.Eq(t), mighty.Neq(t), mighty.Deq(t) 51 | _, _ = eq, neq 52 | 53 | c := sess.DB("").C("users") 54 | 55 | // Insert test documents: 56 | users := []*User{ 57 | {Name: "Aaron", Country: "UK"}, 58 | {Name: "Alice", Country: "US"}, 59 | {Name: "Alice", Country: "US"}, 60 | {Name: "Chloe", Country: "US"}, 61 | {Name: "Dakota", Country: "US"}, 62 | {Name: "Ed", Country: "US"}, 63 | {Name: "Fae", Country: "US"}, 64 | {Name: "Glan", Country: "US"}, 65 | } 66 | 67 | cursorFields := []string{"name", "_id"} 68 | 69 | for _, u := range users { 70 | u.ID = bson.NewObjectId() 71 | eq(nil, c.Insert(u)) 72 | } 73 | 74 | mq := NewWithHint( 75 | sess.DB(""), 76 | "users", 77 | bson.M{"country": "US"}, 78 | map[string]int{"name": 1, "_id": 1}, 79 | ). 80 | Sort("name", "_id").Limit(3) 81 | 82 | var result []*User 83 | 84 | cursor, err := mq.All(&result, cursorFields...) 85 | eq(nil, err) 86 | deq(users[1:4], result) 87 | 88 | cursor, err = mq.Cursor(cursor).All(&result, cursorFields...) 89 | eq(nil, err) 90 | deq(users[4:7], result) 91 | 92 | cursor, err = mq.Cursor(cursor).All(&result, cursorFields...) 93 | eq(nil, err) 94 | deq(users[7:], result) 95 | 96 | _, err = mq.Cursor(cursor).All(&result, cursorFields...) 97 | eq(nil, err) 98 | eq(0, len(result)) 99 | 100 | var parres []bson.M 101 | _, err = mq.Sort("+name", "-_id", "").Select(bson.M{"name": 1, "_id": 1}). 102 | Cursor("").Limit(2).All(&parres, cursorFields...) 103 | eq(nil, err) 104 | deq([]bson.M{ 105 | {"name": "Alice", "_id": users[2].ID}, 106 | {"name": "Alice", "_id": users[1].ID}, 107 | }, parres) 108 | 109 | _, err = mq.CursorCodec(testCodec{}).All(&parres, cursorFields...) 110 | eq(nil, err) 111 | deq([]bson.M{ 112 | {"name": "Alice", "_id": users[2].ID}, 113 | {"name": "Alice", "_id": users[1].ID}, 114 | }, parres) 115 | 116 | // Test cursor error: 117 | _, err = mq.Cursor("(INVALID)").All(&parres, cursorFields...) 118 | neq(nil, err) 119 | mq.Cursor("") 120 | 121 | // Test db.Run() failure 122 | mq.Select("invalid-select-doc") 123 | _, err = mq.All(&result, cursorFields...) 124 | neq(nil, err) 125 | mq.Select(nil) 126 | 127 | // Test cursor creation error 128 | _, err = mq.CursorCodec(testCodec{testError: true}). 129 | All(&parres, cursorFields...) 130 | neq(nil, err) 131 | 132 | // Test first batch unmarshal error: 133 | mq.(*minQuery).testError = true 134 | _, err = mq.CursorCodec(testCodec{testError: true}). 135 | All(&parres, cursorFields...) 136 | neq(nil, err) 137 | mq.(*minQuery).testError = false 138 | } 139 | 140 | type testCodec struct { 141 | testError bool 142 | } 143 | 144 | func (tc testCodec) CreateCursor(cursorData bson.D) (string, error) { 145 | if tc.testError { 146 | return "", errTestValue 147 | } 148 | data, err := bson.Marshal(cursorData) 149 | return hex.EncodeToString(data), err 150 | } 151 | 152 | func (testCodec) ParseCursor(c string) (cursorData bson.D, err error) { 153 | var data []byte 154 | if data, err = hex.DecodeString(c); err != nil { 155 | return 156 | } 157 | err = bson.Unmarshal(data, &cursorData) 158 | return 159 | } 160 | --------------------------------------------------------------------------------