├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE.txt ├── README.md ├── ReleaseChecklist.txt ├── build.sbt ├── docs └── Indexing.md ├── project ├── RogueBuild.scala ├── build.properties └── plugins.sbt ├── rogue-core ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── foursquare │ │ └── rogue │ │ ├── BSONType.scala │ │ ├── MongoHelpers.scala │ │ ├── MongoJavaDriverAdapter.scala │ │ ├── PhantomTypes.scala │ │ ├── Query.scala │ │ ├── QueryClause.scala │ │ ├── QueryExecutor.scala │ │ ├── QueryField.scala │ │ ├── QueryHelpers.scala │ │ ├── QueryOptimizer.scala │ │ ├── Rogue.scala │ │ ├── RogueException.scala │ │ ├── index │ │ ├── IndexChecker.scala │ │ └── IndexEnforcer.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── foursquare │ └── rogue │ └── TrivialORMQueryTest.scala ├── rogue-index ├── build.sbt └── src │ └── main │ └── scala │ └── com │ └── foursquare │ └── index │ ├── IndexModifier.scala │ ├── IndexedRecord.scala │ └── MongoIndex.scala ├── rogue-lift ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── foursquare │ │ └── rogue │ │ ├── ExecutableQuery.scala │ │ ├── HasMongoForeignObjectId.scala │ │ ├── LiftQueryExecutor.scala │ │ ├── LiftRogue.scala │ │ └── ObjectIdKey.scala │ └── test │ └── scala │ └── com │ └── foursquare │ └── rogue │ ├── EndToEndTest.scala │ ├── IndexCheckerTest.scala │ ├── QueryExecutorTest.scala │ ├── QueryTest.scala │ └── TestModels.scala ├── rogue-spindle ├── build.sbt └── src │ ├── main │ └── scala │ │ └── com │ │ └── foursquare │ │ └── rogue │ │ └── spindle │ │ ├── SpindleDBCollectionFactory.scala │ │ ├── SpindleDatabaseService.scala │ │ ├── SpindleQuery.scala │ │ └── SpindleRogueSerializer.scala │ └── test │ ├── scala │ └── com │ │ └── foursquare │ │ └── rogue │ │ └── spindle │ │ └── TestSpindleDBService.scala │ └── thrift │ └── com │ └── foursquare │ └── rogue │ └── spindle │ └── test.thrift ├── sbt └── start-test-mongo.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | *.iml 4 | lib_managed 5 | project/boot/* 6 | project/build/target/* 7 | project/sbt-launch-*.jar 8 | sbtlib 9 | target 10 | out 11 | *~ 12 | *.sw[opn] 13 | mongo-testdb 14 | dependencies 15 | mongo.log 16 | tags 17 | tags.old 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | 3 | scala: 4 | - 2.9.2 5 | - 2.10.2 6 | 7 | services: 8 | - mongodb 9 | 10 | env: 11 | - MONGO_PORT=27017 12 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Release Notes 2 | 3 | # 2.5.1 4 | 5 | - bumped default lift version to 2.6-final 6 | 7 | # 2.5.0 8 | 9 | - dropped support for scala-2.9.x 10 | - bumped scala-2.10 and 2.11 minor versions to latest 11 | - mongo-java-driver 2.12.5 12 | 13 | # 2.4.0 14 | 15 | - REPL-friendly rendering of ObjectIds in .toString 16 | - support for $ projection operator (as $$) 17 | - support for ReadPreference on count() 18 | - scala 2.11 support 19 | 20 | # 2.3.0 21 | 22 | - $setOnInsert support (thanks Konrad!) 23 | - support for eqs on list fields 24 | - support for $push: { $each: [...], $slice: } 25 | - joda.DateTime is a BSONType 26 | - string subtypes are BSONTypes 27 | - support for $not operator expression modifiers 28 | 29 | # 2.2.0 30 | 31 | - ObjectIdKey trait to replace deprecated ObjectId 32 | - overload inc method to take a Double (jdonmez) 33 | 34 | # 2.1.0 35 | 36 | - fixed edge case rendering $or queries (thanks Konrad!) 37 | - support for user defined setFromAny in fields with BsonRecordField and iteratees (thanks Arseny!) 38 | - $elemMatch support 39 | - simplified pullObjectWhere notation 40 | - $nearSphere support 41 | - support for $ in .select() statements 42 | - better support for comparing DateFields against Dates and DateTimes 43 | 44 | # 2.0.0 45 | 46 | - StringQueryField support for subtypes of String 47 | 48 | # 2.0.0-RC4 49 | 50 | - split off index code into rogue-index 51 | 52 | # 2.0.0-RC3 53 | 54 | - long subtypes are BSONTypes 55 | 56 | # 2.0.0-RC2 57 | 58 | - add BSONType type class to eliminate runtime serialization errors 59 | 60 | # 2.0.0-RC1 61 | 62 | - support for type-safe ID fields 63 | 64 | # 2.0.0-beta22 65 | 66 | - support for scala 2.9.2 and 2.10.0 (mattpap) 67 | - support for Model.distinct(_.field) (mattpap) 68 | - sbt 0.12.0 (mattpap) 69 | 70 | # 2.0.0-beta21 71 | 72 | - fix signatures for $or queries 73 | 74 | # 2.0.0-beta20 75 | 76 | - hook up readPreference, remove notion of defaultReadPreference 77 | 78 | # 2.0.0-beta19 79 | 80 | - fixed bug in $or clause serialization 81 | - support for DateTime fields 82 | - defaultReadPreference should be 'secondary' 83 | 84 | # 2.0.0-beta18 85 | 86 | - setTo overload takes an Option (setTo None == unset) 87 | - fixed some implicit conversions 88 | 89 | # 2.0.0-beta17 90 | 91 | - $slice support 92 | - list neqs 93 | - updated Indexing.md 94 | 95 | # 2.0.0-beta15 96 | 97 | - allow Date and List fields to act like any other field (eqs, etc) 98 | 99 | # 2.0.0-beta13 100 | 101 | - Index checker: don't validate indexes if there are no indexes 102 | 103 | # 2.0.0-beta12 104 | 105 | - report cluster name to QueryLogger 106 | 107 | # 2.0.0-beta11 108 | 109 | - fixed bug around count() respecting skip and limit 110 | - added QueryLogger.onExecuteQuery callback 111 | 112 | # 2.0.0-beta10 113 | 114 | - Move index classes to com.foursquare.rogue.index 115 | - Add SeqQueryField and SeqModifyField 116 | 117 | # 2.0.0-beta9 118 | 119 | - Replace Box with Option in rogue-core 120 | 121 | # 2.0.0-beta8 122 | 123 | - Make ObjectIdQueryField more generic 124 | 125 | # 2.0.0-beta7 126 | 127 | - revert change where we always pass a negative limit 128 | - fix weird interaction between negative limits and batchSize 129 | 130 | # 2.0.0-beta6 131 | 132 | - simplified phantom types 133 | - use size() instead of count() to respect skip and limit 134 | - shardkey awareness 135 | - lots of renames, most notably AbstractQuery and ModifyQuery => Query 136 | - pass a negative number to DBCursor.limit() so that the cursor closes 137 | 138 | # 2.0.0-beta5 139 | 140 | - Internal: Use standard Either convention of failure on the Left. 141 | - Use readPreference instead of slaveOk 142 | - Remove generic Field[V, M] => QueryField[V, M] implicit 143 | 144 | # 2.0.0-beta4 145 | 146 | - split off com.foursquare.field into a standalone project 147 | 148 | # 2.0.0-beta3 149 | 150 | - upgrade to v0.6 of gpg plugin 151 | - make QueryExecutor a trait 152 | - rename SelectField#apply so implicits to SelectField don't cause trouble 153 | 154 | # 2.0.0-beta2 155 | 156 | - fix O(N^2) bug in fetchBatch and iterateBatch 157 | 158 | # 2.0.0-beta1 159 | 160 | - total refactor 161 | - separate query building from query execution 162 | - break out lift support into rogue-lift 163 | - core of rogue now in rogue-core, agnostic to model representation 164 | - drop support for scala 2.8.x 165 | 166 | # 1.1.6 167 | 168 | - iteratee support 169 | - default WriteConcern is configurable 170 | - renamed blockingBulkDelete_!! to bulkDelete_!! (takes a WriteConcern) 171 | - moved gt, lt into base QueryField (nsanch) 172 | - fixed the way nested subfield queries work if both fields are ListFields 173 | 174 | # 1.1.5 175 | 176 | - fixed handling of subfields of list fields 177 | - allow nested subfields for BsonRecordFields 178 | 179 | # 1.1.4 180 | 181 | - removed EmptyQuery, fixed handling of upserts on empty queries 182 | - BaseQuery.asDBObject, BaseModifyQuery.asDBObject 183 | - fix for subselecting when the top-level field doesn't exist (nsanch) 184 | - fixes for publishing to sonatype 185 | - bumped mongo java driver version to 2.7.3 186 | 187 | # 1.1.3 188 | 189 | - fixed bug where findAndModify upsert with returnNew = false was returning Some 190 | - fixed bug where $regex query on a field would not allow other queries on that field 191 | - publishing to sonatype instead of scala-tools 192 | 193 | # 1.1.2 194 | 195 | - allow $or queries in modify commands 196 | 197 | # 1.1.1 198 | 199 | - select/selectCase up to 10 fields (davidtaylor) 200 | - only validate lists on $all and $in queries (jliszka) 201 | - pass query object to logging hook (jliszka) 202 | 203 | # 1.1.0 204 | 205 | - compile-time index checking (nsanch) 206 | - stop building select clause from all fields (jliszka) 207 | - QueryLogger.logIndexHit hook (jliszka) 208 | - use distinct values in $in and $all queries (jliszka) 209 | - slaveOk query modifier (nsanch) 210 | 211 | # 1.0.29 212 | 213 | - updated inline documentation (markcc) 214 | - between takes a tuple (davidt) 215 | - end-to-end tests (nsanch) 216 | - subfield select on embedded list (nsanch) 217 | - regex match operator for string fields (jliszka) 218 | 219 | # 1.0.28 220 | 221 | - Support for the $ positional operator 222 | - pullWhere - $pull by query instead of exact match 223 | 224 | # 1.0.27 225 | 226 | - Mongo index checking (see [here](https://github.com/foursquare/rogue/blob/master/docs/Indexing.md) for documentation) 227 | 228 | # 1.0.26 229 | 230 | - $rename support 231 | 232 | # 1.0.25 233 | 234 | - ability to supply a WriteConcern to updateOne, updateMulti and upsertOne. 235 | - select and selectCase can handle 7 and 8 parameters 236 | 237 | # 1.0.24 238 | 239 | - $bit support 240 | 241 | # 1.0.23 242 | 243 | - Add hook for intercepting and transforming queries right before sending request to mongodb. 244 | 245 | # 1.0.22 246 | 247 | - improved support for subfield queries on BsonRecordField 248 | 249 | # 1.0.21 250 | 251 | - support for subfield queries on BsonRecordField 252 | - added "matches" operator (for regexes) on StringFields with explicit index behavior expectations 253 | - fixed some more broken logging 254 | 255 | # 1.0.20 256 | 257 | - sbt 0.10.0 258 | - raw access do BasicDBObjectBuilder in query builder 259 | - fixed some broken logging 260 | 261 | # 1.0.19 262 | 263 | - whereOpt support: Venue.whereOpt(uidOpt)(_.userid eqs _) 264 | - Pass the query signature to the logging hook 265 | 266 | # 1.0.18 267 | 268 | - findAndModify support 269 | - $or query support 270 | - efficient .exists query method (thanks Jorge!) 271 | - support for BsonRecordField and BsonRecordListField (thanks Marc!) 272 | - type-safe foreign key condtions, e.g., Tip.where(_.venueid eqs venueObj) (thanks dtaylor!) 273 | 274 | # 1.0.17 275 | 276 | - blockingBulkDelete_!! which takes a WriteConcern 277 | - more uniform query logging 278 | 279 | # 1.0.16 280 | 281 | - skipOpt query modifier 282 | - use built-in interpreter for type checking tests 283 | 284 | # 1.0.15 285 | 286 | - .toString produces runnable javascript commands for mongodb console 287 | - added tests for constructions that should not compile 288 | - selectCase() builder method for select()ing via case class 289 | - support for $nin (nin) and $ne (notcontains) on list fields 290 | - unchecked warnings cleanup 291 | 292 | # 1.0.14: 293 | 294 | - index hinting support 295 | - support for selecting subfields (MongoMapField and MongoCaseClassField only; no support for MongoCaseClassListField) 296 | - "between" convenience operator (numeric) 297 | - scala 2.9.0 and 2.9.0-1 build support -- thanks eltimn! 298 | 299 | # 1.0.13: 300 | 301 | - fixed ObjectId construction for date ranges by zeroing out machine, pid and counter fields 302 | - support for $maxScan and $comment addSpecial parameters on find() queries 303 | 304 | # 1.0.12: 305 | 306 | - always specify field names to return in the query; if select() was not specified, use all field names from the model 307 | - some code cleanup (use case classes and copy() to save some typing) 308 | 309 | # 1.0.11: 310 | 311 | - explain() method on BaseQuery (thanks tjulien!) 312 | - support for select()ing up to 6 fields 313 | 314 | # 1.0.10: 315 | 316 | - regression fix for 1.0.9 317 | 318 | # 1.0.9 319 | 320 | - added hooks for full query validation 321 | - support for $type and $mod query operators 322 | - query signatures: string version of a query without values 323 | - support for indicating when a query clause is intended to hit an index (for runtime index checking, if you wish to implement it) 324 | 325 | # 1.0.8 326 | 327 | - extra logging around mongo exceptions 328 | 329 | # 1.0.7 330 | 331 | - support for empty (noop) modify queries 332 | 333 | # 1.0.6 334 | 335 | - fetchBatch now uses db cursors 336 | - building against Lift snapshot (thanks Indrajit!) 337 | - support for crossbuilding 2.8.0 & 2.8.1 338 | 339 | # 1.0.5 340 | 341 | - added tiny bit more type safety to unsafeField subfield selector 342 | 343 | # 1.0.4 344 | 345 | - bug fix: alwasy set _id in select() queries 346 | 347 | # 1.0.3 348 | 349 | - fixed setTo serialization 350 | - eqs/neqs support for GeoQueryField 351 | 352 | # 1.0.2 353 | 354 | - support for querying sub-fields of a map 355 | 356 | # 1.0.1 357 | 358 | - added BasePaginatedQuery.setCountPerPage() 359 | 360 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, Foursquare Labs, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## NOTICE - This project has moved. 2 | 3 | It is now part of Foursquare's open source monorepo [Fsq.io](https://github.com/foursquare/fsqio) and all 4 | future work will be published there. 5 | 6 | The project lives on but this Github repo is deprecated. 7 | 8 | 9 | 10 | # Rogue 11 | 12 | Rogue is a type-safe internal Scala DSL for constructing and executing find and modify commands against 13 | MongoDB in the Lift web framework. It is fully expressive with respect to the basic options provided 14 | by MongoDB's native query language, but in a type-safe manner, building on the record types specified in 15 | your Lift models. An example: 16 | 17 | Venue.where(_.mayor eqs 1234).and(_.tags contains "Thai").fetch(10) 18 | 19 | The type system enforces the following constraints: 20 | 21 | - the fields must actually belong to the record (e.g., mayor is a field on the Venue record) 22 | - the field type must match the operand type (e.g., mayor is an IntField) 23 | - the operator must make sense for the field type (e.g., categories is a MongoListField[String]) 24 | 25 | In addition, the type system ensures that certain builder methods are only used in certain circumstances. 26 | For example, take this more complex query: 27 | 28 | Venue.where(_.closed eqs false).orderAsc(_.popularity).limit(10).modify(_.closed setTo true).updateMulti 29 | 30 | This query purportedly finds the 10 least popular open venues and closes them. However, MongoDB 31 | does not (currently) allow you to specify limits on modify queries, so Rogue won't let you either. 32 | The above will generate a compiler error. 33 | 34 | Constructions like this: 35 | 36 | def myMayorships = Venue.where(_.mayor eqs 1234).limit(5) 37 | ... 38 | myMayorships.fetch(10) 39 | 40 | will also not compile, here because a limit is being specified twice. Other similar constraints 41 | are in place to prevent you from accidentally doing things you don't want to do anyway. 42 | 43 | ## Installation 44 | 45 | Because Rogue is designed to work with several versions of lift-mongodb-record, 46 | you'll want to declare your dependency on Rogue as `intransitive` and declare an explicit dependency 47 | on the version of Lift you want to target. In sbt, that would look like the following: 48 | 49 | val rogueField = "com.foursquare" %% "rogue-field" % "2.5.0" intransitive() 50 | val rogueCore = "com.foursquare" %% "rogue-core" % "2.5.1" intransitive() 51 | val rogueLift = "com.foursquare" %% "rogue-lift" % "2.5.1" intransitive() 52 | val rogueIndex = "com.foursquare" %% "rogue-index" % "2.5.1" intransitive() 53 | val liftMongoRecord = "net.liftweb" %% "lift-mongodb-record" % "2.6" 54 | 55 | Rogue 2.5.x requires Lift 2.6-RC1 or later. For support for earlier versions of Lift, use Rogue 2.4.0 or earlier. 56 | If you encounter problems using Rogue with other versions of Lift, please let us know. 57 | 58 | Join the [rogue-users google group](http://groups.google.com/group/rogue-users) for help, bug reports, 59 | feature requests, and general discussion on Rogue. 60 | 61 | ## Setup 62 | 63 | Define your record classes in Lift like you would normally (see [TestModels.scala](https://github.com/foursquare/rogue/blob/master/rogue-lift/src/test/scala/com/foursquare/rogue/TestModels.scala) for examples). 64 | 65 | Then anywhere you want to use rogue queries against these records, import the following: 66 | 67 | import com.foursquare.rogue.LiftRogue._ 68 | 69 | See [EndToEndTest.scala](https://github.com/foursquare/rogue/blob/master/rogue-lift/src/test/scala/com/foursquare/rogue/EndToEndTest.scala) for a complete working example. 70 | 71 | ## More Examples 72 | 73 | [QueryTest.scala](https://github.com/foursquare/rogue/blob/master/rogue-lift/src/test/scala/com/foursquare/rogue/QueryTest.scala) contains sample Records and examples of every kind of query supported by Rogue. 74 | It also indicates what each query translates to in MongoDB's JSON query language. 75 | It's a good place to look when getting started using Rogue. 76 | 77 | NB: The examples in QueryTest only construct query objects; none are actually executed. 78 | Once you have a query object, the following operations are supported (listed here because 79 | they are not demonstrated in QueryTest): 80 | 81 | For "find" query objects 82 | 83 | val query = Venue.where(_.venuename eqs "Starbucks") 84 | query.count() 85 | query.countDistinct(_.mayor) 86 | query.fetch() 87 | query.fetch(n) 88 | query.get() // equivalent to query.fetch(1).headOption 89 | query.exists() // equivalent to query.fetch(1).size > 0 90 | query.foreach{v: Venue => ... } 91 | query.paginate(pageSize) 92 | query.fetchBatch(pageSize){vs: List[Venue] => ...} 93 | query.bulkDelete_!!(WriteConcern.SAFE) 94 | query.findAndDeleteOne() 95 | query.explain() 96 | query.iterate(handler) 97 | query.iterateBatch(batchSize, handler) 98 | 99 | For "modify" query objects 100 | 101 | val modify = query.modify(_.mayor_count inc 1) 102 | modify.updateMulti() 103 | modify.updateOne() 104 | modify.upsertOne() 105 | 106 | for "findAndModify" query objects 107 | 108 | val modify = query.where(_.legacyid eqs 222).findAndModify(_.closed setTo true) 109 | modify.updateOne(returnNew = ...) 110 | modify.upsertOne(returnNew = ...) 111 | 112 | ## Releases 113 | 114 | The latest release is 2.5.1. See the [changelog](https://github.com/foursquare/rogue/blob/master/CHANGELOG.md) for more details. 115 | 116 | ## Dependencies 117 | 118 | lift-mongodb-record, mongodb, joda-time, junit. These dependencies are managed by the build system. 119 | 120 | ## Maintainers 121 | 122 | Rogue was initially developed by Foursquare Labs for internal use -- 123 | nearly all of the MongoDB queries in foursquare's code base go through this library. 124 | The current maintainers are: 125 | 126 | - Jason Liszka jliszka@foursquare.com 127 | - Jorge Ortiz jorge@foursquare.com 128 | - Neil Sanchala nsanch@foursquare.com 129 | 130 | Contributions welcome! 131 | -------------------------------------------------------------------------------- /ReleaseChecklist.txt: -------------------------------------------------------------------------------- 1 | Pre-release checklist: 2 | 3 | [ ] Update CHANGELOG.md with features included in the new release 4 | [ ] Update README.md with latest version number 5 | [ ] Update sbt example in README.md with latest version 6 | [ ] Update "latest release" sentence in README.md with latest version 7 | [ ] Update "New in ..." section in README.md 8 | [ ] Move the existing "New in ..." section to the "Lots of new features since..." section 9 | [ ] Add the features listed in CHANGELOG.md to the "New in ..." section 10 | [ ] Remove the "-SNAPSHOT" from the version in RogueBuild.scala 11 | [ ] Reload sbt 12 | [ ] Run "; +clean; +test" in sbt to make sure everything works. 13 | [ ] Run "+publish-local" 14 | [ ] Commit these changes (but don't push!) 15 | 16 | Release checklist: 17 | 18 | [ ] Run "+rogue-field/publish-signed" in sbt 19 | [ ] Run "+rogue-index/publish-signed" in sbt 20 | [ ] Run "+rogue-core/publish-signed" in sbt 21 | [ ] Run "+rogue-lift/publish-signed" in sbt 22 | [ ] Find the rogue repository on oss.sonatype.org / Staging Repositories 23 | [ ] Select it and click Close 24 | [ ] Select it and click Release 25 | 26 | Post-release checklist: 27 | 28 | [ ] Tag the release version (e.g., "git tag v1.0.0") 29 | [ ] Bump the version in RogueBuild.scala and add "-SNAPSHOT" to it 30 | [ ] Commit these changes 31 | [ ] Push both commits to Github 32 | [ ] Push tags to Github with "git push origin --tags" 33 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | Seq(RogueBuild.defaultSettings: _*) 2 | -------------------------------------------------------------------------------- /docs/Indexing.md: -------------------------------------------------------------------------------- 1 | # Index Checking using the Rogue IndexChecker 2 | 3 | To use index checking, you need to do 3 things: 4 | 5 | 1. Provide information about what indexes you have in Mongo for your collections. 6 | 2. Annotate your queries to indicate how you expect them to use indexes. 7 | 3. Call the rogue index checker methods from the QueryValidator callbacks. 8 | 9 | The index checker methods validate the index behavior expectations annotated in each query against the actual indexes present on the collection by simulating the logic of the Mongo query planner. If the index expectations do not match what Rogue thinks the Mongo query planner will do, you will get a warning (using whatever logging hooks you have set up). 10 | 11 | ## Providing Indexes in MongoRecords 12 | 13 | The way to provide indexing information is to use the trait `com.foursquare.rogue.IndexedRecord`. `IndexedRecord` contains one field, `val mongoIndexList: List[MongoIndex[_]]`. If your record type inherits from `IndexedRecord`, then Rogue will automatically retrieve the index from the record type. 14 | 15 | For example, consider a simple record type: 16 | 17 | class SampleModel extends MongoRecord[SampleModel] with MongoId[SampleModel] { 18 | def meta = SampleModel 19 | object a extends IntField(this) 20 | object b extends StringField(this) 21 | } 22 | 23 | object SampleModel extends SampleModel with MongoMetaRecord[SampleModel] with IndexedRecord[TestModel] { 24 | override def collectionName = "model" 25 | override val mongoIndexList = List( 26 | SampleModel.index(_._id, Asc), 27 | SampleModel.index(_.a, Asc, _.b, Desc)) 28 | } 29 | 30 | The indexes are provided by overriding `val mongoIndexList`, and providing a list of index objects for the indexes that you will have in your mongo database. For simple field expressions, you can use `.index(_.fieldname, dir)`, where "dir" is the index type. For simple types like this, that's basically the sorting direction - `Asc` for ascending, or `Desc` for descending. There are a collection of index-types that are provided by Rogue, which you can find in the source file `Rogue.scala`. You can also add your own, by subclassing the case-class `IndexModifier`. 31 | 32 | You can index more complex field expressions. Basically, if you can write a Mongo field expression in Scala, you can use that field expression for an index. For example, in foursquare code, we have a type that contains a Mongo map field. We can describe an index that uses the map keys: `Type.index(_.mapField.unsafeField[String]("key")`. 33 | 34 | ## Annotating Queries 35 | 36 | Of course, not every query 100% matches an index! For instance, here's an index declared on Campaign: 37 | 38 | { groupids: 1, active: 1 } 39 | 40 | And say you have this query: 41 | 42 | Campaign.where(_.groupids in allGroups.map(_.id)) 43 | .and(_.active eqs true) 44 | .and(_.userid eqs user.id.is) 45 | .fetch() 46 | 47 | The query clearly matches that index! But how does the index checker know to ignore the userid part of the query? It doesn't! Unless you do this: 48 | 49 | Campaign.where(_.groupids in allGroups.map(_.id)) 50 | .and(_.active eqs true) 51 | .scan(_.userid eqs user.id.is) 52 | .fetch() 53 | 54 | If you say `scan` instead of `where` or `and`, the index checker will ignore that clause and try to match the rest of the clauses exactly to an index. It is also a visual indication to the reader which part of the query is hitting an index and which part will result in a scan. 55 | 56 | **NB**: Using `scan` does not _cause_ a scan, it just tells the index checker that you expect that clause to result in a scan. The `where`, `and` and `scan` methods are all functionally equivalent outside of the index checker. 57 | 58 | In addition to the document scan, there is also the notion of an index scan. An index scan can happen when the query can be answered by the index, but a large number of index entries must be scanned in order to find the relevant results. There are a few ways this can happen, which you'll see below. Use `iscan` to indicate to the index checker that you expect an index scan to happen. 59 | 60 | The examples below assume a collection of `Thing`s with fields `a`, `b`, `c` and `d` and an index `{a:1, b:1, c:1}`. 61 | 62 | **Document scan**: the records themselves must be scanned in order to figure out which ones satisfy the query. Use `scan`. 63 | 64 | - An operator is used that cannot be answered by the index. These are `$exists`, `$nin` and `$size`. 65 |
`Thing scan (_.a exists false)` 66 | 67 | - A query field does not appear in the index. 68 |
`Thing where (_.a eqs 1) scan (_.d eqs 4)` 69 | 70 | - The first field in the index does not appear in the query. (compare "a level is skipped" below) 71 |
`Thing scan (_.b eqs 2)` 72 | 73 | **Index scan**: The query can be answered by the index, but multiple (possibly non-matching) index entries need to be examined. Use `iscan`. 74 | 75 | - An operator is used that requires an index scan. These are `$mod` and `$type`. 76 |
`Thing iscan (_.a hastype MongoType.String)` 77 | 78 | - An operator is used that requires a partial index scan. These are the "range" operators `$gt`, `$gte`, `$lt`, `$lte`, `$ne`, `$near` and `$within`. 79 |
`Thing iscan (_.a > 1)` 80 |
`Thing iscan (_.a > 1) iscan (_.b > 2)` 81 |
`Thing where (_.a eqs 1) iscan (_.b > 2)` 82 |
`Thing iscan (_.b > 2) and (_.a eqs 1) // NB: equivalent to previous` 83 | 84 | - A clause that would otherwise hit an index follows (in the index, not in the query) a clause that causes a partial index scan. 85 |
`Thing iscan (_.a > 1) iscan (_.b eqs 2)` 86 | 87 | - A level is skipped in the index. 88 |
`Thing where (_.a eqs 1) iscan (_.c eqs 3)` 89 | 90 | ## Checking Indexes 91 | 92 | To check the indexes for a query, there are two different index checking methods that you can call: 93 | 94 | 1. `MongoIndexChecker.validateIndexExpectations(query)`: this retrieves the list of indexes for the type being queried, and verifies that all of the fields that your query expects to be indexed are, in fact, indexed. 95 | 2. `MongoIndexChecker.validateQueryMatchesSomeIndex(query)`: this retrieves the list of indexes, and confirms not only that the indexes exist, but that Mongo will successfully find the correct indexes to perform the query. (There are conditions where there is an index that could, theoretically, be used by a query, but where Mongo will not recognize that the index is usable.) 96 | 97 | You can call either or both of these from the `validateQuery` and/or `validateModify` callbacks in the `QueryValidator` hook. For example, 98 | when your application boots, set up the hooks like so: 99 | 100 | object MyQueryValidator extends QueryHelpers.DefaultQueryValidator { 101 | override def validateQuery[M](query: Query[M, _, _]) { 102 | if (Props.mode != Props.RunModes.Production) { 103 | MongoIndexChecker.validateIndexExpectations(query) && 104 | MongoIndexChecker.validateQueryMatchesSomeIndex(query) 105 | } 106 | 107 | override def validateModify[M](modify: ModifyQuery[M, _]) { 108 | validateQuery(modify.query) 109 | } 110 | 111 | override def validateFindAndModify[M, R](modify: FindAndModifyQuery[M, R]) { 112 | validateQuery(modify.query) 113 | } 114 | } 115 | 116 | QueryHelpers.validator = MyQueryValidator 117 | 118 | When the index checker has something to warn about, it will call the `QueryHelpers.QueryLogger.logIndexMismatch` callback. 119 | At foursquare we have implemented it like this: 120 | 121 | override def logIndexMismatch(query: Query[_, _, _], msg: => String) { 122 | val stack = currentStackTrace() 123 | val prefix = stack.indexOf("validateQuery(YOURCLASSFILENAME.scala") // a hack 124 | val trimmedStack = stack.drop(prefix).take(800) 125 | LOG.warn(msg + " from " + trimmedStack) 126 | if (services.exists(_.throttleService.throwOnQueryIndexMiss.isShown)) { 127 | throw new Exception(msg) 128 | } 129 | } 130 | 131 | def currentStackTrace() = { 132 | val e = new Exception() 133 | e.fillInStackTrace() 134 | getStackTrace(e) 135 | } 136 | 137 | 138 | -------------------------------------------------------------------------------- /project/RogueBuild.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Foursquare Labs Inc. All Rights Reserved. 2 | import sbt._ 3 | import Keys._ 4 | 5 | object RogueBuild extends Build { 6 | override lazy val projects = 7 | Seq(all, index, core, lift) 8 | 9 | lazy val all: Project = Project("all", file(".")) aggregate( 10 | index, core, lift) 11 | 12 | lazy val index = Project("rogue-index", file("rogue-index/")) dependsOn() 13 | lazy val core = Project("rogue-core", file("rogue-core/")) dependsOn(index % "compile;test->test;runtime->runtime") 14 | lazy val lift = Project("rogue-lift", file("rogue-lift/")) dependsOn(core % "compile;test->test;runtime->runtime") 15 | lazy val IvyDefaultConfiguration = config("default") extend(Compile) 16 | 17 | lazy val defaultSettings: Seq[Setting[_]] = Seq( 18 | version := "2.5.2-SNAPSHOT", 19 | organization := "com.foursquare", 20 | scalaVersion := "2.10.4", 21 | crossScalaVersions := Seq("2.10.4", "2.11.5"), 22 | publishMavenStyle := true, 23 | publishArtifact in Test := false, 24 | pomIncludeRepository := { _ => false }, 25 | publishTo <<= (version) { v => 26 | val nexus = "https://oss.sonatype.org/" 27 | if (v.endsWith("-SNAPSHOT")) 28 | Some("snapshots" at nexus+"content/repositories/snapshots") 29 | else 30 | Some("releases" at nexus+"service/local/staging/deploy/maven2") 31 | }, 32 | pomExtra := ( 33 | http://github.com/foursquare/rogue 34 | 35 | 36 | Apache 37 | http://www.opensource.org/licenses/Apache-2.0 38 | repo 39 | 40 | 41 | 42 | git@github.com:foursquare/rogue.git 43 | scm:git:git@github.com:foursquare/rogue.git 44 | 45 | 46 | 47 | jliszka 48 | Jason Liszka 49 | http://github.com/jliszka 50 | 51 | ), 52 | resolvers ++= Seq( 53 | "Bryan J Swift Repository" at "http://repos.bryanjswift.com/maven2/", 54 | "Releases" at "http://oss.sonatype.org/content/repositories/releases", 55 | "Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots"), 56 | retrieveManaged := true, 57 | ivyConfigurations += IvyDefaultConfiguration, 58 | scalacOptions ++= Seq("-deprecation", "-unchecked"), 59 | scalacOptions <++= scalaVersion map { scalaVersion => 60 | scalaVersion.split('.') match { 61 | case Array(major, minor, _*) if major.toInt >= 2 && minor.toInt >= 10 => Seq("-feature", "-language:_") 62 | case _ => Seq() 63 | } 64 | }, 65 | 66 | // Hack to work around SBT bug generating scaladoc for projects with no dependencies. 67 | // https://github.com/harrah/xsbt/issues/85 68 | unmanagedClasspath in Compile += Attributed.blank(new java.io.File("doesnotexist")), 69 | 70 | testFrameworks += new TestFramework("com.novocode.junit.JUnitFrameworkNoMarker"), 71 | credentials ++= { 72 | val sonatype = ("Sonatype Nexus Repository Manager", "oss.sonatype.org") 73 | def loadMavenCredentials(file: java.io.File) : Seq[Credentials] = { 74 | xml.XML.loadFile(file) \ "servers" \ "server" map (s => { 75 | val host = (s \ "id").text 76 | val realm = if (host == sonatype._2) sonatype._1 else "Unknown" 77 | Credentials(realm, host, (s \ "username").text, (s \ "password").text) 78 | }) 79 | } 80 | val ivyCredentials = Path.userHome / ".ivy2" / ".credentials" 81 | val mavenCredentials = Path.userHome / ".m2" / "settings.xml" 82 | (ivyCredentials.asFile, mavenCredentials.asFile) match { 83 | case (ivy, _) if ivy.canRead => Credentials(ivy) :: Nil 84 | case (_, mvn) if mvn.canRead => loadMavenCredentials(mvn) 85 | case _ => Nil 86 | } 87 | }) 88 | } 89 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.5 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "0.2.1") 2 | 3 | addSbtPlugin("com.typesafe.sbt" % "sbt-pgp" % "0.8.3") 4 | 5 | scalacOptions ++= Seq("-deprecation", "-unchecked") 6 | -------------------------------------------------------------------------------- /rogue-core/build.sbt: -------------------------------------------------------------------------------- 1 | libraryDependencies <++= (scalaVersion) { scalaVersion => 2 | val specsVersion = scalaVersion match { 3 | case "2.11.5" => "2.4.2" 4 | case "2.10.4" => "1.12.3" 5 | } 6 | val liftVersion = "2.6" 7 | def sv(s: String) = s + "_" + (scalaVersion match { 8 | case "2.11.5" => "2.11" 9 | case "2.10.4" => "2.10" 10 | }) 11 | Seq( 12 | "com.foursquare" % sv("rogue-field") % "2.4.0" % "compile", 13 | "net.liftweb" % sv("lift-mongodb") % liftVersion % "compile" intransitive(), 14 | "net.liftweb" % sv("lift-common") % liftVersion % "compile", 15 | "net.liftweb" % sv("lift-json") % liftVersion % "compile", 16 | "net.liftweb" % sv("lift-util") % liftVersion % "compile", 17 | "joda-time" % "joda-time" % "2.1" % "provided", 18 | "org.joda" % "joda-convert" % "1.2" % "provided", 19 | "org.mongodb" % "mongo-java-driver" % "2.12.5" % "compile", 20 | "junit" % "junit" % "4.5" % "test", 21 | "com.novocode" % "junit-interface" % "0.6" % "test", 22 | "ch.qos.logback" % "logback-classic" % "0.9.26" % "provided", 23 | "org.specs2" %% "specs2" % specsVersion % "test", 24 | "org.scala-lang" % "scala-compiler" % scalaVersion % "test" 25 | ) 26 | } 27 | 28 | Seq(RogueBuild.defaultSettings: _*) 29 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/BSONType.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.mongodb.DBObject 6 | import java.util.Date 7 | import java.util.regex.Pattern 8 | import org.bson.types.ObjectId 9 | import org.joda.time.DateTime 10 | 11 | trait BSONType[T] { 12 | def asBSONObject(v: T): AnyRef 13 | } 14 | 15 | object BSONType { 16 | def apply[T: BSONType]: BSONType[T] = implicitly[BSONType[T]] 17 | 18 | implicit object BooleanIsBSONType extends BSONType[Boolean] { 19 | override def asBSONObject(v: Boolean): AnyRef = v: java.lang.Boolean 20 | } 21 | implicit object CharIsBSONType extends BSONType[Char] { 22 | override def asBSONObject(v: Char): AnyRef = v: java.lang.Character 23 | } 24 | implicit object ShortIsBSONType extends BSONType[Short] { 25 | override def asBSONObject(v: Short): AnyRef = v: java.lang.Short 26 | } 27 | implicit object IntIsBSONType extends BSONType[Int] { 28 | override def asBSONObject(v: Int): AnyRef = v: java.lang.Integer 29 | } 30 | implicit object LongIsBSONType extends BSONType[Long] { 31 | override def asBSONObject(v: Long): AnyRef = v: java.lang.Long 32 | } 33 | implicit object FloatIsBSONType extends BSONType[Float] { 34 | override def asBSONObject(v: Float): AnyRef = v: java.lang.Float 35 | } 36 | implicit object DoubleIsBSONType extends BSONType[Double] { 37 | override def asBSONObject(v: Double): AnyRef = v: java.lang.Double 38 | } 39 | implicit object DateIsBSONType extends BSONType[Date] { 40 | override def asBSONObject(v: Date): AnyRef = v 41 | } 42 | implicit object DateTimeIsBSONType extends BSONType[DateTime] { 43 | override def asBSONObject(v: DateTime): AnyRef = v.toDate 44 | } 45 | implicit object PatternIsBSONType extends BSONType[Pattern] { 46 | override def asBSONObject(v: Pattern): AnyRef = v 47 | } 48 | implicit object DBObjectIsBSONType extends BSONType[DBObject] { 49 | override def asBSONObject(v: DBObject): AnyRef = v 50 | } 51 | implicit object StringIsBSONType extends BSONType[String] { 52 | override def asBSONObject(v: String): AnyRef = v 53 | } 54 | implicit object ObjectIdISBSONType extends BSONType[ObjectId] { 55 | override def asBSONObject(v: ObjectId): AnyRef = v 56 | } 57 | 58 | implicit def ObjectIdSubtypesAreBSONTypes[T <: ObjectId]: BSONType[T] = 59 | ObjectIdISBSONType.asInstanceOf[BSONType[T]] 60 | 61 | implicit def LongSubtypesAreBSONTypes[T <: java.lang.Long]: BSONType[T] = 62 | LongIsBSONType.asInstanceOf[BSONType[T]] 63 | 64 | implicit def StringSubtypesAreBSONTypes[T <: String]: BSONType[T] = 65 | StringIsBSONType.asInstanceOf[BSONType[T]] 66 | 67 | class ListsOfBSONTypesAreBSONTypes[T: BSONType] extends BSONType[List[T]] { 68 | override def asBSONObject(v: List[T]): AnyRef = { 69 | val bsonType = BSONType[T] 70 | val ret = new java.util.ArrayList[AnyRef](v.size) 71 | for (x <- v) { 72 | ret.add(bsonType.asBSONObject(x)) 73 | } 74 | ret 75 | } 76 | } 77 | 78 | implicit def ListsOfBSONTypesAreBSONTypes[T: BSONType]: BSONType[List[T]] = new ListsOfBSONTypesAreBSONTypes[T] 79 | 80 | class SeqsOfBSONTypesAreBSONTypes[T: BSONType] extends BSONType[Seq[T]] { 81 | override def asBSONObject(v: Seq[T]): AnyRef = { 82 | val bsonType = BSONType[T] 83 | val ret = new java.util.ArrayList[AnyRef](v.size) 84 | for (x <- v) { 85 | ret.add(bsonType.asBSONObject(x)) 86 | } 87 | ret 88 | } 89 | } 90 | 91 | implicit def SeqsOfBSONTypesAreBSONTypes[T: BSONType]: BSONType[Seq[T]] = new SeqsOfBSONTypesAreBSONTypes[T] 92 | } 93 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/MongoHelpers.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.mongodb.{BasicDBObjectBuilder, DBObject} 6 | import scala.collection.immutable.ListMap 7 | 8 | object MongoHelpers extends Rogue { 9 | case class AndCondition(clauses: List[QueryClause[_]], orCondition: Option[OrCondition]) { 10 | def isEmpty: Boolean = clauses.isEmpty && orCondition.isEmpty 11 | } 12 | 13 | case class OrCondition(conditions: List[AndCondition]) 14 | 15 | sealed case class MongoOrder(terms: List[(String, Boolean)]) 16 | 17 | sealed case class MongoModify(clauses: List[ModifyClause]) 18 | 19 | sealed case class MongoSelect[M, R](fields: List[SelectField[_, M]], transformer: List[Any] => R) 20 | 21 | object MongoBuilder { 22 | def buildCondition(cond: AndCondition, signature: Boolean = false): DBObject = { 23 | buildCondition(cond, BasicDBObjectBuilder.start, signature) 24 | } 25 | 26 | def buildCondition(cond: AndCondition, 27 | builder: BasicDBObjectBuilder, 28 | signature: Boolean): DBObject = { 29 | val (rawClauses, safeClauses) = cond.clauses.partition(_.isInstanceOf[RawQueryClause]) 30 | 31 | // Normal clauses 32 | safeClauses.groupBy(_.fieldName).toList 33 | .sortBy{ case (fieldName, _) => -cond.clauses.indexWhere(_.fieldName == fieldName) } 34 | .foreach{ case (name, cs) => { 35 | // Equality clauses look like { a : 3 } 36 | // but all other clauses look like { a : { $op : 3 }} 37 | // and can be chained like { a : { $gt : 2, $lt: 6 }}. 38 | // So if there is any equality clause, apply it (only) to the builder; 39 | // otherwise, chain the clauses. 40 | cs.filter(_.isInstanceOf[EqClause[_, _]]).headOption match { 41 | case Some(eqClause) => eqClause.extend(builder, signature) 42 | case None => { 43 | builder.push(name) 44 | val (negative, positive) = cs.partition(_.negated) 45 | positive.foreach(_.extend(builder, signature)) 46 | if (negative.nonEmpty) { 47 | builder.push("$not") 48 | negative.foreach(_.extend(builder, signature)) 49 | builder.pop 50 | } 51 | builder.pop 52 | } 53 | } 54 | }} 55 | 56 | // Raw clauses 57 | rawClauses.foreach(_.extend(builder, signature)) 58 | 59 | // Optional $or clause (only one per "and" chain) 60 | cond.orCondition.foreach(or => { 61 | val subclauses = or.conditions 62 | .map(buildCondition(_, signature)) 63 | .filterNot(_.keySet.isEmpty) 64 | builder.add("$or", QueryHelpers.list(subclauses)) 65 | }) 66 | builder.get 67 | } 68 | 69 | def buildOrder(o: MongoOrder): DBObject = { 70 | val builder = BasicDBObjectBuilder.start 71 | o.terms.reverse.foreach { case (field, ascending) => builder.add(field, if (ascending) 1 else -1) } 72 | builder.get 73 | } 74 | 75 | def buildModify(m: MongoModify): DBObject = { 76 | val builder = BasicDBObjectBuilder.start 77 | m.clauses.groupBy(_.operator).foreach{ case (op, cs) => { 78 | builder.push(op.toString) 79 | cs.foreach(_.extend(builder)) 80 | builder.pop 81 | }} 82 | builder.get 83 | } 84 | 85 | def buildSelect[M, R](select: MongoSelect[M, R]): DBObject = { 86 | val builder = BasicDBObjectBuilder.start 87 | // If select.fields is empty, then a MongoSelect clause exists, but has an empty 88 | // list of fields. In this case (used for .exists()), we select just the 89 | // _id field. 90 | if (select.fields.isEmpty) { 91 | builder.add("_id", 1) 92 | } else { 93 | select.fields.foreach(f => { 94 | f.slc match { 95 | case None => builder.add(f.field.name, 1) 96 | case Some((s, None)) => builder.push(f.field.name).add("$slice", s).pop() 97 | case Some((s, Some(e))) => builder.push(f.field.name).add("$slice", QueryHelpers.makeJavaList(List(s, e))).pop() 98 | } 99 | }) 100 | } 101 | builder.get 102 | } 103 | 104 | def buildHint(h: ListMap[String, Any]): DBObject = { 105 | val builder = BasicDBObjectBuilder.start 106 | h.foreach{ case (field, attr) => { 107 | builder.add(field, attr) 108 | }} 109 | builder.get 110 | } 111 | 112 | def stringFromDBObject(dbo: DBObject): String = { 113 | // DBObject.toString renders ObjectIds like { $oid: "..."" }, but we want ObjectId("...") 114 | // because that's the format the Mongo REPL accepts. 115 | dbo.toString.replaceAll("""\{ "\$oid" : "([0-9a-f]{24})"\}""", """ObjectId("$1")""") 116 | } 117 | 118 | def buildQueryString[R, M](operation: String, collectionName: String, query: Query[M, R, _]): String = { 119 | val sb = new StringBuilder("db.%s.%s(".format(collectionName, operation)) 120 | sb.append(stringFromDBObject(buildCondition(query.condition, signature = false))) 121 | query.select.foreach(s => sb.append(", " + buildSelect(s).toString)) 122 | sb.append(")") 123 | query.order.foreach(o => sb.append(".sort(%s)" format buildOrder(o).toString)) 124 | query.lim.foreach(l => sb.append(".limit(%d)" format l)) 125 | query.sk.foreach(s => sb.append(".skip(%d)" format s)) 126 | query.maxScan.foreach(m => sb.append("._addSpecial(\"$maxScan\", %d)" format m)) 127 | query.comment.foreach(c => sb.append("._addSpecial(\"$comment\", \"%s\")" format c)) 128 | query.hint.foreach(h => sb.append(".hint(%s)" format buildHint(h).toString)) 129 | sb.toString 130 | } 131 | 132 | def buildConditionString[R, M](operation: String, collectionName: String, query: Query[M, R, _]): String = { 133 | val sb = new StringBuilder("db.%s.%s(".format(collectionName, operation)) 134 | sb.append(buildCondition(query.condition, signature = false).toString) 135 | sb.append(")") 136 | sb.toString 137 | } 138 | 139 | def buildModifyString[R, M](collectionName: String, modify: ModifyQuery[M, _], 140 | upsert: Boolean = false, multi: Boolean = false): String = { 141 | "db.%s.update(%s, %s, %s, %s)".format( 142 | collectionName, 143 | stringFromDBObject(buildCondition(modify.query.condition, signature = false)), 144 | stringFromDBObject(buildModify(modify.mod)), 145 | upsert, 146 | multi 147 | ) 148 | } 149 | 150 | def buildFindAndModifyString[R, M](collectionName: String, mod: FindAndModifyQuery[M, R], returnNew: Boolean, upsert: Boolean, remove: Boolean): String = { 151 | val query = mod.query 152 | val sb = new StringBuilder("db.%s.findAndModify({ query: %s".format( 153 | collectionName, stringFromDBObject(buildCondition(query.condition)))) 154 | query.order.foreach(o => sb.append(", sort: " + buildOrder(o).toString)) 155 | if (remove) sb.append(", remove: true") 156 | sb.append(", update: " + stringFromDBObject(buildModify(mod.mod))) 157 | sb.append(", new: " + returnNew) 158 | query.select.foreach(s => sb.append(", fields: " + buildSelect(s).toString)) 159 | sb.append(", upsert: " + upsert) 160 | sb.append(" })") 161 | sb.toString 162 | } 163 | 164 | def buildSignature[R, M](collectionName: String, query: Query[M, R, _]): String = { 165 | val sb = new StringBuilder("db.%s.find(".format(collectionName)) 166 | sb.append(buildCondition(query.condition, signature = true).toString) 167 | sb.append(")") 168 | query.order.foreach(o => sb.append(".sort(%s)" format buildOrder(o).toString)) 169 | sb.toString 170 | } 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/MongoJavaDriverAdapter.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.foursquare.index.UntypedMongoIndex 6 | import com.foursquare.rogue.Rogue._ 7 | import com.foursquare.rogue.Iter._ 8 | import com.mongodb.{BasicDBObject, BasicDBObjectBuilder, CommandResult, DBCollection, 9 | DBCursor, DBObject, ReadPreference, WriteConcern} 10 | import scala.collection.mutable.ListBuffer 11 | 12 | trait DBCollectionFactory[MB] { 13 | def getDBCollection[M <: MB](query: Query[M, _, _]): DBCollection 14 | def getPrimaryDBCollection[M <: MB](query: Query[M, _, _]): DBCollection 15 | def getInstanceName[M <: MB](query: Query[M, _, _]): String 16 | // A set of of indexes, which are ordered lists of field names 17 | def getIndexes[M <: MB](query: Query[M, _, _]): Option[List[UntypedMongoIndex]] 18 | } 19 | 20 | class MongoJavaDriverAdapter[MB](dbCollectionFactory: DBCollectionFactory[MB]) { 21 | 22 | import QueryHelpers._ 23 | import MongoHelpers.MongoBuilder._ 24 | 25 | private[rogue] def runCommand[M <: MB, T](description: => String, 26 | query: Query[M, _, _])(f: => T): T = { 27 | // Use nanoTime instead of currentTimeMillis to time the query since 28 | // currentTimeMillis only has 10ms granularity on many systems. 29 | val start = System.nanoTime 30 | val instanceName: String = dbCollectionFactory.getInstanceName(query) 31 | try { 32 | logger.onExecuteQuery(query, instanceName, description, f) 33 | } catch { 34 | case e: Exception => 35 | throw new RogueException("Mongo query on %s [%s] failed after %d ms". 36 | format(instanceName, description, 37 | (System.nanoTime - start) / (1000 * 1000)), e) 38 | } finally { 39 | logger.log(query, instanceName, description, (System.nanoTime - start) / (1000 * 1000)) 40 | } 41 | } 42 | 43 | def count[M <: MB](query: Query[M, _, _], readPreference: Option[ReadPreference]): Long = { 44 | val queryClause = transformer.transformQuery(query) 45 | validator.validateQuery(queryClause, dbCollectionFactory.getIndexes(queryClause)) 46 | val condition: DBObject = buildCondition(queryClause.condition) 47 | val description: String = buildConditionString("count", query.collectionName, queryClause) 48 | 49 | runCommand(description, queryClause) { 50 | val coll = dbCollectionFactory.getDBCollection(query) 51 | val db = coll.getDB 52 | val cmd = new BasicDBObject() 53 | cmd.put("count", query.collectionName) 54 | cmd.put("query", condition) 55 | 56 | queryClause.lim.filter(_ > 0).foreach( cmd.put("limit", _) ) 57 | queryClause.sk.filter(_ > 0).foreach( cmd.put("skip", _) ) 58 | 59 | // 4sq dynamically throttles ReadPreference via an override of 60 | // DBCursor creation. We don't want to override for the whole 61 | // DBCollection because those are cached for the life of the DB 62 | val result: CommandResult = db.command(cmd, coll.getOptions, readPreference.getOrElse(coll.find().getReadPreference)) 63 | if (!result.ok) { 64 | result.getErrorMessage match { 65 | // pretend count is zero craziness from the mongo-java-driver 66 | case "ns does not exist" | "ns missing" => 0L 67 | case _ => 68 | result.throwOnError() 69 | 0L 70 | } 71 | } else { 72 | result.getLong("n") 73 | } 74 | } 75 | } 76 | 77 | def countDistinct[M <: MB](query: Query[M, _, _], 78 | key: String, 79 | readPreference: Option[ReadPreference]): Long = { 80 | val queryClause = transformer.transformQuery(query) 81 | validator.validateQuery(queryClause, dbCollectionFactory.getIndexes(queryClause)) 82 | val cnd = buildCondition(queryClause.condition) 83 | 84 | // TODO: fix this so it looks like the correct mongo shell command 85 | val description = buildConditionString("distinct", query.collectionName, queryClause) 86 | 87 | runCommand(description, queryClause) { 88 | val coll = dbCollectionFactory.getDBCollection(query) 89 | coll.distinct(key, cnd, readPreference.getOrElse(coll.find().getReadPreference)).size() 90 | } 91 | } 92 | 93 | def distinct[M <: MB, R](query: Query[M, _, _], 94 | key: String, 95 | readPreference: Option[ReadPreference]): List[R] = { 96 | val queryClause = transformer.transformQuery(query) 97 | validator.validateQuery(queryClause, dbCollectionFactory.getIndexes(queryClause)) 98 | val cnd = buildCondition(queryClause.condition) 99 | 100 | // TODO: fix this so it looks like the correct mongo shell command 101 | val description = buildConditionString("distinct", query.collectionName, queryClause) 102 | 103 | runCommand(description, queryClause) { 104 | val coll = dbCollectionFactory.getDBCollection(query) 105 | val rv = new ListBuffer[R] 106 | val rj = coll.distinct(key, cnd, readPreference.getOrElse(coll.find().getReadPreference)) 107 | for (i <- 0 until rj.size) rv += rj.get(i).asInstanceOf[R] 108 | rv.toList 109 | } 110 | } 111 | 112 | def delete[M <: MB](query: Query[M, _, _], 113 | writeConcern: WriteConcern): Unit = { 114 | val queryClause = transformer.transformQuery(query) 115 | validator.validateQuery(queryClause, dbCollectionFactory.getIndexes(queryClause)) 116 | val cnd = buildCondition(queryClause.condition) 117 | val description = buildConditionString("remove", query.collectionName, queryClause) 118 | 119 | runCommand(description, queryClause) { 120 | val coll = dbCollectionFactory.getPrimaryDBCollection(query) 121 | coll.remove(cnd, writeConcern) 122 | } 123 | } 124 | 125 | def modify[M <: MB](mod: ModifyQuery[M, _], 126 | upsert: Boolean, 127 | multi: Boolean, 128 | writeConcern: WriteConcern): Unit = { 129 | val modClause = transformer.transformModify(mod) 130 | validator.validateModify(modClause, dbCollectionFactory.getIndexes(modClause.query)) 131 | if (!modClause.mod.clauses.isEmpty) { 132 | val q = buildCondition(modClause.query.condition) 133 | val m = buildModify(modClause.mod) 134 | lazy val description = buildModifyString(mod.query.collectionName, modClause, upsert = upsert, multi = multi) 135 | 136 | runCommand(description, modClause.query) { 137 | val coll = dbCollectionFactory.getPrimaryDBCollection(modClause.query) 138 | coll.update(q, m, upsert, multi, writeConcern) 139 | } 140 | } 141 | } 142 | 143 | def findAndModify[M <: MB, R](mod: FindAndModifyQuery[M, R], 144 | returnNew: Boolean, 145 | upsert: Boolean, 146 | remove: Boolean) 147 | (f: DBObject => R): Option[R] = { 148 | val modClause = transformer.transformFindAndModify(mod) 149 | validator.validateFindAndModify(modClause, dbCollectionFactory.getIndexes(modClause.query)) 150 | if (!modClause.mod.clauses.isEmpty || remove) { 151 | val query = modClause.query 152 | val cnd = buildCondition(query.condition) 153 | val ord = query.order.map(buildOrder) 154 | val sel = query.select.map(buildSelect).getOrElse(BasicDBObjectBuilder.start.get) 155 | val m = buildModify(modClause.mod) 156 | lazy val description = buildFindAndModifyString(mod.query.collectionName, modClause, returnNew, upsert, remove) 157 | 158 | runCommand(description, modClause.query) { 159 | val coll = dbCollectionFactory.getPrimaryDBCollection(query) 160 | val dbObj = coll.findAndModify(cnd, sel, ord.getOrElse(null), remove, m, returnNew, upsert) 161 | if (dbObj == null || dbObj.keySet.isEmpty) None 162 | else Option(dbObj).map(f) 163 | } 164 | } 165 | else None 166 | } 167 | 168 | def query[M <: MB](query: Query[M, _, _], 169 | batchSize: Option[Int], 170 | readPreference: Option[ReadPreference]) 171 | (f: DBObject => Unit): Unit = { 172 | doQuery("find", query, batchSize, readPreference){cursor => 173 | while (cursor.hasNext) 174 | f(cursor.next) 175 | } 176 | } 177 | 178 | def iterate[M <: MB, R, S](query: Query[M, R, _], 179 | initialState: S, 180 | f: DBObject => R, 181 | readPreference: Option[ReadPreference] = None) 182 | (handler: (S, Event[R]) => Command[S]): S = { 183 | def getObject(cursor: DBCursor): Either[Exception, R] = { 184 | try { 185 | Right(f(cursor.next)) 186 | } catch { 187 | case e: Exception => Left(e) 188 | } 189 | } 190 | 191 | @scala.annotation.tailrec 192 | def iter(cursor: DBCursor, curState: S): S = { 193 | if (cursor.hasNext) { 194 | getObject(cursor) match { 195 | case Left(e) => handler(curState, Error(e)).state 196 | case Right(r) => handler(curState, Item(r)) match { 197 | case Continue(s) => iter(cursor, s) 198 | case Return(s) => s 199 | } 200 | } 201 | } else { 202 | handler(curState, EOF).state 203 | } 204 | } 205 | 206 | doQuery("find", query, None, readPreference)(cursor => 207 | iter(cursor, initialState) 208 | ) 209 | } 210 | 211 | def iterateBatch[M <: MB, R, S](query: Query[M, R, _], 212 | batchSize: Int, 213 | initialState: S, 214 | f: DBObject => R, 215 | readPreference: Option[ReadPreference] = None) 216 | (handler: (S, Event[List[R]]) => Command[S]): S = { 217 | val buf = new ListBuffer[R] 218 | 219 | def getBatch(cursor: DBCursor): Either[Exception, List[R]] = { 220 | try { 221 | buf.clear() 222 | // ListBuffer#length is O(1) vs ListBuffer#size is O(N) (true in 2.9.x, fixed in 2.10.x) 223 | while (cursor.hasNext && buf.length < batchSize) { 224 | buf += f(cursor.next) 225 | } 226 | Right(buf.toList) 227 | } catch { 228 | case e: Exception => Left(e) 229 | } 230 | } 231 | 232 | @scala.annotation.tailrec 233 | def iter(cursor: DBCursor, curState: S): S = { 234 | if (cursor.hasNext) { 235 | getBatch(cursor) match { 236 | case Left(e) => handler(curState, Error(e)).state 237 | case Right(Nil) => handler(curState, EOF).state 238 | case Right(rs) => handler(curState, Item(rs)) match { 239 | case Continue(s) => iter(cursor, s) 240 | case Return(s) => s 241 | } 242 | } 243 | } else { 244 | handler(curState, EOF).state 245 | } 246 | } 247 | 248 | doQuery("find", query, Some(batchSize), readPreference)(cursor => { 249 | iter(cursor, initialState) 250 | }) 251 | } 252 | 253 | 254 | def explain[M <: MB](query: Query[M, _, _]): String = { 255 | doQuery("find", query, None, None){cursor => 256 | cursor.explain.toString 257 | } 258 | } 259 | 260 | private def doQuery[M <: MB, T]( 261 | operation: String, 262 | query: Query[M, _, _], 263 | batchSize: Option[Int], 264 | readPreference: Option[ReadPreference] 265 | )( 266 | f: DBCursor => T 267 | ): T = { 268 | val queryClause = transformer.transformQuery(query) 269 | validator.validateQuery(queryClause, dbCollectionFactory.getIndexes(queryClause)) 270 | val cnd = buildCondition(queryClause.condition) 271 | val ord = queryClause.order.map(buildOrder) 272 | val sel = queryClause.select.map(buildSelect).getOrElse(BasicDBObjectBuilder.start.get) 273 | val hnt = queryClause.hint.map(buildHint) 274 | 275 | lazy val description = buildQueryString(operation, query.collectionName, queryClause) 276 | 277 | runCommand(description, queryClause) { 278 | val coll = dbCollectionFactory.getDBCollection(query) 279 | try { 280 | val cursor = coll.find(cnd, sel) 281 | // Always apply batchSize *before* limit. If the caller passes a negative value to limit(), 282 | // the driver applies it instead to batchSize. (A negative batchSize means, return one batch 283 | // and close the cursor.) Then if we set batchSize, the negative "limit" is overwritten, and 284 | // the query executes without a limit. 285 | // http://api.mongodb.org/java/2.7.3/com/mongodb/DBCursor.html#limit(int) 286 | batchSize.foreach(cursor batchSize _) 287 | queryClause.lim.foreach(cursor.limit _) 288 | queryClause.sk.foreach(cursor.skip _) 289 | ord.foreach(cursor.sort _) 290 | readPreference.orElse(queryClause.readPreference).foreach(cursor.setReadPreference _) 291 | queryClause.maxScan.foreach(cursor addSpecial("$maxScan", _)) 292 | queryClause.comment.foreach(cursor addSpecial("$comment", _)) 293 | hnt.foreach(cursor hint _) 294 | val ret = f(cursor) 295 | cursor.close() 296 | ret 297 | } catch { 298 | case e: Exception => 299 | throw new RogueException("Mongo query on %s [%s] failed".format( 300 | coll.getDB().getMongo().toString(), description), e) 301 | } 302 | } 303 | } 304 | } 305 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/PhantomTypes.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import scala.annotation.implicitNotFound 6 | 7 | // *************************************************************************** 8 | // *** Phantom types 9 | // *************************************************************************** 10 | 11 | sealed trait Ordered 12 | sealed trait Unordered 13 | sealed trait Ord extends Ordered with Unordered 14 | 15 | sealed trait Selected 16 | sealed trait SelectedOne extends Selected 17 | sealed trait Unselected 18 | sealed trait Sel extends Selected with SelectedOne with Unselected 19 | 20 | sealed trait Limited 21 | sealed trait Unlimited 22 | sealed trait Lim extends Limited with Unlimited 23 | 24 | sealed trait Skipped 25 | sealed trait Unskipped 26 | sealed trait Sk extends Skipped with Unskipped 27 | 28 | sealed trait HasOrClause 29 | sealed trait HasNoOrClause 30 | sealed trait Or extends HasOrClause with HasNoOrClause 31 | 32 | sealed trait ShardKeyNotSpecified 33 | sealed trait ShardAware 34 | sealed trait ShardKeySpecified extends ShardAware 35 | sealed trait AllShardsOk extends ShardAware 36 | sealed trait Sh extends ShardKeyNotSpecified with ShardKeySpecified with AllShardsOk 37 | 38 | @implicitNotFound(msg = "Query must be Unordered, but it's actually ${In}") 39 | trait AddOrder[-In, +Out] extends Required[In, Unordered] 40 | object AddOrder { 41 | implicit def addOrder[Rest >: Sel with Lim with Sk with Or with Sh]: AddOrder[Rest with Unordered, Rest with Ordered] = null 42 | } 43 | 44 | @implicitNotFound(msg = "Query must be Unselected, but it's actually ${In}") 45 | trait AddSelect[-In, +Out, +One] extends Required[In, Unselected] 46 | object AddSelect { 47 | implicit def addSelect[Rest >: Ord with Lim with Sk with Or with Sh]: AddSelect[Rest with Unselected, Rest with Selected, Rest with SelectedOne] = null 48 | } 49 | 50 | @implicitNotFound(msg = "Query must be Unlimited, but it's actually ${In}") 51 | trait AddLimit[-In, +Out] extends Required[In, Unlimited] 52 | object AddLimit { 53 | implicit def addLimit[Rest >: Ord with Sel with Sk with Or with Sh]: AddLimit[Rest with Unlimited, Rest with Limited] = null 54 | } 55 | 56 | @implicitNotFound(msg = "Query must be Unskipped, but it's actually ${In}") 57 | trait AddSkip[-In, +Out] extends Required[In, Unskipped] 58 | object AddSkip { 59 | implicit def addSkip[Rest >: Ord with Sel with Lim with Or with Sh]: AddSkip[Rest with Unskipped, Rest with Skipped] = null 60 | } 61 | 62 | @implicitNotFound(msg = "Query must be HasNoOrClause, but it's actually ${In}") 63 | trait AddOrClause[-In, +Out] extends Required[In, HasNoOrClause] 64 | object AddOrClause { 65 | implicit def addOrClause[Rest >: Ord with Sel with Lim with Sk with Sh]: AddOrClause[Rest with HasNoOrClause, Rest with HasOrClause] = null 66 | } 67 | 68 | trait AddShardAware[-In, +Specified, +AllOk] extends Required[In, ShardKeyNotSpecified] 69 | object AddShardAware { 70 | implicit def addShardAware[Rest >: Ord with Sel with Lim with Sk with Or]: AddShardAware[Rest with ShardKeyNotSpecified, Rest with ShardKeySpecified, Rest with AllShardsOk] = null 71 | } 72 | 73 | @implicitNotFound(msg = "In order to call this method, ${A} must NOT be a subclass of ${B}.") 74 | sealed trait !<:<[A, B] 75 | object !<:< { 76 | implicit def any[A, B]: A !<:< B = null 77 | implicit def sub1[A, B >: A]: A !<:< B = null 78 | implicit def sub2[A, B >: A]: A !<:< B = null 79 | } 80 | 81 | @implicitNotFound(msg = "Cannot prove that ${A} <: ${B}") 82 | class Required[-A, +B] { 83 | def apply[M, R](q: Query[M, R, A]): Query[M, R, B] = q.asInstanceOf[Query[M, R, B]] 84 | } 85 | object Required { 86 | val default = new Required[Any, Any] 87 | implicit def conforms[A]: Required[A, A] = default.asInstanceOf[Required[A, A]] 88 | } 89 | 90 | @implicitNotFound(msg = "${M} is a sharded collection but the shard key is not specified. Either specify the shard key or add `.allShards` to the query.") 91 | trait ShardingOk[M, -S] 92 | object ShardingOk { 93 | implicit def sharded[M <: Sharded, Sh <: ShardAware]: ShardingOk[M, Sh] = null 94 | implicit def unsharded[M, State](implicit ev: M !<:< Sharded): ShardingOk[M, State] = null 95 | } 96 | 97 | @implicitNotFound(msg = "${M} is a sharded collection. Either specify the shard key or use `.updateMulti()`.") 98 | trait RequireShardKey[M, -S] 99 | object RequireShardKey { 100 | implicit def sharded[M <: Sharded, Sh <: ShardKeySpecified]: RequireShardKey[M, Sh] = null 101 | implicit def unsharded[M, State](implicit ev: M !<:< Sharded): RequireShardKey[M, State] = null 102 | } 103 | 104 | 105 | sealed trait MaybeIndexed 106 | sealed trait Indexable extends MaybeIndexed 107 | sealed trait IndexScannable extends MaybeIndexed 108 | 109 | sealed trait NoIndexInfo extends Indexable with IndexScannable 110 | sealed trait Index extends Indexable with IndexScannable 111 | sealed trait PartialIndexScan extends IndexScannable 112 | sealed trait IndexScan extends IndexScannable 113 | sealed trait DocumentScan extends MaybeIndexed 114 | 115 | case object NoIndexInfo extends NoIndexInfo 116 | case object Index extends Index 117 | case object PartialIndexScan extends PartialIndexScan 118 | case object IndexScan extends IndexScan 119 | case object DocumentScan extends DocumentScan 120 | 121 | sealed trait MaybeUsedIndex 122 | sealed trait UsedIndex extends MaybeUsedIndex 123 | sealed trait HasntUsedIndex extends MaybeUsedIndex 124 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/QueryClause.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.mongodb.DBObject 6 | import com.mongodb.BasicDBObjectBuilder 7 | import java.util.regex.Pattern 8 | 9 | abstract class QueryClause[V](val fieldName: String, val actualIndexBehavior: MaybeIndexed, val conditions: (CondOps.Value, V)*) { 10 | def extend(q: BasicDBObjectBuilder, signature: Boolean) { 11 | conditions foreach { case (op, v) => q.add(op.toString, if (signature) 0 else v) } 12 | } 13 | var negated: Boolean = false 14 | val expectedIndexBehavior: MaybeIndexed = Index 15 | def withExpectedIndexBehavior(b: MaybeIndexed): QueryClause[V] 16 | } 17 | 18 | abstract class IndexableQueryClause[V, Ind <: MaybeIndexed](fname: String, actualIB: Ind, conds: (CondOps.Value, V)*) 19 | extends QueryClause[V](fname, actualIB, conds: _*) 20 | 21 | trait ShardKeyClause 22 | 23 | case class AllQueryClause[V](override val fieldName: String, vs: java.util.List[V], override val expectedIndexBehavior: MaybeIndexed = Index) 24 | extends IndexableQueryClause[java.util.List[V], Index](fieldName, Index, CondOps.All -> vs) { 25 | override def withExpectedIndexBehavior(b: MaybeIndexed): AllQueryClause[V] = this.copy(expectedIndexBehavior = b) 26 | } 27 | 28 | case class InQueryClause[V](override val fieldName: String, vs: java.util.List[V], override val expectedIndexBehavior: MaybeIndexed = Index) 29 | extends IndexableQueryClause[java.util.List[V], Index](fieldName, Index, CondOps.In -> vs) { 30 | override def withExpectedIndexBehavior(b: MaybeIndexed): InQueryClause[V] = this.copy(expectedIndexBehavior = b) 31 | } 32 | 33 | case class GtQueryClause[V](override val fieldName: String, v: V, override val expectedIndexBehavior: MaybeIndexed = Index) 34 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan, CondOps.Gt -> v) { 35 | override def withExpectedIndexBehavior(b: MaybeIndexed): GtQueryClause[V] = this.copy(expectedIndexBehavior = b) 36 | } 37 | 38 | case class GtEqQueryClause[V](override val fieldName: String, v: V, override val expectedIndexBehavior: MaybeIndexed = Index) 39 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan, CondOps.GtEq -> v) { 40 | override def withExpectedIndexBehavior(b: MaybeIndexed): GtEqQueryClause[V] = this.copy(expectedIndexBehavior = b) 41 | } 42 | 43 | case class LtQueryClause[V](override val fieldName: String, v: V, override val expectedIndexBehavior: MaybeIndexed = Index) 44 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan, CondOps.Lt -> v) { 45 | override def withExpectedIndexBehavior(b: MaybeIndexed): LtQueryClause[V] = this.copy(expectedIndexBehavior = b) 46 | } 47 | 48 | case class LtEqQueryClause[V](override val fieldName: String, v: V, override val expectedIndexBehavior: MaybeIndexed = Index) 49 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan, CondOps.LtEq -> v) { 50 | override def withExpectedIndexBehavior(b: MaybeIndexed): LtEqQueryClause[V] = this.copy(expectedIndexBehavior = b) 51 | } 52 | 53 | case class BetweenQueryClause[V](override val fieldName: String, lower: V, upper: V, override val expectedIndexBehavior: MaybeIndexed = Index) 54 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan, CondOps.GtEq -> lower, CondOps.LtEq -> upper) { 55 | override def withExpectedIndexBehavior(b: MaybeIndexed): BetweenQueryClause[V] = this.copy(expectedIndexBehavior = b) 56 | } 57 | 58 | case class StrictBetweenQueryClause[V](override val fieldName: String, lower: V, upper: V, override val expectedIndexBehavior: MaybeIndexed = Index) 59 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan, CondOps.Gt -> lower, CondOps.Lt -> upper) { 60 | override def withExpectedIndexBehavior(b: MaybeIndexed): StrictBetweenQueryClause[V] = this.copy(expectedIndexBehavior = b) 61 | } 62 | 63 | case class NeQueryClause[V](override val fieldName: String, v: V, override val expectedIndexBehavior: MaybeIndexed = Index) 64 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan, CondOps.Ne -> v) { 65 | override def withExpectedIndexBehavior(b: MaybeIndexed): NeQueryClause[V] = this.copy(expectedIndexBehavior = b) 66 | } 67 | 68 | case class NearQueryClause[V](override val fieldName: String, v: V, override val expectedIndexBehavior: MaybeIndexed = Index) 69 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan, CondOps.Near -> v) { 70 | override def withExpectedIndexBehavior(b: MaybeIndexed): NearQueryClause[V] = this.copy(expectedIndexBehavior = b) 71 | } 72 | 73 | case class NearSphereQueryClause[V](override val fieldName: String, lat: Double, lng: Double, radians: Radians, override val expectedIndexBehavior: MaybeIndexed = Index) 74 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan) { 75 | override def extend(q: BasicDBObjectBuilder, signature: Boolean) { 76 | q.add(CondOps.NearSphere.toString, if (signature) 0 else QueryHelpers.list(List(lat, lng))) 77 | q.add(CondOps.MaxDistance.toString, if (signature) 0 else radians.value) 78 | } 79 | override def withExpectedIndexBehavior(b: MaybeIndexed): NearSphereQueryClause[V] = this.copy(expectedIndexBehavior = b) 80 | } 81 | 82 | case class ModQueryClause[V](override val fieldName: String, v: java.util.List[V], override val expectedIndexBehavior: MaybeIndexed = Index) 83 | extends IndexableQueryClause[java.util.List[V], IndexScan](fieldName, IndexScan, CondOps.Mod -> v) { 84 | override def withExpectedIndexBehavior(b: MaybeIndexed): ModQueryClause[V] = this.copy(expectedIndexBehavior = b) 85 | } 86 | 87 | case class TypeQueryClause(override val fieldName: String, v: MongoType.Value, override val expectedIndexBehavior: MaybeIndexed = Index) 88 | extends IndexableQueryClause[Int, IndexScan](fieldName, IndexScan, CondOps.Type -> v.id) { 89 | override def withExpectedIndexBehavior(b: MaybeIndexed): TypeQueryClause = this.copy(expectedIndexBehavior = b) 90 | } 91 | 92 | case class ExistsQueryClause(override val fieldName: String, v: Boolean, override val expectedIndexBehavior: MaybeIndexed = Index) 93 | extends IndexableQueryClause[Boolean, IndexScan](fieldName, IndexScan, CondOps.Exists -> v) { 94 | override def withExpectedIndexBehavior(b: MaybeIndexed): ExistsQueryClause = this.copy(expectedIndexBehavior = b) 95 | } 96 | 97 | case class NinQueryClause[V](override val fieldName: String, vs: java.util.List[V], override val expectedIndexBehavior: MaybeIndexed = Index) 98 | extends IndexableQueryClause[java.util.List[V], DocumentScan](fieldName, DocumentScan, CondOps.Nin -> vs) { 99 | override def withExpectedIndexBehavior(b: MaybeIndexed): NinQueryClause[V] = this.copy(expectedIndexBehavior = b) 100 | } 101 | 102 | case class SizeQueryClause(override val fieldName: String, v: Int, override val expectedIndexBehavior: MaybeIndexed = Index) 103 | extends IndexableQueryClause[Int, DocumentScan](fieldName, DocumentScan, CondOps.Size -> v) { 104 | override def withExpectedIndexBehavior(b: MaybeIndexed): SizeQueryClause = this.copy(expectedIndexBehavior = b) 105 | } 106 | 107 | case class RegexQueryClause[Ind <: MaybeIndexed](override val fieldName: String, actualIB: Ind, p: Pattern, override val expectedIndexBehavior: MaybeIndexed = Index) 108 | extends IndexableQueryClause[Pattern, Ind](fieldName, actualIB) { 109 | val flagMap = Map( 110 | Pattern.CANON_EQ -> "c", 111 | Pattern.CASE_INSENSITIVE -> "i", 112 | Pattern.COMMENTS -> "x", 113 | Pattern.DOTALL -> "s", 114 | Pattern.LITERAL -> "t", 115 | Pattern.MULTILINE -> "m", 116 | Pattern.UNICODE_CASE -> "u", 117 | Pattern.UNIX_LINES -> "d" 118 | ) 119 | 120 | def flagsToString(flags: Int) = { 121 | (for { 122 | (mask, char) <- flagMap 123 | if (flags & mask) != 0 124 | } yield char).mkString 125 | } 126 | 127 | override def extend(q: BasicDBObjectBuilder, signature: Boolean) { 128 | q.add("$regex", if (signature) 0 else p.toString) 129 | q.add("$options", if (signature) 0 else flagsToString(p.flags)) 130 | } 131 | 132 | override def withExpectedIndexBehavior(b: MaybeIndexed): RegexQueryClause[Ind] = this.copy(expectedIndexBehavior = b) 133 | } 134 | 135 | 136 | case class RawQueryClause(f: BasicDBObjectBuilder => Unit, override val expectedIndexBehavior: MaybeIndexed = DocumentScan) extends IndexableQueryClause("raw", DocumentScan) { 137 | override def extend(q: BasicDBObjectBuilder, signature: Boolean) { 138 | f(q) 139 | } 140 | override def withExpectedIndexBehavior(b: MaybeIndexed): RawQueryClause = this.copy(expectedIndexBehavior = b) 141 | } 142 | 143 | case class EmptyQueryClause[V](override val fieldName: String, override val expectedIndexBehavior: MaybeIndexed = Index) 144 | extends IndexableQueryClause[V, Index](fieldName, Index) { 145 | override def extend(q: BasicDBObjectBuilder, signature: Boolean) {} 146 | override def withExpectedIndexBehavior(b: MaybeIndexed): EmptyQueryClause[V] = this.copy(expectedIndexBehavior = b) 147 | } 148 | 149 | case class EqClause[V, Ind <: MaybeIndexed](override val fieldName: String, value: V, override val expectedIndexBehavior: MaybeIndexed = Index) 150 | extends IndexableQueryClause[V, Index](fieldName, Index) { 151 | override def extend(q: BasicDBObjectBuilder, signature: Boolean): Unit = { 152 | q.add(fieldName, if (signature) 0 else value) 153 | } 154 | override def withExpectedIndexBehavior(b: MaybeIndexed): EqClause[V, Ind] = this.copy(expectedIndexBehavior = b) 155 | } 156 | 157 | case class WithinCircleClause[V](override val fieldName: String, lat: Double, lng: Double, radius: Double, override val expectedIndexBehavior: MaybeIndexed = Index) 158 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan) { 159 | override def extend(q: BasicDBObjectBuilder, signature: Boolean): Unit = { 160 | val value = if (signature) 0 else QueryHelpers.list(List(QueryHelpers.list(List(lat, lng)), radius)) 161 | q.push("$within").add("$center", value).pop 162 | } 163 | override def withExpectedIndexBehavior(b: MaybeIndexed): WithinCircleClause[V] = this.copy(expectedIndexBehavior = b) 164 | } 165 | 166 | case class WithinBoxClause[V](override val fieldName: String, lat1: Double, lng1: Double, lat2: Double, lng2: Double, override val expectedIndexBehavior: MaybeIndexed = Index) 167 | extends IndexableQueryClause[V, PartialIndexScan](fieldName, PartialIndexScan) { 168 | override def extend(q: BasicDBObjectBuilder, signature: Boolean): Unit = { 169 | val value = if (signature) 0 else { 170 | QueryHelpers.list(List(QueryHelpers.list(lat1, lng1), QueryHelpers.list(lat2, lng2))) 171 | } 172 | q.push("$within").add("$box", value).pop 173 | } 174 | override def withExpectedIndexBehavior(b: MaybeIndexed): WithinBoxClause[V] = this.copy(expectedIndexBehavior = b) 175 | } 176 | 177 | case class ElemMatchWithPredicateClause[V](override val fieldName: String, clauses: Seq[QueryClause[_]], override val expectedIndexBehavior: MaybeIndexed = Index) 178 | extends IndexableQueryClause[V, DocumentScan](fieldName, DocumentScan) { 179 | override def extend(q: BasicDBObjectBuilder, signature: Boolean): Unit = { 180 | import com.foursquare.rogue.MongoHelpers.AndCondition 181 | val nested = q.push("$elemMatch") 182 | MongoHelpers.MongoBuilder.buildCondition(AndCondition(clauses.toList, None), nested, signature) 183 | nested.pop 184 | } 185 | override def withExpectedIndexBehavior(b: MaybeIndexed): ElemMatchWithPredicateClause[V] = this.copy(expectedIndexBehavior = b) 186 | } 187 | 188 | class ModifyClause(val operator: ModOps.Value, fields: (String, _)*) { 189 | def extend(q: BasicDBObjectBuilder): Unit = { 190 | fields foreach { case (name, value) => q.add(name, value) } 191 | } 192 | } 193 | 194 | class ModifyAddEachClause(fieldName: String, values: Traversable[_]) 195 | extends ModifyClause(ModOps.AddToSet) { 196 | override def extend(q: BasicDBObjectBuilder): Unit = { 197 | q.push(fieldName).add("$each", QueryHelpers.list(values)).pop 198 | } 199 | } 200 | 201 | class ModifyPushEachClause(fieldName: String, values: Traversable[_]) 202 | extends ModifyClause(ModOps.Push) { 203 | override def extend(q: BasicDBObjectBuilder): Unit = { 204 | q.push(fieldName).add("$each", QueryHelpers.list(values)).pop 205 | } 206 | } 207 | 208 | class ModifyPushEachSliceClause(fieldName: String, slice: Int, values: Traversable[_]) 209 | extends ModifyClause(ModOps.Push) { 210 | override def extend(q: BasicDBObjectBuilder): Unit = { 211 | q.push(fieldName).add("$each", QueryHelpers.list(values)).add("$slice", slice).pop 212 | } 213 | } 214 | 215 | class ModifyBitAndClause(fieldName: String, value: Int) extends ModifyClause(ModOps.Bit) { 216 | override def extend(q: BasicDBObjectBuilder): Unit = { 217 | q.push(fieldName).add("and", value).pop 218 | } 219 | } 220 | 221 | class ModifyBitOrClause(fieldName: String, value: Int) extends ModifyClause(ModOps.Bit) { 222 | override def extend(q: BasicDBObjectBuilder): Unit = { 223 | q.push(fieldName).add("or", value).pop 224 | } 225 | } 226 | 227 | class ModifyPullWithPredicateClause[V](fieldName: String, clauses: Seq[QueryClause[_]]) 228 | extends ModifyClause(ModOps.Pull) { 229 | override def extend(q: BasicDBObjectBuilder): Unit = { 230 | import com.foursquare.rogue.MongoHelpers.AndCondition 231 | MongoHelpers.MongoBuilder.buildCondition(AndCondition(clauses.toList, None), q, false) 232 | } 233 | } 234 | 235 | class ModifyPullObjWithPredicateClause[V](fieldName: String, clauses: Seq[QueryClause[_]]) 236 | extends ModifyClause(ModOps.Pull) { 237 | override def extend(q: BasicDBObjectBuilder): Unit = { 238 | import com.foursquare.rogue.MongoHelpers.AndCondition 239 | val nested = q.push(fieldName) 240 | MongoHelpers.MongoBuilder.buildCondition(AndCondition(clauses.toList, None), nested, false) 241 | nested.pop 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/QueryExecutor.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.foursquare.field.Field 6 | import com.foursquare.rogue.MongoHelpers.{MongoModify, MongoSelect} 7 | import com.mongodb.{DBObject, ReadPreference, WriteConcern} 8 | import scala.collection.mutable.ListBuffer 9 | 10 | trait RogueSerializer[R] { 11 | def fromDBObject(dbo: DBObject): R 12 | } 13 | 14 | trait QueryExecutor[MB] extends Rogue { 15 | def adapter: MongoJavaDriverAdapter[MB] 16 | def optimizer: QueryOptimizer 17 | 18 | def defaultWriteConcern: WriteConcern 19 | 20 | protected def serializer[M <: MB, R]( 21 | meta: M, 22 | select: Option[MongoSelect[M, R]] 23 | ): RogueSerializer[R] 24 | 25 | def count[M <: MB, State](query: Query[M, _, State], 26 | readPreference: Option[ReadPreference] = None) 27 | (implicit ev: ShardingOk[M, State]): Long = { 28 | if (optimizer.isEmptyQuery(query)) { 29 | 0L 30 | } else { 31 | adapter.count(query, readPreference) 32 | } 33 | } 34 | 35 | def countDistinct[M <: MB, V, State](query: Query[M, _, State], 36 | readPreference: Option[ReadPreference] = None) 37 | (field: M => Field[V, M]) 38 | (implicit ev: ShardingOk[M, State]): Long = { 39 | if (optimizer.isEmptyQuery(query)) { 40 | 0L 41 | } else { 42 | adapter.countDistinct(query, field(query.meta).name, readPreference) 43 | } 44 | } 45 | 46 | def distinct[M <: MB, V, State](query: Query[M, _, State], 47 | readPreference: Option[ReadPreference] = None) 48 | (field: M => Field[V, M]) 49 | (implicit ev: ShardingOk[M, State]): List[V] = { 50 | if (optimizer.isEmptyQuery(query)) { 51 | Nil 52 | } else { 53 | adapter.distinct(query, field(query.meta).name, readPreference) 54 | } 55 | } 56 | 57 | def fetch[M <: MB, R, State](query: Query[M, R, State], 58 | readPreference: Option[ReadPreference] = None) 59 | (implicit ev: ShardingOk[M, State]): List[R] = { 60 | if (optimizer.isEmptyQuery(query)) { 61 | Nil 62 | } else { 63 | val s = serializer[M, R](query.meta, query.select) 64 | val rv = new ListBuffer[R] 65 | adapter.query(query, None, readPreference)(dbo => rv += s.fromDBObject(dbo)) 66 | rv.toList 67 | } 68 | } 69 | 70 | def fetchOne[M <: MB, R, State, S2](query: Query[M, R, State], 71 | readPreference: Option[ReadPreference] = None) 72 | (implicit ev1: AddLimit[State, S2], ev2: ShardingOk[M, S2]): Option[R] = { 73 | fetch(query.limit(1), readPreference).headOption 74 | } 75 | 76 | def foreach[M <: MB, R, State](query: Query[M, R, State], 77 | readPreference: Option[ReadPreference] = None) 78 | (f: R => Unit) 79 | (implicit ev: ShardingOk[M, State]): Unit = { 80 | if (optimizer.isEmptyQuery(query)) { 81 | () 82 | } else { 83 | val s = serializer[M, R](query.meta, query.select) 84 | adapter.query(query, None, readPreference)(dbo => f(s.fromDBObject(dbo))) 85 | } 86 | } 87 | 88 | private def drainBuffer[A, B]( 89 | from: ListBuffer[A], 90 | to: ListBuffer[B], 91 | f: List[A] => List[B], 92 | size: Int 93 | ): Unit = { 94 | // ListBuffer#length is O(1) vs ListBuffer#size is O(N) (true in 2.9.x, fixed in 2.10.x) 95 | if (from.length >= size) { 96 | to ++= f(from.toList) 97 | from.clear 98 | } 99 | } 100 | 101 | def fetchBatch[M <: MB, R, T, State](query: Query[M, R, State], 102 | batchSize: Int, 103 | readPreference: Option[ReadPreference] = None) 104 | (f: List[R] => List[T]) 105 | (implicit ev: ShardingOk[M, State]): List[T] = { 106 | if (optimizer.isEmptyQuery(query)) { 107 | Nil 108 | } else { 109 | val s = serializer[M, R](query.meta, query.select) 110 | val rv = new ListBuffer[T] 111 | val buf = new ListBuffer[R] 112 | 113 | adapter.query(query, Some(batchSize), readPreference) { dbo => 114 | buf += s.fromDBObject(dbo) 115 | drainBuffer(buf, rv, f, batchSize) 116 | } 117 | drainBuffer(buf, rv, f, 1) 118 | 119 | rv.toList 120 | } 121 | } 122 | 123 | def bulkDelete_!![M <: MB, State](query: Query[M, _, State], 124 | writeConcern: WriteConcern = defaultWriteConcern) 125 | (implicit ev1: Required[State, Unselected with Unlimited with Unskipped], 126 | ev2: ShardingOk[M, State]): Unit = { 127 | if (optimizer.isEmptyQuery(query)) { 128 | () 129 | } else { 130 | adapter.delete(query, writeConcern) 131 | } 132 | } 133 | 134 | def updateOne[M <: MB, State]( 135 | query: ModifyQuery[M, State], 136 | writeConcern: WriteConcern = defaultWriteConcern 137 | )(implicit ev: RequireShardKey[M, State]): Unit = { 138 | if (optimizer.isEmptyQuery(query)) { 139 | () 140 | } else { 141 | adapter.modify(query, upsert = false, multi = false, writeConcern = writeConcern) 142 | } 143 | } 144 | 145 | def upsertOne[M <: MB, State]( 146 | query: ModifyQuery[M, State], 147 | writeConcern: WriteConcern = defaultWriteConcern 148 | )(implicit ev: RequireShardKey[M, State]): Unit = { 149 | if (optimizer.isEmptyQuery(query)) { 150 | () 151 | } else { 152 | adapter.modify(query, upsert = true, multi = false, writeConcern = writeConcern) 153 | } 154 | } 155 | 156 | def updateMulti[M <: MB, State]( 157 | query: ModifyQuery[M, State], 158 | writeConcern: WriteConcern = defaultWriteConcern 159 | ): Unit = { 160 | if (optimizer.isEmptyQuery(query)) { 161 | () 162 | } else { 163 | adapter.modify(query, upsert = false, multi = true, writeConcern = writeConcern) 164 | } 165 | } 166 | 167 | def findAndUpdateOne[M <: MB, R]( 168 | query: FindAndModifyQuery[M, R], 169 | returnNew: Boolean = false, 170 | writeConcern: WriteConcern = defaultWriteConcern 171 | ): Option[R] = { 172 | if (optimizer.isEmptyQuery(query)) { 173 | None 174 | } else { 175 | val s = serializer[M, R](query.query.meta, query.query.select) 176 | adapter.findAndModify(query, returnNew, upsert=false, remove=false)(s.fromDBObject _) 177 | } 178 | } 179 | 180 | def findAndUpsertOne[M <: MB, R]( 181 | query: FindAndModifyQuery[M, R], 182 | returnNew: Boolean = false, 183 | writeConcern: WriteConcern = defaultWriteConcern 184 | ): Option[R] = { 185 | if (optimizer.isEmptyQuery(query)) { 186 | None 187 | } else { 188 | val s = serializer[M, R](query.query.meta, query.query.select) 189 | adapter.findAndModify(query, returnNew, upsert=true, remove=false)(s.fromDBObject _) 190 | } 191 | } 192 | 193 | def findAndDeleteOne[M <: MB, R, State]( 194 | query: Query[M, R, State], 195 | writeConcern: WriteConcern = defaultWriteConcern 196 | )(implicit ev: RequireShardKey[M, State]): Option[R] = { 197 | if (optimizer.isEmptyQuery(query)) { 198 | None 199 | } else { 200 | val s = serializer[M, R](query.meta, query.select) 201 | val mod = FindAndModifyQuery(query, MongoModify(Nil)) 202 | adapter.findAndModify(mod, returnNew=false, upsert=false, remove=true)(s.fromDBObject _) 203 | } 204 | } 205 | 206 | def explain[M <: MB](query: Query[M, _, _]): String = { 207 | adapter.explain(query) 208 | } 209 | 210 | def iterate[S, M <: MB, R, State](query: Query[M, R, State], 211 | state: S, 212 | readPreference: Option[ReadPreference] = None) 213 | (handler: (S, Iter.Event[R]) => Iter.Command[S]) 214 | (implicit ev: ShardingOk[M, State]): S = { 215 | if (optimizer.isEmptyQuery(query)) { 216 | handler(state, Iter.EOF).state 217 | } else { 218 | val s = serializer[M, R](query.meta, query.select) 219 | adapter.iterate(query, state, s.fromDBObject _, readPreference)(handler) 220 | } 221 | } 222 | 223 | def iterateBatch[S, M <: MB, R, State](query: Query[M, R, State], 224 | batchSize: Int, 225 | state: S, 226 | readPreference: Option[ReadPreference] = None) 227 | (handler: (S, Iter.Event[List[R]]) => Iter.Command[S]) 228 | (implicit ev: ShardingOk[M, State]): S = { 229 | if (optimizer.isEmptyQuery(query)) { 230 | handler(state, Iter.EOF).state 231 | } else { 232 | val s = serializer[M, R](query.meta, query.select) 233 | adapter.iterateBatch(query, batchSize, state, s.fromDBObject _, readPreference)(handler) 234 | } 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/QueryHelpers.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.foursquare.index.UntypedMongoIndex 6 | import com.mongodb.{DBObject, WriteConcern} 7 | import net.liftweb.json.{Extraction, Formats, Serializer, TypeInfo} 8 | import net.liftweb.json.JsonAST.{JObject, JValue} 9 | import net.liftweb.mongodb.{JObjectParser, ObjectIdSerializer} 10 | 11 | case class Degrees(value: Double) 12 | case class Radians(value: Double) 13 | case class LatLong(lat: Double, long: Double) 14 | 15 | object QueryHelpers { 16 | class DBObjectSerializer extends Serializer[DBObject] { 17 | val DBObjectClass = classOf[DBObject] 18 | 19 | def deserialize(implicit formats: Formats): PartialFunction[(TypeInfo, JValue), DBObject] = { 20 | case (TypeInfo(klass, _), json : JObject) if DBObjectClass.isAssignableFrom(klass) => 21 | JObjectParser.parse(json) 22 | } 23 | 24 | def serialize(implicit formats: Formats): PartialFunction[Any, JValue] = { 25 | case x: DBObject => 26 | JObjectParser.serialize(x) 27 | } 28 | } 29 | 30 | private implicit val formats = 31 | (net.liftweb.json.DefaultFormats + new ObjectIdSerializer + new DBObjectSerializer) 32 | 33 | trait QueryLogger { 34 | def log(query: Query[_, _, _], instanceName: String, msg: => String, timeMillis: Long): Unit 35 | def onExecuteQuery[T](query: Query[_, _, _], instanceName: String, msg: => String, func: => T): T 36 | def logIndexMismatch(query: Query[_, _, _], msg: => String) 37 | def logIndexHit(query: Query[_, _, _], index: UntypedMongoIndex) 38 | def warn(query: Query[_, _, _], msg: => String): Unit 39 | } 40 | 41 | class DefaultQueryLogger extends QueryLogger { 42 | override def log(query: Query[_, _, _], instanceName: String, msg: => String, timeMillis: Long) {} 43 | override def onExecuteQuery[T](query: Query[_, _, _], instanceName: String, msg: => String, func: => T): T = func 44 | override def logIndexMismatch(query: Query[_, _, _], msg: => String) {} 45 | override def logIndexHit(query: Query[_, _, _], index: UntypedMongoIndex) {} 46 | override def warn(query: Query[_, _, _], msg: => String) {} 47 | } 48 | 49 | object NoopQueryLogger extends DefaultQueryLogger 50 | 51 | var logger: QueryLogger = NoopQueryLogger 52 | 53 | trait QueryValidator { 54 | def validateList[T](xs: Traversable[T]): Unit 55 | def validateRadius(d: Degrees): Degrees 56 | def validateQuery[M](query: Query[M, _, _], indexes: Option[List[UntypedMongoIndex]]): Unit 57 | def validateModify[M](modify: ModifyQuery[M, _], indexes: Option[List[UntypedMongoIndex]]): Unit 58 | def validateFindAndModify[M, R](modify: FindAndModifyQuery[M, R], indexes: Option[List[UntypedMongoIndex]]): Unit 59 | } 60 | 61 | class DefaultQueryValidator extends QueryValidator { 62 | override def validateList[T](xs: Traversable[T]) {} 63 | override def validateRadius(d: Degrees) = d 64 | override def validateQuery[M](query: Query[M, _, _], indexes: Option[List[UntypedMongoIndex]]) {} 65 | override def validateModify[M](modify: ModifyQuery[M, _], indexes: Option[List[UntypedMongoIndex]]) {} // todo possibly validate for update without upsert, yet setOnInsert present -- ktoso 66 | override def validateFindAndModify[M, R](modify: FindAndModifyQuery[M, R], indexes: Option[List[UntypedMongoIndex]]) {} 67 | } 68 | 69 | object NoopQueryValidator extends DefaultQueryValidator 70 | 71 | var validator: QueryValidator = NoopQueryValidator 72 | 73 | trait QueryTransformer { 74 | def transformQuery[M](query: Query[M, _, _]): Query[M, _, _] 75 | def transformModify[M](modify: ModifyQuery[M, _]): ModifyQuery[M, _] 76 | def transformFindAndModify[M, R](modify: FindAndModifyQuery[M, R]): FindAndModifyQuery[M, R] 77 | } 78 | 79 | class DefaultQueryTransformer extends QueryTransformer { 80 | override def transformQuery[M](query: Query[M, _, _]): Query[M, _, _] = { query } 81 | override def transformModify[M](modify: ModifyQuery[M, _]): ModifyQuery[M, _] = { modify } 82 | override def transformFindAndModify[M, R](modify: FindAndModifyQuery[M, R]): FindAndModifyQuery[M, R] = { modify } 83 | } 84 | 85 | object NoopQueryTransformer extends DefaultQueryTransformer 86 | 87 | var transformer: QueryTransformer = NoopQueryTransformer 88 | 89 | trait QueryConfig { 90 | def defaultWriteConcern: WriteConcern 91 | } 92 | 93 | class DefaultQueryConfig extends QueryConfig { 94 | override def defaultWriteConcern = WriteConcern.NONE 95 | } 96 | 97 | object DefaultQueryConfig extends DefaultQueryConfig 98 | 99 | var config: QueryConfig = DefaultQueryConfig 100 | 101 | def makeJavaList[T](sl: Traversable[T]): java.util.List[T] = { 102 | val list = new java.util.ArrayList[T]() 103 | for (id <- sl) list.add(id) 104 | list 105 | } 106 | 107 | def validatedList[T](vs: Traversable[T]): java.util.List[T] = { 108 | validator.validateList(vs) 109 | makeJavaList(vs) 110 | } 111 | 112 | def list[T](vs: Traversable[T]): java.util.List[T] = { 113 | makeJavaList(vs) 114 | } 115 | 116 | def list(vs: Double*): java.util.List[Double] = list(vs) 117 | 118 | def radius(d: Degrees) = { 119 | validator.validateRadius(d).value 120 | } 121 | 122 | def makeJavaMap[K, V](m: Map[K, V]): java.util.Map[K, V] = { 123 | val map = new java.util.HashMap[K, V] 124 | for ((k, v) <- m) map.put(k, v) 125 | map 126 | } 127 | 128 | def inListClause[V](fieldName: String, vs: Traversable[V]) = { 129 | if (vs.isEmpty) 130 | new EmptyQueryClause[java.util.List[V]](fieldName) 131 | else 132 | new InQueryClause(fieldName, QueryHelpers.validatedList(vs.toSet)) 133 | } 134 | 135 | def allListClause[V](fieldName: String, vs: Traversable[V]) = { 136 | if (vs.isEmpty) 137 | new EmptyQueryClause[java.util.List[V]](fieldName) 138 | else 139 | new AllQueryClause(fieldName, QueryHelpers.validatedList(vs.toSet)) 140 | } 141 | 142 | def asDBObject[T](x: T): DBObject = { 143 | JObjectParser.parse(Extraction.decompose(x).asInstanceOf[JObject]) 144 | } 145 | 146 | def orConditionFromQueries(subqueries: List[Query[_, _, _]]) = { 147 | MongoHelpers.OrCondition(subqueries.flatMap(subquery => { 148 | subquery match { 149 | case q: Query[_, _, _] if q.condition.isEmpty => None 150 | case q: Query[_, _, _] => Some(q.condition) 151 | case _ => None 152 | } 153 | })) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/QueryOptimizer.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | package com.foursquare.rogue 3 | 4 | class QueryOptimizer { 5 | def isEmptyClause(clause: QueryClause[_]): Boolean = clause match { 6 | case AllQueryClause(_, vs, _) => vs.isEmpty 7 | case InQueryClause(_, vs, _) => vs.isEmpty 8 | case EmptyQueryClause(_, _) => true 9 | case _ => false 10 | } 11 | 12 | def isEmptyQuery(query: Query[_, _, _]): Boolean = { 13 | query.condition.clauses.exists(isEmptyClause) 14 | } 15 | 16 | def isEmptyQuery(query: ModifyQuery[_, _]): Boolean = 17 | isEmptyQuery(query.query) 18 | 19 | def isEmptyQuery(query: FindAndModifyQuery[_, _]): Boolean = 20 | isEmptyQuery(query.query) 21 | } 22 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/Rogue.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.foursquare.field.{ 6 | Field => RField, 7 | OptionalField => ROptionalField, 8 | RequiredField => RRequiredField} 9 | import com.foursquare.rogue.MongoHelpers.MongoModify 10 | import com.mongodb.DBObject 11 | import java.util.Date 12 | import org.bson.types.ObjectId 13 | import org.joda.time.DateTime 14 | 15 | /** 16 | * A utility trait containing typing shorthands, and a collection of implicit conversions that make query 17 | * syntax much simpler. 18 | * 19 | *@see AbstractQuery for an example of the use of implicit conversions. 20 | */ 21 | trait Rogue { 22 | 23 | // QueryField implicits 24 | implicit def rbooleanFieldtoQueryField[M](f: RField[Boolean, M]): QueryField[Boolean, M] = new QueryField(f) 25 | implicit def rcharFieldtoQueryField[M](f: RField[Char, M]): QueryField[Char, M] = new QueryField(f) 26 | 27 | implicit def rbyteFieldtoNumericQueryField[M](f: RField[Byte, M]): NumericQueryField[Byte, M] = new NumericQueryField(f) 28 | implicit def rshortFieldtoNumericQueryField[M](f: RField[Short, M]): NumericQueryField[Short, M] = new NumericQueryField(f) 29 | implicit def rintFieldtoNumericQueryField[M](f: RField[Int, M]): NumericQueryField[Int, M] = new NumericQueryField(f) 30 | implicit def rlongFieldtoNumericQueryField[F <: Long, M](f: RField[F, M]): NumericQueryField[F, M] = new NumericQueryField(f) 31 | implicit def rjlongFieldtoNumericQueryField[F <: java.lang.Long, M](f: RField[F, M]): NumericQueryField[F, M] = new NumericQueryField(f) 32 | implicit def rfloatFieldtoNumericQueryField[M](f: RField[Float, M]): NumericQueryField[Float, M] = new NumericQueryField(f) 33 | implicit def rdoubleFieldtoNumericQueryField[M](f: RField[Double, M]): NumericQueryField[Double, M] = new NumericQueryField(f) 34 | 35 | implicit def rstringFieldToStringQueryField[F <: String, M](f: RField[F, M]): StringQueryField[F, M] = new StringQueryField(f) 36 | implicit def robjectIdFieldToObjectIdQueryField[F <: ObjectId, M](f: RField[F, M]): ObjectIdQueryField[F, M] = new ObjectIdQueryField[F, M](f) 37 | implicit def rdateFieldToDateQueryField[M](f: RField[Date, M]): DateQueryField[M] = new DateQueryField(f) 38 | implicit def rdatetimeFieldToDateQueryField[M](f: RField[DateTime, M]): DateTimeQueryField[M] = new DateTimeQueryField(f) 39 | implicit def rdbobjectFieldToQueryField[M](f: RField[DBObject, M]): QueryField[DBObject, M] = new QueryField(f) 40 | 41 | implicit def renumNameFieldToEnumNameQueryField[M, F <: Enumeration#Value](f: RField[F, M]): EnumNameQueryField[M, F] = new EnumNameQueryField(f) 42 | implicit def renumerationListFieldToEnumerationListQueryField[M, F <: Enumeration#Value](f: RField[List[F], M]): EnumerationListQueryField[F, M] = new EnumerationListQueryField[F, M](f) 43 | implicit def rlatLongFieldToGeoQueryField[M](f: RField[LatLong, M]): GeoQueryField[M] = new GeoQueryField(f) 44 | implicit def rStringsListFieldToStringsListQueryField[M](f: RField[List[String], M]): StringsListQueryField[M] = new StringsListQueryField[M](f) 45 | implicit def rlistFieldToListQueryField[M, F: BSONType](f: RField[List[F], M]): ListQueryField[F, M] = new ListQueryField[F, M](f) 46 | implicit def rseqFieldToSeqQueryField[M, F: BSONType](f: RField[Seq[F], M]): SeqQueryField[F, M] = new SeqQueryField[F, M](f) 47 | implicit def rmapFieldToMapQueryField[M, F](f: RField[Map[String, F], M]): MapQueryField[F, M] = new MapQueryField[F, M](f) 48 | 49 | /** ModifyField implicits 50 | * 51 | * These are dangerous in the general case, unless the field type can be safely serialized 52 | * or the field class handles necessary serialization. We specialize some safe cases. 53 | **/ 54 | implicit def rfieldToSafeModifyField[M, F](f: RField[F, M]): SafeModifyField[F, M] = new SafeModifyField(f) 55 | implicit def booleanRFieldToModifyField[M](f: RField[Boolean, M]): ModifyField[Boolean, M] = new ModifyField(f) 56 | implicit def charRFieldToModifyField[M](f: RField[Char, M]): ModifyField[Char, M] = new ModifyField(f) 57 | 58 | implicit def byteRFieldToModifyField[M](f: RField[Byte, M]): NumericModifyField[Byte, M] = new NumericModifyField(f) 59 | implicit def shortRFieldToModifyField[M](f: RField[Short, M]): NumericModifyField[Short, M] = new NumericModifyField(f) 60 | implicit def intRFieldToModifyField[M](f: RField[Int, M]): NumericModifyField[Int, M] = new NumericModifyField(f) 61 | implicit def longRFieldToModifyField[M, F <: Long](f: RField[F, M]): NumericModifyField[F, M] = new NumericModifyField(f) 62 | implicit def floatRFieldToModifyField[M](f: RField[Float, M]): NumericModifyField[Float, M] = new NumericModifyField(f) 63 | implicit def doubleRFieldToModifyField[M](f: RField[Double, M]): NumericModifyField[Double, M] = new NumericModifyField(f) 64 | 65 | implicit def stringRFieldToModifyField[M](f: RField[String, M]): ModifyField[String, M] = new ModifyField(f) 66 | implicit def objectidRFieldToModifyField[M, F <: ObjectId](f: RField[F, M]): ModifyField[F, M] = new ModifyField(f) 67 | implicit def dateRFieldToDateModifyField[M](f: RField[Date, M]): DateModifyField[M] = new DateModifyField(f) 68 | implicit def datetimeRFieldToDateModifyField[M](f: RField[DateTime, M]): DateTimeModifyField[M] = new DateTimeModifyField(f) 69 | 70 | implicit def renumerationFieldToEnumerationModifyField[M, F <: Enumeration#Value] 71 | (f: RField[F, M]): EnumerationModifyField[M, F] = 72 | new EnumerationModifyField(f) 73 | 74 | implicit def renumerationListFieldToEnumerationListModifyField[M, F <: Enumeration#Value] 75 | (f: RField[List[F], M]): EnumerationListModifyField[F, M] = 76 | new EnumerationListModifyField[F, M](f) 77 | 78 | implicit def rlatLongFieldToGeoQueryModifyField[M](f: RField[LatLong, M]): GeoModifyField[M] = 79 | new GeoModifyField(f) 80 | 81 | implicit def rlistFieldToListModifyField[M, F: BSONType](f: RField[List[F], M]): ListModifyField[F, M] = 82 | new ListModifyField[F, M](f) 83 | 84 | implicit def rSeqFieldToSeqModifyField[M, F: BSONType](f: RField[Seq[F], M]): SeqModifyField[F, M] = 85 | new SeqModifyField[F, M](f) 86 | 87 | implicit def rmapFieldToMapModifyField[M, F](f: RField[Map[String, F], M]): MapModifyField[F, M] = 88 | new MapModifyField[F, M](f) 89 | 90 | // SelectField implicits 91 | implicit def roptionalFieldToSelectField[M, V]( 92 | f: ROptionalField[V, M] 93 | ): SelectField[Option[V], M] = new OptionalSelectField(f) 94 | 95 | implicit def rrequiredFieldToSelectField[M, V]( 96 | f: RRequiredField[V, M] 97 | ): SelectField[V, M] = new MandatorySelectField(f) 98 | 99 | class Flattened[A, B] 100 | implicit def anyValIsFlattened[A <: AnyVal]: Flattened[A, A] = new Flattened[A, A] 101 | implicit def enumIsFlattened[A <: Enumeration#Value]: Flattened[A, A] = new Flattened[A, A] 102 | implicit val stringIsFlattened = new Flattened[String, String] 103 | implicit val objectIdIsFlattened = new Flattened[ObjectId, ObjectId] 104 | implicit val dateIsFlattened = new Flattened[java.util.Date, java.util.Date] 105 | implicit def recursiveFlattenList[A, B](implicit ev: Flattened[A, B]) = new Flattened[List[A], B] 106 | implicit def recursiveFlattenSeq[A, B](implicit ev: Flattened[A, B]) = new Flattened[Seq[A], B] 107 | } 108 | 109 | object Rogue extends Rogue 110 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/RogueException.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | class RogueException(message: String, cause: Throwable) extends RuntimeException(message, cause) 6 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/index/IndexChecker.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | package com.foursquare.index 3 | 4 | import com.foursquare.rogue.{DocumentScan, Index, IndexScan, PartialIndexScan, MongoHelpers, Query, QueryClause, QueryHelpers} 5 | 6 | trait IndexChecker { 7 | /** 8 | * Verifies that the indexes expected for a query actually exist in the mongo database. 9 | * Logs an error via {@link QueryLogger#logIndexMismatch} if there is no matching index. 10 | * This version of validaateIndexExpectations is intended for use in cases where 11 | * the indexes are not explicitly declared in the class, but the caller knows what set 12 | * of indexes are actually available. 13 | * @param query the query being validated. 14 | * @param indexes a list of the indexes 15 | * @return true if the required indexes are found, false otherwise. 16 | */ 17 | def validateIndexExpectations(query: Query[_, _, _], indexes: List[MongoIndex[_]]): Boolean 18 | 19 | /** 20 | * Verifies that the index expected by a query both exists, and will be used by MongoDB 21 | * to execute that query. (Due to vagaries of the MongoDB implementation, sometimes a 22 | * conceptually usable index won't be found.) 23 | * @param query the query 24 | * @param indexes the list of indexes that exist in the database 25 | */ 26 | def validateQueryMatchesSomeIndex(query: Query[_, _, _], indexes: List[MongoIndex[_]]): Boolean 27 | } 28 | 29 | /** 30 | * A utility object which provides the capability to verify if the set of indexes that 31 | * actually exist for a MongoDB collection match the indexes that are expected by 32 | * a query. 33 | */ 34 | object MongoIndexChecker extends IndexChecker { 35 | 36 | /** 37 | * Flattens an arbitrary query into DNF - that is, into a list of query alternatives 38 | * implicitly joined by logical "or", where each of alternatives consists of a list of query 39 | * clauses implicitly joined by "and". 40 | */ 41 | def flattenCondition(condition: MongoHelpers.AndCondition): List[List[QueryClause[_]]] = { 42 | condition.orCondition match { 43 | case None => List(condition.clauses) 44 | case Some(or) => for { 45 | subconditions <- or.conditions 46 | subclauses <- flattenCondition(subconditions) 47 | } yield (condition.clauses ++ subclauses) 48 | } 49 | } 50 | 51 | def normalizeCondition(condition: MongoHelpers.AndCondition): List[List[QueryClause[_]]] = { 52 | flattenCondition(condition).map(_.filter(_.expectedIndexBehavior != DocumentScan)) 53 | } 54 | 55 | /** 56 | * Verifies that the indexes expected for a query actually exist in the mongo database. 57 | * Logs an error via {@link QueryLogger#logIndexMismatch} if there is no matching index. 58 | * This version of validaateIndexExpectations is intended for use in cases where 59 | * the indexes are not explicitly declared in the class, but the caller knows what set 60 | * of indexes are actually available. 61 | * @param query the query being validated. 62 | * @param indexes a list of the indexes 63 | * @return true if the required indexes are found, false otherwise. 64 | */ 65 | override def validateIndexExpectations(query: Query[_, _, _], indexes: List[MongoIndex[_]]): Boolean = { 66 | val baseConditions = normalizeCondition(query.condition); 67 | val conditions = baseConditions.map(_.filter(_.expectedIndexBehavior != DocumentScan)) 68 | 69 | conditions.forall(clauses => { 70 | clauses.forall(clause => { 71 | // DocumentScan expectations have been filtered out at this point. 72 | // We just have to worry about expectations being more optimistic than actual. 73 | val badExpectations = List( 74 | Index -> List(PartialIndexScan, IndexScan, DocumentScan), 75 | IndexScan -> List(DocumentScan) 76 | ) 77 | badExpectations.forall{ case (expectation, badActual) => { 78 | if (clause.expectedIndexBehavior == expectation && 79 | badActual.exists(_ == clause.actualIndexBehavior)) { 80 | signalError(query, 81 | "Query is expecting %s on %s but actual behavior is %s. query = %s" format 82 | (clause.expectedIndexBehavior, clause.fieldName, clause.actualIndexBehavior, query.toString)) 83 | } else true 84 | }} 85 | }) 86 | }) 87 | } 88 | 89 | /** 90 | * Verifies that the index expected by a query both exists, and will be used by MongoDB 91 | * to execute that query. (Due to vagaries of the MongoDB implementation, sometimes a 92 | * conceptually usable index won't be found.) 93 | * @param query the query 94 | * @param indexes the list of indexes that exist in the database 95 | */ 96 | override def validateQueryMatchesSomeIndex(query: Query[_, _, _], indexes: List[MongoIndex[_]]) = { 97 | val conditions = normalizeCondition(query.condition) 98 | lazy val indexString = indexes.map(idx => "{%s}".format(idx.toString())).mkString(", ") 99 | conditions.forall(clauses => { 100 | clauses.isEmpty || matchesUniqueIndex(clauses) || 101 | indexes.exists(idx => matchesIndex(idx.asListMap.keys.toList, clauses) && logIndexHit(query, idx)) || 102 | signalError(query, "Query does not match an index! query: %s, indexes: %s" format ( 103 | query.toString, indexString)) 104 | }) 105 | } 106 | 107 | private def matchesUniqueIndex(clauses: List[QueryClause[_]]) = { 108 | // Special case for overspecified queries matching on the _id field. 109 | // TODO: Do the same for any overspecified query exactly matching a unique index. 110 | clauses.exists(clause => clause.fieldName == "_id" && clause.actualIndexBehavior == Index) 111 | } 112 | 113 | private def matchesIndex(index: List[String], 114 | clauses: List[QueryClause[_]]) = { 115 | // Unless explicitly hinted, MongoDB will only use an index if the first 116 | // field in the index matches some query field. 117 | clauses.exists(_.fieldName == index.head) && 118 | matchesCompoundIndex(index, clauses, scanning = false) 119 | } 120 | 121 | /** 122 | * Matches a compound index against a list of query clauses, verifying that 123 | * each query clause has its index expectations matched by a field of the 124 | * index. 125 | * @param index the index to be checked. 126 | * @param clauses a list of query clauses joined by logical "and". 127 | * @return true if every clause of the query is matched by a field of the 128 | * index; false otherwise. 129 | */ 130 | private def matchesCompoundIndex(index: List[String], 131 | clauses: List[QueryClause[_]], 132 | scanning: Boolean): Boolean = { 133 | if (clauses.isEmpty) { 134 | // All of the clauses have been matched to an index field. We are done! 135 | true 136 | } else { 137 | index match { 138 | case Nil => { 139 | // Oh no! The index is exhausted but we still have clauses to match. 140 | false 141 | } 142 | case field :: rest => { 143 | val (matchingClauses, remainingClauses) = clauses.partition(_.fieldName == field) 144 | matchingClauses match { 145 | case matchingClause :: _ => { 146 | // If a previous field caused a scan, this field must scan too. 147 | val expectationOk = !scanning || matchingClause.expectedIndexBehavior == IndexScan 148 | // If this field causes a scan, later fields must scan too. 149 | val nowScanning = scanning || 150 | matchingClause.actualIndexBehavior == IndexScan || 151 | matchingClause.actualIndexBehavior == PartialIndexScan 152 | expectationOk && matchesCompoundIndex(rest, remainingClauses, scanning = nowScanning) 153 | } 154 | case Nil => { 155 | // We can skip a field in the index, but everything after it must scan. 156 | matchesCompoundIndex(rest, remainingClauses, scanning = true) 157 | } 158 | } 159 | } 160 | } 161 | } 162 | } 163 | 164 | /** 165 | * Utility method that allows us to signal an error from inside of a disjunctive expression. 166 | * e.g., "blah || blah || black || signalError(....)". 167 | * 168 | * @param query the query involved 169 | * @param msg a message string describing the error. 170 | */ 171 | private def signalError(query: Query[_, _, _], msg: String): Boolean = { 172 | QueryHelpers.logger.logIndexMismatch(query, "Indexing error: " + msg) 173 | false 174 | } 175 | 176 | private def logIndexHit(query: Query[_, _, _], index: MongoIndex[_]): Boolean = { 177 | QueryHelpers.logger.logIndexHit(query, index) 178 | true 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/index/IndexEnforcer.scala: -------------------------------------------------------------------------------- 1 | // // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | // package com.foursquare.index 4 | 5 | // import com.foursquare.rogue.MongoHelpers.AndCondition 6 | // import net.liftweb.mongodb.record.{MongoRecord, MongoMetaRecord} 7 | // import net.liftweb.record.Field 8 | 9 | // // *************************************************************************** 10 | // // *** Indexes 11 | // // *************************************************************************** 12 | 13 | // class IndexEnforcerBuilder[M <: MongoRecord[M]](meta: M with MongoMetaRecord[M] with IndexedRecord[M]) { 14 | // type MetaM = M with MongoMetaRecord[M] with IndexedRecord[M] 15 | 16 | // def useIndex[F1 <: Field[_, M]](i: MongoIndex1[M, F1, _]): IndexEnforcer1[M, NoIndexInfo, F1, HasntUsedIndex] = { 17 | // new IndexEnforcer1[M, NoIndexInfo, F1, HasntUsedIndex](meta, new BaseQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause](meta, None, None, None, None, None, AndCondition(Nil, None), None, None, None)) 18 | // } 19 | 20 | // def useIndex[F1 <: Field[_, M], F2 <: Field[_, M]](i: MongoIndex2[M, F1, _, F2, _]): IndexEnforcer2[M, NoIndexInfo, F1, F2, HasntUsedIndex] = { 21 | // new IndexEnforcer2[M, NoIndexInfo, F1, F2, HasntUsedIndex](meta, new BaseQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause](meta, None, None, None, None, None, AndCondition(Nil, None), None, None, None)) 22 | // } 23 | 24 | // def useIndex[F1 <: Field[_, M], F2 <: Field[_, M], F3 <: Field[_, M]](i: MongoIndex3[M, F1, _, F2, _, F3, _]): IndexEnforcer3[M, NoIndexInfo, F1, F2, F3, HasntUsedIndex] = { 25 | // new IndexEnforcer3[M, NoIndexInfo, F1, F2, F3, HasntUsedIndex](meta, new BaseQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause](meta, None, None, None, None, None, AndCondition(Nil, None), None, None, None)) 26 | // } 27 | 28 | // def useIndex[F1 <: Field[_, M], F2 <: Field[_, M], F3 <: Field[_, M], F4 <: Field[_, M]](i: MongoIndex4[M, F1, _, F2, _, F3, _, F4, _]): IndexEnforcer4[M, NoIndexInfo, F1, F2, F3, F4, HasntUsedIndex] = { 29 | // new IndexEnforcer4[M, NoIndexInfo, F1, F2, F3, F4, HasntUsedIndex](meta, new BaseQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause](meta, None, None, None, None, None, AndCondition(Nil, None), None, None, None)) 30 | // } 31 | 32 | // def useIndex[F1 <: Field[_, M], F2 <: Field[_, M], F3 <: Field[_, M], F4 <: Field[_, M], F5 <: Field[_, M]](i: MongoIndex5[M, F1, _, F2, _, F3, _, F4, _, F5, _]): IndexEnforcer5[M, NoIndexInfo, F1, F2, F3, F4, F5, HasntUsedIndex] = { 33 | // new IndexEnforcer5[M, NoIndexInfo, F1, F2, F3, F4, F5, HasntUsedIndex](meta, new BaseQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause](meta, None, None, None, None, None, AndCondition(Nil, None), None, None, None)) 34 | // } 35 | 36 | // def useIndex[F1 <: Field[_, M], F2 <: Field[_, M], F3 <: Field[_, M], F4 <: Field[_, M], F5 <: Field[_, M], F6 <: Field[_, M]](i: MongoIndex6[M, F1, _, F2, _, F3, _, F4, _, F5, _, F6, _]): IndexEnforcer6[M, NoIndexInfo, F1, F2, F3, F4, F5, F6, HasntUsedIndex] = { 37 | // new IndexEnforcer6[M, NoIndexInfo, F1, F2, F3, F4, F5, F6, HasntUsedIndex](meta, new BaseQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause](meta, None, None, None, None, None, AndCondition(Nil, None), None, None, None)) 38 | // } 39 | // } 40 | 41 | // case class IndexEnforcer1[M <: MongoRecord[M], 42 | // Ind <: MaybeIndexed, 43 | // F1 <: Field[_, M], 44 | // UsedInd <: MaybeUsedIndex](meta: M with MongoMetaRecord[M], 45 | // q: AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause]) { 46 | // def where[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause] = { 47 | // q.where(_ => clause(f1Func(meta))) 48 | // } 49 | 50 | // def and[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause] = { 51 | // q.and(_ => clause(f1Func(meta))) 52 | // } 53 | 54 | // def iscan[F, ClauseInd <: IndexScannable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd]): AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause] = { 55 | // q.iscan(_ => clause(f1Func(meta))) 56 | // } 57 | 58 | // def rangeScan(f1Func: M => F1)(implicit ev: UsedInd <:< UsedIndex): AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause] = q 59 | // } 60 | 61 | // case class IndexEnforcer2[M <: MongoRecord[M], 62 | // Ind <: MaybeIndexed, 63 | // F1 <: Field[_, M], 64 | // F2 <: Field[_, M], 65 | // UsedInd <: MaybeUsedIndex](meta: M with MongoMetaRecord[M], 66 | // q: AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause]) { 67 | // def where[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer1[M, Index, F2, UsedIndex] = { 68 | // new IndexEnforcer1[M, Index, F2, UsedIndex](meta, q.where(_ => clause(f1Func(meta)))) 69 | // } 70 | 71 | // def and[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer1[M, Index, F2, UsedIndex] = { 72 | // new IndexEnforcer1[M, Index, F2, UsedIndex](meta, q.and(_ => clause(f1Func(meta)))) 73 | // } 74 | 75 | // def iscan[F, ClauseInd <: IndexScannable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd]): IndexEnforcer1[M, IndexScan, F2, UsedIndex] = { 76 | // new IndexEnforcer1[M, IndexScan, F2, UsedIndex](meta, q.iscan(_ => clause(f1Func(meta)))) 77 | // } 78 | 79 | // def rangeScan(f1Func: M => F1)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer1[M, IndexScan, F2, UsedIndex] = { 80 | // new IndexEnforcer1[M, IndexScan, F2, UsedIndex](meta, q) 81 | // } 82 | 83 | // def rangeScan(f1Func: M => F1, f2Func: M => F2)(implicit ev: UsedInd <:< UsedIndex): AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause] = q 84 | // } 85 | 86 | // case class IndexEnforcer3[M <: MongoRecord[M], 87 | // Ind <: MaybeIndexed, 88 | // F1 <: Field[_, M], 89 | // F2 <: Field[_, M], 90 | // F3 <: Field[_, M], 91 | // UsedInd <: MaybeUsedIndex](meta: M with MongoMetaRecord[M], 92 | // q: AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause]) { 93 | // def where[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer2[M, Index, F2, F3, UsedIndex] = { 94 | // new IndexEnforcer2[M, Index, F2, F3, UsedIndex](meta, q.where(_ => clause(f1Func(meta)))) 95 | // } 96 | 97 | // def and[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer2[M, Index, F2, F3, UsedIndex] = { 98 | // new IndexEnforcer2[M, Index, F2, F3, UsedIndex](meta, q.and(_ => clause(f1Func(meta)))) 99 | // } 100 | 101 | // def iscan[F, ClauseInd <: IndexScannable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd]): IndexEnforcer2[M, IndexScan, F2, F3, UsedIndex] = { 102 | // new IndexEnforcer2[M, IndexScan, F2, F3, UsedIndex](meta, q.iscan(_ => clause(f1Func(meta)))) 103 | // } 104 | 105 | // def rangeScan(f1Func: M => F1)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer2[M, IndexScan, F2, F3, UsedIndex] = { 106 | // new IndexEnforcer2[M, IndexScan, F2, F3, UsedIndex](meta, q) 107 | // } 108 | 109 | // def rangeScan(f1Func: M => F1, f2Func: M => F2)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer1[M, IndexScan, F3, UsedIndex] = { 110 | // new IndexEnforcer1[M, IndexScan, F3, UsedIndex](meta, q) 111 | // } 112 | 113 | // def rangeScan(f1Func: M => F1, f2Func: M => F2, f3Func: M => F3)(implicit ev: UsedInd <:< UsedIndex): AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause] = q 114 | // } 115 | 116 | // case class IndexEnforcer4[M <: MongoRecord[M], 117 | // Ind <: MaybeIndexed, 118 | // F1 <: Field[_, M], 119 | // F2 <: Field[_, M], 120 | // F3 <: Field[_, M], 121 | // F4 <: Field[_, M], 122 | // UsedInd <: MaybeUsedIndex](meta: M with MongoMetaRecord[M], 123 | // q: AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause]) { 124 | // def where[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer3[M, Index, F2, F3, F4, UsedIndex] = { 125 | // new IndexEnforcer3[M, Index, F2, F3, F4, UsedIndex](meta, q.where(_ => clause(f1Func(meta)))) 126 | // } 127 | 128 | // def and[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer3[M, Index, F2, F3, F4, UsedIndex] = { 129 | // new IndexEnforcer3[M, Index, F2, F3, F4, UsedIndex](meta, q.and(_ => clause(f1Func(meta)))) 130 | // } 131 | 132 | // def iscan[F, ClauseInd <: IndexScannable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd]): IndexEnforcer3[M, IndexScan, F2, F3, F4, UsedIndex] = { 133 | // new IndexEnforcer3[M, IndexScan, F2, F3, F4, UsedIndex](meta, q.iscan(_ => clause(f1Func(meta)))) 134 | // } 135 | 136 | // def rangeScan(f1Func: M => F1)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer3[M, IndexScan, F2, F3, F4, UsedIndex] = { 137 | // new IndexEnforcer3[M, IndexScan, F2, F3, F4, UsedIndex](meta, q) 138 | // } 139 | 140 | // def rangeScan(f1Func: M => F1, f2Func: M => F2)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer2[M, IndexScan, F3, F4, UsedIndex] = { 141 | // new IndexEnforcer2[M, IndexScan, F3, F4, UsedIndex](meta, q) 142 | // } 143 | 144 | // def rangeScan(f1Func: M => F1, f2Func: M => F2, f3Func: M => F3)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer1[M, IndexScan, F4, UsedIndex] = { 145 | // new IndexEnforcer1[M, IndexScan, F4, UsedIndex](meta, q) 146 | // } 147 | 148 | // def rangeScan(f1Func: M => F1, f2Func: M => F2, f3Func: M => F3, f4Func: M => F4)(implicit ev: UsedInd <:< UsedIndex): AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause] = q 149 | // } 150 | 151 | // case class IndexEnforcer5[M <: MongoRecord[M], 152 | // Ind <: MaybeIndexed, 153 | // F1 <: Field[_, M], 154 | // F2 <: Field[_, M], 155 | // F3 <: Field[_, M], 156 | // F4 <: Field[_, M], 157 | // F5 <: Field[_, M], 158 | // UsedInd <: MaybeUsedIndex](meta: M with MongoMetaRecord[M], 159 | // q: AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause]) { 160 | // def where[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer4[M, Index, F2, F3, F4, F5, UsedIndex] = { 161 | // new IndexEnforcer4[M, Index, F2, F3, F4, F5, UsedIndex](meta, q.where(_ => clause(f1Func(meta)))) 162 | // } 163 | 164 | // def and[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer4[M, Index, F2, F3, F4, F5, UsedIndex] = { 165 | // new IndexEnforcer4[M, Index, F2, F3, F4, F5, UsedIndex](meta, q.and(_ => clause(f1Func(meta)))) 166 | // } 167 | 168 | // def iscan[F, ClauseInd <: IndexScannable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd]): IndexEnforcer4[M, IndexScan, F2, F3, F4, F5, UsedIndex] = { 169 | // new IndexEnforcer4[M, IndexScan, F2, F3, F4, F5, UsedIndex](meta, q.iscan(_ => clause(f1Func(meta)))) 170 | // } 171 | 172 | // def rangeScan(f1Func: M => F1)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer4[M, IndexScan, F2, F3, F4, F5, UsedIndex] = { 173 | // new IndexEnforcer4[M, IndexScan, F2, F3, F4, F5, UsedIndex](meta, q) 174 | // } 175 | 176 | // def rangeScan(f1Func: M => F1, f2Func: M => F2)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer3[M, IndexScan, F3, F4, F5, UsedIndex] = { 177 | // new IndexEnforcer3[M, IndexScan, F3, F4, F5, UsedIndex](meta, q) 178 | // } 179 | 180 | // def rangeScan(f1Func: M => F1, f2Func: M => F2, f3Func: M => F3)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer2[M, IndexScan, F4, F5, UsedIndex] = { 181 | // new IndexEnforcer2[M, IndexScan, F4, F5, UsedIndex](meta, q) 182 | // } 183 | 184 | // def rangeScan(f1Func: M => F1, f2Func: M => F2, f3Func: M => F3, f4Func: M => F4)(implicit ev: UsedInd <:< UsedIndex): IndexEnforcer1[M, IndexScan, F5, UsedIndex] = { 185 | // new IndexEnforcer1[M, IndexScan, F5, UsedIndex](meta, q) 186 | // } 187 | // } 188 | 189 | // case class IndexEnforcer6[M <: MongoRecord[M], 190 | // Ind <: MaybeIndexed, 191 | // F1 <: Field[_, M], 192 | // F2 <: Field[_, M], 193 | // F3 <: Field[_, M], 194 | // F4 <: Field[_, M], 195 | // F5 <: Field[_, M], 196 | // F6 <: Field[_, M], 197 | // UsedInd <: MaybeUsedIndex](meta: M with MongoMetaRecord[M], 198 | // q: AbstractQuery[M, M, Unordered, Unselected, Unlimited, Unskipped, HasNoOrClause]) { 199 | // def where[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer5[M, Index, F2, F3, F4, F5, F6, UsedIndex] = { 200 | // new IndexEnforcer5[M, Index, F2, F3, F4, F5, F6, UsedIndex](meta, q.where(_ => clause(f1Func(meta)))) 201 | // } 202 | 203 | // def and[F, ClauseInd <: Indexable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd])(implicit ev: Ind <:< Indexable): IndexEnforcer5[M, Index, F2, F3, F4, F5, F6, UsedIndex] = { 204 | // new IndexEnforcer5[M, Index, F2, F3, F4, F5, F6, UsedIndex](meta, q.and(_ => clause(f1Func(meta)))) 205 | // } 206 | 207 | // def iscan[F, ClauseInd <: IndexScannable](f1Func: M => F1)(clause: F1 => IndexableQueryClause[F, ClauseInd]): IndexEnforcer5[M, IndexScan, F2, F3, F4, F5, F6, UsedIndex] = { 208 | // new IndexEnforcer5[M, IndexScan, F2, F3, F4, F5, F6, UsedIndex](meta, q.iscan(_ => clause(f1Func(meta)))) 209 | // } 210 | 211 | // // IndexEnforcer6 doesn't have methods to scan any later fields in the index 212 | // // because there's no way that we got here via an iscan on a bigger index -- 213 | // // there are no bigger indexes. We require that the first column on the index 214 | // // gets used. 215 | // } 216 | 217 | -------------------------------------------------------------------------------- /rogue-core/src/main/scala/com/foursquare/rogue/package.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare 4 | 5 | package object rogue { 6 | 7 | type InitialState = Unordered with Unselected with Unlimited with Unskipped with HasNoOrClause with ShardKeyNotSpecified 8 | type OrderedState = Ordered with Unselected with Unlimited with Unskipped with HasNoOrClause with ShardKeyNotSpecified 9 | 10 | type SimpleQuery[M] = Query[M, M, InitialState] 11 | type OrderedQuery[M] = Query[M, M, OrderedState] 12 | 13 | trait Sharded 14 | 15 | trait ShardKey[V] { 16 | def name: String 17 | def eqs(v: V) = new EqClause(this.name, v) with ShardKeyClause 18 | def in[L <% Traversable[V]](vs: L) = new InQueryClause(this.name, QueryHelpers.validatedList(vs.toSet)) with ShardKeyClause 19 | } 20 | 21 | /** 22 | * Iteratee helper classes 23 | * @tparam S state type 24 | */ 25 | object Iter { 26 | sealed trait Command[S] { 27 | def state: S 28 | } 29 | case class Continue[S](state: S) extends Command[S] 30 | case class Return[S](state: S) extends Command[S] 31 | 32 | sealed trait Event[+R] 33 | case class Item[R](r: R) extends Event[R] 34 | case class Error(e: Exception) extends Event[Nothing] 35 | case object EOF extends Event[Nothing] 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /rogue-core/src/test/scala/com/foursquare/rogue/TrivialORMQueryTest.scala: -------------------------------------------------------------------------------- 1 | package com.foursquare.rogue 2 | 3 | import com.foursquare.index.UntypedMongoIndex 4 | import com.foursquare.field.{Field, OptionalField} 5 | import com.mongodb.{DB, DBCollection, DBObject, Mongo, ServerAddress, WriteConcern} 6 | import com.foursquare.rogue.MongoHelpers.{AndCondition, MongoModify, MongoSelect} 7 | import org.junit.{Before, Test} 8 | import org.specs2.matcher.JUnitMustMatchers 9 | 10 | /** A trivial ORM layer that implements the interfaces rogue needs. The goal is 11 | * to make sure that rogue-core works without the assistance of rogue-lift. 12 | * Ideally this would be even smaller; as it is, I needed to copy-paste some 13 | * code from the Lift implementations. */ 14 | object TrivialORM { 15 | trait Meta[R] { 16 | def collectionName: String 17 | def fromDBObject(dbo: DBObject): R 18 | } 19 | 20 | val mongo = { 21 | val MongoPort = Option(System.getenv("MONGO_PORT")).map(_.toInt).getOrElse(37648) 22 | new Mongo(new ServerAddress("localhost", MongoPort)) 23 | } 24 | 25 | def disconnectFromMongo = { 26 | mongo.close 27 | } 28 | 29 | type MB = Meta[_] 30 | class MyDBCollectionFactory(db: DB) extends DBCollectionFactory[MB] { 31 | override def getDBCollection[M <: MB](query: Query[M, _, _]): DBCollection = { 32 | db.getCollection(query.meta.collectionName) 33 | } 34 | override def getPrimaryDBCollection[M <: MB](query: Query[M, _, _]): DBCollection = { 35 | db.getCollection(query.meta.collectionName) 36 | } 37 | override def getInstanceName[M <: MB](query: Query[M, _, _]): String = { 38 | db.getName 39 | } 40 | override def getIndexes[M <: MB](query: Query[M, _, _]): Option[List[UntypedMongoIndex]] = { 41 | None 42 | } 43 | } 44 | 45 | class MyQueryExecutor extends QueryExecutor[Meta[_]] { 46 | override val adapter = new MongoJavaDriverAdapter[Meta[_]](new MyDBCollectionFactory(mongo.getDB("test"))) 47 | override val optimizer = new QueryOptimizer 48 | override val defaultWriteConcern: WriteConcern = WriteConcern.SAFE 49 | 50 | protected def serializer[M <: Meta[_], R]( 51 | meta: M, 52 | select: Option[MongoSelect[M, R]] 53 | ): RogueSerializer[R] = new RogueSerializer[R] { 54 | override def fromDBObject(dbo: DBObject): R = select match { 55 | case Some(MongoSelect(Nil, transformer)) => 56 | // A MongoSelect clause exists, but has empty fields. Return null. 57 | // This is used for .exists(), where we just want to check the number 58 | // of returned results is > 0. 59 | transformer(null) 60 | 61 | case Some(MongoSelect(fields, transformer)) => 62 | transformer(fields.map(f => f.valueOrDefault(Option(dbo.get(f.field.name))))) 63 | 64 | case None => 65 | meta.fromDBObject(dbo).asInstanceOf[R] 66 | } 67 | } 68 | } 69 | 70 | object Implicits extends Rogue { 71 | implicit def meta2Query[M <: Meta[R], R](meta: M with Meta[R]): Query[M, R, InitialState] = { 72 | Query[M, R, InitialState]( 73 | meta, meta.collectionName, None, None, None, None, None, AndCondition(Nil, None), None, None, None) 74 | } 75 | } 76 | } 77 | 78 | case class SimpleRecord(a: Int, b: String) 79 | 80 | object SimpleRecord extends TrivialORM.Meta[SimpleRecord] { 81 | val a = new OptionalField[Int, SimpleRecord.type] { override val owner = SimpleRecord; override val name = "a" } 82 | val b = new OptionalField[String, SimpleRecord.type] { override val owner = SimpleRecord; override val name = "b" } 83 | 84 | override val collectionName = "simple_records" 85 | override def fromDBObject(dbo: DBObject): SimpleRecord = { 86 | new SimpleRecord(dbo.get(a.name).asInstanceOf[Int], dbo.get(b.name).asInstanceOf[String]) 87 | } 88 | } 89 | 90 | // TODO(nsanch): Everything in the rogue-lift tests should move here, except for the lift-specific extensions. 91 | class TrivialORMQueryTest extends JUnitMustMatchers { 92 | val executor = new TrivialORM.MyQueryExecutor 93 | 94 | import TrivialORM.Implicits._ 95 | 96 | @Before 97 | def cleanUpMongo = { 98 | executor.bulkDelete_!!(SimpleRecord) 99 | } 100 | 101 | @Test 102 | def canBuildQuery: Unit = { 103 | (SimpleRecord: Query[SimpleRecord.type, SimpleRecord, InitialState]) .toString() must_== """db.simple_records.find({ })""" 104 | SimpleRecord.where(_.a eqs 1) .toString() must_== """db.simple_records.find({ "a" : 1})""" 105 | } 106 | 107 | @Test 108 | def canExecuteQuery: Unit = { 109 | executor.fetch(SimpleRecord.where(_.a eqs 1)) must_== Nil 110 | executor.count(SimpleRecord) must_== 0 111 | } 112 | 113 | @Test 114 | def canUpsertAndGetResults: Unit = { 115 | executor.count(SimpleRecord) must_== 0 116 | 117 | executor.upsertOne(SimpleRecord.modify(_.a setTo 1).and(_.b setTo "foo")) 118 | 119 | executor.count(SimpleRecord) must_== 1 120 | 121 | val results = executor.fetch(SimpleRecord.where(_.a eqs 1)) 122 | results.size must_== 1 123 | results(0).a must_== 1 124 | results(0).b must_== "foo" 125 | 126 | executor.fetch(SimpleRecord.where(_.a eqs 1).select(_.a)) must_== List(Some(1)) 127 | executor.fetch(SimpleRecord.where(_.a eqs 1).select(_.b)) must_== List(Some("foo")) 128 | executor.fetch(SimpleRecord.where(_.a eqs 1).select(_.a, _.b)) must_== List((Some(1), Some("foo"))) 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /rogue-index/build.sbt: -------------------------------------------------------------------------------- 1 | libraryDependencies <++= (scalaVersion) { scalaVersion => 2 | def sv(s: String) = s + "_" + (scalaVersion match { 3 | case "2.11.5" => "2.11" 4 | case "2.10.4" => "2.10" 5 | }) 6 | Seq( 7 | "com.foursquare" % sv("rogue-field") % "2.4.0" % "compile" 8 | ) 9 | } 10 | 11 | Seq(RogueBuild.defaultSettings: _*) 12 | -------------------------------------------------------------------------------- /rogue-index/src/main/scala/com/foursquare/index/IndexModifier.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.index 4 | 5 | case class IndexModifier(value: Any) 6 | 7 | object Asc extends IndexModifier(1) 8 | object Desc extends IndexModifier(-1) 9 | object TwoD extends IndexModifier("2d") 10 | -------------------------------------------------------------------------------- /rogue-index/src/main/scala/com/foursquare/index/IndexedRecord.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.index 4 | 5 | /** 6 | * A trait that represents the fact that a record type includes a list 7 | * of the indexes that exist in MongoDB for that type. 8 | */ 9 | trait IndexedRecord[M] { 10 | val mongoIndexList: List[MongoIndex[_]] = List() 11 | } 12 | -------------------------------------------------------------------------------- /rogue-index/src/main/scala/com/foursquare/index/MongoIndex.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.index 4 | 5 | import com.foursquare.field.Field 6 | import scala.collection.immutable.ListMap 7 | 8 | trait UntypedMongoIndex { 9 | def asListMap: ListMap[String, Any] 10 | 11 | override def toString() = 12 | asListMap.map(fld => "%s:%s".format(fld._1, fld._2)).mkString(", ") 13 | } 14 | 15 | trait MongoIndex[R] extends UntypedMongoIndex 16 | 17 | case class MongoIndex1[R]( 18 | f1: Field[_, R], 19 | m1: IndexModifier 20 | ) extends MongoIndex[R] { 21 | def asListMap = ListMap((f1.name, m1.value)) 22 | } 23 | 24 | case class MongoIndex2[R]( 25 | f1: Field[_, R], 26 | m1: IndexModifier, 27 | f2: Field[_, R], 28 | m2: IndexModifier 29 | ) extends MongoIndex[R] { 30 | def asListMap = ListMap((f1.name, m1.value), (f2.name, m2.value)) 31 | } 32 | 33 | case class MongoIndex3[R]( 34 | f1: Field[_, R], 35 | m1: IndexModifier, 36 | f2: Field[_, R], 37 | m2: IndexModifier, 38 | f3: Field[_, R], 39 | m3: IndexModifier 40 | ) extends MongoIndex[R] { 41 | def asListMap = ListMap((f1.name, m1.value), (f2.name, m2.value), (f3.name, m3.value)) 42 | } 43 | 44 | case class MongoIndex4[R]( 45 | f1: Field[_, R], 46 | m1: IndexModifier, 47 | f2: Field[_, R], 48 | m2: IndexModifier, 49 | f3: Field[_, R], 50 | m3: IndexModifier, 51 | f4: Field[_, R], 52 | m4: IndexModifier 53 | ) extends MongoIndex[R] { 54 | def asListMap = 55 | ListMap( 56 | (f1.name, m1.value), 57 | (f2.name, m2.value), 58 | (f3.name, m3.value), 59 | (f4.name, m4.value)) 60 | } 61 | 62 | case class MongoIndex5[R]( 63 | f1: Field[_, R], 64 | m1: IndexModifier, 65 | f2: Field[_, R], 66 | m2: IndexModifier, 67 | f3: Field[_, R], 68 | m3: IndexModifier, 69 | f4: Field[_, R], 70 | m4: IndexModifier, 71 | f5: Field[_, R], 72 | m5: IndexModifier 73 | ) extends MongoIndex[R] { 74 | def asListMap = 75 | ListMap( 76 | (f1.name, m1.value), 77 | (f2.name, m2.value), 78 | (f3.name, m3.value), 79 | (f4.name, m4.value), 80 | (f5.name, m5.value)) 81 | } 82 | 83 | case class MongoIndex6[R]( 84 | f1: Field[_, R], 85 | m1: IndexModifier, 86 | f2: Field[_, R], 87 | m2: IndexModifier, 88 | f3: Field[_, R], 89 | m3: IndexModifier, 90 | f4: Field[_, R], 91 | m4: IndexModifier, 92 | f5: Field[_, R], 93 | m5: IndexModifier, 94 | f6: Field[_, R], 95 | m6: IndexModifier 96 | ) extends MongoIndex[R] { 97 | def asListMap = 98 | ListMap( 99 | (f1.name, m1.value), 100 | (f2.name, m2.value), 101 | (f3.name, m3.value), 102 | (f4.name, m4.value), 103 | (f5.name, m5.value), 104 | (f6.name, m6.value)) 105 | } 106 | 107 | case class IndexBuilder[M](rec: M) { 108 | def index( 109 | f1: M => Field[_, M], 110 | m1: IndexModifier 111 | ): MongoIndex1[M] = 112 | MongoIndex1[M](f1(rec), m1) 113 | 114 | def index( 115 | f1: M => Field[_, M], 116 | m1: IndexModifier, 117 | f2: M => Field[_, M], 118 | m2: IndexModifier 119 | ): MongoIndex2[M] = 120 | MongoIndex2[M](f1(rec), m1, f2(rec), m2) 121 | 122 | def index( 123 | f1: M => Field[_, M], 124 | m1: IndexModifier, 125 | f2: M => Field[_, M], 126 | m2: IndexModifier, 127 | f3: M => Field[_, M], 128 | m3: IndexModifier 129 | ): MongoIndex3[M] = 130 | MongoIndex3[M](f1(rec), m1, f2(rec), m2, f3(rec), m3) 131 | 132 | def index( 133 | f1: M => Field[_, M], 134 | m1: IndexModifier, 135 | f2: M => Field[_, M], 136 | m2: IndexModifier, 137 | f3: M => Field[_, M], 138 | m3: IndexModifier, 139 | f4: M => Field[_, M], 140 | m4: IndexModifier 141 | ): MongoIndex4[M] = 142 | MongoIndex4[M](f1(rec), m1, f2(rec), m2, f3(rec), m3, f4(rec), m4) 143 | 144 | def index( 145 | f1: M => Field[_, M], 146 | m1: IndexModifier, 147 | f2: M => Field[_, M], 148 | m2: IndexModifier, 149 | f3: M => Field[_, M], 150 | m3: IndexModifier, 151 | f4: M => Field[_, M], 152 | m4: IndexModifier, 153 | f5: M => Field[_, M], 154 | m5: IndexModifier 155 | ): MongoIndex5[M] = 156 | MongoIndex5[M](f1(rec), m1, f2(rec), m2, f3(rec), m3, f4(rec), m4, f5(rec), m5) 157 | 158 | def index( 159 | f1: M => Field[_, M], 160 | m1: IndexModifier, 161 | f2: M => Field[_, M], 162 | m2: IndexModifier, 163 | f3: M => Field[_, M], 164 | m3: IndexModifier, 165 | f4: M => Field[_, M], 166 | m4: IndexModifier, 167 | f5: M => Field[_, M], 168 | m5: IndexModifier, 169 | f6: M => Field[_, M], 170 | m6: IndexModifier 171 | ): MongoIndex6[M] = 172 | MongoIndex6[M](f1(rec), m1, f2(rec), m2, f3(rec), m3, f4(rec), m4, f5(rec), m5, f6(rec), m6) 173 | } 174 | -------------------------------------------------------------------------------- /rogue-lift/build.sbt: -------------------------------------------------------------------------------- 1 | libraryDependencies <++= (scalaVersion) { scalaVersion => 2 | val liftVersion = "2.6" 3 | def sv(s: String) = s + "_" + (scalaVersion match { 4 | case "2.11.5" => "2.11" 5 | case "2.10.4" => "2.10" 6 | }) 7 | Seq( 8 | "net.liftweb" % sv("lift-util") % liftVersion % "compile" intransitive(), 9 | "net.liftweb" % sv("lift-common") % liftVersion % "compile" intransitive(), 10 | "net.liftweb" % sv("lift-record") % liftVersion % "compile" intransitive(), 11 | "net.liftweb" % sv("lift-mongodb-record") % liftVersion % "compile" intransitive(), 12 | "net.liftweb" % sv("lift-mongodb") % liftVersion % "compile" intransitive(), 13 | "net.liftweb" % sv("lift-webkit") % liftVersion % "compile" intransitive(), 14 | "net.liftweb" % sv("lift-json") % liftVersion % "compile", 15 | "joda-time" % "joda-time" % "2.1" % "compile", 16 | "org.joda" % "joda-convert" % "1.2" % "compile", 17 | "org.mongodb" % "mongo-java-driver" % "2.12.5" % "compile") 18 | } 19 | 20 | Seq(RogueBuild.defaultSettings: _*) 21 | -------------------------------------------------------------------------------- /rogue-lift/src/main/scala/com/foursquare/rogue/ExecutableQuery.scala: -------------------------------------------------------------------------------- 1 | package com.foursquare.rogue 2 | 3 | // Copyright 2012 Foursquare Labs Inc. All Rights Reserved. 4 | 5 | import com.foursquare.field.Field 6 | import com.foursquare.rogue.MongoHelpers.MongoSelect 7 | import com.foursquare.rogue.Rogue._ 8 | import com.mongodb.WriteConcern 9 | 10 | case class ExecutableQuery[MB, M <: MB, R, State]( 11 | query: Query[M, R, State], 12 | db: QueryExecutor[MB] 13 | )(implicit ev: ShardingOk[M, State]) { 14 | 15 | /** 16 | * Gets the size of the query result. This should only be called on queries that do not 17 | * have limits or skips. 18 | */ 19 | def count(): Long = 20 | db.count(query) 21 | 22 | /** 23 | * Returns the number of distinct values returned by a query. The query must not have 24 | * limit or skip clauses. 25 | */ 26 | def countDistinct[V](field: M => Field[V, _]): Long = 27 | db.countDistinct(query)(field.asInstanceOf[M => Field[V, M]]) 28 | 29 | /** 30 | * Returns a list of distinct values returned by a query. The query must not have 31 | * limit or skip clauses. 32 | */ 33 | def distinct[V](field: M => Field[V, _]): List[V] = 34 | db.distinct(query)(field.asInstanceOf[M => Field[V, M]]) 35 | 36 | /** 37 | * Checks if there are any records that match this query. 38 | */ 39 | def exists()(implicit ev: State <:< Unlimited with Unskipped): Boolean = { 40 | val q = query.copy(select = Some(MongoSelect[M, Null](Nil, _ => null))) 41 | db.fetch(q.limit(1)).size > 0 42 | } 43 | 44 | /** 45 | * Executes a function on each record value returned by a query. 46 | * @param f a function to be invoked on each fetched record. 47 | * @return nothing. 48 | */ 49 | def foreach(f: R => Unit): Unit = 50 | db.foreach(query)(f) 51 | 52 | /** 53 | * Execute the query, returning all of the records that match the query. 54 | * @return a list containing the records that match the query 55 | */ 56 | def fetch(): List[R] = 57 | db.fetch(query) 58 | 59 | /** 60 | * Execute a query, returning no more than a specified number of result records. The 61 | * query must not have a limit clause. 62 | * @param limit the maximum number of records to return. 63 | */ 64 | def fetch[S2](limit: Int)(implicit ev1: AddLimit[State, S2], ev2: ShardingOk[M, S2]): List[R] = 65 | db.fetch(query.limit(limit)) 66 | 67 | /** 68 | * fetch a batch of results, and execute a function on each element of the list. 69 | * @param f the function to invoke on the records that match the query. 70 | * @return a list containing the results of invoking the function on each record. 71 | */ 72 | def fetchBatch[T](batchSize: Int)(f: List[R] => List[T]): List[T] = 73 | db.fetchBatch(query, batchSize)(f).toList 74 | 75 | 76 | /** 77 | * Fetches the first record that matches the query. The query must not contain a "limited" clause. 78 | * @return an option record containing either the first result that matches the 79 | * query, or None if there are no records that match. 80 | */ 81 | def get[S2]()(implicit ev1: AddLimit[State, S2], ev2: ShardingOk[M, S2]): Option[R] = 82 | db.fetchOne(query) 83 | 84 | /** 85 | * Fetches the records that match the query in paginated form. The query must not contain 86 | * a "limit" clause. 87 | * @param countPerPage the number of records to be contained in each page of the result. 88 | */ 89 | def paginate(countPerPage: Int) 90 | (implicit ev1: Required[State, Unlimited with Unskipped], 91 | ev2: ShardingOk[M, State]) = { 92 | new PaginatedQuery(ev1(query), db, countPerPage) 93 | } 94 | 95 | /** 96 | * Delete all of the records that match the query. The query must not contain any "skip", 97 | * "limit", or "select" clauses. Sends the delete operation to mongo, and returns - does 98 | * not wait for the delete to be finished. 99 | */ 100 | def bulkDelete_!!!() 101 | (implicit ev1: Required[State, Unselected with Unlimited with Unskipped]): Unit = 102 | db.bulkDelete_!!(query) 103 | 104 | /** 105 | * Delete all of the records that match the query. The query must not contain any "skip", 106 | * "limit", or "select" clauses. Sends the delete operation to mongo, and waits for the 107 | * delete operation to complete before returning to the caller. 108 | */ 109 | def bulkDelete_!!(concern: WriteConcern) 110 | (implicit ev1: Required[State, Unselected with Unlimited with Unskipped]): Unit = 111 | db.bulkDelete_!!(query, concern) 112 | 113 | /** 114 | * Finds the first record that matches the query (if any), fetches it, and then deletes it. 115 | * A copy of the deleted record is returned to the caller. 116 | */ 117 | def findAndDeleteOne()(implicit ev: RequireShardKey[M, State]): Option[R] = 118 | db.findAndDeleteOne(query) 119 | 120 | /** 121 | * Return a string containing details about how the query would be executed in mongo. 122 | * In particular, this is useful for finding out what indexes will be used by the query. 123 | */ 124 | def explain(): String = 125 | db.explain(query) 126 | 127 | def iterate[S](state: S)(handler: (S, Iter.Event[R]) => Iter.Command[S]): S = 128 | db.iterate(query, state)(handler) 129 | 130 | def iterateBatch[S](batchSize: Int, state: S)(handler: (S, Iter.Event[List[R]]) => Iter.Command[S]): S = 131 | db.iterateBatch(query, batchSize, state)(handler) 132 | } 133 | 134 | case class ExecutableModifyQuery[MB, M <: MB, State](query: ModifyQuery[M, State], 135 | db: QueryExecutor[MB]) { 136 | def updateMulti(): Unit = 137 | db.updateMulti(query) 138 | 139 | def updateOne()(implicit ev: RequireShardKey[M, State]): Unit = 140 | db.updateOne(query) 141 | 142 | def upsertOne()(implicit ev: RequireShardKey[M, State]): Unit = 143 | db.upsertOne(query) 144 | 145 | def updateMulti(writeConcern: WriteConcern): Unit = 146 | db.updateMulti(query, writeConcern) 147 | 148 | def updateOne(writeConcern: WriteConcern)(implicit ev: RequireShardKey[M, State]): Unit = 149 | db.updateOne(query, writeConcern) 150 | 151 | def upsertOne(writeConcern: WriteConcern)(implicit ev: RequireShardKey[M, State]): Unit = 152 | db.upsertOne(query, writeConcern) 153 | } 154 | 155 | case class ExecutableFindAndModifyQuery[MB, M <: MB, R]( 156 | query: FindAndModifyQuery[M, R], 157 | db: QueryExecutor[MB] 158 | ) { 159 | def updateOne(returnNew: Boolean = false): Option[R] = 160 | db.findAndUpdateOne(query, returnNew) 161 | 162 | def upsertOne(returnNew: Boolean = false): Option[R] = 163 | db.findAndUpsertOne(query, returnNew) 164 | } 165 | 166 | class PaginatedQuery[MB, M <: MB, R, +State <: Unlimited with Unskipped]( 167 | q: Query[M, R, State], 168 | db: QueryExecutor[MB], 169 | val countPerPage: Int, 170 | val pageNum: Int = 1 171 | )(implicit ev: ShardingOk[M, State]) { 172 | def copy() = new PaginatedQuery(q, db, countPerPage, pageNum) 173 | 174 | def setPage(p: Int) = if (p == pageNum) this else new PaginatedQuery(q, db, countPerPage, p) 175 | 176 | def setCountPerPage(c: Int) = if (c == countPerPage) this else new PaginatedQuery(q, db, c, pageNum) 177 | 178 | lazy val countAll: Long = db.count(q) 179 | 180 | def fetch(): List[R] = { 181 | db.fetch(q.skip(countPerPage * (pageNum - 1)).limit(countPerPage)) 182 | } 183 | 184 | def numPages = math.ceil(countAll.toDouble / countPerPage.toDouble).toInt max 1 185 | } 186 | -------------------------------------------------------------------------------- /rogue-lift/src/main/scala/com/foursquare/rogue/HasMongoForeignObjectId.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import net.liftweb.mongodb.record.MongoRecord 6 | 7 | trait HasMongoForeignObjectId[RefType <: MongoRecord[RefType] with ObjectIdKey[RefType]] 8 | -------------------------------------------------------------------------------- /rogue-lift/src/main/scala/com/foursquare/rogue/LiftQueryExecutor.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.foursquare.index.{IndexedRecord, UntypedMongoIndex} 6 | import com.foursquare.rogue.MongoHelpers.MongoSelect 7 | import com.mongodb.{DBCollection, DBObject} 8 | import net.liftweb.common.{Box, Full} 9 | import net.liftweb.mongodb.record.{BsonRecord, BsonMetaRecord, MongoRecord, MongoMetaRecord} 10 | import net.liftweb.mongodb.MongoDB 11 | import net.liftweb.mongodb.record.field.BsonRecordField 12 | import net.liftweb.record.Record 13 | import org.bson.types.BasicBSONList 14 | import sun.reflect.generics.reflectiveObjects.NotImplementedException 15 | 16 | object LiftDBCollectionFactory extends DBCollectionFactory[MongoRecord[_] with MongoMetaRecord[_]] { 17 | override def getDBCollection[M <: MongoRecord[_] with MongoMetaRecord[_]](query: Query[M, _, _]): DBCollection = { 18 | MongoDB.useSession(query.meta.connectionIdentifier){ db => 19 | db.getCollection(query.collectionName) 20 | } 21 | } 22 | override def getPrimaryDBCollection[M <: MongoRecord[_] with MongoMetaRecord[_]](query: Query[M, _, _]): DBCollection = { 23 | MongoDB.useSession(query.meta/* TODO: .master*/.connectionIdentifier){ db => 24 | db.getCollection(query.collectionName) 25 | } 26 | } 27 | override def getInstanceName[M <: MongoRecord[_] with MongoMetaRecord[_]](query: Query[M, _, _]): String = { 28 | query.meta.connectionIdentifier.toString 29 | } 30 | 31 | /** 32 | * Retrieves the list of indexes declared for the record type associated with a 33 | * query. If the record type doesn't declare any indexes, then returns None. 34 | * @param query the query 35 | * @return the list of indexes, or an empty list. 36 | */ 37 | override def getIndexes[M <: MongoRecord[_] with MongoMetaRecord[_]](query: Query[M, _, _]): Option[List[UntypedMongoIndex]] = { 38 | val queryMetaRecord = query.meta 39 | if (queryMetaRecord.isInstanceOf[IndexedRecord[_]]) { 40 | Some(queryMetaRecord.asInstanceOf[IndexedRecord[_]].mongoIndexList) 41 | } else { 42 | None 43 | } 44 | } 45 | } 46 | 47 | class LiftAdapter(dbCollectionFactory: DBCollectionFactory[MongoRecord[_] with MongoMetaRecord[_]]) 48 | extends MongoJavaDriverAdapter(dbCollectionFactory) 49 | 50 | object LiftAdapter extends LiftAdapter(LiftDBCollectionFactory) 51 | 52 | class LiftQueryExecutor(override val adapter: MongoJavaDriverAdapter[MongoRecord[_] with MongoMetaRecord[_]]) extends QueryExecutor[MongoRecord[_] with MongoMetaRecord[_]] { 53 | override def defaultWriteConcern = QueryHelpers.config.defaultWriteConcern 54 | override lazy val optimizer = new QueryOptimizer 55 | 56 | override protected def serializer[M <: MongoRecord[_] with MongoMetaRecord[_], R]( 57 | meta: M, 58 | select: Option[MongoSelect[M, R]] 59 | ): RogueSerializer[R] = { 60 | new RogueSerializer[R] { 61 | override def fromDBObject(dbo: DBObject): R = select match { 62 | case Some(MongoSelect(Nil, transformer)) => 63 | // A MongoSelect clause exists, but has empty fields. Return null. 64 | // This is used for .exists(), where we just want to check the number 65 | // of returned results is > 0. 66 | transformer(null) 67 | case Some(MongoSelect(fields, transformer)) => 68 | val inst = meta.createRecord.asInstanceOf[MongoRecord[_]] 69 | 70 | LiftQueryExecutorHelpers.setInstanceFieldFromDbo(inst, dbo, "_id") 71 | 72 | val values = 73 | fields.map(fld => { 74 | val valueOpt = LiftQueryExecutorHelpers.setInstanceFieldFromDbo(inst, dbo, fld.field.name) 75 | fld.valueOrDefault(valueOpt) 76 | }) 77 | 78 | transformer(values) 79 | case None => 80 | meta.fromDBObject(dbo).asInstanceOf[R] 81 | } 82 | } 83 | } 84 | } 85 | 86 | object LiftQueryExecutor extends LiftQueryExecutor(LiftAdapter) 87 | 88 | object LiftQueryExecutorHelpers { 89 | import net.liftweb.record.{Field => LField} 90 | 91 | def setInstanceFieldFromDboList(instance: BsonRecord[_], dbo: DBObject, fieldNames: List[String]): Option[_] = { 92 | fieldNames match { 93 | case last :: Nil => 94 | val fld: Box[LField[_, _]] = instance.fieldByName(last) 95 | fld.flatMap(setLastFieldFromDbo(_, dbo, last)) 96 | case name :: rest => 97 | val fld: Box[LField[_, _]] = instance.fieldByName(name) 98 | dbo.get(name) match { 99 | case obj: DBObject => fld.flatMap(setFieldFromDbo(_, obj, rest)) 100 | case list: BasicBSONList => fallbackValueFromDbObject(dbo, fieldNames) 101 | case null => None 102 | } 103 | case Nil => throw new UnsupportedOperationException("was called with empty list, shouldn't possibly happen") 104 | } 105 | } 106 | 107 | def setFieldFromDbo(field: LField[_, _], dbo: DBObject, fieldNames: List[String]): Option[_] = { 108 | if (field.isInstanceOf[BsonRecordField[_, _]]) { 109 | val brf = field.asInstanceOf[BsonRecordField[_, _]] 110 | val inner = brf.value.asInstanceOf[BsonRecord[_]] 111 | setInstanceFieldFromDboList(inner, dbo, fieldNames) 112 | } else { 113 | fallbackValueFromDbObject(dbo, fieldNames) 114 | } 115 | } 116 | 117 | def setLastFieldFromDbo(field: LField[_, _], dbo: DBObject, fieldName: String): Option[_] = { 118 | field.setFromAny(dbo.get(fieldName)).toOption 119 | } 120 | 121 | def setInstanceFieldFromDbo(instance: MongoRecord[_], dbo: DBObject, fieldName: String): Option[_] = { 122 | fieldName.contains(".") match { 123 | case true => 124 | val names = fieldName.split("\\.").toList.filter(_ != "$") 125 | setInstanceFieldFromDboList(instance, dbo, names) 126 | case false => 127 | val fld: Box[LField[_, _]] = instance.fieldByName(fieldName) 128 | fld.flatMap (setLastFieldFromDbo(_, dbo, fieldName)) 129 | } 130 | } 131 | 132 | def fallbackValueFromDbObject(dbo: DBObject, fieldNames: List[String]): Option[_] = { 133 | import scala.collection.JavaConversions._ 134 | Box.!!(fieldNames.foldLeft(dbo: Object)((obj: Object, fieldName: String) => { 135 | obj match { 136 | case dbl: BasicBSONList => 137 | dbl.map(_.asInstanceOf[DBObject]).map(_.get(fieldName)).toList 138 | case dbo: DBObject => 139 | dbo.get(fieldName) 140 | case null => null 141 | } 142 | })).toOption 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /rogue-lift/src/main/scala/com/foursquare/rogue/LiftRogue.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.foursquare.field.{ 6 | Field => RField, 7 | OptionalField => ROptionalField, 8 | RequiredField => RRequiredField} 9 | import com.foursquare.index.IndexBuilder 10 | import com.foursquare.rogue.MongoHelpers.{AndCondition, MongoModify} 11 | import java.util.Date 12 | import net.liftweb.json.JsonAST.{JArray, JInt} 13 | import net.liftweb.common.Box.box2Option 14 | import net.liftweb.mongodb.record.{BsonRecord, MongoRecord, MongoMetaRecord} 15 | import net.liftweb.record.{Field, MandatoryTypedField, OptionalTypedField, Record} 16 | import net.liftweb.mongodb.record.field.{ BsonRecordField, BsonRecordListField, MongoCaseClassField, 17 | MongoCaseClassListField} 18 | import org.bson.types.ObjectId 19 | import net.liftweb.record.field.EnumField 20 | 21 | trait LiftRogue extends Rogue { 22 | def OrQuery[M <: MongoRecord[M], R] 23 | (subqueries: Query[M, R, _]*) 24 | : Query[M, R, Unordered with Unselected with Unlimited with Unskipped with HasOrClause] = { 25 | subqueries.toList match { 26 | case Nil => throw new RogueException("No subqueries supplied to OrQuery", null) 27 | case q :: qs => { 28 | val orCondition = QueryHelpers.orConditionFromQueries(q :: qs) 29 | Query[M, R, Unordered with Unselected with Unlimited with Unskipped with HasOrClause]( 30 | q.meta, q.collectionName, None, None, None, None, None, 31 | AndCondition(Nil, Some(orCondition)), None, None, None) 32 | } 33 | } 34 | } 35 | 36 | /* Following are a collection of implicit conversions which take a meta-record and convert it to 37 | * a QueryBuilder. This allows users to write queries as "QueryType where ...". 38 | */ 39 | implicit def metaRecordToQueryBuilder[M <: MongoRecord[M]] 40 | (rec: M with MongoMetaRecord[M]): Query[M, M, InitialState] = 41 | Query[M, M, InitialState]( 42 | rec, rec.collectionName, None, None, None, None, None, AndCondition(Nil, None), None, None, None) 43 | 44 | implicit def metaRecordToIndexBuilder[M <: MongoRecord[M]](rec: M with MongoMetaRecord[M]): IndexBuilder[M] = 45 | IndexBuilder(rec) 46 | 47 | implicit def queryToLiftQuery[M <: MongoRecord[_], R, State] 48 | (query: Query[M, R, State]) 49 | (implicit ev: ShardingOk[M with MongoMetaRecord[_], State]): ExecutableQuery[MongoRecord[_] with MongoMetaRecord[_], M with MongoMetaRecord[_], R, State] = { 50 | ExecutableQuery( 51 | query.asInstanceOf[Query[M with MongoMetaRecord[_], R, State]], 52 | LiftQueryExecutor 53 | ) 54 | } 55 | 56 | implicit def modifyQueryToLiftModifyQuery[M <: MongoRecord[_], State]( 57 | query: ModifyQuery[M, State] 58 | ): ExecutableModifyQuery[MongoRecord[_] with MongoMetaRecord[_], M with MongoMetaRecord[_], State] = { 59 | ExecutableModifyQuery( 60 | query.asInstanceOf[ModifyQuery[M with MongoMetaRecord[_], State]], 61 | LiftQueryExecutor 62 | ) 63 | } 64 | 65 | implicit def findAndModifyQueryToLiftFindAndModifyQuery[M <: MongoRecord[_], R]( 66 | query: FindAndModifyQuery[M, R] 67 | ): ExecutableFindAndModifyQuery[MongoRecord[_] with MongoMetaRecord[_], M with MongoMetaRecord[_], R] = { 68 | ExecutableFindAndModifyQuery( 69 | query.asInstanceOf[FindAndModifyQuery[M with MongoMetaRecord[_], R]], 70 | LiftQueryExecutor 71 | ) 72 | } 73 | 74 | implicit def metaRecordToLiftQuery[M <: MongoRecord[M]]( 75 | rec: M with MongoMetaRecord[M] 76 | ): ExecutableQuery[MongoRecord[_] with MongoMetaRecord[_], M with MongoMetaRecord[_], M, InitialState] = { 77 | val queryBuilder = metaRecordToQueryBuilder(rec) 78 | val liftQuery = queryToLiftQuery(queryBuilder) 79 | liftQuery 80 | } 81 | 82 | implicit def fieldToQueryField[M <: BsonRecord[M], F: BSONType](f: Field[F, M]): QueryField[F, M] = new QueryField(f) 83 | 84 | implicit def bsonRecordFieldToBsonRecordQueryField[ 85 | M <: BsonRecord[M], 86 | B <: BsonRecord[B] 87 | ]( 88 | f: BsonRecordField[M, B] 89 | ): BsonRecordQueryField[M, B] = { 90 | val rec = f.defaultValue // a hack to get at the embedded record 91 | new BsonRecordQueryField[M, B](f, _.asDBObject, rec) 92 | } 93 | 94 | implicit def rbsonRecordFieldToBsonRecordQueryField[ 95 | M <: BsonRecord[M], 96 | B <: BsonRecord[B] 97 | ]( 98 | f: RField[B, M] 99 | ): BsonRecordQueryField[M, B] = { 100 | // a hack to get at the embedded record 101 | val owner = f.owner 102 | if (f.name.indexOf('.') >= 0) { 103 | val fieldName = f.name.takeWhile(_ != '.') 104 | val field = owner.fieldByName(fieldName).openOr(sys.error("Error getting field "+fieldName+" for "+owner)) 105 | val typedField = field.asInstanceOf[BsonRecordListField[M, B]] 106 | // a gross hack to get at the embedded record 107 | val rec: B = typedField.setFromJValue(JArray(JInt(0) :: Nil)).openOrThrowException("Rogue hack failed").head 108 | new BsonRecordQueryField[M, B](f, _.asDBObject, rec) 109 | } else { 110 | val fieldName = f.name 111 | val field = owner.fieldByName(fieldName).openOr(sys.error("Error getting field "+fieldName+" for "+owner)) 112 | val typedField = field.asInstanceOf[BsonRecordField[M, B]] 113 | val rec: B = typedField.defaultValue 114 | new BsonRecordQueryField[M, B](f, _.asDBObject, rec) 115 | } 116 | } 117 | 118 | implicit def bsonRecordListFieldToBsonRecordListQueryField[ 119 | M <: BsonRecord[M], 120 | B <: BsonRecord[B] 121 | ](f: BsonRecordListField[M, B]): BsonRecordListQueryField[M, B] = { 122 | val rec = f.setFromJValue(JArray(JInt(0) :: Nil)).openOrThrowException("Rogue hack failed").head // a gross hack to get at the embedded record 123 | new BsonRecordListQueryField[M, B](f, rec, _.asDBObject) 124 | } 125 | 126 | implicit def dateFieldToDateQueryField[M <: BsonRecord[M]] 127 | (f: Field[java.util.Date, M]): DateQueryField[M] = 128 | new DateQueryField(f) 129 | 130 | implicit def ccFieldToQueryField[M <: BsonRecord[M], F](f: MongoCaseClassField[M, F]): CaseClassQueryField[F, M] = 131 | new CaseClassQueryField[F, M](f) 132 | 133 | implicit def ccListFieldToListQueryField[M <: BsonRecord[M], F] 134 | (f: MongoCaseClassListField[M, F]): CaseClassListQueryField[F, M] = 135 | new CaseClassListQueryField[F, M](liftField2Recordv2Field(f)) 136 | 137 | implicit def doubleFieldtoNumericQueryField[M <: BsonRecord[M], F] 138 | (f: Field[Double, M]): NumericQueryField[Double, M] = 139 | new NumericQueryField(f) 140 | 141 | implicit def enumFieldToEnumNameQueryField[M <: BsonRecord[M], F <: Enumeration#Value] 142 | (f: Field[F, M]): EnumNameQueryField[M, F] = 143 | new EnumNameQueryField(f) 144 | 145 | implicit def enumFieldToEnumQueryField[M <: BsonRecord[M], F <: Enumeration] 146 | (f: EnumField[M, F]): EnumIdQueryField[M, F#Value] = 147 | new EnumIdQueryField(f) 148 | 149 | implicit def enumerationListFieldToEnumerationListQueryField[M <: BsonRecord[M], F <: Enumeration#Value] 150 | (f: Field[List[F], M]): EnumerationListQueryField[F, M] = 151 | new EnumerationListQueryField[F, M](f) 152 | 153 | implicit def foreignObjectIdFieldToForeignObjectIdQueryField[M <: BsonRecord[M], 154 | T <: MongoRecord[T] with ObjectIdKey[T]] 155 | (f: Field[ObjectId, M] with HasMongoForeignObjectId[T]): ForeignObjectIdQueryField[ObjectId, M, T] = 156 | new ForeignObjectIdQueryField[ObjectId, M, T](f, _.id) 157 | 158 | implicit def intFieldtoNumericQueryField[M <: BsonRecord[M], F](f: Field[Int, M]): NumericQueryField[Int, M] = 159 | new NumericQueryField(f) 160 | 161 | implicit def latLongFieldToGeoQueryField[M <: BsonRecord[M]](f: Field[LatLong, M]): GeoQueryField[M] = 162 | new GeoQueryField(f) 163 | 164 | implicit def listFieldToListQueryField[M <: BsonRecord[M], F: BSONType](f: Field[List[F], M]): ListQueryField[F, M] = 165 | new ListQueryField[F, M](f) 166 | 167 | implicit def stringsListFieldToStringsListQueryField[M <: BsonRecord[M]](f: Field[List[String], M]): StringsListQueryField[M] = 168 | new StringsListQueryField[M](f) 169 | 170 | implicit def longFieldtoNumericQueryField[M <: BsonRecord[M], F <: Long](f: Field[F, M]): NumericQueryField[F, M] = 171 | new NumericQueryField(f) 172 | 173 | implicit def objectIdFieldToObjectIdQueryField[M <: BsonRecord[M], F <: ObjectId](f: Field[F, M]): ObjectIdQueryField[F, M] = 174 | new ObjectIdQueryField(f) 175 | 176 | implicit def mapFieldToMapQueryField[M <: BsonRecord[M], F](f: Field[Map[String, F], M]): MapQueryField[F, M] = 177 | new MapQueryField[F, M](f) 178 | 179 | implicit def stringFieldToStringQueryField[F <: String, M <: BsonRecord[M]](f: Field[F, M]): StringQueryField[F, M] = 180 | new StringQueryField(f) 181 | 182 | // ModifyField implicits 183 | implicit def fieldToModifyField[M <: BsonRecord[M], F: BSONType](f: Field[F, M]): ModifyField[F, M] = new ModifyField(f) 184 | implicit def fieldToSafeModifyField[M <: BsonRecord[M], F](f: Field[F, M]): SafeModifyField[F, M] = new SafeModifyField(f) 185 | 186 | implicit def bsonRecordFieldToBsonRecordModifyField[M <: BsonRecord[M], B <: BsonRecord[B]] 187 | (f: BsonRecordField[M, B]): BsonRecordModifyField[M, B] = 188 | new BsonRecordModifyField[M, B](f, _.asDBObject) 189 | 190 | implicit def bsonRecordListFieldToBsonRecordListModifyField[ 191 | M <: BsonRecord[M], 192 | B <: BsonRecord[B] 193 | ]( 194 | f: BsonRecordListField[M, B] 195 | )( 196 | implicit mf: Manifest[B] 197 | ): BsonRecordListModifyField[M, B] = { 198 | val rec = f.setFromJValue(JArray(JInt(0) :: Nil)).openOrThrowException("Rogue hack failed").head // a gross hack to get at the embedded record 199 | new BsonRecordListModifyField[M, B](f, rec, _.asDBObject)(mf) 200 | } 201 | 202 | implicit def dateFieldToDateModifyField[M <: BsonRecord[M]](f: Field[Date, M]): DateModifyField[M] = 203 | new DateModifyField(f) 204 | 205 | implicit def ccListFieldToListModifyField[M <: BsonRecord[M], V] 206 | (f: MongoCaseClassListField[M, V]): CaseClassListModifyField[V, M] = 207 | new CaseClassListModifyField[V, M](liftField2Recordv2Field(f)) 208 | 209 | implicit def doubleFieldToNumericModifyField[M <: BsonRecord[M]] 210 | (f: Field[Double, M]): NumericModifyField[Double, M] = 211 | new NumericModifyField(f) 212 | 213 | implicit def enumerationFieldToEnumerationModifyField[M <: BsonRecord[M], F <: Enumeration#Value] 214 | (f: Field[F, M]): EnumerationModifyField[M, F] = 215 | new EnumerationModifyField(f) 216 | 217 | implicit def enumerationListFieldToEnumerationListModifyField[M <: BsonRecord[M], F <: Enumeration#Value] 218 | (f: Field[List[F], M]): EnumerationListModifyField[F, M] = 219 | new EnumerationListModifyField[F, M](f) 220 | 221 | implicit def intFieldToIntModifyField[M <: BsonRecord[M]] 222 | (f: Field[Int, M]): NumericModifyField[Int, M] = 223 | new NumericModifyField(f) 224 | 225 | implicit def latLongFieldToGeoQueryModifyField[M <: BsonRecord[M]](f: Field[LatLong, M]): GeoModifyField[M] = 226 | new GeoModifyField(f) 227 | 228 | implicit def listFieldToListModifyField[M <: BsonRecord[M], F: BSONType](f: Field[List[F], M]): ListModifyField[F, M] = 229 | new ListModifyField[F, M](f) 230 | 231 | implicit def longFieldToNumericModifyField[M <: BsonRecord[M]](f: Field[Long, M]): NumericModifyField[Long, M] = 232 | new NumericModifyField(f) 233 | 234 | implicit def mapFieldToMapModifyField[M <: BsonRecord[M], F](f: Field[Map[String, F], M]): MapModifyField[F, M] = 235 | new MapModifyField[F, M](f) 236 | 237 | // SelectField implicits 238 | implicit def mandatoryFieldToSelectField[M <: BsonRecord[M], V] 239 | (f: Field[V, M] with MandatoryTypedField[V]): SelectField[V, M] = 240 | new MandatorySelectField(f) 241 | 242 | implicit def optionalFieldToSelectField[M <: BsonRecord[M], V] 243 | (f: Field[V, M] with OptionalTypedField[V]): SelectField[Option[V], M] = 244 | new OptionalSelectField(new ROptionalField[V, M] { 245 | override def name = f.name 246 | override def owner = f.owner 247 | }) 248 | 249 | implicit def mandatoryLiftField2RequiredRecordv2Field[M <: BsonRecord[M], V]( 250 | f: Field[V, M] with MandatoryTypedField[V] 251 | ): com.foursquare.field.RequiredField[V, M] = new com.foursquare.field.RequiredField[V, M] { 252 | override def name = f.name 253 | override def owner = f.owner 254 | override def defaultValue = f.defaultValue 255 | } 256 | 257 | implicit def liftField2Recordv2Field[M <: Record[M], V](f: Field[V, M]): com.foursquare.field.Field[V, M] = new com.foursquare.field.Field[V, M] { 258 | override def name = f.name 259 | override def owner = f.owner 260 | } 261 | 262 | class BsonRecordIsBSONType[T <: BsonRecord[T]] extends BSONType[T] { 263 | override def asBSONObject(v: T): AnyRef = v.asDBObject 264 | } 265 | 266 | object _BsonRecordIsBSONType extends BsonRecordIsBSONType[Nothing] 267 | 268 | implicit def BsonRecordIsBSONType[T <: BsonRecord[T]]: BSONType[T] = _BsonRecordIsBSONType.asInstanceOf[BSONType[T]] 269 | } 270 | 271 | object LiftRogue extends LiftRogue 272 | -------------------------------------------------------------------------------- /rogue-lift/src/main/scala/com/foursquare/rogue/ObjectIdKey.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import net.liftweb.mongodb.record.MongoRecord 6 | import net.liftweb.mongodb.record.field.ObjectIdField 7 | import org.bson.types.ObjectId 8 | 9 | 10 | /** 11 | * Mix this into a Record to add an ObjectIdField 12 | */ 13 | trait ObjectIdKey[OwnerType <: MongoRecord[OwnerType]] { 14 | self: OwnerType => 15 | 16 | object _id extends ObjectIdField(this.asInstanceOf[OwnerType]) 17 | 18 | // convenience method that returns the value of _id 19 | def id: ObjectId = _id.value 20 | } -------------------------------------------------------------------------------- /rogue-lift/src/test/scala/com/foursquare/rogue/EndToEndTest.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | package com.foursquare.rogue 3 | 4 | import com.foursquare.rogue.LiftRogue._ 5 | import com.foursquare.rogue.Iter._ 6 | import com.mongodb.ReadPreference 7 | import java.util.Calendar 8 | import java.util.regex.Pattern 9 | import org.bson.types.ObjectId 10 | import org.junit.{Before, After, Ignore, Test} 11 | import org.specs2.matcher.JUnitMustMatchers 12 | 13 | /** 14 | * Contains tests that test the interaction of Rogue with a real mongo. 15 | */ 16 | class EndToEndTest extends JUnitMustMatchers { 17 | def baseTestVenue(): Venue = { 18 | Venue.createRecord 19 | .legacyid(123) 20 | .userid(456) 21 | .venuename("test venue") 22 | .mayor(789) 23 | .mayor_count(3) 24 | .closed(false) 25 | .popularity(List(1L, 2L, 3L)) 26 | .categories(List(new ObjectId())) 27 | .geolatlng(LatLong(40.73, -73.98)) 28 | .status(VenueStatus.open) 29 | .claims(List(VenueClaimBson.createRecord.userid(1234).status(ClaimStatus.pending), 30 | VenueClaimBson.createRecord.userid(5678).status(ClaimStatus.approved))) 31 | .lastClaim(VenueClaimBson.createRecord.userid(5678).status(ClaimStatus.approved)) 32 | .tags(List("test tag1", "some tag")) 33 | } 34 | 35 | def baseTestVenueClaim(vid: ObjectId): VenueClaim = { 36 | VenueClaim.createRecord 37 | .venueid(vid) 38 | .userid(123) 39 | .status(ClaimStatus.approved) 40 | } 41 | 42 | def baseTestTip(): Tip = { 43 | Tip.createRecord 44 | .legacyid(234) 45 | .counts(Map("foo" -> 1L, 46 | "bar" -> 2L)) 47 | } 48 | 49 | @Before 50 | def setupMongoConnection: Unit = { 51 | RogueTestMongo.connectToMongo 52 | } 53 | 54 | @After 55 | def cleanupTestData: Unit = { 56 | Venue.bulkDelete_!!! 57 | Venue.count must_== 0 58 | 59 | VenueClaim.bulkDelete_!!! 60 | VenueClaim.count must_== 0 61 | 62 | Like.allShards.bulkDelete_!!! 63 | 64 | RogueTestMongo.disconnectFromMongo 65 | } 66 | 67 | @Test 68 | def eqsTests: Unit = { 69 | val v = baseTestVenue().save 70 | val vc = baseTestVenueClaim(v.id).save 71 | 72 | // eqs 73 | metaRecordToQueryBuilder(Venue).where(_._id eqs v.id).fetch().map(_.id) must_== List(v.id) 74 | Venue.where(_.mayor eqs v.mayor.value).fetch().map(_.id) must_== List(v.id) 75 | Venue.where(_.mayor eqs v.mayor.value).fetch().map(_.id) must_== List(v.id) 76 | Venue.where(_.venuename eqs v.venuename.value).fetch().map(_.id) must_== List(v.id) 77 | Venue.where(_.closed eqs false).fetch().map(_.id) must_== List(v.id) 78 | 79 | Venue.where(_.mayor eqs 432432).fetch().map(_.id) must_== Nil 80 | Venue.where(_.closed eqs true).fetch().map(_.id) must_== Nil 81 | 82 | VenueClaim.where(_.status eqs ClaimStatus.approved).fetch().map(_.id) must_== List(vc.id) 83 | VenueClaim.where(_.venueid eqs v.id).fetch().map(_.id) must_== List(vc.id) 84 | VenueClaim.where(_.venueid eqs v).fetch().map(_.id) must_== List(vc.id) 85 | } 86 | 87 | @Test 88 | def testInequalityQueries: Unit = { 89 | val v = baseTestVenue().save 90 | val vc = baseTestVenueClaim(v.id).save 91 | 92 | // neq,lt,gt, where the lone Venue has mayor_count=3, and the only 93 | // VenueClaim has status approved. 94 | Venue.where(_.mayor_count neqs 5).fetch().map(_.id) must_== List(v.id) 95 | Venue.where(_.mayor_count < 5).fetch().map(_.id) must_== List(v.id) 96 | Venue.where(_.mayor_count lt 5).fetch().map(_.id) must_== List(v.id) 97 | Venue.where(_.mayor_count <= 5).fetch().map(_.id) must_== List(v.id) 98 | Venue.where(_.mayor_count lte 5).fetch().map(_.id) must_== List(v.id) 99 | Venue.where(_.mayor_count > 5).fetch().map(_.id) must_== Nil 100 | Venue.where(_.mayor_count gt 5).fetch().map(_.id) must_== Nil 101 | Venue.where(_.mayor_count >= 5).fetch().map(_.id) must_== Nil 102 | Venue.where(_.mayor_count gte 5).fetch().map(_.id) must_== Nil 103 | Venue.where(_.mayor_count between (3, 5)).fetch().map(_.id) must_== List(v.id) 104 | VenueClaim.where (_.status neqs ClaimStatus.approved).fetch().map(_.id) must_== Nil 105 | VenueClaim.where (_.status neqs ClaimStatus.pending).fetch().map(_.id) must_== List(vc.id) 106 | } 107 | 108 | @Test 109 | def selectQueries: Unit = { 110 | val v = baseTestVenue().save 111 | 112 | val base = Venue.where(_._id eqs v.id) 113 | base.select(_.legacyid).fetch() must_== List(v.legacyid.value) 114 | base.select(_.legacyid, _.userid).fetch() must_== List((v.legacyid.value, v.userid.value)) 115 | base.select(_.legacyid, _.userid, _.mayor).fetch() must_== List((v.legacyid.value, v.userid.value, v.mayor.value)) 116 | base.select(_.legacyid, _.userid, _.mayor, _.mayor_count).fetch() must_== List((v.legacyid.value, v.userid.value, v.mayor.value, v.mayor_count.value)) 117 | base.select(_.legacyid, _.userid, _.mayor, _.mayor_count, _.closed).fetch() must_== List((v.legacyid.value, v.userid.value, v.mayor.value, v.mayor_count.value, v.closed.value)) 118 | base.select(_.legacyid, _.userid, _.mayor, _.mayor_count, _.closed, _.tags).fetch() must_== List((v.legacyid.value, v.userid.value, v.mayor.value, v.mayor_count.value, v.closed.value, v.tags.value)) 119 | } 120 | 121 | @Test 122 | def selectEnum: Unit = { 123 | val v = baseTestVenue().save 124 | Venue.where(_._id eqs v.id).select(_.status).fetch() must_== List(VenueStatus.open) 125 | } 126 | 127 | @Test 128 | def selectCaseQueries: Unit = { 129 | val v = baseTestVenue().save 130 | 131 | val base = Venue.where(_._id eqs v.id) 132 | base.selectCase(_.legacyid, V1).fetch() must_== List(V1(v.legacyid.value)) 133 | base.selectCase(_.legacyid, _.userid, V2).fetch() must_== List(V2(v.legacyid.value, v.userid.value)) 134 | base.selectCase(_.legacyid, _.userid, _.mayor, V3).fetch() must_== List(V3(v.legacyid.value, v.userid.value, v.mayor.value)) 135 | base.selectCase(_.legacyid, _.userid, _.mayor, _.mayor_count, V4).fetch() must_== List(V4(v.legacyid.value, v.userid.value, v.mayor.value, v.mayor_count.value)) 136 | base.selectCase(_.legacyid, _.userid, _.mayor, _.mayor_count, _.closed, V5).fetch() must_== List(V5(v.legacyid.value, v.userid.value, v.mayor.value, v.mayor_count.value, v.closed.value)) 137 | base.selectCase(_.legacyid, _.userid, _.mayor, _.mayor_count, _.closed, _.tags, V6).fetch() must_== List(V6(v.legacyid.value, v.userid.value, v.mayor.value, v.mayor_count.value, v.closed.value, v.tags.value)) 138 | } 139 | 140 | @Test 141 | def selectSubfieldQueries: Unit = { 142 | val v = baseTestVenue().save 143 | val t = baseTestTip().save 144 | 145 | // select subfields 146 | Tip.where(_._id eqs t.id).select(_.counts at "foo").fetch() must_== List(Some(1L)) 147 | 148 | Venue.where(_._id eqs v.id).select(_.geolatlng.unsafeField[Double]("lat")).fetch() must_== List(Some(40.73)) 149 | 150 | val subuserids: List[Option[List[Long]]] = Venue.where(_._id eqs v.id).select(_.claims.subselect(_.userid)).fetch() 151 | subuserids must_== List(Some(List(1234, 5678))) 152 | 153 | val subclaims: List[List[VenueClaimBson]] = Venue.where(_.claims.subfield(_.userid) eqs 1234).select(_.claims.$$).fetch() 154 | subclaims.size must_== 1 155 | subclaims.head.size must_== 1 156 | subclaims.head.head.userid.value must_== 1234 157 | subclaims.head.head.status.value must_== ClaimStatus.pending 158 | 159 | // selecting a claims.userid when there is no top-level claims list should 160 | // have one element in the List for the one Venue, but an Empty for that 161 | // Venue since there's no list of claims there. 162 | Venue.where(_._id eqs v.id).modify(_.claims unset).and(_.lastClaim unset).updateOne() 163 | Venue.where(_._id eqs v.id).select(_.lastClaim.subselect(_.userid)).fetch() must_== List(None) 164 | Venue.where(_._id eqs v.id).select(_.claims.subselect(_.userid)).fetch() must_== List(None) 165 | } 166 | 167 | @Ignore("These tests are broken because DummyField doesn't know how to convert a String to an Enum") 168 | def testSelectEnumSubfield: Unit = { 169 | val v = baseTestVenue().save 170 | 171 | // This behavior is broken because we get a String back from mongo, and at 172 | // that point we only have a DummyField for the subfield, and that doesn't 173 | // know how to convert the String to an Enum. 174 | 175 | val statuses: List[Option[VenueClaimBson.status.MyType]] = 176 | Venue.where(_._id eqs v.id).select(_.lastClaim.subselect(_.status)) .fetch() 177 | // This assertion works. 178 | statuses must_== List(Some("Approved")) 179 | // This assertion is what we want, and it fails. 180 | // statuses must_== List(Some(ClaimStatus.approved)) 181 | 182 | val subuseridsAndStatuses: List[(Option[List[Long]], Option[List[VenueClaimBson.status.MyType]])] = 183 | Venue.where(_._id eqs v.id) 184 | .select(_.claims.subselect(_.userid), _.claims.subselect(_.status)) 185 | .fetch() 186 | // This assertion works. 187 | subuseridsAndStatuses must_== List((Some(List(1234, 5678)), Some(List("Pending approval", "Approved")))) 188 | 189 | // This assertion is what we want, and it fails. 190 | // subuseridsAndStatuses must_== List((Some(List(1234, 5678)), Some(List(ClaimStatus.pending, ClaimStatus.approved)))) 191 | } 192 | 193 | @Test 194 | def testReadPreference: Unit = { 195 | // Note: this isn't a real test of readpreference because the test mongo setup 196 | // doesn't have replicas. This basically just makes sure that readpreference 197 | // doesn't break everything. 198 | val v = baseTestVenue().save 199 | 200 | // eqs 201 | Venue.where(_._id eqs v.id).fetch().map(_.id) must_== List(v.id) 202 | Venue.where(_._id eqs v.id).setReadPreference(ReadPreference.secondary).fetch().map(_.id) must_== List(v.id) 203 | Venue.where(_._id eqs v.id).setReadPreference(ReadPreference.primary).fetch().map(_.id) must_== List(v.id) 204 | } 205 | 206 | @Test 207 | def testFindAndModify { 208 | val v1 = Venue.where(_.venuename eqs "v1") 209 | .findAndModify(_.userid setTo 5) 210 | .upsertOne(returnNew = false) 211 | v1 must_== None 212 | 213 | val v2 = Venue.where(_.venuename eqs "v2") 214 | .findAndModify(_.userid setTo 5) 215 | .upsertOne(returnNew = true) 216 | v2.map(_.userid.value) must_== Some(5) 217 | 218 | val v3 = Venue.where(_.venuename eqs "v2") 219 | .findAndModify(_.userid setTo 6) 220 | .upsertOne(returnNew = false) 221 | v3.map(_.userid.value) must_== Some(5) 222 | 223 | val v4 = Venue.where(_.venuename eqs "v2") 224 | .findAndModify(_.userid setTo 7) 225 | .upsertOne(returnNew = true) 226 | v4.map(_.userid.value) must_== Some(7) 227 | } 228 | 229 | @Test 230 | def testRegexQuery { 231 | val v = baseTestVenue().save 232 | Venue.where(_._id eqs v.id).and(_.venuename startsWith "test v").count must_== 1 233 | Venue.where(_._id eqs v.id).and(_.venuename matches ".es. v".r).count must_== 1 234 | Venue.where(_._id eqs v.id).and(_.venuename matches "Tes. v".r).count must_== 0 235 | Venue.where(_._id eqs v.id).and(_.venuename matches Pattern.compile("Tes. v", Pattern.CASE_INSENSITIVE)).count must_== 1 236 | Venue.where(_._id eqs v.id).and(_.venuename matches "test .*".r).and(_.legacyid in List(v.legacyid.value)).count must_== 1 237 | Venue.where(_._id eqs v.id).and(_.venuename matches "test .*".r).and(_.legacyid nin List(v.legacyid.value)).count must_== 0 238 | Venue.where(_.tags matches """some\s.*""".r).count must_== 1 239 | } 240 | 241 | @Test 242 | def testIteratees { 243 | // Insert some data 244 | val vs = for (i <- 1 to 10) yield { 245 | baseTestVenue().legacyid(i).save 246 | } 247 | val ids = vs.map(_.id) 248 | 249 | val items1 = Venue.where(_._id in ids) 250 | .iterate[List[Venue]](Nil){ case (accum, event) => { 251 | if (accum.length >= 3) { 252 | Return(accum) 253 | } else { 254 | event match { 255 | case Item(i) if i.legacyid.value % 2 == 0 => Continue(i :: accum) 256 | case Item(_) => Continue(accum) 257 | case EOF => Return(accum) 258 | case Error(e) => Return(accum) 259 | } 260 | } 261 | }} 262 | 263 | items1.map(_.legacyid.value) must_== List(6, 4, 2) 264 | 265 | val items2 = Venue.where(_._id in ids) 266 | .iterateBatch[List[Venue]](2, Nil){ case (accum, event) => { 267 | if (accum.length >= 3) { 268 | Return(accum) 269 | } else { 270 | event match { 271 | case Item(items) => { 272 | Continue(accum ++ items.filter(_.legacyid.value % 3 == 1)) 273 | } 274 | case EOF => Return(accum) 275 | case Error(e) => Return(accum) 276 | } 277 | } 278 | }} 279 | 280 | items2.map(_.legacyid.value) must_== List(1, 4, 7) 281 | 282 | def findIndexOfWithLimit(id: Long, limit: Int) = { 283 | Venue.where(_._id in ids).iterate(1){ case (idx, event) => { 284 | if (idx >= limit) { 285 | Return(-1) 286 | } else { 287 | event match { 288 | case Item(i) if i.legacyid.value == id => Return(idx) 289 | case Item(i) => Continue(idx+1) 290 | case EOF => Return(-2) 291 | case Error(e) => Return(-3) 292 | } 293 | } 294 | }} 295 | } 296 | 297 | findIndexOfWithLimit(5, 2) must_== -1 298 | findIndexOfWithLimit(5, 7) must_== 5 299 | findIndexOfWithLimit(11, 12) must_== -2 300 | } 301 | 302 | @Test 303 | def testDeserializationWithIteratee() { 304 | val inner = CalendarInner.createRecord.date(Calendar.getInstance()) 305 | CalendarFld.createRecord.inner(inner).save 306 | 307 | val q = CalendarFld select(_.inner.subfield(_.date)) 308 | val cnt = q.count() 309 | val list = q.iterate(List[Calendar]()) { 310 | case (list, Iter.Item(cal)) => 311 | val c: Calendar = cal.get //class cast exception was here 312 | c.set(Calendar.HOUR_OF_DAY, 0) 313 | Iter.Continue(c :: list) 314 | case (list, Iter.Error(e)) => e.printStackTrace(); Iter.Continue(list) 315 | case (list, _) => Iter.Return(list) 316 | } 317 | list.length must_== (cnt) 318 | 319 | CalendarFld.bulkDelete_!!!() 320 | } 321 | 322 | @Test 323 | def testSharding { 324 | val l1 = Like.createRecord.userid(1).checkin(111).save 325 | val l2 = Like.createRecord.userid(2).checkin(111).save 326 | val l3 = Like.createRecord.userid(2).checkin(333).save 327 | val l4 = Like.createRecord.userid(2).checkin(444).save 328 | 329 | // Find 330 | Like.where(_.checkin eqs 111).allShards.count() must_== 2 331 | Like.where(_.checkin eqs 111).withShardKey(_.userid eqs 1).count() must_== 1 332 | Like.withShardKey(_.userid eqs 1).where(_.checkin eqs 111).count() must_== 1 333 | Like.withShardKey(_.userid eqs 1).count() must_== 1 334 | Like.withShardKey(_.userid eqs 2).count() must_== 3 335 | Like.where(_.checkin eqs 333).withShardKey(_.userid eqs 2).count() must_== 1 336 | 337 | // Modify 338 | Like.withShardKey(_.userid eqs 2).and(_.checkin eqs 333).modify(_.checkin setTo 334).updateOne() 339 | Like.find(l3.id).open_!.checkin.value must_== 334 340 | Like.where(_.checkin eqs 334).allShards.count() must_== 1 341 | 342 | Like.where(_.checkin eqs 111).allShards.modify(_.checkin setTo 112).updateMulti() 343 | Like.where(_.checkin eqs 112).withShardKey(_.userid in List(1L, 2L)).count() must_== 2 344 | 345 | val l5 = Like.where(_.checkin eqs 112).withShardKey(_.userid eqs 1).findAndModify(_.checkin setTo 113).updateOne(returnNew = true) 346 | l5.get.id must_== l1.id 347 | l5.get.checkin.value must_== 113 348 | } 349 | 350 | @Test 351 | def testLimitAndBatch { 352 | (1 to 50).foreach(_ => baseTestVenue().save) 353 | 354 | val q = Venue.select(_._id) 355 | q.limit(10).fetch().length must_== 10 356 | q.limit(-10).fetch().length must_== 10 357 | q.fetchBatch(20)(x => List(x.length)) must_== List(20, 20, 10) 358 | q.limit(35).fetchBatch(20)(x => List(x.length)) must_== List(20, 15) 359 | q.limit(-35).fetchBatch(20)(x => List(x.length)) must_== List(20, 15) 360 | } 361 | 362 | @Test 363 | def testCount { 364 | (1 to 10).foreach(_ => baseTestVenue().save) 365 | val q = Venue.select(_._id) 366 | q.count() must_== 10 367 | q.limit(3).count() must_== 3 368 | q.limit(15).count() must_== 10 369 | q.skip(5).count() must_== 5 370 | q.skip(12).count() must_== 0 371 | q.skip(3).limit(5).count() must_== 5 372 | q.skip(8).limit(4).count() must_== 2 373 | } 374 | 375 | @Test 376 | def testDistinct { 377 | (1 to 5).foreach(_ => baseTestVenue().userid(1).save) 378 | (1 to 5).foreach(_ => baseTestVenue().userid(2).save) 379 | (1 to 5).foreach(_ => baseTestVenue().userid(3).save) 380 | Venue.where(_.mayor eqs 789).distinct(_.userid).length must_== 3 381 | Venue.where(_.mayor eqs 789).countDistinct(_.userid) must_== 3 382 | } 383 | 384 | @Test 385 | def testSlice { 386 | baseTestVenue().tags(List("1", "2", "3", "4")).save 387 | Venue.select(_.tags.slice(2)).get() must_== Some(List("1", "2")) 388 | Venue.select(_.tags.slice(-2)).get() must_== Some(List("3", "4")) 389 | Venue.select(_.tags.slice(1, 2)).get() must_== Some(List("2", "3")) 390 | } 391 | } 392 | -------------------------------------------------------------------------------- /rogue-lift/src/test/scala/com/foursquare/rogue/IndexCheckerTest.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue 4 | 5 | import com.foursquare.index.{Asc, IndexedRecord, MongoIndexChecker, TwoD} 6 | import com.foursquare.rogue.LiftRogue._ 7 | import net.liftweb.mongodb.record._ 8 | import net.liftweb.mongodb.record.field._ 9 | import net.liftweb.record._ 10 | import net.liftweb.record.field.{BooleanField, IntField} 11 | import scala.collection.immutable.ListMap 12 | 13 | import org.bson.types.ObjectId 14 | import org.joda.time.DateTime 15 | import org.junit._ 16 | import org.specs2.matcher.JUnitMustMatchers 17 | 18 | 19 | class TestModel extends MongoRecord[TestModel] with ObjectIdKey[TestModel] { 20 | def meta = TestModel 21 | object a extends IntField(this) 22 | object b extends IntField(this) 23 | object c extends IntField(this) 24 | object d extends IntField(this) 25 | object m extends MongoMapField[TestModel, Int](this) 26 | object n extends MongoMapField[TestModel, Int](this) 27 | object ll extends MongoCaseClassField[TestModel, LatLong](this) 28 | object l extends MongoListField[TestModel, Int](this) 29 | } 30 | 31 | object TestModel extends TestModel with MongoMetaRecord[TestModel] with IndexedRecord[TestModel] { 32 | override def collectionName = "model" 33 | 34 | override val mongoIndexList = List( 35 | TestModel.index(_._id, Asc), 36 | TestModel.index(_.a, Asc, _.b, Asc, _.c, Asc), 37 | TestModel.index(_.m, Asc, _.a, Asc), 38 | TestModel.index(_.l, Asc), 39 | TestModel.index(_.ll, TwoD, _.b, Asc)) 40 | } 41 | 42 | class MongoIndexCheckerTest extends JUnitMustMatchers { 43 | 44 | @Test 45 | def testIndexExpectations { 46 | def test(query: Query[_, _, _]) = { 47 | val q = query.asInstanceOf[Query[_, _, _]] 48 | val indexes = q.meta.asInstanceOf[IndexedRecord[_]].mongoIndexList 49 | MongoIndexChecker.validateIndexExpectations(q, indexes) 50 | } 51 | 52 | def yes(query: Query[_, _, _]) = 53 | test(query) must beTrue 54 | def no(query: Query[_, _, _]) = 55 | test(query) must beFalse 56 | 57 | yes(TestModel where (_.a eqs 1)) 58 | yes(TestModel iscan (_.a eqs 1)) 59 | yes(TestModel scan (_.a eqs 1)) 60 | 61 | no(TestModel where (_.a > 1)) 62 | 63 | yes(TestModel iscan (_.a > 1)) 64 | yes(TestModel scan (_.a > 1)) 65 | 66 | no(TestModel where (_.a neqs 1)) 67 | yes(TestModel iscan (_.a neqs 1)) 68 | yes(TestModel scan (_.a neqs 1)) 69 | 70 | no(TestModel where (_.a exists true)) 71 | yes(TestModel iscan (_.a exists true)) 72 | yes(TestModel scan (_.a exists true)) 73 | 74 | no(TestModel where (_.l size 1)) 75 | no(TestModel iscan (_.l size 1)) 76 | yes(TestModel scan (_.l size 1)) 77 | 78 | no(TestModel where (_.ll near (1.0, 2.0, Degrees(1.0)))) 79 | yes(TestModel iscan (_.ll near (1.0, 2.0, Degrees(1.0)))) 80 | yes(TestModel scan (_.ll near (1.0, 2.0, Degrees(1.0)))) 81 | 82 | // $or queries 83 | yes(TestModel where (_.a eqs 1) or (_.where(_.b eqs 2), _.where(_.b eqs 2))) 84 | no(TestModel where (_.a eqs 1) or (_.where(_.b > 2), _.where(_.b < 2))) 85 | yes(TestModel where (_.a eqs 1) or (_.iscan(_.b > 2), _.iscan(_.b < 2))) 86 | yes(TestModel where (_.a eqs 1) or (_.scan(_.b > 2), _.scan(_.b < 2))) 87 | no(TestModel where (_.a eqs 1) or (_.where(_.b exists true), _.where(_.b eqs 0))) 88 | yes(TestModel where (_.a eqs 1) or (_.iscan(_.b exists true), _.where(_.b eqs 0))) 89 | yes(TestModel where (_.a eqs 1) or (_.scan(_.b exists true), _.where(_.b eqs 0))) 90 | no(TestModel where (_.a eqs 1) or (_.where(_.l size 1), _.where(_.b eqs 0))) 91 | no(TestModel where (_.a eqs 1) or (_.iscan(_.l size 1), _.where(_.b eqs 0))) 92 | yes(TestModel where (_.a eqs 1) or (_.scan(_.l size 1), _.where(_.b eqs 0))) 93 | } 94 | 95 | @Test 96 | def testMatchesIndex { 97 | def test(query: Query[_, _, _]) = { 98 | val q = query.asInstanceOf[Query[_, _, _]] 99 | val indexes = q.meta.asInstanceOf[IndexedRecord[_]].mongoIndexList 100 | MongoIndexChecker.validateIndexExpectations(q, indexes) && 101 | MongoIndexChecker.validateQueryMatchesSomeIndex(q, indexes) 102 | } 103 | 104 | def yes(query: Query[_, _, _]) = 105 | test(query) must beTrue 106 | 107 | def no(query: Query[_, _, _]) = 108 | test(query) must beFalse 109 | 110 | yes(TestModel where (_.a eqs 1)) 111 | yes(TestModel where (_.a eqs 1) and (_.b eqs 2)) 112 | yes(TestModel where (_.a eqs 1) and (_.b eqs 2) and (_.c eqs 3)) 113 | yes(TestModel where (_.a eqs 1) and (_.c eqs 3) and (_.b eqs 2)) 114 | 115 | // Skip level 116 | yes(TestModel where (_.a eqs 1) iscan (_.c eqs 3)) 117 | yes(TestModel where (_.a eqs 1) scan (_.c eqs 3)) 118 | no(TestModel where (_.a eqs 1) and (_.c eqs 3)) 119 | 120 | // Missing initial 121 | yes(TestModel scan (_.b eqs 2)) 122 | no(TestModel where (_.b eqs 2)) 123 | no(TestModel iscan (_.b eqs 2)) 124 | 125 | // Range 126 | yes(TestModel iscan (_.a > 1) iscan (_.b eqs 2)) 127 | yes(TestModel scan (_.a > 1) scan (_.b eqs 2)) 128 | no(TestModel where (_.a > 1) and (_.b eqs 2)) 129 | no(TestModel where (_.a > 1) iscan (_.b eqs 2)) 130 | yes(TestModel where (_.a eqs 1) and (_.b eqs 2) iscan (_.c > 3)) 131 | 132 | // Range placement 133 | yes(TestModel where (_.a eqs 1) iscan (_.b eqs 2) iscan (_.c > 3)) 134 | yes(TestModel where (_.a eqs 1) iscan (_.b > 2) iscan (_.c eqs 3)) 135 | yes(TestModel iscan (_.a > 1) iscan (_.b eqs 2) iscan (_.c eqs 3)) 136 | 137 | // Double range 138 | yes(TestModel iscan (_.a > 1) iscan (_.b > 2) iscan (_.c eqs 3)) 139 | no(TestModel where (_.a > 1) and (_.b > 2) and (_.c eqs 3)) 140 | no(TestModel where (_.a > 1) and (_.b > 2) iscan (_.c eqs 3)) 141 | no(TestModel where (_.a > 1) iscan (_.b > 2) iscan (_.c eqs 3)) 142 | no(TestModel where (_.a > 1) iscan (_.b > 2) and (_.c eqs 3)) 143 | yes(TestModel iscan (_.a > 1) scan (_.b > 2) scan (_.c eqs 3)) 144 | yes(TestModel iscan (_.a > 1) scan (_.b > 2) iscan (_.c eqs 3)) 145 | yes(TestModel where (_.a eqs 1) iscan (_.b > 2) iscan (_.c > 3)) 146 | no(TestModel where (_.a eqs 1) and (_.b > 2) iscan (_.c > 3)) 147 | 148 | // Index scan only 149 | yes(TestModel scan (_.a exists true)) 150 | no(TestModel where (_.a exists true)) 151 | yes(TestModel iscan (_.a exists true)) 152 | 153 | yes(TestModel scan (_.a exists true) scan (_.b eqs 3)) 154 | no(TestModel scan (_.a exists true) iscan (_.b eqs 3)) 155 | 156 | // Unindexable 157 | yes(TestModel scan (_.l size 1)) 158 | no(TestModel where (_.l size 1)) 159 | no(TestModel iscan (_.l size 1)) 160 | 161 | // Not in index 162 | yes(TestModel where (_.a eqs 1) scan (_.d eqs 4)) 163 | no(TestModel where (_.a eqs 1) and (_.d eqs 4)) 164 | no(TestModel where (_.a eqs 1) iscan (_.d eqs 4)) 165 | 166 | // 2d indexes 167 | yes(TestModel iscan (_.ll near (1.0, 2.0, Degrees(1.0))) iscan (_.b eqs 2)) 168 | no(TestModel where (_.ll near (1.0, 2.0, Degrees(1.0))) and (_.b eqs 2)) 169 | no(TestModel where (_.ll near (1.0, 2.0, Degrees(1.0))) iscan (_.b eqs 2)) 170 | no(TestModel iscan (_.ll near (1.0, 2.0, Degrees(1.0))) and (_.b eqs 2)) 171 | yes(TestModel iscan (_.ll near (1.0, 2.0, Degrees(1.0))) scan (_.c eqs 2)) 172 | no(TestModel iscan (_.ll near (1.0, 2.0, Degrees(1.0))) iscan (_.c eqs 2)) 173 | 174 | // Overspecifed queries 175 | val id = new ObjectId 176 | val d = new DateTime 177 | yes(TestModel where (_._id eqs id) and (_.d eqs 4)) 178 | yes(TestModel where (_._id in List(id)) and (_.d eqs 4)) 179 | no(TestModel where (_._id after d) scan (_.d eqs 4)) 180 | no(TestModel iscan (_._id after d) iscan (_.d eqs 4)) 181 | yes(TestModel iscan (_._id after d) scan (_.d eqs 4)) 182 | 183 | // Multikeys 184 | yes(TestModel scan (_.m at "foo" eqs 2)) 185 | no(TestModel where (_.m at "foo" eqs 2)) 186 | no(TestModel iscan (_.m at "foo" eqs 2)) 187 | 188 | //TODO(markcc) yes(TestModel where (_.n at "foo" eqs 2)) 189 | no(TestModel where (_.n at "fo" eqs 2)) 190 | no(TestModel where (_.n at "foot" eqs 2)) 191 | no(TestModel where (_.n at "bar" eqs 2)) 192 | 193 | // $or queries 194 | yes(TestModel where (_.a eqs 1) or (_.where(_.b eqs 2), _.where(_.b eqs 2))) 195 | no(TestModel where (_.a eqs 1) or (_.where(_.d eqs 4), _.where(_.d eqs 4))) 196 | no(TestModel where (_.a eqs 1) or (_.iscan(_.d eqs 4), _.iscan(_.d eqs 4))) 197 | yes(TestModel where (_.a eqs 1) or (_.scan(_.d eqs 4), _.scan(_.d eqs 4))) 198 | no(TestModel where (_.a eqs 1) or (_.where(_.b eqs 2), _.where(_.d eqs 4))) 199 | yes(TestModel where (_.a eqs 1) or (_.where(_.b eqs 2), _.scan(_.d eqs 4))) 200 | no(TestModel where (_.a eqs 1) or (_.where(_.c eqs 3), _.where(_.c eqs 3))) 201 | yes(TestModel where (_.a eqs 1) or (_.iscan(_.c eqs 3), _.iscan(_.c eqs 3))) 202 | no(TestModel where (_.a eqs 1) and (_.b between (2, 2)) or (_.where(_.c eqs 3), _.where(_.c eqs 3))) 203 | yes(TestModel where (_.a eqs 1) iscan (_.b between (2, 2)) or (_.iscan(_.c eqs 3), _.iscan(_.c eqs 3))) 204 | yes(TestModel where (_.a eqs 1) or (_.where(_.b eqs 2), _.where(_.b eqs 2)) and (_.c eqs 3)) 205 | no(TestModel where (_.a eqs 1) or (_.where(_.b eqs 2), _.where(_.b > 2)) and (_.c eqs 3)) 206 | no(TestModel where (_.a eqs 1) or (_.where(_.b eqs 2), _.iscan(_.b > 2)) and (_.c eqs 3)) 207 | yes(TestModel where (_.a eqs 1) or (_.where(_.b eqs 2), _.iscan(_.b > 2)) iscan (_.c eqs 3)) 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /rogue-lift/src/test/scala/com/foursquare/rogue/QueryExecutorTest.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | package com.foursquare.rogue 3 | 4 | import com.foursquare.rogue.LiftRogue._ 5 | import com.foursquare.rogue.MongoHelpers.AndCondition 6 | import org.junit._ 7 | import org.specs2.matcher.JUnitMustMatchers 8 | import net.liftweb.mongodb.record.{MongoMetaRecord, MongoRecord} 9 | 10 | class LegacyQueryExecutorTest extends JUnitMustMatchers { 11 | 12 | @Test 13 | def testExeptionInRunCommandIsDecorated { 14 | val query = Venue.where(_.tags contains "test").asInstanceOf[Query[MongoRecord[_] with MongoMetaRecord[_], _, _]] 15 | (LiftAdapter.runCommand("hello", query){ 16 | throw new RuntimeException("bang") 17 | "hi" 18 | }) must throwA[RogueException] 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /rogue-lift/src/test/scala/com/foursquare/rogue/TestModels.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2011 Foursquare Labs Inc. All Rights Reserved. 2 | package com.foursquare.rogue 3 | 4 | import com.foursquare.index.{Asc, Desc, IndexedRecord, IndexModifier, TwoD} 5 | import com.foursquare.rogue.LiftRogue._ 6 | import com.mongodb.{Mongo, ServerAddress} 7 | import net.liftweb.mongodb.{MongoDB, MongoIdentifier} 8 | import net.liftweb.mongodb.record._ 9 | import net.liftweb.mongodb.record.field._ 10 | import net.liftweb.record.field._ 11 | import net.liftweb.record._ 12 | import org.bson.types.ObjectId 13 | 14 | ///////////////////////////////////////////////// 15 | // Sample records for testing 16 | ///////////////////////////////////////////////// 17 | 18 | object RogueTestMongo extends MongoIdentifier { 19 | 20 | override def jndiName = "rogue_mongo" 21 | 22 | private var mongo: Option[Mongo] = None 23 | 24 | def connectToMongo = { 25 | val MongoPort = Option(System.getenv("MONGO_PORT")).map(_.toInt).getOrElse(37648) 26 | mongo = Some(new Mongo(new ServerAddress("localhost", MongoPort))) 27 | MongoDB.defineDb(RogueTestMongo, mongo.get, "rogue-test") 28 | } 29 | 30 | def disconnectFromMongo = { 31 | mongo.foreach(_.close) 32 | MongoDB.close 33 | mongo = None 34 | } 35 | } 36 | 37 | object VenueStatus extends Enumeration { 38 | val open = Value("Open") 39 | val closed = Value("Closed") 40 | } 41 | 42 | class Venue extends MongoRecord[Venue] with ObjectIdKey[Venue] with IndexedRecord[Venue] { 43 | def meta = Venue 44 | object legacyid extends LongField(this) { override def name = "legid" } 45 | object userid extends LongField(this) 46 | object venuename extends StringField(this, 255) 47 | object mayor extends LongField(this) 48 | object mayor_count extends LongField(this) 49 | object closed extends BooleanField(this) 50 | object tags extends MongoListField[Venue, String](this) 51 | object popularity extends MongoListField[Venue, Long](this) 52 | object categories extends MongoListField[Venue, ObjectId](this) 53 | object geolatlng extends MongoCaseClassField[Venue, LatLong](this) { override def name = "latlng" } 54 | object last_updated extends DateField(this) 55 | object status extends EnumNameField(this, VenueStatus) { override def name = "status" } 56 | object claims extends BsonRecordListField(this, VenueClaimBson) 57 | object lastClaim extends BsonRecordField(this, VenueClaimBson) 58 | } 59 | object Venue extends Venue with MongoMetaRecord[Venue] { 60 | override def collectionName = "venues" 61 | override def mongoIdentifier = RogueTestMongo 62 | 63 | object CustomIndex extends IndexModifier("custom") 64 | val idIdx = Venue.index(_._id, Asc) 65 | val mayorIdIdx = Venue.index(_.mayor, Asc, _._id, Asc) 66 | val mayorIdClosedIdx = Venue.index(_.mayor, Asc, _._id, Asc, _.closed, Asc) 67 | val legIdx = Venue.index(_.legacyid, Desc) 68 | val geoIdx = Venue.index(_.geolatlng, TwoD) 69 | val geoCustomIdx = Venue.index(_.geolatlng, CustomIndex, _.tags, Asc) 70 | override val mongoIndexList = List(idIdx, mayorIdIdx, mayorIdClosedIdx, legIdx, geoIdx, geoCustomIdx) 71 | 72 | trait FK[T <: FK[T]] extends MongoRecord[T] { 73 | self: T=> 74 | object venueid extends ObjectIdField[T](this) with HasMongoForeignObjectId[Venue] { 75 | override def name = "vid" 76 | } 77 | } 78 | } 79 | 80 | object ClaimStatus extends Enumeration { 81 | val pending = Value("Pending approval") 82 | val approved = Value("Approved") 83 | } 84 | 85 | object RejectReason extends Enumeration { 86 | val tooManyClaims = Value("too many claims") 87 | val cheater = Value("cheater") 88 | val wrongCode = Value("wrong code") 89 | } 90 | 91 | class VenueClaim extends MongoRecord[VenueClaim] with ObjectIdKey[VenueClaim] with Venue.FK[VenueClaim] { 92 | def meta = VenueClaim 93 | object userid extends LongField(this) { override def name = "uid" } 94 | object status extends EnumNameField(this, ClaimStatus) 95 | object reason extends EnumField(this, RejectReason) 96 | object date extends DateField(this) 97 | } 98 | object VenueClaim extends VenueClaim with MongoMetaRecord[VenueClaim] { 99 | override def fieldOrder = List(status, _id, userid, venueid, reason) 100 | override def collectionName = "venueclaims" 101 | override def mongoIdentifier = RogueTestMongo 102 | } 103 | 104 | class VenueClaimBson extends BsonRecord[VenueClaimBson] { 105 | def meta = VenueClaimBson 106 | object userid extends LongField(this) { override def name = "uid" } 107 | object status extends EnumNameField(this, ClaimStatus) 108 | object source extends BsonRecordField(this, SourceBson) 109 | object date extends DateField(this) 110 | } 111 | object VenueClaimBson extends VenueClaimBson with BsonMetaRecord[VenueClaimBson] { 112 | override def fieldOrder = List(status, userid, source, date) 113 | } 114 | 115 | class SourceBson extends BsonRecord[SourceBson] { 116 | def meta = SourceBson 117 | object name extends StringField(this, 100) 118 | object url extends StringField(this, 200) 119 | } 120 | object SourceBson extends SourceBson with BsonMetaRecord[SourceBson] { 121 | override def fieldOrder = List(name, url) 122 | } 123 | 124 | case class OneComment(timestamp: String, userid: Long, comment: String) 125 | class Comment extends MongoRecord[Comment] with ObjectIdKey[Comment] { 126 | def meta = Comment 127 | object comments extends MongoCaseClassListField[Comment, OneComment](this) 128 | } 129 | object Comment extends Comment with MongoMetaRecord[Comment] { 130 | override def collectionName = "comments" 131 | override def mongoIdentifier = RogueTestMongo 132 | 133 | val idx1 = Comment.index(_._id, Asc) 134 | } 135 | 136 | class Tip extends MongoRecord[Tip] with ObjectIdKey[Tip] { 137 | def meta = Tip 138 | object legacyid extends LongField(this) { override def name = "legid" } 139 | object counts extends MongoMapField[Tip, Long](this) 140 | object userid extends LongField(this) 141 | } 142 | object Tip extends Tip with MongoMetaRecord[Tip] { 143 | override def collectionName = "tips" 144 | override def mongoIdentifier = RogueTestMongo 145 | } 146 | 147 | class Like extends MongoRecord[Like] with ObjectIdKey[Like] with Sharded { 148 | def meta = Like 149 | object userid extends LongField(this) with ShardKey[Long] 150 | object checkin extends LongField(this) 151 | object tip extends ObjectIdField(this) 152 | } 153 | object Like extends Like with MongoMetaRecord[Like] { 154 | override def collectionName = "likes" 155 | override def mongoIdentifier = RogueTestMongo 156 | } 157 | 158 | object ConsumerPrivilege extends Enumeration { 159 | val awardBadges = Value("Award badges") 160 | } 161 | 162 | class OAuthConsumer extends MongoRecord[OAuthConsumer] with ObjectIdKey[OAuthConsumer] { 163 | def meta = OAuthConsumer 164 | object privileges extends MongoListField[OAuthConsumer, ConsumerPrivilege.Value](this) 165 | } 166 | object OAuthConsumer extends OAuthConsumer with MongoMetaRecord[OAuthConsumer] { 167 | override def collectionName = "oauthconsumers" 168 | override def mongoIdentifier = RogueTestMongo 169 | } 170 | 171 | // Used for selectCase tests. 172 | case class V1(legacyid: Long) 173 | case class V2(legacyid: Long, userid: Long) 174 | case class V3(legacyid: Long, userid: Long, mayor: Long) 175 | case class V4(legacyid: Long, userid: Long, mayor: Long, mayor_count: Long) 176 | case class V5(legacyid: Long, userid: Long, mayor: Long, mayor_count: Long, closed: Boolean) 177 | case class V6(legacyid: Long, userid: Long, mayor: Long, mayor_count: Long, closed: Boolean, tags: List[String]) 178 | 179 | class CalendarFld private() extends MongoRecord[CalendarFld] with ObjectIdPk[CalendarFld] { 180 | def meta = CalendarFld 181 | 182 | object inner extends BsonRecordField(this, CalendarInner) 183 | } 184 | 185 | object CalendarFld extends CalendarFld with MongoMetaRecord[CalendarFld] { 186 | override def mongoIdentifier = RogueTestMongo 187 | } 188 | 189 | class CalendarInner private() extends BsonRecord[CalendarInner] { 190 | def meta = CalendarInner 191 | 192 | object date extends DateTimeField(this) //actually calendar field, not joda DateTime 193 | } 194 | 195 | object CalendarInner extends CalendarInner with BsonMetaRecord[CalendarInner] 196 | 197 | -------------------------------------------------------------------------------- /rogue-spindle/build.sbt: -------------------------------------------------------------------------------- 1 | libraryDependencies <++= (scalaVersion) { scalaVersion => 2 | val spindleVersion = "1.7.0" 3 | Seq( 4 | "com.foursquare" % "common-thrift-bson" % spindleVersion 5 | ) 6 | } 7 | 8 | Seq(RogueBuild.defaultSettings: _*) 9 | 10 | Seq(thriftSettings: _*) 11 | -------------------------------------------------------------------------------- /rogue-spindle/src/main/scala/com/foursquare/rogue/spindle/SpindleDBCollectionFactory.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue.spindle 4 | 5 | import com.foursquare.index.UntypedMongoIndex 6 | import com.foursquare.rogue.{DBCollectionFactory, Query => RogueQuery} 7 | import com.foursquare.spindle.{IndexParser, UntypedMetaRecord} 8 | import com.mongodb.{DB, DBCollection} 9 | import scala.collection.immutable.ListMap 10 | import scala.collection.mutable.ConcurrentMap 11 | 12 | trait SpindleDBCollectionFactory extends DBCollectionFactory[UntypedMetaRecord] { 13 | def getDB(meta: UntypedMetaRecord): DB = { 14 | getPrimaryDB(meta) 15 | } 16 | 17 | def getPrimaryDB(meta: UntypedMetaRecord): DB 18 | 19 | override def getDBCollection[M <: UntypedMetaRecord](query: RogueQuery[M, _, _]): DBCollection = 20 | getDB(query.meta).getCollection(query.collectionName) 21 | 22 | override def getPrimaryDBCollection[M <: UntypedMetaRecord](query: RogueQuery[M, _, _]): DBCollection = { 23 | getPrimaryDBCollection(query.meta) 24 | } 25 | 26 | def getPrimaryDBCollection(meta: UntypedMetaRecord): DBCollection = { 27 | getPrimaryDB(meta).getCollection(getCollection(meta)) 28 | } 29 | 30 | override def getInstanceName[M <: UntypedMetaRecord](query: RogueQuery[M, _, _]): String = { 31 | getIdentifier(query.meta) 32 | } 33 | 34 | def getIdentifier(meta: UntypedMetaRecord): String = { 35 | meta.annotations.get("mongo_identifier").getOrElse { 36 | throw new Exception("Add a mongo_identifier annotation to the Thrift definition for this class.") 37 | } 38 | } 39 | 40 | def getCollection(meta: UntypedMetaRecord): String = { 41 | meta.annotations.get("mongo_collection").getOrElse { 42 | throw new Exception("Add a mongo_collection annotation to the Thrift definition for this class.") 43 | } 44 | } 45 | 46 | protected def indexCache: Option[ConcurrentMap[UntypedMetaRecord, List[UntypedMongoIndex]]] 47 | 48 | /** 49 | * Retrieves the list of indexes declared for the record type associated with a 50 | * query. If the record type doesn't declare any indexes, then returns None. 51 | * @param query the query 52 | * @return the list of indexes, or an empty list. 53 | */ 54 | override def getIndexes[M <: UntypedMetaRecord](query: RogueQuery[M, _, _]): Option[List[UntypedMongoIndex]] = { 55 | val cachedIndexes = indexCache.flatMap(_.get(query.meta)) 56 | if (cachedIndexes.isDefined) { 57 | cachedIndexes 58 | } else { 59 | val rv = 60 | for (indexes <- IndexParser.parse(query.meta.annotations).right.toOption) yield { 61 | for (index <- indexes.toList) yield { 62 | val entries = index.map(entry => (entry.fieldName, entry.indexType)) 63 | new SpindleMongoIndex(ListMap(entries: _*)) 64 | } 65 | } 66 | 67 | // Update the cache 68 | for { 69 | indexes <- rv 70 | cache <- indexCache 71 | } { 72 | cache.put(query.meta, indexes) 73 | } 74 | 75 | rv 76 | } 77 | } 78 | } 79 | 80 | private[spindle] class SpindleMongoIndex(override val asListMap: ListMap[String, Any]) extends UntypedMongoIndex 81 | -------------------------------------------------------------------------------- /rogue-spindle/src/main/scala/com/foursquare/rogue/spindle/SpindleDatabaseService.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue.spindle 4 | 5 | import com.foursquare.rogue.{DBCollectionFactory, MongoJavaDriverAdapter, QueryExecutor, QueryOptimizer} 6 | import com.foursquare.spindle.{UntypedMetaRecord, UntypedRecord} 7 | import com.foursquare.rogue.MongoHelpers.MongoSelect 8 | import com.mongodb.WriteConcern 9 | 10 | class SpindleDatabaseService(val dbCollectionFactory: SpindleDBCollectionFactory) extends QueryExecutor[UntypedMetaRecord] { 11 | override def serializer[M <: UntypedMetaRecord, R](meta: M, select: Option[MongoSelect[M, R]]): 12 | SpindleRogueSerializer[M, R] = { 13 | new SpindleRogueSerializer(meta, select) 14 | } 15 | override def defaultWriteConcern: WriteConcern = WriteConcern.SAFE 16 | override val adapter: MongoJavaDriverAdapter[UntypedMetaRecord] = new MongoJavaDriverAdapter(dbCollectionFactory) 17 | override val optimizer = new QueryOptimizer 18 | 19 | def save[R <: UntypedRecord](record: R, writeConcern: WriteConcern = defaultWriteConcern): R = { 20 | if (record.meta.annotations.contains("nosave")) 21 | throw new IllegalArgumentException("Cannot save a %s record".format(record.meta.recordName)) 22 | val s = serializer[UntypedMetaRecord, R](record.meta, None) 23 | val dbo = s.toDBObject(record) 24 | val collection = dbCollectionFactory.getPrimaryDBCollection(record.meta) 25 | collection.save(dbo, writeConcern) 26 | record 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /rogue-spindle/src/main/scala/com/foursquare/rogue/spindle/SpindleQuery.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue.spindle 4 | 5 | import com.foursquare.rogue.{InitialState, Query => RogueQuery} 6 | import com.foursquare.rogue.MongoHelpers.AndCondition 7 | import com.foursquare.spindle.{MetaRecord, Record} 8 | 9 | object SpindleQuery { 10 | def apply[R <: Record[R], M <: MetaRecord[R]]( 11 | model: M with MetaRecord[R] 12 | ): RogueQuery[M, R, InitialState] = { 13 | val collection = model.annotations.get("mongo_collection").getOrElse { 14 | throw new Exception("Add a mongo_collection annotation to the Thrift definition for this class.") 15 | } 16 | RogueQuery[M, R, InitialState]( 17 | model, collection, None, None, None, None, None, AndCondition(Nil, None), None, None, None) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /rogue-spindle/src/main/scala/com/foursquare/rogue/spindle/SpindleRogueSerializer.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue.spindle 4 | 5 | import com.foursquare.common.thrift.bson.TBSONObjectProtocol 6 | import com.foursquare.rogue.MongoHelpers.MongoSelect 7 | import com.foursquare.rogue.RogueSerializer 8 | import com.foursquare.spindle.{UntypedFieldDescriptor, UntypedMetaRecord, UntypedRecord} 9 | import com.mongodb.DBObject 10 | 11 | class SpindleRogueSerializer[M <: UntypedMetaRecord, R](meta: M, select: Option[MongoSelect[M, R]]) 12 | extends RogueSerializer[R] { 13 | 14 | private def getValueFromRecord(metaRecord: UntypedMetaRecord, record: Any, fieldName: String): Option[Any] = { 15 | val fieldList: List[UntypedFieldDescriptor] = metaRecord.fields.toList 16 | val fieldDescriptor = fieldList.find(fd => fd.name == fieldName).getOrElse( 17 | throw new Exception("The meta record does not have a definition for field %s".format(fieldName)) 18 | ) 19 | fieldDescriptor.unsafeGetterOption(record) 20 | } 21 | 22 | private def getValueFromAny(sourceObj: Option[Any], fieldName: String): Option[Any] = sourceObj.flatMap(_ match { 23 | case (map: Map[_, _]) => map.find({case(key, value) => key.toString == fieldName}).map(_._2) 24 | case (seq: Seq[_]) => Some(seq.map(v => getValueFromAny(Some(v), fieldName))) 25 | case (rec: UntypedRecord) => getValueFromRecord(rec.meta, rec, fieldName) 26 | case _ => throw new Exception("Rogue bug: unepected object type") 27 | }) 28 | 29 | override def fromDBObject(dbo: DBObject): R = select match { 30 | case Some(MongoSelect(Nil, transformer)) => { 31 | // A MongoSelect clause exists, but has empty fields. Return null. 32 | // This is used for .exists(), where we just want to check the number 33 | // of returned results is > 0. 34 | transformer(null) 35 | } 36 | case Some(MongoSelect(fields, transformer)) => { 37 | val record = meta.createRawRecord 38 | val protocolFactory = new TBSONObjectProtocol.ReaderFactory 39 | val protocol = protocolFactory.getProtocol 40 | protocol.setSource(dbo) 41 | record.read(protocol) 42 | 43 | val values = { 44 | fields.map(fld => { 45 | if (fld.field.isInstanceOf[UntypedFieldDescriptor]) { 46 | val valueOpt = fld.field.asInstanceOf[UntypedFieldDescriptor].unsafeGetterOption(record) 47 | fld.valueOrDefault(valueOpt) 48 | } else { 49 | // We need to handle a request for a subrecord, such as foo.x.y 50 | val (rootFieldName :: subPath) = fld.field.name.split('.').toList 51 | val rootValueOpt = getValueFromRecord(fld.field.owner.asInstanceOf[UntypedMetaRecord], record, rootFieldName) 52 | val valueOpt = subPath.foldLeft(rootValueOpt)(getValueFromAny) 53 | fld.valueOrDefault(valueOpt) 54 | } 55 | }) 56 | } 57 | transformer(values) 58 | } 59 | case None => { 60 | val record = meta.createRawRecord 61 | val protocolFactory = new TBSONObjectProtocol.ReaderFactory 62 | val protocol = protocolFactory.getProtocol 63 | protocol.setSource(dbo) 64 | record.read(protocol) 65 | record.asInstanceOf[R] 66 | } 67 | } 68 | 69 | def toDBObject(record: R with UntypedRecord): DBObject = { 70 | val protocolFactory = new TBSONObjectProtocol.WriterFactoryForDBObject 71 | val protocol = protocolFactory.getProtocol 72 | record.write(protocol) 73 | protocol.getOutput.asInstanceOf[DBObject] 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /rogue-spindle/src/test/scala/com/foursquare/rogue/spindle/TestSpindleDBService.scala: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Foursquare Labs Inc. All Rights Reserved. 2 | 3 | package com.foursquare.rogue.spindle 4 | 5 | import com.foursquare.rogue.Rogue._ 6 | import com.foursquare.rogue.spindle.gen.TestStruct 7 | import com.foursquare.spindle.UntypedMetaRecord 8 | import com.mongodb.{Mongo, ServerAddress} 9 | import org.junit.Test 10 | import org.junit.Assert._ 11 | 12 | class TestSpindleDBService { 13 | @Test 14 | def testSimpleStruct { 15 | val MongoPort = Option(System.getenv("MONGO_PORT")).map(_.toInt).getOrElse(37648) 16 | val mongo = new Mongo(new ServerAddress("localhost", MongoPort)) 17 | 18 | val dbService = new SpindleDatabaseService( 19 | new SpindleDBCollectionFactory { 20 | override def getPrimaryDB(meta: UntypedMetaRecord) = mongo.getDB("test") 21 | override def indexCache = None 22 | } 23 | ) 24 | 25 | val record = TestStruct.newBuilder 26 | .id(1) 27 | .info("hi") 28 | .result 29 | 30 | dbService.save(record) 31 | 32 | val q = SpindleQuery(TestStruct).where(_.id eqs 1) 33 | 34 | assertEquals("query string", "db.test_structs.find({ \"_id\" : 1})", q.toString) 35 | 36 | val res = dbService.fetch(q) 37 | assertEquals("result length", 1, res.length) 38 | assertEquals("result id ", 1, res.head.idOrNull) 39 | assertEquals("result info", "hi", res.head.infoOrNull) 40 | 41 | // delete the record 42 | dbService.bulkDelete_!!(q) 43 | 44 | // ensure the record no longer exists 45 | assertEquals("result length post-delete", 0, dbService.fetch(q).length) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /rogue-spindle/src/test/thrift/com/foursquare/rogue/spindle/test.thrift: -------------------------------------------------------------------------------- 1 | namespace java com.foursquare.rogue.spindle.gen 2 | 3 | struct TestStruct { 4 | 1: optional i32 id (wire_name="_id") 5 | 2: optional string info 6 | } ( 7 | mongo_collection="test_structs" 8 | mongo_identifier="core" 9 | ) 10 | -------------------------------------------------------------------------------- /sbt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Internal options, always specified 4 | INTERNAL_OPTS="-Dfile.encoding=UTF-8 -Xss8M -Xmx1G -noverify -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M" 5 | 6 | # Default options, if nothing is specified 7 | DEFAULT_OPTS="" 8 | 9 | SBT_VERSION="0.13.5" 10 | SBT_LAUNCHER="$(dirname $0)/project/sbt-launch-$SBT_VERSION.jar" 11 | 12 | if [ ! -e "$SBT_LAUNCHER" ]; 13 | then 14 | URL="http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/$SBT_VERSION/sbt-launch.jar" 15 | curl -o $SBT_LAUNCHER $URL 16 | fi 17 | 18 | # Call with INTERNAL_OPTS followed by SBT_OPTS (or DEFAULT_OPTS). java aways takes the last option when duplicate. 19 | exec java ${INTERNAL_OPTS} ${SBT_OPTS:-${DEFAULT_OPTS}} -jar $SBT_LAUNCHER "$@" 20 | -------------------------------------------------------------------------------- /start-test-mongo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Usage: ./start-test-mongo.sh 3 | 4 | # EROGU 5 | MONGO_PORT=37648 6 | 7 | MY_DIR=$(dirname $0) 8 | 9 | # download mongodb if necessary and place it inside the dependencies/ subdirectory. 10 | MONGO_DIR=osx 11 | MONGO_VERSION=2.4.8 12 | MONGO=mongodb-osx-x86_64-$MONGO_VERSION 13 | 14 | if [ $(uname) = 'Linux' ]; then 15 | MONGO_DIR=linux 16 | MONGO=mongodb-linux-x86_64-$MONGO_VERSION 17 | fi 18 | 19 | mkdir -p dependencies 20 | 21 | if [ ! -e dependencies/$MONGO ]; then 22 | echo "Fetching MongoDB: $MONGO" 23 | curl -# http://fastdl.mongodb.org/$MONGO_DIR/$MONGO.tgz | tar -xz -C dependencies/ 24 | fi 25 | 26 | # ln -sf doesn't work on mac os x, so we need to rm and recreate. 27 | rm -f dependencies/mongodb 28 | ln -sf $MONGO dependencies/mongodb 29 | 30 | if ! [ -d mongo-testdb ]; then 31 | echo "creating mongo-testdb directly for tests that use mongo" 32 | mkdir mongo-testdb 33 | fi 34 | 35 | if ! ./dependencies/mongodb/bin/mongo --port $MONGO_PORT --eval "db.serverStatus()" 2>&1 > /dev/null; then 36 | echo "automatically starting up local mongo on $MONGO_PORT so we can use it for tests" 37 | ./dependencies/mongodb/bin/mongod --dbpath mongo-testdb --maxConns 800 --port $MONGO_PORT $@ 2>&1 | tee mongo.log 38 | else 39 | echo "great, you have a local mongo running for tests already" 40 | fi 41 | --------------------------------------------------------------------------------