├── .gitignore ├── Justfile ├── LICENSE.txt ├── README.md ├── dev-requirements.in ├── dev-requirements.txt ├── examples ├── list_followers.py ├── readme_example.py ├── simple_example.py └── why │ ├── Justfile │ ├── requirements-dev.in │ ├── requirements-dev.txt │ ├── requirements.in │ ├── requirements.txt │ ├── scripts │ ├── playground-1.mongodb.js │ └── populate │ └── why │ └── __init__.py ├── pyproject.toml ├── src └── docbridge │ └── __init__.py └── tests ├── conftest.py ├── test_asyncio.py └── test_docbridge.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/python,osx 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=python,osx 3 | 4 | ### OSX ### 5 | # General 6 | .DS_Store 7 | .AppleDouble 8 | .LSOverride 9 | 10 | # Icon must end with two \r 11 | Icon 12 | 13 | # Thumbnails 14 | ._* 15 | 16 | # Files that might appear in the root of a volume 17 | .DocumentRevisions-V100 18 | .fseventsd 19 | .Spotlight-V100 20 | .TemporaryItems 21 | .Trashes 22 | .VolumeIcon.icns 23 | .com.apple.timemachine.donotpresent 24 | 25 | # Directories potentially created on remote AFP share 26 | .AppleDB 27 | .AppleDesktop 28 | Network Trash Folder 29 | Temporary Items 30 | .apdisk 31 | 32 | ### Python ### 33 | # Byte-compiled / optimized / DLL files 34 | __pycache__/ 35 | *.py[cod] 36 | *$py.class 37 | 38 | # C extensions 39 | *.so 40 | 41 | # Distribution / packaging 42 | .Python 43 | build/ 44 | develop-eggs/ 45 | dist/ 46 | downloads/ 47 | eggs/ 48 | .eggs/ 49 | lib/ 50 | lib64/ 51 | parts/ 52 | sdist/ 53 | var/ 54 | wheels/ 55 | share/python-wheels/ 56 | *.egg-info/ 57 | .installed.cfg 58 | *.egg 59 | MANIFEST 60 | 61 | # PyInstaller 62 | # Usually these files are written by a python script from a template 63 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 64 | *.manifest 65 | *.spec 66 | 67 | # Installer logs 68 | pip-log.txt 69 | pip-delete-this-directory.txt 70 | 71 | # Unit test / coverage reports 72 | htmlcov/ 73 | .tox/ 74 | .nox/ 75 | .coverage 76 | .coverage.* 77 | .cache 78 | nosetests.xml 79 | coverage.xml 80 | *.cover 81 | *.py,cover 82 | .hypothesis/ 83 | .pytest_cache/ 84 | cover/ 85 | 86 | # Translations 87 | *.mo 88 | *.pot 89 | 90 | # Django stuff: 91 | *.log 92 | local_settings.py 93 | db.sqlite3 94 | db.sqlite3-journal 95 | 96 | # Flask stuff: 97 | instance/ 98 | .webassets-cache 99 | 100 | # Scrapy stuff: 101 | .scrapy 102 | 103 | # Sphinx documentation 104 | docs/_build/ 105 | 106 | # PyBuilder 107 | .pybuilder/ 108 | target/ 109 | 110 | # Jupyter Notebook 111 | .ipynb_checkpoints 112 | 113 | # IPython 114 | profile_default/ 115 | ipython_config.py 116 | 117 | # pyenv 118 | # For a library or package, you might want to ignore these files since the code is 119 | # intended to run in multiple environments; otherwise, check them in: 120 | # .python-version 121 | 122 | # pipenv 123 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 124 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 125 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 126 | # install all needed dependencies. 127 | #Pipfile.lock 128 | 129 | # poetry 130 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 131 | # This is especially recommended for binary packages to ensure reproducibility, and is more 132 | # commonly ignored for libraries. 133 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 134 | #poetry.lock 135 | 136 | # pdm 137 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 138 | #pdm.lock 139 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 140 | # in version control. 141 | # https://pdm.fming.dev/#use-with-ide 142 | .pdm.toml 143 | 144 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 145 | __pypackages__/ 146 | 147 | # Celery stuff 148 | celerybeat-schedule 149 | celerybeat.pid 150 | 151 | # SageMath parsed files 152 | *.sage.py 153 | 154 | # Environments 155 | .env 156 | .venv 157 | env/ 158 | venv/ 159 | ENV/ 160 | env.bak/ 161 | venv.bak/ 162 | 163 | # Spyder project settings 164 | .spyderproject 165 | .spyproject 166 | 167 | # Rope project settings 168 | .ropeproject 169 | 170 | # mkdocs documentation 171 | /site 172 | 173 | # mypy 174 | .mypy_cache/ 175 | .dmypy.json 176 | dmypy.json 177 | 178 | # Pyre type checker 179 | .pyre/ 180 | 181 | # pytype static type analyzer 182 | .pytype/ 183 | 184 | # Cython debug symbols 185 | cython_debug/ 186 | 187 | # PyCharm 188 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 189 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 190 | # and can be added to the global gitignore or merged into this file. For a more nuclear 191 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 192 | #.idea/ 193 | 194 | ### Python Patch ### 195 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 196 | poetry.toml 197 | 198 | # ruff 199 | .ruff_cache/ 200 | 201 | # LSP config files 202 | pyrightconfig.json 203 | 204 | # End of https://www.toptal.com/developers/gitignore/api/python,osx 205 | .vscode 206 | -------------------------------------------------------------------------------- /Justfile: -------------------------------------------------------------------------------- 1 | build: 2 | python -m build 3 | 4 | publish: 5 | twine 6 | 7 | update-deps: 8 | pip-compile dev-requirements.in 9 | 10 | develop: 11 | python -m pip install -e . 12 | python -m pip install -r dev-requirements.txt 13 | 14 | test: 15 | python -m pytest 16 | 17 | clean: 18 | rm -rf dist -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Docbridge 2 | 3 | This is an **experimental** Object-Document Mapping library for MongoDB. 4 | You can watch it being developed *live* on [MongoDB's YouTube channel](https://www.youtube.com/@MongoDB)! 5 | 6 | ## Mission Statement 7 | 8 | * Managing large amounts of data in MongoDB while keeping a data schema flexible is challenging. 9 | * This ODM is not an active record implementation, mapping documents in the database directly into similar objects in code. 10 | * This ODM *is* designed to abstract underlying documents, mapping potentially multiple document schemata into a shared object representation. 11 | * It should also simplify the evolution of documents in the database, automatically migrating individual documents' schemas either on-read or on-write. 12 | * There should be "escape hatches" so that unforeseen mappings can be implemented, hiding away the implementation code behind hopefully reuseable components. 13 | 14 | ## What Does It Do? 15 | 16 | The library currently doesn't interact directly with MongoDB - what it _does_ do is wrap BSON documents returned by [PyMongo] or [Motor]. 17 | 18 | For example, let's say you have a BSON document like this: 19 | 20 | ```python 21 | user_data_bson = {'_id': ObjectId('657072b56731c9e580e9dd70'), 22 | 'bio': 'Music conference able doctor degree debate. Participant usually above ' 23 | 'relate.', 24 | 'birth_date': datetime.datetime(1999, 7, 6, 0, 0), 25 | 'email': 'deanjacob@yahoo.com', 26 | 'follower_count': 59, 27 | 'full_name': 'Deborah White', 28 | 'user_id': '4', 29 | 'user_name': '@tanya15', 30 | 'followers': [{'_id': ObjectId('657072b66731c9e580e9dda6'), 31 | 'bio': 'Rich beautiful color life. Relationship instead win ' 32 | 'join enough board successful.', 33 | 'user_id': '58', 34 | 'user_name': '@rduncan'}, 35 | {'_id': ObjectId('657072b66731c9e580e9dd99'), 36 | 'bio': 'Picture day couple democratic morning. Environment ' 37 | 'manage opportunity option star food she. Occur imagine ' 38 | 'population single avoid.', 39 | 'user_id': '45', 40 | 'user_name': '@paynericky'}, 41 | ]} 42 | ``` 43 | 44 | You can define a wrapper for it like this: 45 | 46 | ```python 47 | from docbridge import Document 48 | 49 | 50 | class UserProfile(Document): 51 | pass 52 | ``` 53 | 54 | The wrapper doesn't currently do very much - it just makes the `dict` returned by PyMongo look more like a regular Python class: 55 | 56 | ```python 57 | profile = UserProfile(user_data_bson, db=None) 58 | print(repr(profile._id)) # ObjectId('657072b56731c9e580e9dd70') 59 | print(repr(profile.user_id)) # "4" 60 | ``` 61 | 62 | The real power of the library (like with most [ODM]s) comes from attaching field definitions to the class, to transform the way data is looked up on the underlying document. 63 | 64 | Here is how the `Field` class can be used to configure mappings to different field names in the underlying document, or to transform the data in the underlying field, to convert a string to an int: 65 | 66 | ```python 67 | from docbridge import Document, Field 68 | 69 | 70 | class UserProfile(Document): 71 | id = Field(field_name="_id") # id maps to the _id doc field. 72 | user_id = Field(transform=int) # user_id transforms the field value to an int 73 | 74 | 75 | profile = UserProfile(user_data_bson, db=None) 76 | print(repr(profile.id)) # ObjectId('657072b56731c9e580e9dd70') 77 | print(repr(profile.user_id)) # 4 <- This is an int now! 78 | print( 79 | repr(profile.follower_count) 80 | ) # 59 <- You can still access other doc fields as attributes. 81 | ``` 82 | 83 | ## Fallthrough Fields 84 | 85 | There are other types of field, though. 86 | FallthroughField is one of them. 87 | It allows you to _try_ to look up a field by one name, 88 | and if the field is missing, 89 | it will try other names that it's been configured with. 90 | 91 | **Note:** This field type will _probably_ disappear, as I may merge its 92 | functionality into `Field`. 93 | 94 | ```python 95 | from docbridge import Document, FallthroughField 96 | 97 | class UserProfile(Document): 98 | # The `name` attribute will look up the "full_name" field, 99 | # and fall back to the "name" if it's missing. 100 | name = FallthroughField( 101 | field_names=[ 102 | "full_name", # v2 103 | "name", # v1 104 | ] 105 | ) 106 | 107 | profile = UserProfile({"full_name", "Mark Smith"}) 108 | assert profile.name == "Mark Smith" # Works 109 | 110 | profile = UserProfile({"name", "Mark Smith"}) 111 | assert profile.name == "Mark Smith" # Also works! 112 | ``` 113 | 114 | ## The Subset Pattern 115 | 116 | Some support already exists for abstracting [MongoDB Design Patterns][mongodb-patterns], 117 | like the [Subset Pattern][subset]. 118 | The subset pattern preserves document size at a reasonable level by only embedding a subset of related data - for example, only the first 10 followers on a social media profile. The rest of the followers would be stored in their own collection, and loaded only when necessary. 119 | 120 | ```python 121 | class Follower(Document): 122 | _id = Field(transform=str) 123 | 124 | class Profile(Document): 125 | _id = Field(transform=str) 126 | followers = SequenceField( 127 | type=Follower, 128 | superset_collection="followers", 129 | # The following query will be executed on "followers" if the field 130 | # is iterated past the embedded follower subdocuments. 131 | superset_query=lambda ob: [ 132 | { 133 | "$match": {"user_id": ob.user_id}, 134 | }, 135 | {"$unwind": "$followers"}, 136 | {"$replaceRoot": {"newRoot": "$followers"}}, 137 | ], 138 | ) 139 | 140 | # Print all the profile's followers to the screen, 141 | # including those in the followers collection: 142 | profile = Profile(user_data_bson, db=test_db) 143 | for follower in profile: 144 | print(follower.id) 145 | ``` 146 | 147 | # Live Streams on YouTube 148 | 149 | I've been developing docbridge on YouTube. You can catch the live streams at 2pm GMT on Wednesdays, or you can view the recordings: 150 | 151 | ## Episode 1: Building a Simple Data Access Layer 152 | 153 | Introducing my plans for the library, and building out the `Document` class, and the `Simple` and `Fallthrough` classes. (The latter two get renamed later to `Field` and `FallthroughField`) 154 | 155 | [![Building a Simple Data Access Layer](https://img.youtube.com/vi/dXXkuLjjHBA/0.jpg)](https://www.youtube.com/watch?v=dXXkuLjjHBA) 156 | 157 | 158 | ## Episode 2: Testing and Publishing a Python Module 159 | 160 | Writing some Pytest test fixtures that will run tests in a transaction, and roll back any changes to the database. Then (attempting to) publish my module to PyPI! 161 | 162 | [![Testing and Publishing a Python Module](https://img.youtube.com/vi/X9QqA0alA8Q/0.jpg)](https://www.youtube.com/watch?v=X9QqA0alA8Q) 163 | 164 | ## Episode 3: Subsets & Joins - Part 1 165 | 166 | Joins are a fundamental part of data modeling in MongoDB! This episode adds a field type for embedded arrays, and in the next episode it'll be extended to look up data in other collections! 167 | 168 | [![Subsets & Joins: Part 1](https://img.youtube.com/vi/YvZeA_jvYrY/0.jpg)](https://www.youtube.com/watch?v=YvZeA_jvYrY) 169 | 170 | ## Episode 4: Subsets & Joins - Part 2 171 | 172 | More metaprogramming to turn a sequence of items that is split across documents and collections into a single Python sequence. 173 | 174 | [![Subsets & Joins: Part 2](https://img.youtube.com/vi/TJVLkVUUzGk/0.jpg)](https://www.youtube.com/watch?v=TJVLkVUUzGk) 175 | 176 | ## Episode 5: Updating Data - Part 1 177 | 178 | It's all very well reading data from the database, but it's also nice to be able 179 | to update it! 180 | 181 | [![Updating Data - Part 1](https://img.youtube.com/vi/Ab_NmiKP2_w/0.jpg)](https://www.youtube.com/watch?v=Ab_NmiKP2_w) 182 | 183 | ## Episode 6: Updating Data - Part 2 184 | 185 | It turns out there's quite a lot of work to record and replay updates. 186 | Let's get on with it! 187 | 188 | [![Updating Data - Part 2](https://img.youtube.com/vi/2kIrKr0n9WY/0.jpg)](https://www.youtube.com/watch?v=2kIrKr0n9WY) 189 | 190 | ## Episode 7: Updating Data - Part 3 191 | 192 | It turns out there's quite a lot of work to record and replay updates. 193 | Let's get on with it! 194 | 195 | [![Updating Data - Part 3](https://img.youtube.com/vi/3bW8Zzm8dpE/0.jpg)](https://www.youtube.com/watch?v=3bW8Zzm8dpE) 196 | 197 | 198 | 199 | [PyMongo]: https://pymongo.readthedocs.io/en/stable/ 200 | [Motor]: https://motor.readthedocs.io/en/stable/ 201 | [ODM]: https://www.mongodb.com/developer/products/mongodb/mongodb-orms-odms-libraries/ 202 | [subset]: https://www.mongodb.com/blog/post/building-with-patterns-the-subset-pattern 203 | [mongodb-patterns]: https://www.mongodb.com/blog/post/building-with-patterns-a-summary -------------------------------------------------------------------------------- /dev-requirements.in: -------------------------------------------------------------------------------- 1 | pip-tools 2 | pytest 3 | pytest-asyncio 4 | ruff 5 | twine -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.12 3 | # by the following command: 4 | # 5 | # pip-compile dev-requirements.in 6 | # 7 | build==1.2.1 8 | # via pip-tools 9 | certifi==2024.2.2 10 | # via requests 11 | charset-normalizer==3.3.2 12 | # via requests 13 | click==8.1.7 14 | # via pip-tools 15 | docutils==0.21.2 16 | # via readme-renderer 17 | idna==3.7 18 | # via requests 19 | importlib-metadata==7.1.0 20 | # via twine 21 | iniconfig==2.0.0 22 | # via pytest 23 | jaraco-classes==3.4.0 24 | # via keyring 25 | jaraco-context==5.3.0 26 | # via keyring 27 | jaraco-functools==4.0.1 28 | # via keyring 29 | keyring==25.1.0 30 | # via twine 31 | markdown-it-py==3.0.0 32 | # via rich 33 | mdurl==0.1.2 34 | # via markdown-it-py 35 | more-itertools==10.2.0 36 | # via 37 | # jaraco-classes 38 | # jaraco-functools 39 | nh3==0.2.17 40 | # via readme-renderer 41 | packaging==24.0 42 | # via 43 | # build 44 | # pytest 45 | pip-tools==7.4.1 46 | # via -r dev-requirements.in 47 | pkginfo==1.10.0 48 | # via twine 49 | pluggy==1.5.0 50 | # via pytest 51 | pygments==2.17.2 52 | # via 53 | # readme-renderer 54 | # rich 55 | pyproject-hooks==1.0.0 56 | # via 57 | # build 58 | # pip-tools 59 | pytest==8.1.1 60 | # via 61 | # -r dev-requirements.in 62 | # pytest-asyncio 63 | pytest-asyncio==0.23.6 64 | # via -r dev-requirements.in 65 | readme-renderer==43.0 66 | # via twine 67 | requests==2.31.0 68 | # via 69 | # requests-toolbelt 70 | # twine 71 | requests-toolbelt==1.0.0 72 | # via twine 73 | rfc3986==2.0.0 74 | # via twine 75 | rich==13.7.1 76 | # via twine 77 | ruff==0.4.1 78 | # via -r dev-requirements.in 79 | twine==5.0.0 80 | # via -r dev-requirements.in 81 | urllib3==2.2.1 82 | # via 83 | # requests 84 | # twine 85 | wheel==0.43.0 86 | # via pip-tools 87 | zipp==3.18.1 88 | # via importlib-metadata 89 | 90 | # The following packages are considered to be unsafe in a requirements file: 91 | # pip 92 | # setuptools 93 | -------------------------------------------------------------------------------- /examples/list_followers.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import docbridge 5 | import pymongo 6 | 7 | mdb_uri = os.environ["MDB_URI"] 8 | 9 | 10 | class Follower(docbridge.Document): 11 | pass 12 | 13 | 14 | class Profile(docbridge.Document): 15 | id = docbridge.Field(field_name="user_id", transform=int) 16 | followers = docbridge.SequenceField( 17 | type=Follower, 18 | superset_collection="followers", 19 | superset_query=lambda ob: [ 20 | { 21 | "$match": {"user_id": ob.user_id}, 22 | }, 23 | {"$unwind": "$followers"}, 24 | {"$replaceRoot": {"newRoot": "$followers"}}, 25 | ], 26 | ) 27 | 28 | 29 | def main(argv=sys.argv[1:]): 30 | client = pymongo.MongoClient(mdb_uri) 31 | db = client.get_database("why") 32 | profiles = db.get_collection("profiles") 33 | 34 | if db.command("ping")["ok"] < 0.5: 35 | raise Exception("Problem connected to database cluster.") 36 | 37 | profile = Profile(profiles.find_one({"user_id": "4"}), db) 38 | print(profile.id) 39 | print(type(profile.id)) 40 | for index, follower in enumerate(profile.followers): 41 | # print(f"{index}: {follower['user_name']}") 42 | print(f"{index}: {follower.user_name}") 43 | 44 | 45 | if __name__ == "__main__": 46 | main() 47 | -------------------------------------------------------------------------------- /examples/readme_example.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pprint import pprint 3 | from pymongo import MongoClient 4 | 5 | collection = ( 6 | MongoClient(os.environ["MDB_URI"]).get_database("why").get_collection("profiles") 7 | ) 8 | 9 | user_data_bson = collection.find_one({"user_id": "4"}) 10 | pprint(user_data_bson) 11 | 12 | # Example 1: 13 | from docbridge import Document 14 | 15 | 16 | class UserProfile(Document): 17 | pass 18 | 19 | 20 | profile = UserProfile(user_data_bson, db=None) 21 | print(repr(profile._id)) # ObjectId('657072b56731c9e580e9dd70') 22 | print(repr(profile.user_id)) # "4" 23 | 24 | 25 | # Example 2: 26 | from docbridge import Document, Field 27 | 28 | 29 | class UserProfile(Document): 30 | id = Field(field_name="_id") # id maps to the _id doc field. 31 | user_id = Field(transform=int) # user_id transforms the field value to an int 32 | 33 | 34 | profile = UserProfile(user_data_bson, db=None) 35 | print(repr(profile.id)) # ObjectId('657072b56731c9e580e9dd70') 36 | print(repr(profile.user_id)) # 4 <- This is an int now! 37 | print( 38 | repr(profile.follower_count) 39 | ) # 59 <- You can still access other doc fields as attributes. 40 | 41 | # Example 3: 42 | from docbridge import Document, FallthroughField 43 | 44 | 45 | class UserProfile(Document): 46 | id = Field(field_name="_id") # id maps to the _id doc field. 47 | name = FallthroughField( 48 | field_names=[ 49 | "full_name", # v2 50 | "name", # v1 51 | ] 52 | ) 53 | # The `name` attribute will look up the "full_name" field, 54 | # and fall back to the "name" if it's missing. 55 | 56 | 57 | profile = UserProfile({"full_name", "Mark Smith"}) 58 | assert profile.name == "Mark Smith" # Works 59 | 60 | profile = UserProfile({"name", "Mark Smith"}) 61 | assert profile.name == "Mark Smith" # Also works! 62 | -------------------------------------------------------------------------------- /examples/simple_example.py: -------------------------------------------------------------------------------- 1 | import os 2 | from docbridge import Document, Field, FallthroughField 3 | from pymongo import MongoClient 4 | 5 | collection = ( 6 | MongoClient(os.environ["MDB_URI"]) 7 | .get_database("docbridge_test") 8 | .get_collection("people") 9 | ) 10 | 11 | collection.delete_many({}) # Clean up any leftover documents. 12 | # Insert a couple of sample documents: 13 | collection.insert_many( 14 | [ 15 | { 16 | "name": "Mark Smith", 17 | "schema_version": 1, 18 | }, 19 | { 20 | "full_name": "Mark Smith", 21 | "first_name": "Mark", 22 | "last_name": "Smith", 23 | "schema_version": 2, 24 | }, 25 | ] 26 | ) 27 | 28 | 29 | # Define a mapping for "person" documents: 30 | class Person(Document): 31 | version = Field("schema_version") 32 | name = FallthroughField( 33 | [ 34 | "name", # v1 35 | "full_name", # v2 36 | ] 37 | ) 38 | 39 | 40 | # This finds all the documents in the collection, but wraps each BSON document with a Person wrapper: 41 | people = (Person(doc, None) for doc in collection.find()) 42 | for person in people: 43 | print( 44 | "Name:", 45 | person.name, 46 | ) # The name (or full_name) of the underlying document. 47 | print( 48 | "Document version:", 49 | person.version, # The schema_version field of the underlying document. 50 | ) 51 | 52 | 53 | session = mongodb.start_session() 54 | session.start_transaction() 55 | try: 56 | my_collection.insert_one( 57 | {"this document": "will be erased"}, 58 | session=session, 59 | ) 60 | finally: 61 | session.abort_transaction() 62 | -------------------------------------------------------------------------------- /examples/why/Justfile: -------------------------------------------------------------------------------- 1 | update-deps: 2 | pip-compile requirements.in 3 | pip-compile requirements-dev.in 4 | 5 | install: 6 | python -m pip install -r requirements.txt -r requirements-dev.txt 7 | 8 | run: 9 | uvicorn why:app --reload -------------------------------------------------------------------------------- /examples/why/requirements-dev.in: -------------------------------------------------------------------------------- 1 | faker 2 | tqdm -------------------------------------------------------------------------------- /examples/why/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.12 3 | # by the following command: 4 | # 5 | # pip-compile requirements-dev.in 6 | # 7 | faker==24.11.0 8 | # via -r requirements-dev.in 9 | python-dateutil==2.9.0.post0 10 | # via faker 11 | six==1.16.0 12 | # via python-dateutil 13 | tqdm==4.66.2 14 | # via -r requirements-dev.in 15 | -------------------------------------------------------------------------------- /examples/why/requirements.in: -------------------------------------------------------------------------------- 1 | fastapi 2 | motor[srv] 3 | uvicorn 4 | jinja2 5 | beanie -------------------------------------------------------------------------------- /examples/why/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.12 3 | # by the following command: 4 | # 5 | # pip-compile requirements.in 6 | # 7 | annotated-types==0.6.0 8 | # via pydantic 9 | anyio==4.3.0 10 | # via starlette 11 | beanie==1.25.0 12 | # via -r requirements.in 13 | click==8.1.7 14 | # via 15 | # beanie 16 | # uvicorn 17 | dnspython==2.6.1 18 | # via pymongo 19 | fastapi==0.110.2 20 | # via -r requirements.in 21 | h11==0.14.0 22 | # via uvicorn 23 | idna==3.7 24 | # via anyio 25 | jinja2==3.1.3 26 | # via -r requirements.in 27 | lazy-model==0.2.0 28 | # via beanie 29 | markupsafe==2.1.5 30 | # via jinja2 31 | motor[srv]==3.4.0 32 | # via 33 | # -r requirements.in 34 | # beanie 35 | pydantic==2.7.1 36 | # via 37 | # beanie 38 | # fastapi 39 | # lazy-model 40 | pydantic-core==2.18.2 41 | # via pydantic 42 | pymongo[srv]==4.6.3 43 | # via motor 44 | sniffio==1.3.1 45 | # via anyio 46 | starlette==0.37.2 47 | # via fastapi 48 | toml==0.10.2 49 | # via beanie 50 | typing-extensions==4.11.0 51 | # via 52 | # fastapi 53 | # pydantic 54 | # pydantic-core 55 | uvicorn==0.29.0 56 | # via -r requirements.in 57 | -------------------------------------------------------------------------------- /examples/why/scripts/playground-1.mongodb.js: -------------------------------------------------------------------------------- 1 | // Playground 2 | use('why'); 3 | 4 | profiles = db.getCollection("profiles"); 5 | followers = db.getCollection("followers"); 6 | 7 | profiles.find({ "user_id": "4" }) -------------------------------------------------------------------------------- /examples/why/scripts/populate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from datetime import datetime 4 | from itertools import islice 5 | import os 6 | import random 7 | import sys 8 | 9 | from faker import Faker 10 | from pymongo import MongoClient 11 | from tqdm import tqdm 12 | 13 | 14 | def take(i, n): 15 | try: 16 | for _ in range(n): 17 | yield next(i) 18 | except StopIteration: 19 | pass 20 | 21 | 22 | def main(argv=sys.argv[1:]): 23 | client = MongoClient(os.environ["MDB_URI"]) 24 | db = client.get_database("why") 25 | profiles = db.get_collection("profiles") 26 | followers = db.get_collection("followers") 27 | 28 | profiles.drop() 29 | profiles.create_index({"user_id": 1}) 30 | profiles.create_index({"user_name": 1}) 31 | followers.drop() 32 | followers.create_index({"user_id": 1}) 33 | 34 | NUM_PROFILES = 100 35 | 36 | f = Faker() 37 | print("Creating profiles:", file=sys.stderr) 38 | for i in tqdm(range(NUM_PROFILES)): 39 | p = f.simple_profile() 40 | profiles.insert_one( 41 | { 42 | "user_id": str(i), 43 | "user_name": "@" + p["username"], 44 | "full_name": p["name"], 45 | "birth_date": datetime.combine(p["birthdate"], datetime.min.time()), 46 | "email": p["mail"], 47 | "bio": f.paragraph(), 48 | } 49 | ) 50 | 51 | print("Adding followers:", file=sys.stderr) 52 | # Loop through all the created profiles, and add followers: 53 | for profile in tqdm(profiles.find()): 54 | follower_count = random.randint(0, min(200, NUM_PROFILES - 1)) 55 | pipeline = [ 56 | { 57 | "$match": { 58 | "user_id": { 59 | "$ne": profile["user_id"], 60 | } 61 | } 62 | }, 63 | { 64 | "$sample": { 65 | "size": follower_count, 66 | }, 67 | }, 68 | { 69 | "$project": { 70 | "user_id": 1, 71 | "user_name": 1, 72 | "bio": 1, 73 | } 74 | }, 75 | ] 76 | selected_followers = profiles.aggregate(pipeline) 77 | embedded_followers = list(take(selected_followers, 20)) 78 | profiles.update_one( 79 | { 80 | "user_id": profile["user_id"], 81 | }, 82 | { 83 | "$set": { 84 | "followers": embedded_followers, 85 | "follower_count": follower_count, 86 | }, 87 | }, 88 | ) 89 | while True: 90 | joined_followers = list(take(selected_followers, 20)) 91 | print(len(joined_followers)) 92 | if len(joined_followers) == 0: 93 | break 94 | followers.insert_one( 95 | {"user_id": profile["user_id"], "followers": joined_followers} 96 | ) 97 | 98 | 99 | if __name__ == "__main__": 100 | main() 101 | -------------------------------------------------------------------------------- /examples/why/why/__init__.py: -------------------------------------------------------------------------------- 1 | from contextlib import asynccontextmanager 2 | from datetime import datetime 3 | import os 4 | 5 | from fastapi import FastAPI 6 | 7 | from motor.motor_asyncio import AsyncIOMotorClient 8 | # from docbridge import Document, Field, SequenceField 9 | from beanie import Document, init_beanie 10 | from pydantic import BaseModel, Field 11 | 12 | 13 | 14 | CONNECTION_STRING = os.environ["MDB_URI"] 15 | 16 | @asynccontextmanager 17 | async def db_lifespan(app: FastAPI): 18 | # Startup 19 | app.mongodb_client = motor = AsyncIOMotorClient(CONNECTION_STRING) 20 | app.database = db = motor.get_database("why") 21 | ping_response = await db.command("ping") 22 | if int(ping_response["ok"]) != 1: 23 | raise Exception("Problem connecting to database cluster.") 24 | else: 25 | print("Connected to database cluster.") 26 | 27 | await init_beanie(database=db, document_models=[Profile]) 28 | 29 | yield 30 | 31 | # Shutdown 32 | app.mongodb_client.close() 33 | 34 | 35 | app = FastAPI(lifespan=db_lifespan) 36 | 37 | 38 | class Follower(BaseModel): 39 | user_id: str 40 | 41 | class Profile(Document): 42 | user_id: str 43 | user_name: str 44 | full_name: str 45 | birth_date: datetime 46 | email: str 47 | followers: list[Follower] 48 | 49 | class Settings: 50 | name = "profiles" 51 | 52 | 53 | 54 | @app.get("/profiles/{user_id}") 55 | async def read_item(user_id: str) -> Profile: 56 | print(dir(Profile)) 57 | profile = await Profile.find_one({"user_id": user_id}) 58 | 59 | return profile 60 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "docbridge" 7 | version = "0.0.3" 8 | authors = [{ name = "Mark Smith", email = "mark.smith@mongodb.com" }] 9 | description = "A thin abstraction layer over BSON documents, to keep your data model agile." 10 | readme = "README.md" 11 | requires-python = ">=3.7" 12 | dependencies = ["pymongo[srv]==4.6.0"] 13 | classifiers = [ 14 | "Programming Language :: Python :: 3", 15 | "License :: OSI Approved :: Apache Software License", 16 | "Operating System :: OS Independent", 17 | "Development Status :: 1 - Planning", 18 | "Intended Audience :: Developers", 19 | "Topic :: Database", 20 | ] 21 | 22 | [project.urls] 23 | Homepage = "https://github.com/mongodb-developer/docbridge" 24 | Issues = "https://github.com/mongodb-developer/docbridge/issues" 25 | 26 | # [tool.ruff] 27 | # exclude = ["examples/*.py"] -------------------------------------------------------------------------------- /src/docbridge/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023-present MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | docbridge - An experimental Object-Document Mapper library, primarily designed for teaching. 17 | """ 18 | 19 | from typing import Any, Sequence, Mapping, Iterable, Callable 20 | 21 | __all__ = ["Document", "FallthroughField", "Field", "SequenceField"] 22 | 23 | _SENTINEL = object() 24 | NO_DEFAULT = object() 25 | 26 | 27 | class DocumentMeta(type): 28 | def __new__(cls, name, bases, dict, strict=False, **kwds): 29 | result = super().__new__(cls, name, bases, dict, **kwds) 30 | result._strict = strict 31 | return result 32 | 33 | 34 | class Document: 35 | """ 36 | An object wrapper for a BSON document. 37 | 38 | This class is designed to be subclassed, so that different Fields can be 39 | configured for attribute lookup. 40 | """ 41 | 42 | _doc = None 43 | _db = None 44 | _modified_fields = None 45 | _strict = False 46 | 47 | def __init__(self, doc, db): 48 | self._doc = doc 49 | self._db = db 50 | self._modified_fields = {} 51 | 52 | def __getattr__(self, attr): 53 | if attr == "_doc": 54 | return object.__getattribute__(self, attr) 55 | if not self._strict: 56 | return self._wrap(self._doc[attr]) 57 | 58 | else: 59 | raise AttributeError( 60 | f"{self.__class__.__name__!r} object has no attribute {attr!r}" 61 | ) 62 | 63 | def _wrap(self, value): 64 | if isinstance(value, dict): 65 | return Document(value, self._db) 66 | elif isinstance(value, list): 67 | result = [self._wrap(item) for item in value] 68 | return result 69 | else: 70 | return value 71 | 72 | def __setattr__(self, name: str, value: Any) -> None: 73 | if hasattr(self.__class__, name): 74 | super().__setattr__(name, value) 75 | elif not self._strict: 76 | self._doc[name] = value 77 | self._modified_fields[name] = value 78 | else: 79 | raise AttributeError( 80 | f"{self.__class__.__name__!r} cannot have instance attributes dynamically assigned." 81 | ) 82 | 83 | async def save(self, collection, match_criteria=None, session=None): 84 | if match_criteria is None: 85 | try: 86 | match_criteria = {"_id": self._doc["_id"]} 87 | except Exception: 88 | raise Exception( 89 | "Attempt to update a document without _id, without providing `match_criteria`." 90 | ) 91 | await self._db.get_collection(collection).update_one( 92 | match_criteria, {"$set": self._modified_fields}, session=session 93 | ) 94 | self._modified_fields = {} 95 | # TODO: Return something that details the update - error if no document updated? 96 | 97 | def __init_subclass__(cls, /, strict=False): 98 | cls._strict = strict 99 | 100 | 101 | def identity(val): 102 | return val 103 | 104 | 105 | class Field: 106 | """ 107 | Field is designed to configure attribute lookup for a `Document` attribute. 108 | 109 | Currently it can be configured to map to a different field name in the 110 | underlying BSON document, and can apply an optional transformation to 111 | convert the field value to a desired type. 112 | """ 113 | 114 | def __init__(self, field_name=None, default=NO_DEFAULT, transform=None): 115 | self.field_name = field_name 116 | self.transform = identity if transform is None else transform 117 | 118 | def __set_name__(self, owner, name): 119 | self.name = name 120 | if self.field_name is None: 121 | self.field_name = name 122 | 123 | def __get__(self, ob, cls): 124 | if ob is not None: 125 | try: 126 | return self.transform(ob._doc[self.field_name]) 127 | except KeyError as ke: 128 | raise ValueError( 129 | f"Attribute {self.name!r} is mapped to missing document property {self.field_name!r}." 130 | ) from ke 131 | 132 | return self 133 | 134 | def __set__(self, ob, value: Any) -> None: 135 | transformed_value = self.transform(value) 136 | ob._doc[self.field_name] = transformed_value 137 | print(f"Setting configured field {self.field_name} to {transformed_value}") 138 | ob._modified_fields[self.field_name] = transformed_value 139 | 140 | 141 | class FallthroughField: 142 | """ 143 | FallthroughField allows a series of different field names to be tried when looking up the attribute. 144 | The first field name that exists in the underlying document will be the value that is returned. 145 | 146 | This class's functionality will probably be rolled into `Field` instead of being its own class. 147 | """ 148 | 149 | def __init__(self, field_names: Sequence[str]) -> None: 150 | self.field_names = field_names 151 | 152 | def __get__(self, ob, cls): 153 | for field_name in self.field_names: 154 | try: 155 | return ob._doc[field_name] 156 | except KeyError: 157 | pass 158 | else: 159 | raise ValueError( 160 | f"Attribute {self.name!r} references the field names {', '.join([repr(fn) for fn in self.field_names])} which are not present." 161 | ) 162 | 163 | def __set_name__(self, owner, name): 164 | self.name = name 165 | 166 | 167 | class SequenceField: 168 | """ 169 | Allows an underlying array to have its elements wrapped in `Document` instances. 170 | """ 171 | 172 | def __init__( 173 | self, 174 | type, 175 | field_name=None, 176 | superset_collection=None, 177 | superset_query: Callable = None, 178 | ): 179 | self._type = type 180 | self.field_name = field_name 181 | self.superset_collection = superset_collection 182 | self.superset_query = superset_query 183 | 184 | def __get__(self, ob, cls): 185 | if self.superset_query is None: 186 | # Use an empty sequence if there are no extra items. 187 | # It's still iterable, like a cursor, but immediately exits. 188 | superset = [] 189 | else: 190 | # Call the superset_query callable to obtain the generated query: 191 | query = self.superset_query(ob) 192 | 193 | # If the query is a mapping, it's a `find` query, otherwise it's an 194 | # aggregation pipeline. 195 | if isinstance(query, Mapping): 196 | superset = ob._db.get_collection(self.superset_collection).find(query) 197 | elif isinstance(query, Iterable): 198 | superset = ob._db.get_collection(self.superset_collection).aggregate( 199 | query 200 | ) 201 | else: 202 | raise Exception("Returned was not a mapping or iterable.") 203 | 204 | try: 205 | # Return an iterable that first yields all the embedded items, and 206 | # then once that is exhausted, queries the database for more. 207 | return self.superset_iterator( 208 | ob, 209 | ob._doc[self.field_name], 210 | superset, 211 | ) 212 | except KeyError as ke: 213 | raise ValueError( 214 | f"Attribute {self.name!r} is mapped to missing document property {self.field_name!r}." 215 | ) from ke 216 | 217 | async def superset_iterator(self, ob, embedded, related): 218 | for item in embedded: 219 | yield self._type(item, ob._db) 220 | if isinstance(related, list): 221 | for item in related: 222 | yield self._type(item, ob._db) 223 | else: 224 | async for item in related: 225 | yield self._type(item, ob._db) 226 | 227 | def __set_name__(self, owner, name): 228 | self.name = name 229 | if self.field_name is None: 230 | self.field_name = name 231 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest_asyncio 3 | from motor.motor_asyncio import AsyncIOMotorClient as MotorClient 4 | 5 | 6 | @pytest_asyncio.fixture(scope="session") 7 | async def motor(): 8 | client = MotorClient(os.environ["MDB_URI"]) 9 | result = await client.admin.command("ping") 10 | assert result["ok"] > 0.5 11 | 12 | return client 13 | 14 | 15 | @pytest_asyncio.fixture(scope="session") 16 | async def rollback_session(motor: MotorClient): 17 | """ 18 | This fixture provides a session that will be aborted at the end of the test, to clean up any written data. 19 | """ 20 | session = await motor.start_session() 21 | session.start_transaction() 22 | try: 23 | yield session 24 | finally: 25 | await session.abort_transaction() 26 | -------------------------------------------------------------------------------- /tests/test_asyncio.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from docbridge import Document, Field, SequenceField 4 | 5 | 6 | @pytest.mark.asyncio(scope="session") 7 | async def test_motor_connection(motor): 8 | assert motor is not None 9 | 10 | 11 | @pytest.mark.asyncio(scope="session") 12 | async def test_embedded_sequence(motor): 13 | class Follower(Document): 14 | _id = Field(transform=str) 15 | 16 | class Profile(Document): 17 | _id = Field(transform=str) 18 | followers = SequenceField(type=Follower) 19 | 20 | db = motor.get_database("why") 21 | profiles = db.get_collection("profiles") 22 | 23 | bson = await profiles.find_one( 24 | {"user_id": "4"}, 25 | ) 26 | profile = Profile( 27 | bson, 28 | db, 29 | ) 30 | 31 | followers = [{"db_id": follower._id} async for follower in profile.followers] 32 | assert len(followers) == 20 33 | 34 | 35 | @pytest.mark.asyncio(scope="session") 36 | async def test_related_sequence(motor): 37 | class Follower(Document): 38 | _id = Field(transform=str) 39 | 40 | class Profile(Document): 41 | _id = Field(transform=str) 42 | followers = SequenceField( 43 | type=Follower, 44 | superset_collection="followers", 45 | superset_query=lambda ob: [ 46 | { 47 | "$match": {"user_id": ob.user_id}, 48 | }, 49 | {"$unwind": "$followers"}, 50 | {"$replaceRoot": {"newRoot": "$followers"}}, 51 | ], 52 | ) 53 | 54 | db = motor.get_database("why") 55 | profiles = db.get_collection("profiles") 56 | 57 | bson = await profiles.find_one( 58 | {"user_id": "4"}, 59 | ) 60 | profile = Profile( 61 | bson, 62 | db, 63 | ) 64 | 65 | followers = [{"db_id": follower._id} async for follower in profile.followers] 66 | assert len(followers) == 59 67 | -------------------------------------------------------------------------------- /tests/test_docbridge.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023-present MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pytest 16 | from pytest import fail 17 | import sys 18 | 19 | from docbridge import Document, Field, FallthroughField, SequenceField 20 | 21 | manhattan_data = { 22 | "_id": {"$oid": "63177d736c36240b38778162"}, 23 | "cocktail_name": "Manhattan", 24 | "description": "A classic cocktail consisting of Whiskey, Sweet Vermouth, and Angostura Bitters.", 25 | "created": {"$date": {"$numberLong": "1562176800000"}}, 26 | "modified": {"$date": {"$numberLong": "1586167200000"}}, 27 | "ingredients": [ 28 | {"name": "Bourbon", "quantity": {"$numberInt": "60"}, "unit": "ml"}, 29 | {"name": "Sweet Vermouth", "quantity": {"$numberInt": "30"}, "unit": "ml"}, 30 | {"name": "Angostura Bitters", "quantity": {"$numberInt": "1"}, "unit": "dash"}, 31 | { 32 | "name": "Spiced Cherry Bitters", 33 | "quantity": {"$numberInt": "1"}, 34 | "unit": "dash", 35 | }, 36 | ], 37 | "instructions": "Stir with ice. Serve in a martini glass with a maraschino cherry.", 38 | "garnish": "Maraschino Cherry", 39 | "favourite": True, 40 | "comments": ["I love this cocktail", "Meh. It's not for me."], 41 | "comments_length": 2, 42 | "schema_version": {"$numberInt": "2"}, 43 | } 44 | 45 | 46 | def test_cocktails(): 47 | class Cocktail(Document): 48 | name = FallthroughField(field_names=["name", "cocktail_name"]) 49 | 50 | manhattan = Cocktail(manhattan_data, None) 51 | assert manhattan.name == "Manhattan" 52 | 53 | 54 | def test_fallthrough(): 55 | class FallthroughClass(Document): 56 | a = FallthroughField(["a", "b"]) 57 | 58 | myc = FallthroughClass({"a": "the_a_value"}, None) 59 | assert myc.a == "the_a_value" 60 | 61 | myc = FallthroughClass({"a": None}, None) 62 | assert myc.a is None 63 | 64 | myc = FallthroughClass({"a": "the_a_value", "b": "the_b_value"}, None) 65 | assert myc.a == "the_a_value" 66 | 67 | myc = FallthroughClass({"b": "the_b_value"}, None) 68 | assert myc.a == "the_b_value" 69 | 70 | try: 71 | myc = FallthroughClass({"c": "not_in_the_cascade"}, None) 72 | assert myc.a == "should not be evaluated" 73 | fail() 74 | except ValueError as v: 75 | assert ( 76 | str(v) 77 | == """Attribute 'a' references the field names 'a', 'b' which are not present.""" 78 | ) 79 | 80 | 81 | @pytest.mark.asyncio(scope="session") 82 | async def test_mongodb_client(motor): 83 | assert (await motor.admin.command("ping"))["ok"] > 0.5 84 | 85 | 86 | @pytest.mark.asyncio(scope="session") 87 | async def test_update_mongodb(motor, rollback_session): 88 | await motor.docbridge.tests.insert_one( 89 | { 90 | "_id": "bad_document", 91 | "description": "If this still exists, then transactions aren't working.", 92 | }, 93 | session=rollback_session, 94 | ) 95 | assert ( 96 | await motor.docbridge.tests.find_one( 97 | {"_id": "bad_document"}, session=rollback_session 98 | ) 99 | is not None 100 | ) 101 | 102 | 103 | @pytest.mark.asyncio(scope="session") 104 | async def test_sequence_field(motor): 105 | sample_profile = { 106 | "_id": {"$oid": "657072b56731c9e580e9dd6f"}, 107 | "user_id": "4", 108 | "user_name": "@tara86", 109 | "full_name": "Bradley Olsen", 110 | "birth_date": {"$date": {"$numberLong": "1502064000000"}}, 111 | "email": "elizabeth92@yahoo.com", 112 | "bio": "Discussion maintain watch computer impact tree situation. Vote know dream strong cause recently.", 113 | "follower_count": {"$numberInt": "11"}, 114 | "followers": [ 115 | { 116 | "_id": {"$oid": "657072b76731c9e580e9ddc5"}, 117 | "user_id": "89", 118 | "user_name": "@christopherespinoza", 119 | "bio": "Require father citizen during. Nearly set of.", 120 | }, 121 | { 122 | "_id": {"$oid": "657072b56731c9e580e9dd72"}, 123 | "user_id": "6", 124 | "user_name": "@karenwilkins", 125 | "bio": "Each right different describe indicate scientist short look. Turn town either decade.", 126 | }, 127 | { 128 | "_id": {"$oid": "657072b76731c9e580e9ddb7"}, 129 | "user_id": "75", 130 | "user_name": "@tonymartinez", 131 | "bio": "Structure stage religious fund test. How eight large participant will morning first.", 132 | }, 133 | ], 134 | } 135 | 136 | class Follower(Document): 137 | _id = Field(transform=str) 138 | 139 | class Profile(Document): 140 | _id = Field(transform=str) 141 | followers = SequenceField(type=Follower) 142 | 143 | profile = Profile(sample_profile, None) 144 | assert isinstance(await anext(profile.followers), Follower) 145 | 146 | 147 | async def aenumerate(aiterable): 148 | i = 0 149 | async for x in aiterable: 150 | yield i, x 151 | i += 1 152 | 153 | 154 | async def aislice(aiterable, *args): 155 | s = slice(*args) 156 | it = iter(range(s.start or 0, s.stop or sys.maxsize, s.step or 1)) 157 | try: 158 | nexti = next(it) 159 | except StopIteration: 160 | return 161 | async for i, element in aenumerate(aiterable): 162 | if i == nexti: 163 | yield element 164 | try: 165 | nexti = next(it) 166 | except StopIteration: 167 | return 168 | 169 | 170 | @pytest.mark.asyncio(scope="session") 171 | async def test_sequence_field_superset(motor): 172 | class Follower(Document): 173 | _id = Field(transform=str) 174 | 175 | class Profile(Document): 176 | _id = Field(transform=str) 177 | followers = SequenceField( 178 | type=Follower, 179 | superset_collection="followers", 180 | superset_query=lambda ob: [ 181 | { 182 | "$match": {"user_id": ob.user_id}, 183 | }, 184 | {"$unwind": "$followers"}, 185 | {"$replaceRoot": {"newRoot": "$followers"}}, 186 | ], 187 | ) 188 | 189 | db = motor.get_database("why") 190 | profile = Profile( 191 | await db.get_collection("profiles").find_one({"user_id": "4"}), db 192 | ) 193 | assert profile.user_id == "4" 194 | assert profile.full_name == "Deborah White" 195 | follower_boundary = aislice(profile.followers, 19, 21) 196 | last_embed = await anext(follower_boundary) 197 | print(last_embed) 198 | assert last_embed.user_name == "@nbrown" 199 | first_related = await anext(follower_boundary) 200 | assert first_related.user_name == "@hooperchristopher" 201 | 202 | 203 | @pytest.mark.asyncio(scope="session") 204 | async def test_update_field(motor, rollback_session): 205 | class Profile(Document): 206 | user_id = Field(transform=str.lower) 207 | 208 | db = motor.get_database("why") 209 | profile = Profile( 210 | await db.get_collection("profiles").find_one({"user_id": "4"}), db 211 | ) 212 | 213 | assert isinstance(Profile.user_id, Field) 214 | 215 | # Test that storing a configured value stores the (transformed) value on _doc: 216 | profile.user_id = "TEST_VALUE_4" 217 | assert profile.user_id == "test_value_4" 218 | assert profile._doc["user_id"] == "test_value_4" 219 | assert profile._modified_fields["user_id"] == "test_value_4" 220 | assert len(profile._modified_fields) == 1 221 | 222 | # Test that storing dynamic attributes stores the value in _doc: 223 | profile.non_existant = "new value" 224 | profile.non_existant == "new value" 225 | assert profile._doc["non_existant"] == "new value" 226 | assert profile._modified_fields["non_existant"] == "new value" 227 | assert len(profile._modified_fields) == 2 228 | 229 | 230 | @pytest.mark.asyncio(scope="session") 231 | async def test_update_strict_document(motor, rollback_session): 232 | class Profile(Document, strict=True): 233 | user_id = Field(transform=str.lower) 234 | 235 | db = motor.get_database("why") 236 | profile = Profile( 237 | await db.get_collection("profiles").find_one({"user_id": "4"}), db 238 | ) 239 | 240 | # Pre-defined field: 241 | profile.user_id = "TEST_VALUE_4" 242 | 243 | assert profile.user_id == "test_value_4" 244 | assert profile._doc["user_id"] == "test_value_4" 245 | assert profile._modified_fields["user_id"] == "test_value_4" 246 | assert len(profile._modified_fields) == 1 247 | 248 | try: 249 | profile.non_existant = "new value" 250 | fail("Should not be able to set dynamic value") 251 | except Exception: 252 | pass 253 | 254 | 255 | @pytest.mark.asyncio(scope="session") 256 | async def test_save(motor, rollback_session): 257 | class Profile(Document): 258 | user_id = Field(transform=str.lower) 259 | 260 | db = motor.get_database("why") 261 | profile = Profile( 262 | await db.get_collection("profiles").find_one( 263 | {"user_id": "4"}, session=rollback_session 264 | ), 265 | db, 266 | ) 267 | 268 | # This is a dynamic field: 269 | assert profile.user_name == "@tanya15" 270 | profile.user_name = "new name value" 271 | assert "user_name" in profile._modified_fields 272 | 273 | # This is a configured field: 274 | assert profile.user_id == "4" 275 | profile.user_id = "new id value" 276 | assert "user_id" in profile._modified_fields 277 | 278 | await profile.save("profiles", session=rollback_session) 279 | 280 | doc = await db.get_collection("profiles").find_one( 281 | {"user_id": "new id value"}, session=rollback_session 282 | ) 283 | assert doc is not None 284 | assert doc["user_id"] == "new id value" 285 | assert doc["user_name"] == "new name value" 286 | 287 | assert profile._modified_fields == {} 288 | 289 | 290 | def test_meta(): 291 | class StrictProfile(Document, strict=True): 292 | user_id = Field(transform=str.lower) 293 | 294 | assert StrictProfile._strict is True 295 | 296 | class Profile(Document): 297 | user_id = Field(transform=str.lower) 298 | 299 | assert Profile._strict is False 300 | --------------------------------------------------------------------------------