├── .github ├── tag-changelog-config.js └── workflows │ ├── deploy.yml │ └── tests.yml ├── .gitignore ├── LICENSE.txt ├── MAINTAINERS.md ├── MANIFEST.in ├── README.rst ├── dpath ├── __init__.py ├── exceptions.py ├── options.py ├── py.typed ├── segments.py ├── types.py ├── util.py └── version.py ├── flake8.ini ├── maintainers_log.md ├── setup.py ├── tests ├── __init__.py ├── test_broken_afilter.py ├── test_delete.py ├── test_get_values.py ├── test_merge.py ├── test_new.py ├── test_path_get.py ├── test_path_paths.py ├── test_paths.py ├── test_search.py ├── test_segments.py ├── test_set.py ├── test_types.py └── test_unicode.py └── tox.ini /.github/tag-changelog-config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | types: [ 3 | { types: ["other"], label: "Commits" }, 4 | ], 5 | 6 | renderTypeSection: function (label, commits) { 7 | let text = `\n## ${label}\n`; 8 | 9 | commits.forEach((commit) => { 10 | text += `- ${commit.subject}\n`; 11 | }); 12 | 13 | return text; 14 | }, 15 | 16 | renderChangelog: function (release, changes) { 17 | const now = new Date(); 18 | return `# ${release} - ${now.toISOString().substr(0, 10)}\n` + changes + "\n\n"; 19 | }, 20 | }; 21 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy and Release 2 | 3 | # Controls when the workflow will run 4 | on: 5 | # Triggers the workflow on version change 6 | push: 7 | branches: 8 | - master 9 | paths: 10 | - dpath/version.py 11 | 12 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 13 | jobs: 14 | # This workflow contains a single job called "deploy" 15 | deploy: 16 | # The type of runner that the job will run on 17 | runs-on: ubuntu-latest 18 | 19 | # Steps represent a sequence of tasks that will be executed as part of the job 20 | steps: 21 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 22 | - uses: actions/checkout@v2 23 | 24 | - name: Get Version 25 | id: get-version 26 | run: | 27 | python -c "from dpath.version import VERSION; print(f'::set-output name=version::v{VERSION}');" 28 | 29 | - name: Check Tag 30 | uses: mukunku/tag-exists-action@v1.0.0 31 | id: check-tag 32 | with: 33 | tag: ${{ steps.get-version.outputs.version }} 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | 37 | - name: Create Tag 38 | if: steps.check-tag.outputs.exists == 'false' 39 | uses: negz/create-tag@v1 40 | with: 41 | version: ${{ steps.get-version.outputs.version }} 42 | token: ${{ secrets.GITHUB_TOKEN }} 43 | 44 | - name: Generate Changelog 45 | id: generate-changelog 46 | uses: loopwerk/tag-changelog@v1 47 | with: 48 | token: ${{ secrets.GITHUB_TOKEN }} 49 | config_file: .github/tag-changelog-config.js 50 | 51 | - name: PyPI Deployment 52 | uses: casperdcl/deploy-pypi@v2 53 | with: 54 | # PyPI username 55 | user: ${{ secrets.PYPI_USER }} 56 | # PyPI password or API token 57 | password: ${{ secrets.PYPI_PASS }} 58 | # `setup.py` command to run ("true" is a shortcut for "clean sdist -d bdist_wheel -d ") 59 | build: clean sdist -d dist/ 60 | # `pip` command to run ("true" is a shortcut for "wheel -w --no-deps .") 61 | pip: true 62 | 63 | - name: Github Release 64 | uses: softprops/action-gh-release@v1 65 | with: 66 | tag_name: ${{ steps.get-version.outputs.version }} 67 | body: ${{ steps.generate-changelog.outputs.changes }} 68 | files: dist/* 69 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | # Controls when the workflow will run 4 | on: 5 | # Triggers the workflow on push or pull request events but only for important files 6 | push: 7 | branches: 8 | - master 9 | paths: 10 | - "dpath/" 11 | - "**.py" 12 | - "tox.ini" 13 | pull_request: 14 | paths: 15 | - "dpath/" 16 | - "**.py" 17 | - "tox.ini" 18 | 19 | # Allows you to run this workflow manually from the Actions tab 20 | workflow_dispatch: 21 | 22 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 23 | jobs: 24 | 25 | # Run flake8 linter 26 | flake8: 27 | runs-on: ubuntu-latest 28 | 29 | steps: 30 | - name: Check out code 31 | uses: actions/checkout@main 32 | 33 | - name: Set up Python 3.12 34 | uses: actions/setup-python@main 35 | with: 36 | python-version: "3.12" 37 | 38 | - name: Setup flake8 annotations 39 | uses: TrueBrain/actions-flake8@v2.3 40 | with: 41 | path: setup.py dpath/ tests/ 42 | 43 | # Generate a common hashseed for all tests 44 | generate-hashseed: 45 | runs-on: ubuntu-latest 46 | 47 | outputs: 48 | hashseed: ${{ steps.generate.outputs.hashseed }} 49 | 50 | steps: 51 | - name: Generate Hashseed 52 | id: generate 53 | run: | 54 | python -c "import os 55 | from random import randint 56 | hashseed = randint(0, 4294967295) 57 | print(f'{hashseed=}') 58 | open(os.environ['GITHUB_OUTPUT'], 'a').write(f'hashseed={hashseed}')" 59 | 60 | # Tests job 61 | tests: 62 | # The type of runner that the job will run on 63 | runs-on: ubuntu-latest 64 | 65 | needs: [generate-hashseed, flake8] 66 | 67 | strategy: 68 | matrix: 69 | # Match versions specified in tox.ini 70 | python-version: ['3.8', '3.9', '3.10', '3.11', 'pypy-3.7', '3.12'] 71 | 72 | # Steps represent a sequence of tasks that will be executed as part of the job 73 | steps: 74 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 75 | - name: Check out code 76 | uses: actions/checkout@main 77 | 78 | - name: Set up Python ${{ matrix.python-version }} 79 | uses: actions/setup-python@main 80 | with: 81 | python-version: ${{ matrix.python-version }} 82 | 83 | - name: Run tox with tox-gh-actions 84 | uses: ymyzk/run-tox-gh-actions@main 85 | with: 86 | tox-args: -vv --hashseed=${{ needs.generate-hashseed.outputs.hashseed }} 87 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /MANIFEST 2 | /.tox 3 | /build 4 | /env 5 | .hypothesis 6 | *.pyc 7 | .vscode 8 | venv_39 9 | .idea/ 10 | dpath.egg-info/ 11 | dist/ 12 | tests/.hypothesis -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Andrew Kesterson , Caleb Case 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | Who Maintains DPATH 2 | =================== 3 | 4 | dpath was created by and originally maintained by Andrew Kesterson and Caleb Case . In July 5 | of 2020 they put out a call for new maintainers. [@bigsablept](https://github.com/bigsablept) and 6 | [@moomoohk](https://github.com/moomoohk) stepped up to become the new maintainers. 7 | 8 | There are several individuals in the community who have taken an active role in helping to maintain the project and submit fixes. Those individuals are shown in the git changelog. 9 | 10 | Where and How do we communicate 11 | =============================== 12 | 13 | The dpath maintainers communicate in 3 primary ways: 14 | 15 | 1. Email, directly to each other. 16 | 2. Github via issue and pull request comments 17 | 3. A monthly maintainers meeting via Zoom 18 | 19 | The remainder of this document is subject to change after further discussion among the new maintainers. 20 | 21 | What is the roadmap 22 | =================== 23 | 24 | dpath has 3 major series: 1.x, 2.x, and 3.x. 25 | 26 | 1.x is the original dpath release from way way back. It has a util library with a C-like calling convention, lots of assumptions about how it would be used (it was built originally to solve a somewhat narrow use case), and very bad unicode support. 27 | 28 | 2.x is a transitional branch that intends to fix the unicode support and to introduce some newer concepts (such as the segments library) while still being backwards compatible with 1.x. 29 | 30 | 3.x is a total reconstruction of the library that does not guarantee backwards compatibility with 1.x. 31 | 32 | Finding and Prioritizing Work 33 | ============================= 34 | 35 | There are GitHub project boards which show the work to be done for a given series: 36 | 37 | https://github.com/akesterson/dpath-python/projects/ 38 | 39 | Each series has a board with 4 columns: 40 | 41 | * Backlog. New work for this series appears here. 42 | * To Do. This column represents work that has been prioritized and someone has agreed to do the work when they have an available time slot. Each maintainer should never have more than 1 or 2 things in To Do. 43 | * In Progress. Maintainers are actively working on these issues. 44 | * Done. These issues have been recently completed. 45 | 46 | Work is prioritized depending on: 47 | 48 | 1. The type of work. Bugs almost always get worked before features. 49 | 2. The versions impacted by the work. Versions which are already in use get worked first (so 1.x before 2.x before 3.x etc) 50 | 3. The relative importance/usefulness of the work. "Really useful" tends to get worked before "nice to have". 51 | 4. The amount of time to complete the work. Quick issues tend to get worked sooner than issues that will take a long time to resolve. 52 | 53 | There is no specific SLA around dpath, for features or bugs. However, generally speaking: 54 | 55 | * All issues get triaged within 1 calendar month 56 | * High priority bugs get addressed on the monthly maintainers call 57 | * Very severe bugs are often fixed out of cycle in less than 30 days 58 | 59 | Note that we have not always had anything remotely resembling a rigorous process around this, so there are some bugs that have lingered for several years. This is not something we intend to repeat. 60 | 61 | Taking and Completing Work 62 | ========================== 63 | 64 | Anyone who wants to is welcome to submit a pull request against a given issue. You do not need any special maintainer permissions to say "hey, I know how to solve that, let me send up a PR". 65 | 66 | The more complete process goes: 67 | 68 | 1. Decide what issue(s) you will be working on 69 | 2. On the Projects tab on Github, move those items to the To Do column on the appropriate board 70 | 3. For the item you are ACTIVELY WORKING, move that item to "In Progress" 71 | 4. Create a fork of dpath-python, and name your branch for the work. We name bugfixes as "bugfix/ISSUENUMBER_shortname"; features are named "feature/ISSUENUMBER_shortname". 72 | 5. Complete and push your work on your fork. Use tox to test your work against the test suites. Features MUST ship with at least one new unit test that covers the new functionality. Bugfixes MUST ship with one new test (or an updated old test) that guards against regression. 73 | 6. Send your pull request 74 | 7. If accepted, the maintainers will merge your pull request and close the issue. 75 | 76 | Branching Strategy 77 | ================== 78 | 79 | We run a clean bleeding edge master. Long term support for major version numbers are broken out into version branches. 80 | 81 | * master : Current 3.x (bleeding edge) development 82 | * version/1.x : 1.x series bugfixes 83 | * version/2.x : 2.x series features and bugfixes 84 | 85 | We name bugfixes as "bugfix/ISSUENUMBER_shortname"; features are named "feature/ISSUENUMBER_shortname". All branches representing work against an issue must have the issue number in the branch name. 86 | 87 | Cutting a New Release 88 | ===================== 89 | 90 | Releases for dpath occur automatically from Github Actions based on version changes on the master branch. 91 | 92 | Due to legacy reasons older tag names do not follow a uniform format: 93 | 94 | akesterson@akesterson:~/dpath-python$ git tag 95 | 1.0-0 96 | 1.1 97 | 1.2-66 98 | 1.2-68 99 | 1.2-70 100 | build,1.2,70 101 | build,1.2,71 102 | build,1.2,72 103 | build,1.3,0 104 | build,1.3,1 105 | build,1.3,2 106 | build,1.3,3 107 | build,1.4,0 108 | build,1.4,1 109 | build,1.4,3 110 | build,1.5,0 111 | build,2.0,0 112 | 113 | Moving forward version numbers and tag names will be identical and follow the standard semver format. 114 | 115 | The version string is stored in `dpath/version.py` and tag names/release versions are generated using this string. 116 | 117 | akesterson@akesterson:~/dpath-python$ cat dpath/version.py 118 | VERSION = "2.0.0" 119 | 120 | To cut a new release, follow this procedure: 121 | 122 | 1. Commit a new `dpath/version.py` on the appropriate branch with the format "MAJOR.MINOR.RELEASE". 123 | 2. Github Actions SHOULD push the new release to PyPI on merge to `master`. 124 | 125 | See `.github/workflows/deploy.yml` for more information. 126 | 127 | If the Github workflow fails to update pypi, follow the instructions on manually creating a release, here: 128 | 129 | https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives 130 | 131 | Deployment CI was previously implemented using [Travis CI](https://travis-ci.org/github/akesterson/dpath-python). 132 | 133 | Running Tests 134 | ============= 135 | 136 | Tests are managed using [tox](https://tox.readthedocs.io/en/latest/). 137 | 138 | Environment creation and dependency installation is managed by this tool, all one has to do is install it with `pip` and run `tox` in this repo's root directory. 139 | 140 | Tests can also be run with Github Actions via the [tests.yml](https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml) workflow. 141 | 142 | This workflow will run automatically on pretty much any commit to any branch of this repo but manual runs are also available. 143 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt 2 | include README.md 3 | include README.rst 4 | recursive-include tests * 5 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | dpath-python 2 | ============ 3 | 4 | |PyPI| 5 | |Python Version| 6 | |Build Status| 7 | |Gitter| 8 | 9 | A python library for accessing and searching dictionaries via 10 | /slashed/paths ala xpath 11 | 12 | Basically it lets you glob over a dictionary as if it were a filesystem. 13 | It allows you to specify globs (ala the bash eglob syntax, through some 14 | advanced fnmatch.fnmatch magic) to access dictionary elements, and 15 | provides some facility for filtering those results. 16 | 17 | sdists are available on pypi: http://pypi.python.org/pypi/dpath 18 | 19 | Installing 20 | ========== 21 | 22 | The best way to install dpath is via easy\_install or pip. 23 | 24 | :: 25 | 26 | easy_install dpath 27 | pip install dpath 28 | 29 | Using Dpath 30 | =========== 31 | 32 | .. code-block:: python 33 | 34 | import dpath 35 | 36 | Separators 37 | ========== 38 | 39 | All of the functions in this library (except 'merge') accept a 40 | 'separator' argument, which is the character that should separate path 41 | components. The default is '/', but you can set it to whatever you want. 42 | 43 | Searching 44 | ========= 45 | 46 | Suppose we have a dictionary like this: 47 | 48 | .. code-block:: python 49 | 50 | x = { 51 | "a": { 52 | "b": { 53 | "3": 2, 54 | "43": 30, 55 | "c": [], 56 | "d": ['red', 'buggy', 'bumpers'], 57 | } 58 | } 59 | } 60 | 61 | ... And we want to ask a simple question, like "Get me the value of the 62 | key '43' in the 'b' hash which is in the 'a' hash". That's easy. 63 | 64 | .. code-block:: pycon 65 | 66 | >>> help(dpath.get) 67 | Help on function get in module dpath: 68 | 69 | get(obj, glob, separator='/') 70 | Given an object which contains only one possible match for the given glob, 71 | return the value for the leaf matching the given glob. 72 | 73 | If more than one leaf matches the glob, ValueError is raised. If the glob is 74 | not found, KeyError is raised. 75 | 76 | >>> dpath.get(x, '/a/b/43') 77 | 30 78 | 79 | Or you could say "Give me a new dictionary with the values of all 80 | elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. 81 | 82 | .. code-block:: pycon 83 | 84 | >>> help(dpath.search) 85 | Help on function search in module dpath: 86 | 87 | search(obj, glob, yielded=False) 88 | Given a path glob, return a dictionary containing all keys 89 | that matched the given glob. 90 | 91 | If 'yielded' is true, then a dictionary will not be returned. 92 | Instead tuples will be yielded in the form of (path, value) for 93 | every element in the document that matched the glob. 94 | 95 | ... Sounds easy! 96 | 97 | .. code-block:: pycon 98 | 99 | >>> result = dpath.search(x, "a/b/[cd]") 100 | >>> print(json.dumps(result, indent=4, sort_keys=True)) 101 | { 102 | "a": { 103 | "b": { 104 | "c": [], 105 | "d": [ 106 | "red", 107 | "buggy", 108 | "bumpers" 109 | ] 110 | } 111 | } 112 | } 113 | 114 | ... Wow that was easy. What if I want to iterate over the results, and 115 | not get a merged view? 116 | 117 | .. code-block:: pycon 118 | 119 | >>> for x in dpath.search(x, "a/b/[cd]", yielded=True): print(x) 120 | ... 121 | ('a/b/c', []) 122 | ('a/b/d', ['red', 'buggy', 'bumpers']) 123 | 124 | ... Or what if I want to just get all the values back for the glob? I 125 | don't care about the paths they were found at: 126 | 127 | .. code-block:: pycon 128 | 129 | >>> help(dpath.values) 130 | Help on function values in module dpath: 131 | 132 | values(obj, glob, separator='/', afilter=None, dirs=True) 133 | Given an object and a path glob, return an array of all values which match 134 | the glob. The arguments to this function are identical to those of search(), 135 | and it is primarily a shorthand for a list comprehension over a yielded 136 | search call. 137 | 138 | >>> dpath.values(x, '/a/b/d/*') 139 | ['red', 'buggy', 'bumpers'] 140 | 141 | Example: Setting existing keys 142 | ============================== 143 | 144 | Let's use that same dictionary, and set keys like 'a/b/[cd]' to the 145 | value 'Waffles'. 146 | 147 | .. code-block:: pycon 148 | 149 | >>> help(dpath.set) 150 | Help on function set in module dpath: 151 | 152 | set(obj, glob, value) 153 | Given a path glob, set all existing elements in the document 154 | to the given value. Returns the number of elements changed. 155 | 156 | >>> dpath.set(x, 'a/b/[cd]', 'Waffles') 157 | 2 158 | >>> print(json.dumps(x, indent=4, sort_keys=True)) 159 | { 160 | "a": { 161 | "b": { 162 | "3": 2, 163 | "43": 30, 164 | "c": "Waffles", 165 | "d": "Waffles" 166 | } 167 | } 168 | } 169 | 170 | Example: Adding new keys 171 | ======================== 172 | 173 | Let's make a new key with the path 'a/b/e/f/g', set it to "Roffle". This 174 | behaves like 'mkdir -p' in that it makes all the intermediate paths 175 | necessary to get to the terminus. 176 | 177 | .. code-block:: pycon 178 | 179 | >>> help(dpath.new) 180 | Help on function new in module dpath: 181 | 182 | new(obj, path, value) 183 | Set the element at the terminus of path to value, and create 184 | it if it does not exist (as opposed to 'set' that can only 185 | change existing keys). 186 | 187 | path will NOT be treated like a glob. If it has globbing 188 | characters in it, they will become part of the resulting 189 | keys 190 | 191 | >>> dpath.new(x, 'a/b/e/f/g', "Roffle") 192 | >>> print(json.dumps(x, indent=4, sort_keys=True)) 193 | { 194 | "a": { 195 | "b": { 196 | "3": 2, 197 | "43": 30, 198 | "c": "Waffles", 199 | "d": "Waffles", 200 | "e": { 201 | "f": { 202 | "g": "Roffle" 203 | } 204 | } 205 | } 206 | } 207 | } 208 | 209 | This works the way we expect with lists, as well. If you have a list 210 | object and set index 10 of that list object, it will grow the list 211 | object with None entries in order to make it big enough: 212 | 213 | .. code-block:: pycon 214 | 215 | >>> dpath.new(x, 'a/b/e/f/h', []) 216 | >>> dpath.new(x, 'a/b/e/f/h/13', 'Wow this is a big array, it sure is lonely in here by myself') 217 | >>> print(json.dumps(x, indent=4, sort_keys=True)) 218 | { 219 | "a": { 220 | "b": { 221 | "3": 2, 222 | "43": 30, 223 | "c": "Waffles", 224 | "d": "Waffles", 225 | "e": { 226 | "f": { 227 | "g": "Roffle", 228 | "h": [ 229 | null, 230 | null, 231 | null, 232 | null, 233 | null, 234 | null, 235 | null, 236 | null, 237 | null, 238 | null, 239 | null, 240 | null, 241 | null, 242 | "Wow this is a big array, it sure is lonely in here by myself" 243 | ] 244 | } 245 | } 246 | } 247 | } 248 | } 249 | 250 | Handy! 251 | 252 | Example: Deleting Existing Keys 253 | =============================== 254 | 255 | To delete keys in an object, use dpath.delete, which accepts the same globbing syntax as the other methods. 256 | 257 | .. code-block:: pycon 258 | 259 | >>> help(dpath.delete) 260 | 261 | delete(obj, glob, separator='/', afilter=None): 262 | Given a path glob, delete all elements that match the glob. 263 | 264 | Returns the number of deleted objects. Raises PathNotFound if 265 | no paths are found to delete. 266 | 267 | Example: Merging 268 | ================ 269 | 270 | Also, check out dpath.merge. The python dict update() method is 271 | great and all but doesn't handle merging dictionaries deeply. This one 272 | does. 273 | 274 | .. code-block:: pycon 275 | 276 | >>> help(dpath.merge) 277 | Help on function merge in module dpath: 278 | 279 | merge(dst, src, afilter=None, flags=4, _path='') 280 | Merge source into destination. Like dict.update() but performs 281 | deep merging. 282 | 283 | flags is an OR'ed combination of MergeType enum members. 284 | * ADDITIVE : List objects are combined onto one long 285 | list (NOT a set). This is the default flag. 286 | * REPLACE : Instead of combining list objects, when 287 | 2 list objects are at an equal depth of merge, replace 288 | the destination with the source. 289 | * TYPESAFE : When 2 keys at equal levels are of different 290 | types, raise a TypeError exception. By default, the source 291 | replaces the destination in this situation. 292 | 293 | >>> y = {'a': {'b': { 'e': {'f': {'h': [None, 0, 1, None, 13, 14]}}}, 'c': 'RoffleWaffles'}} 294 | >>> print(json.dumps(y, indent=4, sort_keys=True)) 295 | { 296 | "a": { 297 | "b": { 298 | "e": { 299 | "f": { 300 | "h": [ 301 | null, 302 | 0, 303 | 1, 304 | null, 305 | 13, 306 | 14 307 | ] 308 | } 309 | } 310 | }, 311 | "c": "RoffleWaffles" 312 | } 313 | } 314 | >>> dpath.merge(x, y) 315 | >>> print(json.dumps(x, indent=4, sort_keys=True)) 316 | { 317 | "a": { 318 | "b": { 319 | "3": 2, 320 | "43": 30, 321 | "c": "Waffles", 322 | "d": "Waffles", 323 | "e": { 324 | "f": { 325 | "g": "Roffle", 326 | "h": [ 327 | null, 328 | 0, 329 | 1, 330 | null, 331 | 13, 332 | 14, 333 | null, 334 | null, 335 | null, 336 | null, 337 | null, 338 | null, 339 | null, 340 | "Wow this is a big array, it sure is lonely in here by myself" 341 | ] 342 | } 343 | } 344 | }, 345 | "c": "RoffleWaffles" 346 | } 347 | } 348 | 349 | Now that's handy. You shouldn't try to use this as a replacement for the 350 | deepcopy method, however - while merge does create new dict and list 351 | objects inside the target, the terminus objects (strings and ints) are 352 | not copied, they are just re-referenced in the merged object. 353 | 354 | Filtering 355 | ========= 356 | 357 | All of the methods in this library (except new()) support a 'afilter' 358 | argument. This can be set to a function that will return True or False 359 | to say 'yes include that value in my result set' or 'no don't include 360 | it'. 361 | 362 | Filtering functions receive every terminus node in a search - e.g., 363 | anything that is not a dict or a list, at the very end of the path. For 364 | each value, they return True to include that value in the result set, or 365 | False to exclude it. 366 | 367 | Consider this example. Given the source dictionary, we want to find ALL 368 | keys inside it, but we only really want the ones that contain "ffle" in 369 | them: 370 | 371 | .. code-block:: pycon 372 | 373 | >>> print(json.dumps(x, indent=4, sort_keys=True)) 374 | { 375 | "a": { 376 | "b": { 377 | "3": 2, 378 | "43": 30, 379 | "c": "Waffles", 380 | "d": "Waffles", 381 | "e": { 382 | "f": { 383 | "g": "Roffle" 384 | } 385 | } 386 | } 387 | } 388 | } 389 | >>> def afilter(x): 390 | ... if "ffle" in str(x): 391 | ... return True 392 | ... return False 393 | ... 394 | >>> result = dpath.search(x, '**', afilter=afilter) 395 | >>> print(json.dumps(result, indent=4, sort_keys=True)) 396 | { 397 | "a": { 398 | "b": { 399 | "c": "Waffles", 400 | "d": "Waffles", 401 | "e": { 402 | "f": { 403 | "g": "Roffle" 404 | } 405 | } 406 | } 407 | } 408 | } 409 | 410 | Obviously filtering functions can perform more advanced tests (regular 411 | expressions, etc etc). 412 | 413 | Key Names 414 | ========= 415 | 416 | By default, dpath only understands dictionary keys that are integers or 417 | strings. String keys must be non-empty. You can change this behavior by 418 | setting a library-wide dpath option: 419 | 420 | .. code-block:: python 421 | 422 | import dpath.options 423 | dpath.options.ALLOW_EMPTY_STRING_KEYS = True 424 | 425 | Again, by default, this behavior is OFF, and empty string keys will 426 | result in ``dpath.exceptions.InvalidKeyName`` being thrown. 427 | 428 | Separator got you down? Use lists as paths 429 | ========================================== 430 | 431 | The default behavior in dpath is to assume that the path given is a string, which must be tokenized by splitting at the separator to yield a distinct set of path components against which dictionary keys can be individually glob tested. However, this presents a problem when you want to use paths that have a separator in their name; the tokenizer cannot properly understand what you mean by '/a/b/c' if it is possible for '/' to exist as a valid character in a key name. 432 | 433 | To get around this, you can sidestep the whole "filesystem path" style, and abandon the separator entirely, by using lists as paths. All of the methods in dpath.* support the use of a list instead of a string as a path. So for example: 434 | 435 | .. code-block:: python 436 | 437 | >>> x = { 'a': {'b/c': 0}} 438 | >>> dpath.get(['a', 'b/c']) 439 | 0 440 | 441 | dpath.segments : The Low-Level Backend 442 | ====================================== 443 | 444 | dpath is where you want to spend your time: this library has the friendly 445 | functions that will understand simple string globs, afilter functions, etc. 446 | 447 | dpath.segments is the backend pathing library. It passes around tuples of path 448 | components instead of string globs. 449 | 450 | .. |PyPI| image:: https://img.shields.io/pypi/v/dpath.svg?style=flat 451 | :target: https://pypi.python.org/pypi/dpath/ 452 | :alt: PyPI: Latest Version 453 | 454 | .. |Python Version| image:: https://img.shields.io/pypi/pyversions/dpath?style=flat 455 | :target: https://pypi.python.org/pypi/dpath/ 456 | :alt: Supported Python Version 457 | 458 | .. |Build Status| image:: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml/badge.svg 459 | :target: https://github.com/dpath-maintainers/dpath-python/actions/workflows/tests.yml 460 | 461 | .. |Gitter| image:: https://badges.gitter.im/dpath-python/chat.svg 462 | :target: https://gitter.im/dpath-python/chat?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge 463 | :alt: Gitter 464 | 465 | Contributors 466 | ============ 467 | 468 | We would like to thank the community for their interest and involvement. You 469 | have all made this project significantly better than the sum of its parts, and 470 | your continued feedback makes it better every day. Thank you so much! 471 | 472 | The following authors have contributed to this project, in varying capacities: 473 | 474 | + Caleb Case 475 | + Andrew Kesterson 476 | + Marc Abramowitz 477 | + Richard Han 478 | + Stanislav Ochotnicky 479 | + Misja Hoebe 480 | + Gagandeep Singh 481 | + Alan Gibson 482 | 483 | And many others! If we've missed you please open an PR and add your name here. 484 | -------------------------------------------------------------------------------- /dpath/__init__.py: -------------------------------------------------------------------------------- 1 | # Needed for pre-3.10 versions 2 | from __future__ import annotations 3 | 4 | __all__ = [ 5 | "new", 6 | "delete", 7 | "set", 8 | "get", 9 | "values", 10 | "search", 11 | "merge", 12 | "exceptions", 13 | "options", 14 | "segments", 15 | "types", 16 | "version", 17 | "MergeType", 18 | "PathSegment", 19 | "Filter", 20 | "Glob", 21 | "Path", 22 | "Hints", 23 | "Creator", 24 | ] 25 | 26 | from collections.abc import MutableMapping, MutableSequence 27 | from typing import Union, List, Any, Callable, Optional 28 | 29 | from dpath import segments, options 30 | from dpath.exceptions import InvalidKeyName, PathNotFound 31 | from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, Path, Hints 32 | 33 | _DEFAULT_SENTINEL = object() 34 | 35 | 36 | def _split_path(path: Path, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: 37 | """ 38 | Given a path and separator, return a tuple of segments. If path is 39 | already a non-leaf thing, return it. 40 | 41 | Note that a string path with the separator at index[0] will have the 42 | separator stripped off. If you pass a list path, the separator is 43 | ignored, and is assumed to be part of each key glob. It will not be 44 | stripped. 45 | """ 46 | if not segments.leaf(path): 47 | split_segments = path 48 | else: 49 | split_segments = path.lstrip(separator).split(separator) 50 | 51 | return split_segments 52 | 53 | 54 | def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator | None = None) -> MutableMapping: 55 | """ 56 | Set the element at the terminus of path to value, and create 57 | it if it does not exist (as opposed to 'set' that can only 58 | change existing keys). 59 | 60 | path will NOT be treated like a glob. If it has globbing 61 | characters in it, they will become part of the resulting 62 | keys 63 | 64 | creator allows you to pass in a creator method that is 65 | responsible for creating missing keys at arbitrary levels of 66 | the path (see the help for dpath.path.set) 67 | """ 68 | split_segments = _split_path(path, separator) 69 | if creator: 70 | return segments.set(obj, split_segments, value, creator=creator) 71 | return segments.set(obj, split_segments, value) 72 | 73 | 74 | def delete(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter | None = None) -> int: 75 | """ 76 | Given a obj, delete all elements that match the glob. 77 | 78 | Returns the number of deleted objects. Raises PathNotFound if no paths are 79 | found to delete. 80 | """ 81 | globlist = _split_path(glob, separator) 82 | 83 | def f(obj, pair, counter): 84 | (path_segments, value) = pair 85 | 86 | # Skip segments if they no longer exist in obj. 87 | if not segments.has(obj, path_segments): 88 | return 89 | 90 | matched = segments.match(path_segments, globlist) 91 | selected = afilter and segments.leaf(value) and afilter(value) 92 | 93 | if (matched and not afilter) or selected: 94 | key = path_segments[-1] 95 | parent = segments.get(obj, path_segments[:-1]) 96 | 97 | # Deletion behavior depends on parent type 98 | if isinstance(parent, MutableMapping): 99 | del parent[key] 100 | 101 | else: 102 | # Handle sequence types 103 | # TODO: Consider cases where type isn't a simple list (e.g. set) 104 | 105 | if len(parent) - 1 == key: 106 | # Removing the last element of a sequence. It can be 107 | # truly removed without affecting the ordering of 108 | # remaining items. 109 | # 110 | # Note: In order to achieve proper behavior we are 111 | # relying on the reverse iteration of 112 | # non-dictionaries from segments.kvs(). 113 | # Otherwise we'd be unable to delete all the tails 114 | # of a list and end up with None values when we 115 | # don't need them. 116 | del parent[key] 117 | 118 | else: 119 | # This key can't be removed completely because it 120 | # would affect the order of items that remain in our 121 | # result. 122 | parent[key] = None 123 | 124 | counter[0] += 1 125 | 126 | [deleted] = segments.foldm(obj, f, [0]) 127 | if not deleted: 128 | raise PathNotFound(f"Could not find {glob} to delete it") 129 | 130 | return deleted 131 | 132 | 133 | def set(obj: MutableMapping, glob: Glob, value, separator="/", afilter: Filter | None = None) -> int: 134 | """ 135 | Given a path glob, set all existing elements in the document 136 | to the given value. Returns the number of elements changed. 137 | """ 138 | globlist = _split_path(glob, separator) 139 | 140 | def f(obj, pair, counter): 141 | (path_segments, found) = pair 142 | 143 | # Skip segments if they no longer exist in obj. 144 | if not segments.has(obj, path_segments): 145 | return 146 | 147 | matched = segments.match(path_segments, globlist) 148 | selected = afilter and segments.leaf(found) and afilter(found) 149 | 150 | if (matched and not afilter) or (matched and selected): 151 | segments.set(obj, path_segments, value, creator=None) 152 | counter[0] += 1 153 | 154 | [changed] = segments.foldm(obj, f, [0]) 155 | return changed 156 | 157 | 158 | def get( 159 | obj: MutableMapping, 160 | glob: Glob, 161 | separator="/", 162 | default: Any = _DEFAULT_SENTINEL 163 | ) -> Union[MutableMapping, object, Callable]: 164 | """ 165 | Given an object which contains only one possible match for the given glob, 166 | return the value for the leaf matching the given glob. 167 | If the glob is not found and a default is provided, 168 | the default is returned. 169 | 170 | If more than one leaf matches the glob, ValueError is raised. If the glob is 171 | not found and a default is not provided, KeyError is raised. 172 | """ 173 | if isinstance(glob, str) and glob == "/" or len(glob) == 0: 174 | return obj 175 | 176 | globlist = _split_path(glob, separator) 177 | 178 | def f(_, pair, results): 179 | (path_segments, found) = pair 180 | 181 | if segments.match(path_segments, globlist): 182 | results.append(found) 183 | if len(results) > 1: 184 | return False 185 | 186 | results = segments.fold(obj, f, []) 187 | 188 | if len(results) == 0: 189 | if default is not _DEFAULT_SENTINEL: 190 | return default 191 | 192 | raise KeyError(glob) 193 | elif len(results) > 1: 194 | raise ValueError(f"dpath.get() globs must match only one leaf: {glob}") 195 | 196 | return results[0] 197 | 198 | 199 | def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter | None = None, dirs=True): 200 | """ 201 | Given an object and a path glob, return an array of all values which match 202 | the glob. The arguments to this function are identical to those of search(). 203 | """ 204 | yielded = True 205 | 206 | return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] 207 | 208 | 209 | def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilter: Filter | None = None, dirs=True): 210 | """ 211 | Given a path glob, return a dictionary containing all keys 212 | that matched the given glob. 213 | 214 | If 'yielded' is true, then a dictionary will not be returned. 215 | Instead, tuples will be yielded in the form of (path, value) for 216 | every element in the document that matched the glob. 217 | """ 218 | 219 | split_glob = _split_path(glob, separator) 220 | 221 | def keeper(path, found): 222 | """ 223 | Generalized test for use in both yielded and folded cases. 224 | Returns True if we want this result. Otherwise, returns False. 225 | """ 226 | if not dirs and not segments.leaf(found): 227 | return False 228 | 229 | matched = segments.match(path, split_glob) 230 | selected = afilter and afilter(found) 231 | 232 | return (matched and not afilter) or (matched and selected) 233 | 234 | if yielded: 235 | def yielder(): 236 | for path, found in segments.walk(obj): 237 | if keeper(path, found): 238 | yield separator.join(map(segments.int_str, path)), found 239 | 240 | return yielder() 241 | else: 242 | def f(obj, pair, result): 243 | (path, found) = pair 244 | 245 | if keeper(path, found): 246 | segments.set(result, path, found, hints=segments.types(obj, path)) 247 | 248 | return segments.fold(obj, f, {}) 249 | 250 | 251 | def merge( 252 | dst: MutableMapping, 253 | src: MutableMapping, 254 | separator="/", 255 | afilter: Filter | None = None, 256 | flags=MergeType.ADDITIVE 257 | ): 258 | """ 259 | Merge source into destination. Like dict.update() but performs deep 260 | merging. 261 | 262 | NOTE: This does not do a deep copy of the source object. Applying merge 263 | will result in references to src being present in the dst tree. If you do 264 | not want src to potentially be modified by other changes in dst (e.g. more 265 | merge calls), then use a deep copy of src. 266 | 267 | NOTE that merge() does NOT copy objects - it REFERENCES. If you merge 268 | take these two dictionaries: 269 | 270 | >>> a = {'a': [0] } 271 | >>> b = {'a': [1] } 272 | 273 | ... and you merge them into an empty dictionary, like so: 274 | 275 | >>> d = {} 276 | >>> dpath.merge(d, a) 277 | >>> dpath.merge(d, b) 278 | 279 | ... you might be surprised to find that a['a'] now contains [0, 1]. 280 | This is because merge() says (d['a'] = a['a']), and thus creates a reference. 281 | This reference is then modified when b is merged, causing both d and 282 | a to have ['a'][0, 1]. To avoid this, make your own deep copies of source 283 | objects that you intend to merge. For further notes see 284 | https://github.com/akesterson/dpath-python/issues/58 285 | 286 | flags is an OR'ed combination of MergeType enum members. 287 | """ 288 | filtered_src = search(src, '**', afilter=afilter, separator='/') 289 | 290 | def are_both_mutable(o1, o2): 291 | mapP = isinstance(o1, MutableMapping) and isinstance(o2, MutableMapping) 292 | seqP = isinstance(o1, MutableSequence) and isinstance(o2, MutableSequence) 293 | 294 | if mapP or seqP: 295 | return True 296 | 297 | return False 298 | 299 | def merger(dst, src, _segments=()): 300 | for key, found in segments.make_walkable(src): 301 | # Our current path in the source. 302 | current_path = _segments + (key,) 303 | 304 | if len(key) == 0 and not options.ALLOW_EMPTY_STRING_KEYS: 305 | raise InvalidKeyName("Empty string keys not allowed without " 306 | "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " 307 | f"{current_path}") 308 | 309 | # Validate src and dst types match. 310 | if flags & MergeType.TYPESAFE: 311 | if segments.has(dst, current_path): 312 | target = segments.get(dst, current_path) 313 | tt = type(target) 314 | ft = type(found) 315 | if tt != ft: 316 | path = separator.join(current_path) 317 | raise TypeError(f"Cannot merge objects of type {tt} and {ft} at {path}") 318 | 319 | # Path not present in destination, create it. 320 | if not segments.has(dst, current_path): 321 | segments.set(dst, current_path, found) 322 | continue 323 | 324 | # Retrieve the value in the destination. 325 | target = segments.get(dst, current_path) 326 | 327 | # If the types don't match, replace it. 328 | if type(found) is not type(target) and not are_both_mutable(found, target): 329 | segments.set(dst, current_path, found) 330 | continue 331 | 332 | # If target is a leaf, the replace it. 333 | if segments.leaf(target): 334 | segments.set(dst, current_path, found) 335 | continue 336 | 337 | # At this point we know: 338 | # 339 | # * The target exists. 340 | # * The types match. 341 | # * The target isn't a leaf. 342 | # 343 | # Pretend we have a sequence and account for the flags. 344 | try: 345 | if flags & MergeType.ADDITIVE: 346 | target += found 347 | continue 348 | 349 | if flags & MergeType.REPLACE: 350 | try: 351 | target[""] 352 | except TypeError: 353 | segments.set(dst, current_path, found) 354 | continue 355 | except Exception: 356 | raise 357 | except Exception: 358 | # We have a dictionary like thing and we need to attempt to 359 | # recursively merge it. 360 | merger(dst, found, current_path) 361 | 362 | merger(dst, filtered_src) 363 | 364 | return dst 365 | -------------------------------------------------------------------------------- /dpath/exceptions.py: -------------------------------------------------------------------------------- 1 | class InvalidGlob(Exception): 2 | """The glob passed is invalid.""" 3 | pass 4 | 5 | 6 | class PathNotFound(Exception): 7 | """One or more elements of the requested path did not exist in the object""" 8 | pass 9 | 10 | 11 | class InvalidKeyName(Exception): 12 | """This key contains the separator character or another invalid character""" 13 | pass 14 | 15 | 16 | class FilteredValue(Exception): 17 | """Unable to return a value, since the filter rejected it""" 18 | pass 19 | -------------------------------------------------------------------------------- /dpath/options.py: -------------------------------------------------------------------------------- 1 | ALLOW_EMPTY_STRING_KEYS = False 2 | -------------------------------------------------------------------------------- /dpath/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dpath-maintainers/dpath-python/c8722e6b815bedf4e6aaeea9ccc7d6ff3e9b4f84/dpath/py.typed -------------------------------------------------------------------------------- /dpath/segments.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | from fnmatch import fnmatchcase 3 | from typing import Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping, MutableSequence 4 | 5 | from dpath import options 6 | from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound 7 | from dpath.types import PathSegment, Creator, Hints, Glob, Path, ListIndex 8 | 9 | 10 | def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: 11 | """ 12 | Returns an iterator which yields tuple pairs of (node index, node value), regardless of node type. 13 | 14 | * For dict nodes `node.items()` will be returned. 15 | * For sequence nodes (lists/tuples/etc.) a zip between index number and index value will be returned. 16 | * Edge cases will result in an empty iterator being returned. 17 | 18 | make_walkable(node) -> (generator -> (key, value)) 19 | """ 20 | try: 21 | return iter(node.items()) 22 | except AttributeError: 23 | try: 24 | indices = range(len(node)) 25 | # Convert all list indices to objects so negative indices are supported. 26 | indices = map(lambda i: ListIndex(i, len(node)), indices) 27 | return zip(indices, node) 28 | except TypeError: 29 | # This can happen in cases where the node isn't leaf(node) == True, 30 | # but also isn't actually iterable. Instead of this being an error 31 | # we will treat this node as if it has no children. 32 | return enumerate([]) 33 | 34 | 35 | def leaf(thing): 36 | """ 37 | Return True if thing is a leaf, otherwise False. 38 | """ 39 | leaves = (bytes, str, int, float, bool, type(None)) 40 | 41 | return isinstance(thing, leaves) 42 | 43 | 44 | def leafy(thing): 45 | """ 46 | Same as leaf(thing), but also treats empty sequences and 47 | dictionaries as True. 48 | """ 49 | 50 | try: 51 | return leaf(thing) or len(thing) == 0 52 | except TypeError: 53 | # In case thing has no len() 54 | return False 55 | 56 | 57 | def walk(obj, location=()): 58 | """ 59 | Yield all valid (segments, value) pairs (from a breadth-first 60 | search, right-to-left on sequences). 61 | 62 | walk(obj) -> (generator -> (segments, value)) 63 | """ 64 | if not leaf(obj): 65 | for k, v in make_walkable(obj): 66 | length = None 67 | 68 | try: 69 | length = len(k) 70 | except TypeError: 71 | pass 72 | 73 | if length is not None and length == 0 and not options.ALLOW_EMPTY_STRING_KEYS: 74 | raise InvalidKeyName("Empty string keys not allowed without " 75 | "dpath.options.ALLOW_EMPTY_STRING_KEYS=True: " 76 | f"{location + (k,)}") 77 | yield (location + (k,)), v 78 | 79 | for k, v in make_walkable(obj): 80 | for found in walk(v, location + (k,)): 81 | yield found 82 | 83 | 84 | def get(obj, segments: Path): 85 | """ 86 | Return the value at the path indicated by segments. 87 | 88 | get(obj, segments) -> value 89 | """ 90 | current = obj 91 | for i, segment in enumerate(segments): 92 | if leaf(current): 93 | raise PathNotFound(f"Path: {segments}[{i}]") 94 | 95 | if isinstance(current, Sequence) and isinstance(segment, str) and segment.isdecimal(): 96 | segment = int(segment) 97 | 98 | current = current[segment] 99 | return current 100 | 101 | 102 | def has(obj, segments): 103 | """ 104 | Return True if the path exists in the obj. Otherwise return False. 105 | 106 | has(obj, segments) -> bool 107 | """ 108 | try: 109 | get(obj, segments) 110 | return True 111 | except: 112 | return False 113 | 114 | 115 | def expand(segments): 116 | """ 117 | Yield a tuple of segments for each possible length of segments. 118 | Starting from the shortest length of segments and increasing by 1. 119 | 120 | expand(keys) -> (..., keys[:-2], keys[:-1]) 121 | """ 122 | index = 0 123 | for _ in segments: 124 | index += 1 125 | yield segments[:index] 126 | 127 | 128 | def types(obj, segments): 129 | """ 130 | For each segment produce a tuple of (segment, type(value)). 131 | 132 | types(obj, segments) -> ((segment[0], type0), (segment[1], type1), ...) 133 | """ 134 | result = [] 135 | for depth in expand(segments): 136 | result.append((depth[-1], type(get(obj, depth)))) 137 | return tuple(result) 138 | 139 | 140 | def leaves(obj): 141 | """ 142 | Yield all leaves as (segment, value) pairs. 143 | 144 | leaves(obj) -> (generator -> (segment, value)) 145 | """ 146 | return filter(lambda p: leafy(p[1]), walk(obj)) 147 | 148 | 149 | def int_str(segment: PathSegment) -> PathSegment: 150 | """ 151 | If the segment is an integer, return the string conversion. 152 | Otherwise return the segment unchanged. The conversion uses 'str'. 153 | 154 | int_str(segment) -> str 155 | """ 156 | if isinstance(segment, int): 157 | return str(segment) 158 | return segment 159 | 160 | 161 | class Star(object): 162 | """ 163 | Used to create a global STAR symbol for tracking stars added when 164 | expanding star-star globs. 165 | """ 166 | pass 167 | 168 | 169 | STAR = Star() 170 | 171 | 172 | def match(segments: Path, glob: Glob): 173 | """ 174 | Return True if the segments match the given glob, otherwise False. 175 | 176 | For the purposes of matching, integers are converted to their string 177 | equivalent (via str(segment)). This conversion happens on both the 178 | segments and the glob. This implies you cannot (with this function) 179 | differentiate a list index 0 from a dictionary key '0'. 180 | 181 | Star-star segments are a special case in that they will expand to 0 182 | or more star segments and the type will be coerced to match that of 183 | the segment. 184 | 185 | A segment is considered to match a glob if the function 186 | fnmatch.fnmatchcase returns True. If fnmatchcase returns False or 187 | throws an exception the result will be False. 188 | 189 | match(segments, glob) -> bool 190 | """ 191 | segments = tuple(segments) 192 | glob = tuple(glob) 193 | 194 | path_len = len(segments) 195 | glob_len = len(glob) 196 | 197 | # The star-star normalized glob ('**' has been removed). 198 | ss_glob = glob 199 | 200 | if '**' in glob: 201 | # Index of the star-star in the glob. 202 | ss = glob.index('**') 203 | 204 | if '**' in glob[ss + 1:]: 205 | raise InvalidGlob(f"Invalid glob. Only one '**' is permitted per glob: {glob}") 206 | 207 | # Convert '**' segment into multiple '*' segments such that the 208 | # lengths of the path and glob match. '**' also can collapse and 209 | # result in the removal of 1 segment. 210 | if path_len >= glob_len: 211 | # Path and glob have the same number of stars or the glob 212 | # needs more stars (which we add). 213 | more_stars = (STAR,) * (path_len - glob_len + 1) 214 | ss_glob = glob[:ss] + more_stars + glob[ss + 1:] 215 | elif path_len == glob_len - 1: 216 | # Glob has one more segment than the path. Here we remove 217 | # the '**' segment altogether to match the lengths up. 218 | ss_glob = glob[:ss] + glob[ss + 1:] 219 | 220 | # If we were successful in matching up the lengths, then we can 221 | # compare them using fnmatch. 222 | if path_len == len(ss_glob): 223 | i = zip(segments, ss_glob) 224 | for s, g in i: 225 | # Match the stars we added to the glob to the type of the 226 | # segment itself. 227 | if g is STAR: 228 | if isinstance(s, bytes): 229 | g = b'*' 230 | else: 231 | g = '*' 232 | 233 | try: 234 | # If search path segment (s) is an int then assume currently evaluated index (g) might be a sequence 235 | # index as well. Try converting it to an int. 236 | if isinstance(s, int) and s == int(g): 237 | continue 238 | except: 239 | # Will reach this point if g can't be converted to an int (e.g. when g is a RegEx pattern). 240 | # In this case convert s to a str so fnmatch can work on it. 241 | s = str(s) 242 | 243 | try: 244 | # Let's see if the glob matches. We will turn any kind of 245 | # exception while attempting to match into a False for the 246 | # match. 247 | if not fnmatchcase(s, g): 248 | return False 249 | except: 250 | return False 251 | 252 | # All of the segments matched so we have a complete match. 253 | return True 254 | 255 | # Otherwise the lengths aren't the same and we couldn't have a 256 | # match. 257 | return False 258 | 259 | 260 | def extend(thing: MutableSequence, index: int, value=None): 261 | """ 262 | Extend a sequence like thing such that it contains at least index + 263 | 1 many elements. The extension values will be None (default). 264 | 265 | extend(thing, int) -> [thing..., None, ...] 266 | """ 267 | try: 268 | expansion = type(thing)() 269 | 270 | # Using this rather than the multiply notation in order to support a 271 | # wider variety of sequence like things. 272 | extra = (index + 1) - len(thing) 273 | for i in range(extra): 274 | expansion += [value] 275 | thing.extend(expansion) 276 | except TypeError: 277 | # We attempted to extend something that doesn't support it. In 278 | # this case we assume thing is actually more like a dictionary 279 | # and doesn't need to be extended. 280 | pass 281 | 282 | return thing 283 | 284 | 285 | def _default_creator( 286 | current: Union[MutableMapping, Sequence], 287 | segments: Sequence[PathSegment], 288 | i: int, 289 | hints: Sequence[Tuple[PathSegment, type]] = () 290 | ): 291 | """ 292 | Create missing path components. If the segment is an int, then it will 293 | create a list. Otherwise a dictionary is created. 294 | 295 | set(obj, segments, value) -> obj 296 | """ 297 | segment = segments[i] 298 | length = len(segments) 299 | 300 | if isinstance(current, Sequence): 301 | segment = int(segment) 302 | 303 | if isinstance(current, MutableSequence): 304 | extend(current, segment) 305 | 306 | # Infer the type from the hints provided. 307 | if i < len(hints): 308 | current[segment] = hints[i][1]() 309 | else: 310 | # Peek at the next segment to determine if we should be 311 | # creating an array for it to access or dictionary. 312 | if i + 1 < length: 313 | segment_next = segments[i + 1] 314 | else: 315 | segment_next = None 316 | 317 | if isinstance(segment_next, int) or (isinstance(segment_next, str) and segment_next.isdecimal()): 318 | current[segment] = [] 319 | else: 320 | current[segment] = {} 321 | 322 | 323 | def set( 324 | obj: MutableMapping, 325 | segments: Sequence[PathSegment], 326 | value, 327 | creator: Optional[Creator] = _default_creator, 328 | hints: Hints = () 329 | ) -> MutableMapping: 330 | """ 331 | Set the value in obj at the place indicated by segments. If creator is not 332 | None (default _default_creator), then call the creator function to 333 | create any missing path components. 334 | 335 | set(obj, segments, value) -> obj 336 | """ 337 | current = obj 338 | length = len(segments) 339 | 340 | # For everything except the last value, walk down the path and 341 | # create if creator is set. 342 | for (i, segment) in enumerate(segments[:-1]): 343 | 344 | # If segment is non-int but supposed to be a sequence index 345 | if isinstance(segment, str) and isinstance(current, Sequence) and segment.isdecimal(): 346 | segment = int(segment) 347 | 348 | try: 349 | # Optimistically try to get the next value. This makes the 350 | # code agnostic to whether current is a list or a dict. 351 | # Unfortunately, for our use, 'x in thing' for lists checks 352 | # values, not keys whereas dicts check keys. 353 | current[segment] 354 | except: 355 | if creator is not None: 356 | creator(current, segments, i, hints) 357 | else: 358 | raise 359 | 360 | current = current[segment] 361 | if i != length - 1 and leaf(current): 362 | raise PathNotFound(f"Path: {segments}[{i}]") 363 | 364 | last_segment = segments[-1] 365 | 366 | # Resolve ambiguity of last segment 367 | if isinstance(last_segment, str) and isinstance(current, Sequence) and last_segment.isdecimal(): 368 | last_segment = int(last_segment) 369 | 370 | if isinstance(last_segment, int): 371 | extend(current, last_segment) 372 | 373 | current[last_segment] = value 374 | 375 | return obj 376 | 377 | 378 | def fold(obj, f, acc): 379 | """ 380 | Walk obj applying f to each path and returning accumulator acc. 381 | 382 | The function f will be called, for each result in walk(obj): 383 | 384 | f(obj, (segments, value), acc) 385 | 386 | If the function f returns False (exactly False), then processing 387 | will stop. Otherwise processing will continue with the next value 388 | retrieved from the walk. 389 | 390 | fold(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc 391 | """ 392 | for pair in walk(obj): 393 | if f(obj, pair, acc) is False: 394 | break 395 | return acc 396 | 397 | 398 | def foldm(obj, f, acc): 399 | """ 400 | Same as fold(), but permits mutating obj. 401 | 402 | This requires all paths in walk(obj) to be loaded into memory 403 | (whereas fold does not). 404 | 405 | foldm(obj, f(obj, (segments, value), acc) -> bool, acc) -> acc 406 | """ 407 | pairs = tuple(walk(obj)) 408 | for pair in pairs: 409 | if f(obj, pair, acc) is False: 410 | break 411 | return acc 412 | 413 | 414 | def view(obj: MutableMapping, glob: Glob): 415 | """ 416 | Return a view of the object where the glob matches. A view retains 417 | the same form as the obj, but is limited to only the paths that 418 | matched. Views are new objects (a deepcopy of the matching values). 419 | 420 | view(obj, glob) -> obj' 421 | """ 422 | 423 | def f(obj, pair, result): 424 | (segments, value) = pair 425 | if match(segments, glob): 426 | if not has(result, segments): 427 | set(result, segments, deepcopy(value), hints=types(obj, segments)) 428 | 429 | return fold(obj, f, type(obj)()) 430 | -------------------------------------------------------------------------------- /dpath/types.py: -------------------------------------------------------------------------------- 1 | from enum import IntFlag, auto 2 | from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping 3 | 4 | 5 | class ListIndex(int): 6 | """Same as a normal int but mimics the behavior of list indices (can be compared to a negative number).""" 7 | 8 | def __new__(cls, value: int, list_length: int, *args, **kwargs): 9 | if value >= list_length: 10 | raise TypeError( 11 | f"Tried to initiate a {cls.__name__} with a value ({value}) " 12 | f"greater than the provided max value ({list_length})" 13 | ) 14 | 15 | obj = super().__new__(cls, value) 16 | obj.list_length = list_length 17 | 18 | return obj 19 | 20 | def __eq__(self, other): 21 | if not isinstance(other, int): 22 | return False 23 | 24 | # Based on how Python sequences handle negative indices as described in footnote (3) of https://docs.python.org/3/library/stdtypes.html#common-sequence-operations 25 | return other == int(self) or self.list_length + other == int(self) 26 | 27 | def __repr__(self): 28 | return f"<{self.__class__.__name__} {int(self)}/{self.list_length}>" 29 | 30 | def __str__(self): 31 | return str(int(self)) 32 | 33 | 34 | class MergeType(IntFlag): 35 | ADDITIVE = auto() 36 | """List objects are combined onto one long list (NOT a set). This is the default flag.""" 37 | 38 | REPLACE = auto() 39 | """Instead of combining list objects, when 2 list objects are at an equal depth of merge, replace the destination \ 40 | with the source.""" 41 | 42 | TYPESAFE = auto() 43 | """When 2 keys at equal levels are of different types, raise a TypeError exception. By default, the source \ 44 | replaces the destination in this situation.""" 45 | 46 | 47 | PathSegment = Union[int, str, bytes] 48 | """Type alias for dict path segments where integers are explicitly casted.""" 49 | 50 | Filter = Callable[[Any], bool] 51 | """Type alias for filter functions. 52 | 53 | (Any) -> bool""" 54 | 55 | Glob = Union[str, Sequence[str]] 56 | """Type alias for glob parameters.""" 57 | 58 | Path = Union[str, Sequence[PathSegment]] 59 | """Type alias for path parameters.""" 60 | 61 | Hints = Sequence[Tuple[PathSegment, type]] 62 | """Type alias for creator function hint sequences.""" 63 | 64 | Creator = Callable[[Union[MutableMapping, List], Path, int, Optional[Hints]], None] 65 | """Type alias for creator functions. 66 | 67 | Example creator function signature: 68 | 69 | def creator( 70 | current: Union[MutableMapping, List], 71 | segments: Sequence[PathSegment], 72 | i: int, 73 | hints: Sequence[Tuple[PathSegment, type]] = () 74 | )""" 75 | -------------------------------------------------------------------------------- /dpath/util.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | import dpath 4 | from dpath import _DEFAULT_SENTINEL 5 | from dpath.types import MergeType 6 | 7 | 8 | def deprecated(func): 9 | message = \ 10 | "The dpath.util package is being deprecated. All util functions have been moved to dpath package top level." 11 | 12 | def wrapper(*args, **kwargs): 13 | warnings.warn(message, DeprecationWarning, stacklevel=2) 14 | return func(*args, **kwargs) 15 | 16 | return wrapper 17 | 18 | 19 | @deprecated 20 | def new(obj, path, value, separator="/", creator=None): 21 | return dpath.new(obj, path, value, separator, creator) 22 | 23 | 24 | @deprecated 25 | def delete(obj, glob, separator="/", afilter=None): 26 | return dpath.delete(obj, glob, separator, afilter) 27 | 28 | 29 | @deprecated 30 | def set(obj, glob, value, separator="/", afilter=None): 31 | return dpath.set(obj, glob, value, separator, afilter) 32 | 33 | 34 | @deprecated 35 | def get(obj, glob, separator="/", default=_DEFAULT_SENTINEL): 36 | return dpath.get(obj, glob, separator, default) 37 | 38 | 39 | @deprecated 40 | def values(obj, glob, separator="/", afilter=None, dirs=True): 41 | return dpath.values(obj, glob, separator, afilter, dirs) 42 | 43 | 44 | @deprecated 45 | def search(obj, glob, yielded=False, separator="/", afilter=None, dirs=True): 46 | return dpath.search(obj, glob, yielded, separator, afilter, dirs) 47 | 48 | 49 | @deprecated 50 | def merge(dst, src, separator="/", afilter=None, flags=MergeType.ADDITIVE): 51 | return dpath.merge(dst, src, separator, afilter, flags) 52 | -------------------------------------------------------------------------------- /dpath/version.py: -------------------------------------------------------------------------------- 1 | VERSION = "2.2.0" 2 | -------------------------------------------------------------------------------- /flake8.ini: -------------------------------------------------------------------------------- 1 | [flake8] 2 | filename= 3 | setup.py, 4 | dpath/, 5 | tests/ 6 | -------------------------------------------------------------------------------- /maintainers_log.md: -------------------------------------------------------------------------------- 1 | # 03/29/2020 2 | 3 | Attendees : Caleb, Andrew 4 | 5 | ## Old business : 6 | 7 | * Need to onboard new member Vladimir Ulogov 8 | * No movement 9 | * Need to make project board for 1.5 open bugs 10 | * Done 11 | 12 | ## New business : 13 | 14 | * Andrew to define maintainers meeting process and establish log of decisions, process for filing open action items 15 | * Andrew to forward maintainers invite to Vladimir and include in next monthly maintainers meeting 16 | * Andrew to set followup for 1wk from now to check for comments on PRs and cut release version for 1.x / 2.x 17 | * Andrew to rename LTS branches from version/1.0 version/2.0 to version/1.x and version/2.x 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | 4 | import dpath.version 5 | 6 | long_description = open( 7 | os.path.join( 8 | os.path.dirname(__file__), 9 | 'README.rst' 10 | ) 11 | ).read() 12 | 13 | if __name__ == "__main__": 14 | setup( 15 | name="dpath", 16 | url="https://github.com/dpath-maintainers/dpath-python", 17 | version=dpath.version.VERSION, 18 | description="Filesystem-like pathing and searching for dictionaries", 19 | long_description=long_description, 20 | author=("Caleb Case, " 21 | "Andrew Kesterson"), 22 | author_email="calebcase@gmail.com, andrew@aklabs.net", 23 | license="MIT", 24 | install_requires=[], 25 | scripts=[], 26 | packages=["dpath"], 27 | data_files=[], 28 | package_data={"dpath": ["py.typed"]}, 29 | 30 | # Type hints are great. 31 | # Function annotations were added in Python 3.0. 32 | # Typing module was added in Python 3.5. 33 | # Variable annotations were added in Python 3.6. 34 | # Python versions that are >=3.6 are more popular. 35 | # (Source: https://github.com/hugovk/pypi-tools/blob/master/README.md) 36 | # 37 | # Conclusion: In order to accommodate type hinting support must be limited to Python versions >=3.6. 38 | # 3.6 was dropped because of EOL and this issue: https://github.com/actions/setup-python/issues/544 39 | python_requires=">=3.7", 40 | classifiers=[ 41 | 'Development Status :: 5 - Production/Stable', 42 | 'Environment :: Console', 43 | 'Intended Audience :: Developers', 44 | 'License :: OSI Approved :: MIT License', 45 | 'Natural Language :: English', 46 | 'Programming Language :: Python :: 3', 47 | 'Programming Language :: Python :: 3.7', 48 | 'Programming Language :: Python :: 3.8', 49 | 'Programming Language :: Python :: 3.9', 50 | 'Programming Language :: Python :: 3.10', 51 | 'Programming Language :: Python :: 3.11', 52 | 'Programming Language :: Python :: 3.12', 53 | 'Topic :: Software Development :: Libraries :: Python Modules', 54 | 'Typing :: Typed', 55 | ], 56 | ) 57 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | warnings.simplefilter("always", DeprecationWarning) 4 | -------------------------------------------------------------------------------- /tests/test_broken_afilter.py: -------------------------------------------------------------------------------- 1 | import dpath 2 | import sys 3 | 4 | 5 | def test_broken_afilter(): 6 | def afilter(x): 7 | if x in [1, 2]: 8 | return True 9 | return False 10 | 11 | dict = { 12 | "a": { 13 | "view_failure": "a", 14 | "b": { 15 | "c": { 16 | "d": 0, 17 | "e": 1, 18 | "f": 2, 19 | }, 20 | }, 21 | }, 22 | } 23 | paths = [ 24 | 'a/b/c/e', 25 | 'a/b/c/f', 26 | ] 27 | 28 | for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): 29 | assert path in paths 30 | assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] 31 | assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] 32 | 33 | for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): 34 | assert path in paths 35 | assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] 36 | assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] 37 | 38 | def filter(x): 39 | sys.stderr.write(str(x)) 40 | if hasattr(x, 'get'): 41 | return x.get('type', None) == 'correct' 42 | return False 43 | 44 | a = { 45 | 'actions': [ 46 | { 47 | 'type': 'correct' 48 | }, 49 | { 50 | 'type': 'incorrect' 51 | }, 52 | ], 53 | } 54 | 55 | results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', yielded=True)] 56 | results = [[x[0], x[1]] for x in dpath.search(a, 'actions/*', afilter=filter, yielded=True)] 57 | assert len(results) == 1 58 | assert results[0][1]['type'] == 'correct' 59 | -------------------------------------------------------------------------------- /tests/test_delete.py: -------------------------------------------------------------------------------- 1 | from nose2.tools.such import helper 2 | 3 | import dpath 4 | import dpath.exceptions 5 | 6 | 7 | def test_delete_separator(): 8 | dict = { 9 | "a": { 10 | "b": 0, 11 | }, 12 | } 13 | 14 | dpath.delete(dict, ';a;b', separator=";") 15 | assert 'b' not in dict['a'] 16 | 17 | 18 | def test_delete_existing(): 19 | dict = { 20 | "a": { 21 | "b": 0, 22 | }, 23 | } 24 | 25 | dpath.delete(dict, '/a/b') 26 | assert 'b' not in dict['a'] 27 | 28 | 29 | def test_delete_missing(): 30 | dict = { 31 | "a": { 32 | }, 33 | } 34 | 35 | with helper.assertRaises(dpath.exceptions.PathNotFound): 36 | dpath.delete(dict, '/a/b') 37 | 38 | 39 | def test_delete_filter(): 40 | def afilter(x): 41 | if int(x) == 31: 42 | return True 43 | return False 44 | 45 | dict = { 46 | "a": { 47 | "b": 0, 48 | "c": 1, 49 | "d": 31, 50 | }, 51 | } 52 | 53 | dpath.delete(dict, '/a/*', afilter=afilter) 54 | assert dict['a']['b'] == 0 55 | assert dict['a']['c'] == 1 56 | assert 'd' not in dict['a'] 57 | -------------------------------------------------------------------------------- /tests/test_get_values.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import decimal 3 | import time 4 | 5 | from unittest import mock 6 | 7 | from nose2.tools.such import helper 8 | 9 | import dpath 10 | 11 | 12 | def test_util_get_root(): 13 | x = {'p': {'a': {'t': {'h': 'value'}}}} 14 | 15 | ret = dpath.get(x, '/p/a/t/h') 16 | assert ret == 'value' 17 | 18 | ret = dpath.get(x, '/') 19 | assert ret == x 20 | 21 | ret = dpath.get(x, []) 22 | assert ret == x 23 | 24 | 25 | def test_get_explicit_single(): 26 | ehash = { 27 | "a": { 28 | "b": { 29 | "c": { 30 | "d": 0, 31 | "e": 1, 32 | "f": 2, 33 | }, 34 | }, 35 | }, 36 | } 37 | 38 | assert dpath.get(ehash, '/a/b/c/f') == 2 39 | assert dpath.get(ehash, ['a', 'b', 'c', 'f']) == 2 40 | assert dpath.get(ehash, ['a', 'b', 'c', 'f'], default=5) == 2 41 | assert dpath.get(ehash, ['does', 'not', 'exist'], default=None) is None 42 | assert dpath.get(ehash, ['doesnt', 'exist'], default=5) == 5 43 | 44 | 45 | def test_get_glob_single(): 46 | ehash = { 47 | "a": { 48 | "b": { 49 | "c": { 50 | "d": 0, 51 | "e": 1, 52 | "f": 2, 53 | }, 54 | }, 55 | }, 56 | } 57 | 58 | assert dpath.get(ehash, '/a/b/*/f') == 2 59 | assert dpath.get(ehash, ['a', 'b', '*', 'f']) == 2 60 | assert dpath.get(ehash, ['a', 'b', '*', 'f'], default=5) == 2 61 | assert dpath.get(ehash, ['doesnt', '*', 'exist'], default=6) == 6 62 | 63 | 64 | def test_get_glob_multiple(): 65 | ehash = { 66 | "a": { 67 | "b": { 68 | "c": { 69 | "d": 0, 70 | }, 71 | "e": { 72 | "d": 0, 73 | }, 74 | }, 75 | }, 76 | } 77 | 78 | helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') 79 | helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) 80 | helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd'], default=3) 81 | 82 | 83 | def test_get_absent(): 84 | ehash = {} 85 | 86 | helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') 87 | helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) 88 | 89 | 90 | def test_values(): 91 | ehash = { 92 | "a": { 93 | "b": { 94 | "c": { 95 | "d": 0, 96 | "e": 1, 97 | "f": 2, 98 | }, 99 | }, 100 | }, 101 | } 102 | 103 | ret = dpath.values(ehash, '/a/b/c/*') 104 | assert isinstance(ret, list) 105 | assert 0 in ret 106 | assert 1 in ret 107 | assert 2 in ret 108 | 109 | ret = dpath.values(ehash, ['a', 'b', 'c', '*']) 110 | assert isinstance(ret, list) 111 | assert 0 in ret 112 | assert 1 in ret 113 | assert 2 in ret 114 | 115 | 116 | @mock.patch('dpath.search') 117 | def test_values_passes_through(searchfunc): 118 | searchfunc.return_value = [] 119 | 120 | def y(): 121 | return False 122 | 123 | dpath.values({}, '/a/b', ':', y, False) 124 | searchfunc.assert_called_with({}, '/a/b', True, ':', y, False) 125 | 126 | dpath.values({}, ['a', 'b'], ':', y, False) 127 | searchfunc.assert_called_with({}, ['a', 'b'], True, ':', y, False) 128 | 129 | 130 | def test_none_values(): 131 | d = {'p': {'a': {'t': {'h': None}}}} 132 | 133 | v = dpath.get(d, 'p/a/t/h') 134 | assert v is None 135 | 136 | 137 | def test_values_list(): 138 | a = { 139 | 'actions': [ 140 | { 141 | 'type': 'correct', 142 | }, 143 | { 144 | 'type': 'incorrect', 145 | }, 146 | ], 147 | } 148 | 149 | ret = dpath.values(a, 'actions/*') 150 | assert isinstance(ret, list) 151 | assert len(ret) == 2 152 | 153 | 154 | def test_non_leaf_leaf(): 155 | # The leaves in this test aren't leaf(thing) == True, but we should still 156 | # be able to get them. They should also not prevent fetching other values. 157 | 158 | def func(x): 159 | return x 160 | 161 | testdict = { 162 | 'a': func, 163 | 'b': lambda x: x, 164 | 'c': [ 165 | { 166 | 'a', 167 | 'b', 168 | }, 169 | ], 170 | 'd': [ 171 | decimal.Decimal(1.5), 172 | decimal.Decimal(2.25), 173 | ], 174 | 'e': datetime.datetime(2020, 1, 1), 175 | 'f': { 176 | 'config': 'something', 177 | }, 178 | } 179 | 180 | # It should be possible to get the callables: 181 | assert dpath.get(testdict, 'a') == func 182 | assert dpath.get(testdict, 'b')(42) == 42 183 | 184 | # It should be possible to get other values: 185 | assert dpath.get(testdict, 'c/0') == testdict['c'][0] 186 | assert dpath.get(testdict, 'd')[0] == testdict['d'][0] 187 | assert dpath.get(testdict, 'd/0') == testdict['d'][0] 188 | assert dpath.get(testdict, 'd/1') == testdict['d'][1] 189 | assert dpath.get(testdict, 'e') == testdict['e'] 190 | 191 | # Values should also still work: 192 | assert dpath.values(testdict, 'f/config') == ['something'] 193 | 194 | # Data classes should also be retrievable: 195 | try: 196 | import dataclasses 197 | except: 198 | return 199 | 200 | @dataclasses.dataclass 201 | class Connection: 202 | group_name: str 203 | channel_name: str 204 | last_seen: float 205 | 206 | testdict['g'] = { 207 | 'my-key': Connection( 208 | group_name='foo', 209 | channel_name='bar', 210 | last_seen=time.time(), 211 | ), 212 | } 213 | 214 | assert dpath.search(testdict, 'g/my*')['g']['my-key'] == testdict['g']['my-key'] 215 | -------------------------------------------------------------------------------- /tests/test_merge.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from nose2.tools.such import helper 4 | 5 | 6 | import dpath 7 | from dpath import MergeType 8 | 9 | 10 | def test_merge_typesafe_and_separator(): 11 | src = { 12 | "dict": { 13 | "integer": 0, 14 | }, 15 | } 16 | dst = { 17 | "dict": { 18 | "integer": "3", 19 | }, 20 | } 21 | 22 | try: 23 | dpath.merge(dst, src, flags=(dpath.MergeType.ADDITIVE | dpath.MergeType.TYPESAFE), separator=";") 24 | except TypeError as e: 25 | assert str(e).endswith("dict;integer") 26 | 27 | return 28 | raise Exception("MERGE_TYPESAFE failed to raise an exception when merging between str and int!") 29 | 30 | 31 | def test_merge_simple_int(): 32 | src = { 33 | "integer": 0, 34 | } 35 | dst = { 36 | "integer": 3, 37 | } 38 | 39 | dpath.merge(dst, src) 40 | assert dst["integer"] == src["integer"], "%r != %r" % (dst["integer"], src["integer"]) 41 | 42 | 43 | def test_merge_simple_string(): 44 | src = { 45 | "string": "lol I am a string", 46 | } 47 | dst = { 48 | "string": "lol I am a string", 49 | } 50 | 51 | dpath.merge(dst, src) 52 | assert dst["string"] == src["string"], "%r != %r" % (dst["string"], src["string"]) 53 | 54 | 55 | def test_merge_simple_list_additive(): 56 | src = { 57 | "list": [7, 8, 9, 10], 58 | } 59 | dst = { 60 | "list": [0, 1, 2, 3], 61 | } 62 | 63 | dpath.merge(dst, src, flags=MergeType.ADDITIVE) 64 | assert dst["list"] == [0, 1, 2, 3, 7, 8, 9, 10], "%r != %r" % (dst["list"], [0, 1, 2, 3, 7, 8, 9, 10]) 65 | 66 | 67 | def test_merge_simple_list_replace(): 68 | src = { 69 | "list": [7, 8, 9, 10], 70 | } 71 | dst = { 72 | "list": [0, 1, 2, 3], 73 | } 74 | 75 | dpath.merge(dst, src, flags=dpath.MergeType.REPLACE) 76 | assert dst["list"] == [7, 8, 9, 10], "%r != %r" % (dst["list"], [7, 8, 9, 10]) 77 | 78 | 79 | def test_merge_simple_dict(): 80 | src = { 81 | "dict": { 82 | "key": "WEHAW", 83 | }, 84 | } 85 | dst = { 86 | "dict": { 87 | "key": "", 88 | }, 89 | } 90 | 91 | dpath.merge(dst, src) 92 | assert dst["dict"]["key"] == src["dict"]["key"], "%r != %r" % (dst["dict"]["key"], src["dict"]["key"]) 93 | 94 | 95 | def test_merge_filter(): 96 | def afilter(x): 97 | if "rubber" not in str(x): 98 | return False 99 | return True 100 | 101 | src = { 102 | "key": "metal", 103 | "key2": "rubber", 104 | "otherdict": { 105 | "key3": "I shouldn't be here", 106 | }, 107 | } 108 | dst = {} 109 | 110 | dpath.merge(dst, src, afilter=afilter) 111 | assert "key2" in dst 112 | assert "key" not in dst 113 | assert "otherdict" not in dst 114 | 115 | 116 | def test_merge_typesafe(): 117 | src = { 118 | "dict": { 119 | }, 120 | } 121 | dst = { 122 | "dict": [ 123 | ], 124 | } 125 | 126 | helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) 127 | 128 | 129 | def test_merge_mutables(): 130 | class tcid(dict): 131 | pass 132 | 133 | class tcis(list): 134 | pass 135 | 136 | src = { 137 | "mm": { 138 | "a": "v1", 139 | }, 140 | "ms": [ 141 | 0, 142 | ], 143 | } 144 | dst = { 145 | "mm": tcid([ 146 | ("a", "v2"), 147 | ("casserole", "this should keep"), 148 | ]), 149 | "ms": tcis(['a', 'b', 'c']), 150 | } 151 | 152 | dpath.merge(dst, src) 153 | print(dst) 154 | assert dst["mm"]["a"] == src["mm"]["a"] 155 | assert dst['ms'][2] == 'c' 156 | assert "casserole" in dst["mm"] 157 | 158 | helper.assertRaises(TypeError, dpath.merge, dst, src, flags=dpath.MergeType.TYPESAFE) 159 | 160 | 161 | def test_merge_replace_1(): 162 | dct_a = {"a": {"b": [1, 2, 3]}} 163 | dct_b = {"a": {"b": [1]}} 164 | dpath.merge(dct_a, dct_b, flags=dpath.MergeType.REPLACE) 165 | assert len(dct_a['a']['b']) == 1 166 | 167 | 168 | def test_merge_replace_2(): 169 | d1 = {'a': [0, 1, 2]} 170 | d2 = {'a': ['a']} 171 | dpath.merge(d1, d2, flags=dpath.MergeType.REPLACE) 172 | assert len(d1['a']) == 1 173 | assert d1['a'][0] == 'a' 174 | 175 | 176 | def test_merge_list(): 177 | src = {"l": [1]} 178 | p1 = {"l": [2], "v": 1} 179 | p2 = {"v": 2} 180 | 181 | dst1 = {} 182 | for d in [copy.deepcopy(src), copy.deepcopy(p1)]: 183 | dpath.merge(dst1, d) 184 | dst2 = {} 185 | for d in [copy.deepcopy(src), copy.deepcopy(p2)]: 186 | dpath.merge(dst2, d) 187 | assert dst1["l"] == [1, 2] 188 | assert dst2["l"] == [1] 189 | 190 | dst1 = {} 191 | for d in [src, p1]: 192 | dpath.merge(dst1, d) 193 | dst2 = {} 194 | for d in [src, p2]: 195 | dpath.merge(dst2, d) 196 | assert dst1["l"] == [1, 2] 197 | assert dst2["l"] == [1, 2] 198 | -------------------------------------------------------------------------------- /tests/test_new.py: -------------------------------------------------------------------------------- 1 | import dpath 2 | 3 | 4 | def test_set_new_separator(): 5 | dict = { 6 | "a": { 7 | }, 8 | } 9 | 10 | dpath.new(dict, ';a;b', 1, separator=";") 11 | assert dict['a']['b'] == 1 12 | 13 | dpath.new(dict, ['a', 'b'], 1, separator=";") 14 | assert dict['a']['b'] == 1 15 | 16 | 17 | def test_set_new_dict(): 18 | dict = { 19 | "a": { 20 | }, 21 | } 22 | 23 | dpath.new(dict, '/a/b', 1) 24 | assert dict['a']['b'] == 1 25 | 26 | dpath.new(dict, ['a', 'b'], 1) 27 | assert dict['a']['b'] == 1 28 | 29 | 30 | def test_set_new_list(): 31 | dict = { 32 | "a": [ 33 | ], 34 | } 35 | 36 | dpath.new(dict, '/a/1', 1) 37 | assert dict['a'][1] == 1 38 | assert dict['a'][0] is None 39 | 40 | dpath.new(dict, ['a', 1], 1) 41 | assert dict['a'][1] == 1 42 | assert dict['a'][0] is None 43 | 44 | 45 | def test_set_list_with_dict_int_ambiguity(): 46 | d = {"list": [{"root": {"1": {"k": None}}}]} 47 | 48 | dpath.new(d, "list/0/root/1/k", "new") 49 | 50 | expected = {"list": [{"root": {"1": {"k": "new"}}}]} 51 | 52 | assert d == expected 53 | 54 | 55 | def test_int_segment_list_type_check(): 56 | d = {} 57 | dpath.new(d, "a/b/0/c/0", "hello") 58 | assert 'b' in d.get("a", {}) 59 | assert isinstance(d["a"]["b"], list) 60 | assert len(d["a"]["b"]) == 1 61 | assert 'c' in d["a"]["b"][0] 62 | assert isinstance(d["a"]["b"][0]["c"], list) 63 | assert len(d["a"]["b"][0]["c"]) == 1 64 | 65 | 66 | def test_int_segment_dict_type_check(): 67 | d = {"a": {"b": {"0": {}}}} 68 | dpath.new(d, "a/b/0/c/0", "hello") 69 | assert "b" in d.get("a", {}) 70 | assert isinstance(d["a"]["b"], dict) 71 | assert '0' in d["a"]["b"] 72 | assert 'c' in d["a"]["b"]["0"] 73 | assert isinstance(d["a"]["b"]["0"]["c"], list) 74 | 75 | 76 | def test_set_new_list_path_with_separator(): 77 | # This test kills many birds with one stone, forgive me 78 | dict = { 79 | "a": { 80 | }, 81 | } 82 | 83 | dpath.new(dict, ['a', 'b/c/d', 0], 1) 84 | assert len(dict['a']) == 1 85 | assert len(dict['a']['b/c/d']) == 1 86 | assert dict['a']['b/c/d'][0] == 1 87 | 88 | 89 | def test_set_new_list_integer_path_with_creator(): 90 | d = {} 91 | 92 | def mycreator(obj, pathcomp, nextpathcomp, hints): 93 | print(hints) 94 | print(pathcomp) 95 | print(nextpathcomp) 96 | print("...") 97 | 98 | target = pathcomp[0] 99 | if isinstance(obj, list) and (target.isdigit()): 100 | target = int(target) 101 | 102 | if ((nextpathcomp is not None) and (isinstance(nextpathcomp, int) or str(nextpathcomp).isdigit())): 103 | obj[target] = [None] * (int(nextpathcomp) + 1) 104 | print("Created new list in target") 105 | else: 106 | print("Created new dict in target") 107 | obj[target] = {} 108 | print(obj) 109 | 110 | dpath.new(d, '/a/2', 3, creator=mycreator) 111 | print(d) 112 | assert isinstance(d['a'], list) 113 | assert len(d['a']) == 3 114 | assert d['a'][2] == 3 115 | -------------------------------------------------------------------------------- /tests/test_path_get.py: -------------------------------------------------------------------------------- 1 | import dpath.segments 2 | import dpath.exceptions 3 | 4 | 5 | def test_path_get_list_of_dicts(): 6 | tdict = { 7 | "a": { 8 | "b": [ 9 | {0: 0}, 10 | {0: 1}, 11 | {0: 2}, 12 | ], 13 | }, 14 | } 15 | segments = ['a', 'b', 0, 0] 16 | 17 | res = dpath.segments.view(tdict, segments) 18 | assert isinstance(res['a']['b'], list) 19 | assert len(res['a']['b']) == 1 20 | assert res['a']['b'][0][0] == 0 21 | -------------------------------------------------------------------------------- /tests/test_path_paths.py: -------------------------------------------------------------------------------- 1 | from nose2.tools.such import helper 2 | 3 | import dpath.segments 4 | import dpath.exceptions 5 | import dpath.options 6 | 7 | 8 | def test_path_paths_empty_key_disallowed(): 9 | tdict = { 10 | "Empty": { 11 | "": { 12 | "Key": "" 13 | } 14 | } 15 | } 16 | 17 | with helper.assertRaises(dpath.exceptions.InvalidKeyName): 18 | for x in dpath.segments.walk(tdict): 19 | pass 20 | 21 | 22 | def test_path_paths_empty_key_allowed(): 23 | tdict = { 24 | "Empty": { 25 | "": { 26 | "Key": "" 27 | } 28 | } 29 | } 30 | 31 | segments = [] 32 | dpath.options.ALLOW_EMPTY_STRING_KEYS = True 33 | 34 | for segments, value in dpath.segments.leaves(tdict): 35 | pass 36 | 37 | dpath.options.ALLOW_EMPTY_STRING_KEYS = False 38 | assert "/".join(segments) == "Empty//Key" 39 | -------------------------------------------------------------------------------- /tests/test_paths.py: -------------------------------------------------------------------------------- 1 | import dpath 2 | 3 | 4 | def test_util_safe_path_list(): 5 | res = dpath._split_path(["Ignore", "the/separator"], None) 6 | 7 | assert len(res) == 2 8 | assert res[0] == "Ignore" 9 | assert res[1] == "the/separator" 10 | -------------------------------------------------------------------------------- /tests/test_search.py: -------------------------------------------------------------------------------- 1 | import dpath 2 | 3 | 4 | def test_search_paths_with_separator(): 5 | dict = { 6 | "a": { 7 | "b": { 8 | "c": { 9 | "d": 0, 10 | "e": 1, 11 | "f": 2, 12 | }, 13 | }, 14 | }, 15 | } 16 | paths = [ 17 | 'a', 18 | 'a;b', 19 | 'a;b;c', 20 | 'a;b;c;d', 21 | 'a;b;c;e', 22 | 'a;b;c;f', 23 | ] 24 | 25 | for (path, value) in dpath.search(dict, '/**', yielded=True, separator=";"): 26 | assert path in paths 27 | 28 | for (path, value) in dpath.search(dict, ['**'], yielded=True, separator=";"): 29 | assert path in paths 30 | 31 | 32 | def test_search_paths(): 33 | dict = { 34 | "a": { 35 | "b": { 36 | "c": { 37 | "d": 0, 38 | "e": 1, 39 | "f": 2, 40 | }, 41 | }, 42 | }, 43 | } 44 | paths = [ 45 | 'a', 46 | 'a/b', 47 | 'a/b/c', 48 | 'a/b/c/d', 49 | 'a/b/c/e', 50 | 'a/b/c/f', 51 | ] 52 | 53 | for (path, value) in dpath.search(dict, '/**', yielded=True): 54 | assert path in paths 55 | 56 | for (path, value) in dpath.search(dict, ['**'], yielded=True): 57 | assert path in paths 58 | 59 | 60 | def test_search_afilter(): 61 | def afilter(x): 62 | if x in [1, 2]: 63 | return True 64 | return False 65 | 66 | dict = { 67 | "a": { 68 | "view_failure": "a", 69 | "b": { 70 | "c": { 71 | "d": 0, 72 | "e": 1, 73 | "f": 2, 74 | }, 75 | }, 76 | }, 77 | } 78 | paths = [ 79 | 'a/b/c/e', 80 | 'a/b/c/f', 81 | ] 82 | 83 | for (path, value) in dpath.search(dict, '/**', yielded=True, afilter=afilter): 84 | assert path in paths 85 | assert "view_failure" not in dpath.search(dict, '/**', afilter=afilter)['a'] 86 | assert "d" not in dpath.search(dict, '/**', afilter=afilter)['a']['b']['c'] 87 | 88 | for (path, value) in dpath.search(dict, ['**'], yielded=True, afilter=afilter): 89 | assert path in paths 90 | assert "view_failure" not in dpath.search(dict, ['**'], afilter=afilter)['a'] 91 | assert "d" not in dpath.search(dict, ['**'], afilter=afilter)['a']['b']['c'] 92 | 93 | 94 | def test_search_globbing(): 95 | dict = { 96 | "a": { 97 | "b": { 98 | "c": { 99 | "d": 0, 100 | "e": 1, 101 | "f": 2, 102 | }, 103 | }, 104 | }, 105 | } 106 | paths = [ 107 | 'a/b/c/d', 108 | 'a/b/c/f', 109 | ] 110 | 111 | for (path, value) in dpath.search(dict, '/a/**/[df]', yielded=True): 112 | assert path in paths 113 | 114 | for (path, value) in dpath.search(dict, ['a', '**', '[df]'], yielded=True): 115 | assert path in paths 116 | 117 | 118 | def test_search_return_dict_head(): 119 | tdict = { 120 | "a": { 121 | "b": { 122 | 0: 0, 123 | 1: 1, 124 | 2: 2, 125 | }, 126 | }, 127 | } 128 | res = dpath.search(tdict, '/a/b') 129 | assert isinstance(res['a']['b'], dict) 130 | assert len(res['a']['b']) == 3 131 | assert res['a']['b'] == {0: 0, 1: 1, 2: 2} 132 | 133 | res = dpath.search(tdict, ['a', 'b']) 134 | assert isinstance(res['a']['b'], dict) 135 | assert len(res['a']['b']) == 3 136 | assert res['a']['b'] == {0: 0, 1: 1, 2: 2} 137 | 138 | 139 | def test_search_return_dict_globbed(): 140 | tdict = { 141 | "a": { 142 | "b": { 143 | 0: 0, 144 | 1: 1, 145 | 2: 2, 146 | }, 147 | }, 148 | } 149 | 150 | res = dpath.search(tdict, '/a/b/[02]') 151 | assert isinstance(res['a']['b'], dict) 152 | assert len(res['a']['b']) == 2 153 | assert res['a']['b'] == {0: 0, 2: 2} 154 | 155 | res = dpath.search(tdict, ['a', 'b', '[02]']) 156 | assert isinstance(res['a']['b'], dict) 157 | assert len(res['a']['b']) == 2 158 | assert res['a']['b'] == {0: 0, 2: 2} 159 | 160 | 161 | def test_search_return_list_head(): 162 | tdict = { 163 | "a": { 164 | "b": [ 165 | 0, 166 | 1, 167 | 2, 168 | ], 169 | }, 170 | } 171 | 172 | res = dpath.search(tdict, '/a/b') 173 | assert isinstance(res['a']['b'], list) 174 | assert len(res['a']['b']) == 3 175 | assert res['a']['b'] == [0, 1, 2] 176 | 177 | res = dpath.search(tdict, ['a', 'b']) 178 | assert isinstance(res['a']['b'], list) 179 | assert len(res['a']['b']) == 3 180 | assert res['a']['b'] == [0, 1, 2] 181 | 182 | 183 | def test_search_return_list_globbed(): 184 | tdict = { 185 | "a": { 186 | "b": [ 187 | 0, 188 | 1, 189 | 2, 190 | ] 191 | } 192 | } 193 | 194 | res = dpath.search(tdict, '/a/b/[02]') 195 | assert isinstance(res['a']['b'], list) 196 | assert len(res['a']['b']) == 3 197 | assert res['a']['b'] == [0, None, 2] 198 | 199 | res = dpath.search(tdict, ['a', 'b', '[02]']) 200 | assert isinstance(res['a']['b'], list) 201 | assert len(res['a']['b']) == 3 202 | assert res['a']['b'] == [0, None, 2] 203 | 204 | 205 | def test_search_list_key_with_separator(): 206 | tdict = { 207 | "a": { 208 | "b": { 209 | "d": 'failure', 210 | }, 211 | "/b/d": 'success', 212 | }, 213 | } 214 | 215 | res = dpath.search(tdict, ['a', '/b/d']) 216 | assert 'b' not in res['a'] 217 | assert res['a']['/b/d'] == 'success' 218 | 219 | 220 | def test_search_multiple_stars(): 221 | testdata = { 222 | 'a': [ 223 | { 224 | 'b': [ 225 | {'c': 1}, 226 | {'c': 2}, 227 | {'c': 3}, 228 | ], 229 | }, 230 | ], 231 | } 232 | testpath = 'a/*/b/*/c' 233 | 234 | res = dpath.search(testdata, testpath) 235 | assert len(res['a'][0]['b']) == 3 236 | assert res['a'][0]['b'][0]['c'] == 1 237 | assert res['a'][0]['b'][1]['c'] == 2 238 | assert res['a'][0]['b'][2]['c'] == 3 239 | 240 | 241 | def test_search_negative_index(): 242 | d = {'a': {'b': [1, 2, 3]}} 243 | res = dpath.search(d, 'a/b/-1') 244 | 245 | assert res == dpath.search(d, "a/b/2") 246 | -------------------------------------------------------------------------------- /tests/test_segments.py: -------------------------------------------------------------------------------- 1 | import os 2 | from unittest import TestCase 3 | 4 | import hypothesis.strategies as st 5 | from hypothesis import given, assume, settings, HealthCheck 6 | 7 | import dpath.segments as api 8 | from dpath import options 9 | 10 | settings.register_profile("default", suppress_health_check=(HealthCheck.too_slow,)) 11 | settings.load_profile(os.getenv(u'HYPOTHESIS_PROFILE', 'default')) 12 | 13 | random_key_int = st.integers(0, 1000) 14 | random_key_str = st.binary() | st.text() 15 | random_key = random_key_str | random_key_int 16 | random_segments = st.lists(random_key) 17 | random_leaf = st.integers() | st.floats() | st.booleans() | st.binary() | st.text() | st.none() 18 | 19 | random_thing = st.recursive( 20 | random_leaf, 21 | lambda children: st.lists(children) | st.tuples(children) | st.dictionaries(st.binary() | st.text(), children), 22 | max_leaves=100 23 | ) 24 | random_node = random_thing.filter(lambda thing: isinstance(thing, (list, tuple, dict))) 25 | 26 | random_mutable_thing = st.recursive( 27 | random_leaf, 28 | lambda children: st.lists(children) | st.dictionaries(st.binary() | st.text(), children) 29 | ) 30 | random_mutable_node = random_mutable_thing.filter(lambda thing: isinstance(thing, (list, dict))) 31 | 32 | 33 | @st.composite 34 | def mutate(draw, segment): 35 | # Convert number segments. 36 | segment = api.int_str(segment) 37 | 38 | # Infer the type constructor for the result. 39 | kind = type(segment) 40 | 41 | # Produce a valid kind conversion for our wildcards. 42 | if isinstance(segment, bytes): 43 | def to_kind(v): 44 | try: 45 | return bytes(v, 'utf-8') 46 | except: 47 | return kind(v) 48 | else: 49 | def to_kind(v): 50 | return kind(v) 51 | 52 | # Convert to an list of single values. 53 | converted = [] 54 | for i in range(len(segment)): 55 | # This carefully constructed nonsense to get a single value 56 | # is necessary to work around limitations in the bytes type 57 | # iteration returning integers instead of byte strings of 58 | # length 1. 59 | c = segment[i:i + 1] 60 | 61 | # Check for values that need to be escaped. 62 | if c in tuple(map(to_kind, ('*', '?', '[', ']'))): 63 | c = to_kind('[') + c + to_kind(']') 64 | 65 | converted.append(c) 66 | 67 | # Start with a non-mutated result. 68 | result = converted 69 | 70 | # 50/50 chance we will attempt any mutation. 71 | change = draw(st.sampled_from((True, False))) 72 | if change: 73 | result = [] 74 | 75 | # For every value in segment maybe mutate, maybe not. 76 | for c in converted: 77 | # If the length isn't 1 then, we know this value is already 78 | # an escaped special character. We will not mutate these. 79 | if len(c) != 1: 80 | result.append(c) 81 | else: 82 | result.append(draw(st.sampled_from((c, to_kind('?'), to_kind('*'))))) 83 | 84 | combined = kind().join(result) 85 | 86 | # If we by chance produce the star-star result, then just revert 87 | # back to the original converted segment. This is not the mutation 88 | # you are looking for. 89 | if combined == to_kind('**'): 90 | combined = kind().join(converted) 91 | 92 | return combined 93 | 94 | 95 | @st.composite 96 | def random_segments_with_glob(draw): 97 | segments = draw(random_segments) 98 | glob = list(map(lambda x: draw(mutate(x)), segments)) 99 | 100 | # 50/50 chance we will attempt to add a star-star to the glob. 101 | use_ss = draw(st.sampled_from((True, False))) 102 | if use_ss: 103 | # Decide if we are inserting a new segment or replacing a range. 104 | insert_ss = draw(st.sampled_from((True, False))) 105 | if insert_ss: 106 | index = draw(st.integers(0, len(glob))) 107 | glob.insert(index, '**') 108 | else: 109 | start = draw(st.integers(0, len(glob))) 110 | stop = draw(st.integers(start, len(glob))) 111 | glob[start:stop] = ['**'] 112 | 113 | return segments, glob 114 | 115 | 116 | @st.composite 117 | def random_segments_with_nonmatching_glob(draw): 118 | (segments, glob) = draw(random_segments_with_glob()) 119 | 120 | # Generate a segment that is not in segments. 121 | invalid = draw(random_key.filter(lambda x: x not in segments and x not in ('*', '**'))) 122 | 123 | # Do we just have a star-star glob? It matches everything, so we 124 | # need to replace it entirely. 125 | if len(glob) == 1 and glob[0] == '**': 126 | glob = [invalid] 127 | # Do we have a star glob and only one segment? It matches anything 128 | # in the segment, so we need to replace it entirely. 129 | elif len(glob) == 1 and glob[0] == '*' and len(segments) == 1: 130 | glob = [invalid] 131 | # Otherwise we can add something we know isn't in the segments to 132 | # the glob. 133 | else: 134 | index = draw(st.integers(0, len(glob))) 135 | glob.insert(index, invalid) 136 | 137 | return (segments, glob) 138 | 139 | 140 | @st.composite 141 | def random_walk(draw): 142 | node = draw(random_mutable_node) 143 | found = tuple(api.walk(node)) 144 | assume(len(found) > 0) 145 | return (node, draw(st.sampled_from(found))) 146 | 147 | 148 | @st.composite 149 | def random_leaves(draw): 150 | node = draw(random_mutable_node) 151 | found = tuple(api.leaves(node)) 152 | assume(len(found) > 0) 153 | return (node, draw(st.sampled_from(found))) 154 | 155 | 156 | class TestSegments(TestCase): 157 | @classmethod 158 | def setUpClass(cls): 159 | # Allow empty strings in segments. 160 | options.ALLOW_EMPTY_STRING_KEYS = True 161 | 162 | @classmethod 163 | def tearDownClass(cls): 164 | # Revert back to default. 165 | options.ALLOW_EMPTY_STRING_KEYS = False 166 | 167 | @given(random_node) 168 | def test_kvs(self, node): 169 | ''' 170 | Given a node, kvs should produce a key that when used to extract 171 | from the node renders the exact same value given. 172 | ''' 173 | for k, v in api.make_walkable(node): 174 | assert node[k] is v 175 | 176 | @given(random_leaf) 177 | def test_leaf_with_leaf(self, leaf): 178 | ''' 179 | Given a leaf, leaf should return True. 180 | ''' 181 | assert api.leaf(leaf) is True 182 | 183 | @given(random_node) 184 | def test_leaf_with_node(self, node): 185 | ''' 186 | Given a node, leaf should return False. 187 | ''' 188 | assert api.leaf(node) is False 189 | 190 | @given(random_thing) 191 | def test_walk(self, thing): 192 | ''' 193 | Given a thing to walk, walk should yield key, value pairs where key 194 | is a tuple of non-zero length. 195 | ''' 196 | for k, v in api.walk(thing): 197 | assert isinstance(k, tuple) 198 | assert len(k) > 0 199 | 200 | @given(random_node) 201 | def test_get(self, node): 202 | ''' 203 | Given a node, get should return the exact value given a key for all 204 | key, value pairs in the node. 205 | ''' 206 | for k, v in api.walk(node): 207 | assert api.get(node, k) is v 208 | 209 | @given(random_node) 210 | def test_has(self, node): 211 | ''' 212 | Given a node, has should return True for all paths, False otherwise. 213 | ''' 214 | for k, v in api.walk(node): 215 | assert api.has(node, k) is True 216 | 217 | # If we are at a leaf, then we can create a value that isn't 218 | # present easily. 219 | if api.leaf(v): 220 | assert api.has(node, k + (0,)) is False 221 | 222 | @given(random_segments) 223 | def test_expand(self, segments): 224 | ''' 225 | Given segments expand should produce as many results are there were 226 | segments and the last result should equal the given segments. 227 | ''' 228 | count = len(segments) 229 | result = list(api.expand(segments)) 230 | 231 | assert count == len(result) 232 | 233 | if count > 0: 234 | assert segments == result[-1] 235 | 236 | @given(random_node) 237 | def test_types(self, node): 238 | ''' 239 | Given a node, types should yield a tuple of key, type pairs and the 240 | type indicated should equal the type of the value. 241 | ''' 242 | for k, v in api.walk(node): 243 | ts = api.types(node, k) 244 | ta = () 245 | for tk, tt in ts: 246 | ta += (tk,) 247 | assert type(api.get(node, ta)) is tt 248 | 249 | @given(random_node) 250 | def test_leaves(self, node): 251 | ''' 252 | Given a node, leaves should yield only leaf key, value pairs. 253 | ''' 254 | for k, v in api.leaves(node): 255 | assert api.leafy(v) 256 | 257 | @given(random_segments_with_glob()) 258 | def test_match(self, pair): 259 | ''' 260 | Given segments and a known good glob, match should be True. 261 | ''' 262 | (segments, glob) = pair 263 | assert api.match(segments, glob) is True 264 | 265 | @given(random_segments_with_nonmatching_glob()) 266 | def test_match_nonmatching(self, pair): 267 | ''' 268 | Given segments and a known bad glob, match should be False. 269 | ''' 270 | (segments, glob) = pair 271 | assert api.match(segments, glob) is False 272 | 273 | @given(walkable=random_walk(), value=random_thing) 274 | def test_set_walkable(self, walkable, value): 275 | ''' 276 | Given a walkable location, set should be able to update any value. 277 | ''' 278 | (node, (segments, found)) = walkable 279 | api.set(node, segments, value) 280 | assert api.get(node, segments) is value 281 | 282 | @given(walkable=random_leaves(), 283 | kstr=random_key_str, 284 | kint=random_key_int, 285 | value=random_thing, 286 | extension=random_segments) 287 | def test_set_create_missing(self, walkable, kstr, kint, value, extension): 288 | ''' 289 | Given a walkable non-leaf, set should be able to create missing 290 | nodes and set a new value. 291 | ''' 292 | (node, (segments, found)) = walkable 293 | assume(api.leaf(found)) 294 | 295 | parent_segments = segments[:-1] 296 | parent = api.get(node, parent_segments) 297 | 298 | if isinstance(parent, list): 299 | assume(len(parent) < kint) 300 | destination = parent_segments + (kint,) + tuple(extension) 301 | elif isinstance(parent, dict): 302 | assume(kstr not in parent) 303 | destination = parent_segments + (kstr,) + tuple(extension) 304 | else: 305 | raise Exception('mad mad world') 306 | 307 | api.set(node, destination, value) 308 | assert api.get(node, destination) is value 309 | 310 | @given(thing=random_thing) 311 | def test_fold(self, thing): 312 | ''' 313 | Given a thing, count paths with fold. 314 | ''' 315 | 316 | def f(o, p, a): 317 | a[0] += 1 318 | 319 | [count] = api.fold(thing, f, [0]) 320 | assert count == len(tuple(api.walk(thing))) 321 | 322 | @given(walkable=random_walk()) 323 | def test_view(self, walkable): 324 | ''' 325 | Given a walkable location, view that location. 326 | ''' 327 | (node, (segments, found)) = walkable 328 | assume(found == found) # Hello, nan! We don't want you here. 329 | 330 | view = api.view(node, segments) 331 | assert api.get(view, segments) == api.get(node, segments) 332 | -------------------------------------------------------------------------------- /tests/test_set.py: -------------------------------------------------------------------------------- 1 | import dpath 2 | 3 | 4 | def test_set_existing_separator(): 5 | dict = { 6 | "a": { 7 | "b": 0, 8 | }, 9 | } 10 | 11 | dpath.set(dict, ';a;b', 1, separator=";") 12 | assert dict['a']['b'] == 1 13 | 14 | dict['a']['b'] = 0 15 | dpath.set(dict, ['a', 'b'], 1, separator=";") 16 | assert dict['a']['b'] == 1 17 | 18 | 19 | def test_set_existing_dict(): 20 | dict = { 21 | "a": { 22 | "b": 0, 23 | }, 24 | } 25 | 26 | dpath.set(dict, '/a/b', 1) 27 | assert dict['a']['b'] == 1 28 | 29 | dict['a']['b'] = 0 30 | dpath.set(dict, ['a', 'b'], 1) 31 | assert dict['a']['b'] == 1 32 | 33 | 34 | def test_set_existing_list(): 35 | dict = { 36 | "a": [ 37 | 0, 38 | ], 39 | } 40 | 41 | dpath.set(dict, '/a/0', 1) 42 | assert dict['a'][0] == 1 43 | 44 | dict['a'][0] = 0 45 | dpath.set(dict, ['a', '0'], 1) 46 | assert dict['a'][0] == 1 47 | 48 | 49 | def test_set_filter(): 50 | def afilter(x): 51 | if int(x) == 31: 52 | return True 53 | return False 54 | 55 | dict = { 56 | "a": { 57 | "b": 0, 58 | "c": 1, 59 | "d": 31, 60 | } 61 | } 62 | 63 | dpath.set(dict, '/a/*', 31337, afilter=afilter) 64 | assert dict['a']['b'] == 0 65 | assert dict['a']['c'] == 1 66 | assert dict['a']['d'] == 31337 67 | 68 | dict = { 69 | "a": { 70 | "b": 0, 71 | "c": 1, 72 | "d": 31, 73 | } 74 | } 75 | 76 | dpath.set(dict, ['a', '*'], 31337, afilter=afilter) 77 | assert dict['a']['b'] == 0 78 | assert dict['a']['c'] == 1 79 | assert dict['a']['d'] == 31337 80 | 81 | 82 | def test_set_existing_path_with_separator(): 83 | dict = { 84 | "a": { 85 | 'b/c/d': 0, 86 | }, 87 | } 88 | 89 | dpath.set(dict, ['a', 'b/c/d'], 1) 90 | assert len(dict['a']) == 1 91 | assert dict['a']['b/c/d'] == 1 92 | -------------------------------------------------------------------------------- /tests/test_types.py: -------------------------------------------------------------------------------- 1 | from collections.abc import MutableSequence, MutableMapping 2 | 3 | from nose2.tools.such import helper 4 | 5 | import dpath 6 | from dpath import MergeType 7 | 8 | 9 | class TestMapping(MutableMapping): 10 | def __init__(self, data=None): 11 | if data is None: 12 | data = {} 13 | 14 | self._mapping = {} 15 | self._mapping.update(data) 16 | 17 | def __len__(self): 18 | return len(self._mapping) 19 | 20 | def __iter__(self): 21 | return iter(self._mapping) 22 | 23 | def __contains__(self, key): 24 | return key in self._mapping 25 | 26 | def __getitem__(self, key): 27 | return self._mapping[key] 28 | 29 | def __setitem__(self, key, value): 30 | self._mapping[key] = value 31 | 32 | def __delitem__(self, key): 33 | del self._mapping[key] 34 | 35 | 36 | class TestSequence(MutableSequence): 37 | def __init__(self, data=None): 38 | if data is None: 39 | data = list() 40 | 41 | self._list = [] + data 42 | 43 | def __len__(self): 44 | return len(self._list) 45 | 46 | def __getitem__(self, idx): 47 | return self._list[idx] 48 | 49 | def __delitem__(self, idx): 50 | del self._list[idx] 51 | 52 | def __setitem__(self, idx, value): 53 | self._list[idx] = value 54 | 55 | def __str__(self): 56 | return str(self._list) 57 | 58 | def __eq__(self, other): 59 | return self._list == other._list 60 | 61 | def __ne__(self, other): 62 | return not self.__eq__(other) 63 | 64 | def insert(self, idx, value): 65 | self._list.insert(idx, value) 66 | 67 | def append(self, value): 68 | self.insert(len(self._list), value) 69 | 70 | 71 | def test_types_set(): 72 | data = TestMapping({"a": TestSequence([0])}) 73 | 74 | dpath.set(data, '/a/0', 1) 75 | assert data['a'][0] == 1 76 | 77 | data['a'][0] = 0 78 | 79 | dpath.set(data, ['a', '0'], 1) 80 | assert data['a'][0] == 1 81 | 82 | 83 | def test_types_get_list_of_dicts(): 84 | tdict = TestMapping({ 85 | "a": TestMapping({ 86 | "b": TestSequence([ 87 | {0: 0}, 88 | {0: 1}, 89 | {0: 2}, 90 | ]), 91 | }), 92 | }) 93 | 94 | res = dpath.segments.view(tdict, ['a', 'b', 0, 0]) 95 | 96 | assert isinstance(res['a']['b'], TestSequence) 97 | assert len(res['a']['b']) == 1 98 | assert res['a']['b'][0][0] == 0 99 | 100 | 101 | def test_types_merge_simple_list_replace(): 102 | src = TestMapping({ 103 | "list": TestSequence([7, 8, 9, 10]) 104 | }) 105 | dst = TestMapping({ 106 | "list": TestSequence([0, 1, 2, 3]) 107 | }) 108 | 109 | dpath.merge(dst, src, flags=MergeType.REPLACE) 110 | assert dst["list"] == TestSequence([7, 8, 9, 10]), "%r != %r" % (dst["list"], TestSequence([7, 8, 9, 10])) 111 | 112 | 113 | def test_types_get_absent(): 114 | ehash = TestMapping() 115 | helper.assertRaises(KeyError, dpath.get, ehash, '/a/b/c/d/f') 116 | helper.assertRaises(KeyError, dpath.get, ehash, ['a', 'b', 'c', 'd', 'f']) 117 | 118 | 119 | def test_types_get_glob_multiple(): 120 | ehash = TestMapping({ 121 | "a": TestMapping({ 122 | "b": TestMapping({ 123 | "c": TestMapping({ 124 | "d": 0, 125 | }), 126 | "e": TestMapping({ 127 | "d": 0, 128 | }), 129 | }), 130 | }), 131 | }) 132 | 133 | helper.assertRaises(ValueError, dpath.get, ehash, '/a/b/*/d') 134 | helper.assertRaises(ValueError, dpath.get, ehash, ['a', 'b', '*', 'd']) 135 | 136 | 137 | def test_delete_filter(): 138 | def afilter(x): 139 | if int(x) == 31: 140 | return True 141 | return False 142 | 143 | data = TestMapping({ 144 | "a": TestMapping({ 145 | "b": 0, 146 | "c": 1, 147 | "d": 31, 148 | }), 149 | }) 150 | 151 | dpath.delete(data, '/a/*', afilter=afilter) 152 | assert data['a']['b'] == 0 153 | assert data['a']['c'] == 1 154 | assert 'd' not in data['a'] 155 | -------------------------------------------------------------------------------- /tests/test_unicode.py: -------------------------------------------------------------------------------- 1 | import dpath 2 | 3 | 4 | def test_unicode_merge(): 5 | a = {'中': 'zhong'} 6 | b = {'文': 'wen'} 7 | 8 | dpath.merge(a, b) 9 | assert len(a.keys()) == 2 10 | assert a['中'] == 'zhong' 11 | assert a['文'] == 'wen' 12 | 13 | 14 | def test_unicode_search(): 15 | a = {'中': 'zhong'} 16 | 17 | results = [[x[0], x[1]] for x in dpath.search(a, '*', yielded=True)] 18 | assert len(results) == 1 19 | assert results[0][0] == '中' 20 | assert results[0][1] == 'zhong' 21 | 22 | 23 | def test_unicode_str_hybrid(): 24 | a = {'first': u'1'} 25 | b = {u'second': '2'} 26 | 27 | dpath.merge(a, b) 28 | assert len(a.keys()) == 2 29 | assert a[u'second'] == '2' 30 | assert a['second'] == u'2' 31 | assert a[u'first'] == '1' 32 | assert a['first'] == u'1' 33 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [flake8] 7 | ignore = E501,E722 8 | 9 | [tox] 10 | envlist = pypy37, py38, py39, py310, py311, py312 11 | 12 | [gh-actions] 13 | python = 14 | pypy-3.7: pypy37 15 | 3.8: py38 16 | 3.9: py39 17 | 3.10: py310 18 | 3.11: py311 19 | 3.12: py312 20 | 21 | [testenv] 22 | deps = 23 | hypothesis 24 | nose2 25 | commands = nose2 {posargs} 26 | --------------------------------------------------------------------------------