├── .github └── workflows │ ├── docker.yml │ └── pythonpackage.yml ├── .gitignore ├── .readthedocs.yml ├── CITATION.cff ├── COPYING ├── Dockerfile ├── README.rst ├── docs ├── Makefile ├── conf.py ├── index.rst ├── make.bat ├── requirements.txt └── src │ ├── authors.rst │ ├── configuration.rst │ ├── examples.rst │ ├── future.rst │ ├── install.rst │ ├── misc_pkgs │ ├── client_side.rst │ ├── concurrent_fetcher.rst │ ├── misc_pkgs.rst │ ├── pub_sub.rst │ └── storage.rst │ ├── readme.rst │ └── specification │ ├── check.rst │ ├── delete.rst │ ├── encoding.rst │ ├── insert.rst │ ├── security.rst │ ├── specification.rst │ ├── sync.rst │ ├── sync_join.rst │ ├── sync_leave.rst │ └── tcp_bulk.rst ├── examples ├── command_checker.py ├── delfile.py ├── getfile.py ├── leavesync.py ├── publisher.py ├── putfile.py ├── subscriber.py └── sync.py ├── ndn_python_repo ├── __init__.py ├── clients │ ├── __init__.py │ ├── command_checker.py │ ├── delete.py │ ├── getfile.py │ ├── putfile.py │ └── sync.py ├── cmd │ ├── __init__.py │ ├── install.py │ ├── main.py │ └── port.py ├── command │ ├── __init__.py │ └── repo_commands.py ├── config.py ├── handle │ ├── __init__.py │ ├── command_handle_base.py │ ├── delete_command_handle.py │ ├── read_handle.py │ ├── sync_command_handle.py │ ├── tcp_bulk_insert_handle.py │ ├── utils.py │ └── write_command_handle.py ├── ndn-python-repo.conf.sample ├── ndn-python-repo.service ├── repo.py ├── storage │ ├── __init__.py │ ├── leveldb.py │ ├── mongodb.py │ ├── sqlite.py │ ├── storage_base.py │ └── storage_factory.py └── utils │ ├── __init__.py │ ├── concurrent_fetcher.py │ ├── passive_svs.py │ └── pubsub.py ├── poetry.lock ├── pyproject.toml └── tests ├── concurrent_fetcher_test.py ├── integration ├── integration_test.py └── pytest.ini └── storage_test.py /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | on: 3 | push: 4 | tags: 5 | - '*' 6 | schedule: 7 | # twice a month 8 | - cron: '20 8 5,20 * *' 9 | workflow_dispatch: 10 | 11 | permissions: 12 | attestations: write 13 | packages: write 14 | id-token: write 15 | 16 | jobs: 17 | ndn-python-repo: 18 | uses: named-data/actions/.github/workflows/docker-image.yml@v1 19 | with: 20 | name: ndn-python-repo 21 | target: ndn-python-repo 22 | -------------------------------------------------------------------------------- /.github/workflows/pythonpackage.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | # Allows you to run this workflow manually from the Actions tab 9 | workflow_dispatch: 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | build: 16 | # The type of runner that the job will run on 17 | runs-on: ${{ matrix.os }} 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | os: [ubuntu-latest] 22 | python-version: ['3.12', 'pypy-3.10'] 23 | 24 | services: 25 | nfd: 26 | image: ghcr.io/named-data/nfd:latest 27 | volumes: 28 | - /run/nfd:/run/nfd 29 | 30 | steps: 31 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 32 | - uses: actions/checkout@v4 33 | - name: Set up Python ${{ matrix.python-version }} 34 | uses: actions/setup-python@v5 35 | with: 36 | python-version: ${{ matrix.python-version }} 37 | - name: Install dependencies 38 | run: | 39 | python -m pip install --upgrade pip 40 | pip3 install ".[dev]" 41 | - name: Setup PIB and TPM 42 | run: | 43 | pyndnsec Init-Pib 44 | pyndnsec New-Item /test 45 | - name: Run tests 46 | run: pytest tests 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # ProtoBuf generated files 107 | *_pb2.py 108 | 109 | # PyCharm 110 | .idea/ 111 | 112 | **/.DS_Store 113 | .vscode/ 114 | .leveldb/ 115 | repo.db 116 | NDN_Repo.egg-info/ 117 | 118 | # tianyuan's helper 119 | feature.* 120 | testrepo.* 121 | 122 | # Temporary file 123 | pib.db 124 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: docs/conf.py 11 | 12 | # Build documentation with MkDocs 13 | #mkdocs: 14 | # configuration: mkdocs.yml 15 | 16 | # Optionally build your docs in additional formats such as PDF 17 | formats: 18 | - pdf 19 | 20 | build: 21 | os: ubuntu-22.04 22 | tools: 23 | python: "3.12" 24 | 25 | # Optionally set the version of Python and requirements required to build your docs 26 | python: 27 | install: 28 | - requirements: docs/requirements.txt 29 | - method: pip 30 | path: . 31 | extra_requirements: 32 | - docs 33 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: "Yu" 5 | given-names: "Tianyuan" 6 | orcid: "https://orcid.org/0000-0002-6722-1938" 7 | - family-names: "Kong" 8 | given-names: "Zhaoning" 9 | orcid: "https://orcid.org/0009-0009-8887-1698" 10 | - family-names: "Ma" 11 | given-names: "Xinyu" 12 | orcid: "https://orcid.org/0000-0002-7575-1058" 13 | - family-names: "Wang" 14 | given-names: "Lan" 15 | orcid: "https://orcid.org/0000-0001-7925-7957" 16 | - family-names: "Zhang" 17 | given-names: "Lixia" 18 | orcid: "https://orcid.org/0000-0003-0701-757X" 19 | title: "YaNFD" 20 | version: 0.4 21 | doi: 10.1109/ICNC59896.2024.10556243 22 | date-released: 2024-02-19 23 | url: "https://github.com/UCLA-IRL/ndn-python-repo" 24 | preferred-citation: 25 | type: conference-paper 26 | authors: 27 | - family-names: "Yu" 28 | given-names: "Tianyuan" 29 | orcid: "https://orcid.org/0000-0002-6722-1938" 30 | - family-names: "Kong" 31 | given-names: "Zhaoning" 32 | orcid: "https://orcid.org/0009-0009-8887-1698" 33 | - family-names: "Ma" 34 | given-names: "Xinyu" 35 | orcid: "https://orcid.org/0000-0002-7575-1058" 36 | - family-names: "Wang" 37 | given-names: "Lan" 38 | orcid: "https://orcid.org/0000-0001-7925-7957" 39 | - family-names: "Zhang" 40 | given-names: "Lixia" 41 | orcid: "https://orcid.org/0000-0003-0701-757X" 42 | doi: "10.1109/ICNC59896.2024.10556243" 43 | conference: 44 | name: "2024 International Conference on Computing, Networking and Communications (ICNC)" 45 | month: 2 46 | start: 927 # First page number 47 | end: 931 # Last page number 48 | title: "PythonRepo: Persistent In-Network Storage for Named Data Networking" 49 | year: 2024 50 | publisher: "IEEE" 51 | url: "https://doi.org/10.1109/ICNC59896.2024.10556243" 52 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | https://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-alpine AS ndn-python-repo 2 | 3 | COPY . /repo 4 | 5 | RUN pip install --disable-pip-version-check -e /repo[mongodb] 6 | 7 | ENV HOME=/config 8 | VOLUME /config 9 | VOLUME /run/nfd 10 | 11 | ENTRYPOINT ["/usr/local/bin/ndn-python-repo"] 12 | CMD ["-c", "/config/repo.conf"] -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ndn-python-repo 2 | =============== 3 | 4 | |Test Badge| 5 | |Release Badge| 6 | |Docs Badge| 7 | 8 | A Named Data Networking (NDN) Repo implementation using python-ndn_. 9 | 10 | Please see our documentation_ if you have any issues. 11 | 12 | .. |Test Badge| image:: https://github.com/UCLA-IRL/ndn-python-repo/actions/workflows/pythonpackage.yml/badge.svg 13 | :target: https://github.com/UCLA-IRL/ndn-python-repo/actions/workflows/pythonpackage.yml 14 | :alt: Test Status 15 | 16 | .. |Release Badge| image:: https://badge.fury.io/py/ndn-python-repo.svg 17 | :target: https://pypi.org/project/ndn-python-repo/ 18 | :alt: Release Status 19 | 20 | .. |Docs Badge| image:: https://readthedocs.org/projects/ndn-python-repo/badge/?version=latest 21 | :target: https://ndn-python-repo.readthedocs.io/en/latest/ 22 | :alt: Docs Status 23 | 24 | .. _python-ndn: https://github.com/named-data/python-ndn 25 | 26 | .. _documentation: https://ndn-python-repo.readthedocs.io/en/latest 27 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('..')) 16 | import sphinx_rtd_theme 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = 'ndn-python-repo' 22 | copyright = '2020, Zhaoning Kong' 23 | author = 'Zhaoning Kong' 24 | 25 | 26 | # -- General configuration --------------------------------------------------- 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx_rtd_theme', 34 | 'sphinx_autodoc_typehints', 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # List of patterns, relative to source directory, that match files and 41 | # directories to ignore when looking for source files. 42 | # This pattern also affects html_static_path and html_extra_path. 43 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | autoclass_content = 'both' 49 | 50 | # -- Options for HTML output ------------------------------------------------- 51 | 52 | # The theme to use for HTML and HTML Help pages. See the documentation for 53 | # a list of builtin themes. 54 | # 55 | html_theme = 'sphinx_rtd_theme' 56 | pygments_style = 'sphinx' 57 | 58 | # Add any paths that contain custom static files (such as style sheets) here, 59 | # relative to this directory. They are copied after the builtin static files, 60 | # so a file named "default.css" will overwrite the builtin "default.css". 61 | html_static_path = ['_static'] -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. ndn-python-repo documentation master file, created by 2 | sphinx-quickstart on Tue May 12 16:51:55 2020. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to ndn-python-repo's documentation! 7 | =========================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | src/readme.rst 14 | src/install.rst 15 | src/configuration.rst 16 | src/specification/specification.rst 17 | src/misc_pkgs/misc_pkgs.rst 18 | src/examples.rst 19 | src/authors.rst 20 | src/future.rst 21 | 22 | 23 | 24 | Indices and tables 25 | ================== 26 | 27 | * :ref:`genindex` 28 | * :ref:`modindex` 29 | * :ref:`search` 30 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx-autodoc-typehints>=1.10.2 2 | sphinx-rtd-theme>=0.4.3 3 | python-ndn>=0.2b2.post1 4 | PyYAML>=5.1.2 5 | plyvel>=1.2.0 6 | pymongo>=3.10.1 -------------------------------------------------------------------------------- /docs/src/authors.rst: -------------------------------------------------------------------------------- 1 | Authors 2 | ======= 3 | 4 | * Zhaoning Kong 5 | * Xinyu Ma 6 | * Yufeng Zhang 7 | * Zhiyi Zhang 8 | * Davide Pesavento 9 | * Susmit Shannigrahi 10 | * Saurab Dulal 11 | * Junxiao Shi 12 | -------------------------------------------------------------------------------- /docs/src/configuration.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ============= 3 | 4 | You can configure ndn-python-repo with a config file, by specifying the path to the file when 5 | starting a repo instance: 6 | 7 | .. code-block:: bash 8 | 9 | $ ndn-python-repo -c 10 | 11 | A sample config file is provided at ``ndn_python_repo/ndn-python-repo.conf.sample``. 12 | 13 | If no config file is given on the command line, this sample config file will be used by default. 14 | 15 | 16 | Repo namespace 17 | -------------- 18 | 19 | Specify the name of a repo in the config file. For example:: 20 | 21 | repo_config: 22 | # the repo's routable prefix 23 | repo_name: 'testrepo' 24 | 25 | Another option is to specify the repo name when starting a repo on the command line. 26 | This overrides the repo name in the config file:: 27 | 28 | $ ndn-python-repo -r "/name_foo" 29 | 30 | 31 | Repo prefix registration 32 | ------------------------ 33 | By default, the repo registers the root prefix ``/``. 34 | 35 | Alternatively, you can configure repo such that it doesn't register the root prefix:: 36 | 37 | repo_config: 38 | register_root: False 39 | 40 | If ``register_root`` is set to ``False``, the client is responsible of telling the 41 | repo which prefix to register or unregister every time in ``RepoCommandParameter``. 42 | See :ref:`specification-insert-label` and :ref:`specification-delete-label` for details. 43 | 44 | 45 | Choose the backend database 46 | --------------------------- 47 | 48 | The ndn-python-repo uses one of the three backend databases: 49 | 50 | * SQLite3 (default) 51 | * leveldb 52 | * MongoDB 53 | 54 | To use non-default databases, perform the following steps: 55 | 56 | #. Install ndn-python-repo with additional database support that you need:: 57 | 58 | $ /usr/bin/pip3 install ndn-python-repo[leveldb] 59 | $ /usr/bin/pip3 install ndn-python-repo[mongodb] 60 | 61 | #. Specify the database selection and database file in the config file. For example:: 62 | 63 | db_config: 64 | # choose one among sqlite3, leveldb, and mongodb 65 | db_type: 'mongodb' 66 | 67 | # only the chosen db's config will be read 68 | mongodb: 69 | 'uri': 'mongodb://127.0.0.1:27017/' 70 | 'db': 'repo' 71 | 'collection': 'data' 72 | 73 | 74 | TCP bulk insert 75 | --------------- 76 | 77 | By default, the repo listens on ``0.0.0.0:7376`` for TCP bulk insert. 78 | You can configure in the config file which address the repo listens on. For example:: 79 | 80 | tcp_bulk_insert: 81 | 'addr': '127.0.0.1' 82 | 'port': '7377' 83 | 84 | 85 | Logging 86 | ------- 87 | 88 | Repo uses the python logging module, and by default logs all messages of and above 89 | level ``INFO`` to ``stdout``. 90 | You can override the default options in the config file. For example:: 91 | 92 | logging_config: 93 | 'level': 'WARNING' 94 | 'file': '/var/log/ndn/ndn-python-repo/repo.log' 95 | 96 | 97 | systemd 98 | ---------------- 99 | 100 | To run ndn-python-repo with systemd on Linux, perform the following steps: 101 | 102 | #. Run the provided script to install the systemd script to ``/etc/systemd/system/``:: 103 | 104 | $ sudo ndn-python-repo-install 105 | 106 | #. Then, start, stop, and monitor a repo instance with systemd:: 107 | 108 | $ sudo systemctl start ndn-python-repo 109 | $ sudo systemctl stop ndn-python-repo 110 | $ sudo systemctl status ndn-python-repo 111 | 112 | #. Examine logs:: 113 | 114 | $ sudo journalctl -u ndn-python-repo.service 115 | 116 | -------------------------------------------------------------------------------- /docs/src/examples.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | TBD 5 | -------------------------------------------------------------------------------- /docs/src/future.rst: -------------------------------------------------------------------------------- 1 | Future Plans 2 | ============ 3 | 4 | * Fix knowing bugs and problems 5 | * Make better tests 6 | * Make a Docker container for test, since installing nfd takes too long. 7 | -------------------------------------------------------------------------------- /docs/src/install.rst: -------------------------------------------------------------------------------- 1 | Install and Run 2 | =============== 3 | 4 | Install 5 | ------- 6 | 7 | Install the latest release with pip: 8 | 9 | .. code-block:: bash 10 | 11 | $ /usr/bin/pip3 install ndn-python-repo 12 | 13 | Optionally, you can install the latest development version from source: 14 | 15 | .. code-block:: bash 16 | 17 | $ git clone https://github.com/JonnyKong/ndn-python-repo.git 18 | $ cd ndn-python-repo && /usr/bin/pip3 install -e . 19 | 20 | 21 | Migrate from repo-ng 22 | -------------------- 23 | 24 | ndn-python-repo provides a script to migrate existing data from repo-ng:: 25 | 26 | $ ndn-python-repo-port -d \ 27 | -a \ 28 | -p 29 | 30 | It takes as input a repo-ng database file, reads the Data packets and pipe them through TCP bulk insert into the new repo. 31 | 32 | 33 | Instruction for developers 34 | -------------------------- 35 | 36 | For development, `poetry `_ is recommended. 37 | 38 | .. code-block:: bash 39 | 40 | $ poetry install --all-extras 41 | 42 | To setup a traditional python3 virtual environment with editable installation: 43 | 44 | .. code-block:: bash 45 | 46 | python3 -m venv venv 47 | . venv/bin/activate 48 | pip3 install -e ".[dev,docs]" 49 | 50 | Run all tests: 51 | 52 | .. code-block:: bash 53 | 54 | $ nfd-start 55 | $ pytest tests 56 | 57 | Compile the documentation with Sphinx: 58 | 59 | .. code-block:: bash 60 | 61 | $ poetry run make -C docs html 62 | $ open docs/_build/html/index.html 63 | -------------------------------------------------------------------------------- /docs/src/misc_pkgs/client_side.rst: -------------------------------------------------------------------------------- 1 | Client-side packages 2 | ==================== 3 | 4 | Introduction 5 | ------------ 6 | 7 | Application built with python-ndn can make use of the client packages provided. 8 | 9 | There are four parts: 10 | 11 | #. **PutfileClient**: insert files into the repo. 12 | #. **GetfileClient**: get files from the repo. 13 | #. **DeleteClient**: delete data packets from the repo. 14 | #. **CommandChecker**: check process status from the repo. 15 | 16 | The example programs in :mod:`examples/` illustrate how to use these packages. 17 | 18 | Note that the type ``Union[Iterable[Union[bytes, bytearray, memoryview, str]], str, bytes, bytearray, memoryview]`` 19 | in the documentation is equivalent to the ``ndn.name.NonStrictName`` type. 20 | 21 | 22 | 23 | Reference 24 | --------- 25 | 26 | .. automodule:: ndn_python_repo.clients.putfile 27 | 28 | .. autoclass:: PutfileClient 29 | :members: 30 | 31 | .. automodule:: ndn_python_repo.clients.getfile 32 | 33 | .. autoclass:: GetfileClient 34 | :members: 35 | 36 | .. automodule:: ndn_python_repo.clients.delete 37 | 38 | .. autoclass:: DeleteClient 39 | :members: 40 | 41 | .. automodule:: ndn_python_repo.clients.command_checker 42 | 43 | .. autoclass:: CommandChecker 44 | :members: -------------------------------------------------------------------------------- /docs/src/misc_pkgs/concurrent_fetcher.rst: -------------------------------------------------------------------------------- 1 | ``ConcurrentFetcher`` package 2 | ============================= 3 | 4 | Introduction 5 | ------------ 6 | 7 | Fetch data packets in parallel using a fixed window size. 8 | 9 | Note that the type ``Union[Iterable[Union[bytes, bytearray, memoryview, str]], str, bytes, bytearray, memoryview]`` 10 | in the documentation is equivalent to the ``ndn.name.NonStrictName`` type. 11 | 12 | 13 | Reference 14 | --------- 15 | 16 | .. autofunction:: ndn_python_repo.utils.concurrent_fetcher -------------------------------------------------------------------------------- /docs/src/misc_pkgs/misc_pkgs.rst: -------------------------------------------------------------------------------- 1 | Miscellaneous packages 2 | ====================== 3 | 4 | .. toctree:: 5 | 6 | Client-side packages 7 | ConcurrentFetcher 8 | PubSub 9 | Storage -------------------------------------------------------------------------------- /docs/src/misc_pkgs/pub_sub.rst: -------------------------------------------------------------------------------- 1 | ``PubSub`` package 2 | ================== 3 | 4 | Introduction 5 | ------------ 6 | 7 | The ``PubSub`` package provides a pub-sub API with best-effort, at-most-once delivery guarantee. 8 | 9 | If there are no subscribers reachable when a message is published, this message will not be 10 | re-transmitted. 11 | 12 | If there are multiple subscribers reachable, the nearest subscriber will be notified of the 13 | published message in an any-cast style. 14 | 15 | Note that the type ``Union[Iterable[Union[bytes, bytearray, memoryview, str]], str, bytes, bytearray, memoryview]`` 16 | in the documentation is equivalent to the ``ndn.name.NonStrictName`` type. 17 | 18 | 19 | Process 20 | ------- 21 | 22 | Under the hood the ``PubSub`` module transmits a series of Interest and Data packets: 23 | 24 | 1. The subscriber calls ``subscribe(topic, cb)``. This makes the subscriber listen on 25 | ``"//notify"``. 26 | 27 | 2. The publisher invokes ``publish(topic, msg)``. This method sends an Interest with name 28 | ``"//notify"``, which will be routed to a subscriber. The interest carries the following fields in its application parameters: 29 | 30 | * Publisher prefix: used by the subscriber to reach the publisher in the next step 31 | * NotifyNonce: a random bytes string, used by the publisher to de-multiplex among different publications 32 | * Forwarding hint (optional): if publisher prefix is not announced in the routing system, publisher can provide a forwarding hint 33 | 34 | Meanwhile, ``msg`` is wrapped into a Data packet named ``"//msg//"``. Here, the data name contains ``topic`` to establish a binding between topic and nonce, to prevent man-in-the-middle attacks that changes the topic. 35 | 36 | 3. The subscriber receives the notification interest, constructs a new Interest 37 | ``"//msg//"`` and send it to the publisher. 38 | 39 | 4. The publisher receives the interest ``"//msg//"``, and returns the 40 | corresponding data. 41 | 42 | 5. The subscriber receives the data, and invokes ``cb(data.content)`` to hand the message to the 43 | application. 44 | 45 | 6. The publisher receives the acknowledgement Data packet, and erases the soft state. 46 | 47 | 48 | Encoding 49 | -------- 50 | 51 | The notify Interest's application parameter is encoded as follows: 52 | 53 | .. code-block:: 54 | 55 | NotifyAppParam = DATA-TYPE TLV-LENGTH 56 | [PublisherPrefix] 57 | [NotifyNonce] 58 | [PublisherFwdHint] 59 | 60 | PublisherPrefix = Name 61 | 62 | NotifyNonce = NOTIFY-NONCE-TYPE TLV-LENGTH Bytes 63 | 64 | PublisherFwdHint = PUBLISHER-FWD-HINT-TYPE TLV-LENGTH Name 65 | 66 | The type number assignments are as follows: 67 | 68 | +---------------------------+----------------------------+--------------------------------+ 69 | | type | Assigned number (decimal) | Assigned number (hexadecimal) | 70 | +===========================+============================+================================+ 71 | | NOTIFY-NONCE-TYPE | 128 | 0x80 | 72 | +---------------------------+----------------------------+--------------------------------+ 73 | | PUBLISHER-FWD-HINT-TYPE | 211 | 0xD3 | 74 | +---------------------------+----------------------------+--------------------------------+ 75 | 76 | 77 | Reference 78 | --------- 79 | 80 | .. autoclass:: ndn_python_repo.utils.PubSub 81 | :members: 82 | -------------------------------------------------------------------------------- /docs/src/misc_pkgs/storage.rst: -------------------------------------------------------------------------------- 1 | ``Storage`` package 2 | =================== 3 | 4 | ndn-python-repo supports 3 types of databases as backends. 5 | The ``Storage`` package provides a unified key-value storage API with the following features: 6 | 7 | * Supports ``MustBeFresh`` 8 | * Supports ``CanBePrefix`` 9 | * Batched writes with periodic writebacks to improve performance 10 | 11 | The ``Storage`` class provides an interface, and is implemented by: 12 | 13 | * ``SqliteStorage`` 14 | * ``LevelDBStorage`` 15 | * ``MongoDBStorage`` 16 | 17 | Note that the type ``Union[Iterable[Union[bytes, bytearray, memoryview, str]], str, bytes, bytearray, memoryview]`` 18 | in the documentation is equivalent to the ``ndn.name.NonStrictName`` type. 19 | 20 | Reference 21 | --------- 22 | 23 | .. autoclass:: ndn_python_repo.storage.Storage 24 | :members: 25 | 26 | .. autoclass:: ndn_python_repo.storage.SqliteStorage 27 | :members: 28 | 29 | .. autoclass:: ndn_python_repo.storage.LevelDBStorage 30 | :members: 31 | 32 | .. autoclass:: ndn_python_repo.storage.MongoDBStorage 33 | :members: -------------------------------------------------------------------------------- /docs/src/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../README.rst 2 | -------------------------------------------------------------------------------- /docs/src/specification/check.rst: -------------------------------------------------------------------------------- 1 | Check 2 | ===== 3 | 4 | The check protocol is used by clients to check the progress of a insertion or deletion process. 5 | 6 | 1. The check prefix for ```` is ``// check``. 7 | For example, the check prefix for insertion is ``//insert check``, 8 | and deletion is ``//delete check``. 9 | 2. Status check Interests are send to the check prefix directly. No Pub-Sub is used here. 10 | 3. The check Interest should carry an application parameter ``RepoStatQuery``, 11 | which contains the request number of the command. 12 | The request number of the command is always the SHA256 hash of the command data published in Pub-Sub. 13 | 4. After receiving the query Interest, the repo responds with a Data packet containing ``RepoCommandRes``. 14 | 5. The status is only kept for 60s after the operation finishes. 15 | After that time, all queries will be responded with ``NOT-FOUND``. 16 | 17 | RepoCommandRes 18 | ============== 19 | 20 | * The ``RepoCommandRes`` Data contains a status code for the whole command, with the following rules: 21 | 22 | * ``MALFORMED``: If the command cannot be parsed. 23 | * ``NOT-FOUND``: If the given request no is not associated with a valid command. 24 | This is also returned when the repo has not finish fetching the command from Pub-Sub, 25 | or the command has finished for more than 60s. 26 | * ``COMPLETED``: If all operations (for all objects) completed. 27 | * ``IN-PROGRESS``: The command is received and being executed. 28 | * ``FAILED``: If one or more operation in the command fails. 29 | If the is insertion, this means some or all objects requested to insert cannot be completely fetched. 30 | However, fetched objects or segments are still inserted into the repo. 31 | Only the objects with ``insert_num=0`` are not inserted. 32 | 33 | * For each ``ObjStatus`` contained in the ``RepoCommandRes``, the status code can be one of the following: 34 | 35 | * ``ROGER``: The whole command is received and the operation on this object will be started in the future. 36 | * ``MALFORMED``: If the object has wrong parameter. 37 | * ``FAILED``: If the operation on this object failed to execute. 38 | For example, not all segments specified can be fetched. 39 | Note that even for a failed object, fetched segments are still put into the repo and can be fetched. 40 | * ``COMPLETED``: If the operation on this object succeeded. 41 | -------------------------------------------------------------------------------- /docs/src/specification/delete.rst: -------------------------------------------------------------------------------- 1 | .. _specification-delete-label: 2 | 3 | Delete 4 | ====== 5 | 6 | Repo deletion process makes use of the :doc:`../misc_pkgs/pub_sub`. 7 | 8 | 1. The repo subscribes to the topic ``//delete``. 9 | 10 | 2. The client publishes a message to the topic ``//delete``. 11 | The message payload is ``RepoCommandParam`` containing one or more ``ObjParam`` with the following fields: 12 | 13 | * ``name``: either a Data packet name, or a name prefix of Data packets. 14 | * ``start_block_id`` (Optional): inclusive start segment number. 15 | * ``end_block_id`` (Optional): inclusive end segment number. 16 | * ``register_prefix`` (Optional): if repo doesn't register the root prefix (:doc:`../configuration` ``register_root`` is disabled), client can tell repo to unregister this prefix. 17 | 18 | 3. The repo deletes Data packets according to given parameters. 19 | 20 | * If both ``start_block_id`` and ``end_block_id`` are omitted, the repo deletes a single packet identified in ``name`` parameter. 21 | The deletion process succeeds when this packet is deleted. 22 | * If ``start_block_id`` is specified but ``end_block_id`` is omitted, the repo starts deleting segments starting from ``/name/start_block_id``, and increments segment number after each packet. 23 | When a name query does not find an existing segment, the deletion process stops and is considered successful. 24 | * Otherwise, the repo fetches all segments between ``/name/start_block_id`` and ``/name/end_block_id``. 25 | If ``start_block_id`` is omitted, it defaults to 0. 26 | The deletion process succeeds when all packets are deleted. 27 | * Segment numbers are encoded in accordance with `NDN naming conventions rev2 `_. 28 | 29 | 30 | .. warning:: 31 | Please use exactly the same parameters as you inserted the Data to delete them. 32 | The current maintainer is not sure whether there will be problems if you provide 33 | a wrong ``register_prefix`` or only delete partial of the segments (i.e. provide different block ids). 34 | Also, using single packet deletion command to delete a segment Data object or vice versa will 35 | always fail, with ``delete_num`` being 0. 36 | 37 | 38 | Delete status check 39 | ------------------- 40 | 41 | The client can use the :doc:`check` protocol to check the progress of an deletion process. 42 | The deletion check response message payload is ``RepoCommandRes`` containing zero or more 43 | ``ObjStatus`` with the following fields: 44 | 45 | * ``status_code``: status code, as defined on :doc:`check`. 46 | Both the command itself and objects has a status code. 47 | * ``name``: the name of object to delete. 48 | * ``delete_num``: number of Data packets deleted by the repo so far. 49 | * The number of ``ObjStatus`` in the result should be either: 50 | * =0, which means the command is malformed or not allowed. 51 | * equals to the number of ``ObjParam`` in the deletion command. 52 | -------------------------------------------------------------------------------- /docs/src/specification/encoding.rst: -------------------------------------------------------------------------------- 1 | Encoding 2 | ======== 3 | 4 | Most repo commands and status reports are Data packets whose Content contains 5 | ``RepoCommandParam`` or ``RepoCommandRes`` structure. 6 | These Data are issued via Pub-Sub protocol. 7 | Each ``RepoCommandParam`` and ``RepoCommandRes`` contains 8 | multiple ``ObjParam`` and ``ObjStatus``, resp. 9 | 10 | Current protocol does not support compatibility among different versions. All TLV-TYPE numbers are critical. 11 | 12 | These structures are defined as follows: 13 | 14 | .. code-block:: abnf 15 | 16 | ObjParam = 17 | Name 18 | [ForwardingHint] 19 | [StartBlockId] 20 | [EndBlockId] 21 | [RegisterPrefix] 22 | 23 | SyncParam = 24 | SyncPrefix 25 | [RegisterPrefix] 26 | [DataNameDedupe] 27 | [Reset] 28 | 29 | ObjStatus = 30 | Name 31 | StatusCode 32 | [InsertNum] 33 | [DeleteNum] 34 | 35 | SyncStatus = 36 | Name 37 | StatusCode 38 | 39 | RepoCommandParam = 40 | 0* (OBJECT-PARAM-TYPE TLV-LENGTH ObjParam) 41 | 0* (SYNC-PARAM-TYPE TLV-LENGTH SyncParam) 42 | 43 | RepoCommandRes = 44 | StatusCode 45 | 0* (OBJECT-RESULT-TYPE TLV-LENGTH ObjStatus) 46 | 0* (SYNC-RESULT-TYPE TLV-LENGTH SyncStatus) 47 | 48 | RepoStatQuery = 49 | RequestNo 50 | 51 | ForwardingHint = FORWARDING-HINT-TYPE TLV-LENGTH Name 52 | 53 | StartBlockId = START-BLOCK-ID-TYPE TLV-LENGTH NonNegativeInteger 54 | 55 | EndBlockId = END-BLOCK-ID-TYPE TLV-LENGTH NonNegativeInteger 56 | 57 | RegisterPrefix = REGISTER-PREFIX-TYPE TLV-LENGTH Name 58 | 59 | SyncPrefix = SYNC-PREFIX-TYPE TLV-LENGTH Name 60 | 61 | DataNameDedupe = SYNC-DATA-NAME-DEDUPE-TYPE TLV-LENGTH ; TLV-LENGTH = 0 62 | 63 | Reset = SYNC-RESET-TYPE TLV-LENGTH ; TLV-LENGTH = 0 64 | 65 | StatusCode = STATUS-CODE-TYPE TLV-LENGTH NonNegativeInteger 66 | 67 | InsertNum = INSERT-NUM-TYPE TLV-LENGTH NonNegativeInteger 68 | 69 | DeleteNum = DELETE-NUM-TYPE TLV-LENGTH NonNegativeInteger 70 | 71 | RequestNo = REQUEST-NO-TYPE TLV-LENGTH 1*OCTET 72 | 73 | The type number assignments are as follows: 74 | 75 | +----------------------------+----------------------------+--------------------------------+ 76 | | type | Assigned number (decimal) | Assigned number (hexadecimal) | 77 | +============================+============================+================================+ 78 | | START-BLOCK-ID-TYPE | 204 | 0xCC | 79 | +----------------------------+----------------------------+--------------------------------+ 80 | | END-BLOCK-ID-TYPE | 205 | 0xCD | 81 | +----------------------------+----------------------------+--------------------------------+ 82 | | REQUEST-NO-TYPE | 206 | 0xCE | 83 | +----------------------------+----------------------------+--------------------------------+ 84 | | STATUS-CODE-TYPE | 208 | 0xD0 | 85 | +----------------------------+----------------------------+--------------------------------+ 86 | | INSERT-NUM-TYPE | 209 | 0xD1 | 87 | +----------------------------+----------------------------+--------------------------------+ 88 | | DELETE-NUM-TYPE | 210 | 0xD2 | 89 | +----------------------------+----------------------------+--------------------------------+ 90 | | FORWARDING-HINT-TYPE | 211 | 0xD3 | 91 | +----------------------------+----------------------------+--------------------------------+ 92 | | REGISTER-PREFIX-TYPE | 212 | 0xD4 | 93 | +----------------------------+----------------------------+--------------------------------+ 94 | | OBJECT-PARAM-TYPE | 301 | 0x12D | 95 | +----------------------------+----------------------------+--------------------------------+ 96 | | OBJECT-RESULT-TYPE | 302 | 0x12E | 97 | +----------------------------+----------------------------+--------------------------------+ 98 | | SYNC-PARAM-TYPE | 401 | 0x191 | 99 | +----------------------------+----------------------------+--------------------------------+ 100 | | SYNC-RESULT-TYPE | 402 | 0x192 | 101 | +----------------------------+----------------------------+--------------------------------+ 102 | | SYNC-DATA-NAME-DEDUPE-TYPE | 403 | 0x193 | 103 | +----------------------------+----------------------------+--------------------------------+ 104 | | SYNC-RESET-TYPE | 404 | 0x194 | 105 | +----------------------------+----------------------------+--------------------------------+ 106 | | SYNC-PREFIX-TYPE | 405 | 0x195 | 107 | +----------------------------+----------------------------+--------------------------------+ 108 | 109 | 110 | Status Code Definition 111 | ---------------------- 112 | 113 | The status codes are defined as follows: 114 | 115 | +---------------+-------+-----------------------------------------------+ 116 | | Code name | Value | Explanation | 117 | +===============+=======+===============================================+ 118 | | ROGER | 100 | Command received but not been executed yet | 119 | +---------------+-------+-----------------------------------------------+ 120 | | COMPLETED | 200 | Command completed | 121 | +---------------+-------+-----------------------------------------------+ 122 | | IN-PROGRESS | 300 | Command working in progress | 123 | +---------------+-------+-----------------------------------------------+ 124 | | FAILED | 400 | Command or parts of it failed | 125 | +---------------+-------+-----------------------------------------------+ 126 | | MALFORMED | 403 | Command is malformed | 127 | +---------------+-------+-----------------------------------------------+ 128 | | NOT-FOUND | 404 | Queried command not found | 129 | +---------------+-------+-----------------------------------------------+ 130 | -------------------------------------------------------------------------------- /docs/src/specification/insert.rst: -------------------------------------------------------------------------------- 1 | .. _specification-insert-label: 2 | 3 | Insert 4 | ====== 5 | 6 | Repo insertion process makes use of the :doc:`../misc_pkgs/pub_sub`. 7 | 8 | 1. The repo subscribes to the topic ``//insert``. 9 | 10 | 2. The client publishes a message to the topic ``//insert``. 11 | The message payload is ``RepoCommandParam`` containing one or more ``ObjParam`` with the following fields: 12 | 13 | * ``name``: either a Data packet name, or a name prefix of segmented Data packets. 14 | * ``start_block_id`` (Optional): inclusive start segment number. 15 | * ``end_block_id`` (Optional): inclusive end segment number. 16 | * ``forwarding_hint`` (Optional): forwarding hint for Data fetching. 17 | This is useful in two scenarios: 18 | 19 | * The producer choose not to announce its name prefix, but only allow the repo to reach it via forwarding hint. 20 | * The name prefix is already announced by repo node(s), but the producer in another node wants to insert to the repo. 21 | 22 | * ``register_prefix`` (Optional): if repo doesn't register the root prefix (:doc:`../configuration` ``register_root`` is disabled), client can tell repo to register this prefix. 23 | 24 | 3. The repo fetches and inserts single or segmented Data packets according to given parameters. 25 | 26 | * If neither ``start_block_id`` nor ``end_block_id`` are given, the repo fetches a single packet identified in ``name`` parameter. 27 | The insertion process succeeds when this packet is received. 28 | * If only ``end_block_id`` is given, ``start_block_id`` is considered 0. 29 | * If only ``start_block_id`` is given, ``end_block_id`` is auto detected, i.e. infinity. 30 | * If both block ids are given, the command is considered as correct only if ``end_block_id >= start_block_id``. 31 | * Whenever the repo cannot fetch a segment, it will stop, no matter what ``end_block_id`` is. 32 | * Segment numbers are encoded in accordance with `NDN naming conventions rev2 `_. 33 | 34 | 35 | Insert status check 36 | ------------------- 37 | 38 | The client can use the :doc:`check` protocol to check the progress of an insertion process. 39 | The insertion check response message payload is ``RepoCommandRes`` containing zero or more 40 | ``ObjStatus`` with the following fields: 41 | 42 | * ``status_code``: status code, as defined on :doc:`check`. 43 | Both the command itself and objects has a status code. 44 | * ``name``: the name of object to insert. 45 | * ``insert_num``: number of Data packets received by the repo so far. 46 | * The number of ``ObjStatus`` in the result should be either: 47 | * =0, which means the command is malformed or not allowed. 48 | * equals to the number of ``ObjParam`` in the insertion command. 49 | -------------------------------------------------------------------------------- /docs/src/specification/security.rst: -------------------------------------------------------------------------------- 1 | Overall 2 | ======= 3 | 4 | The repo is considered as part of the NDN network infrastructure. 5 | Therefore, when deployed for production, 6 | the security requirements are supposed to be specified by the authority, 7 | such as the network operator or the project manager. 8 | 9 | For development deployment or internal use, the settings described in this section are recommended. 10 | 11 | Required Settings 12 | ----------------- 13 | 14 | - All Interests with application parameters are required to be signed. Otherwise the Repo must drop the Interest. 15 | Currently including the check Interest ``// check`` and the publication notification Interest 16 | ``"//notify"``. 17 | - Check Interests are required to have at least one of ``SignatureTime``, ``SignatureNonce``, or ``SignatureSeqNum``. 18 | Otherwise, Check Interests' result is undefined behavior. 19 | This is to make sure these check Interests are different to avoid cache invalidation. 20 | 21 | .. warning:: 22 | Unfortunately current implementation does not follow these requirements by default. 23 | This may cause some potential vulnerabilities. Will be fixed in future versions. 24 | 25 | 26 | Recommended Settings 27 | -------------------- 28 | 29 | - Packet signatures should be signed by asymmetric algorithms. Use ECDSA or Ed25519 when in doubt. 30 | - Signed Interests should use ``SignatureNonce``. 31 | 32 | - Since there is no replay attack, the Repo does not have to remember the list of ``SignatureNonce``. 33 | 34 | - If the Repo is provided as a network service, the certificate obtained from the network provider should be used. 35 | In this case, the prefix registration command and repo's publication data are signed using the same key. 36 | 37 | - For example, if one employs a Repo as a public service of NDN testbed, 38 | then both the client and the Repo server should use their testbed certificates. 39 | - The Repo should use the same verification method as its local NFD node to verify the register. 40 | - The client should use the same verification method as how it verifies the prefix registration command. 41 | 42 | - If the Repo is provided as an application service, it should either obtain an identity from the application namespace, 43 | or runs on its own trust domain and holds the trust anchor. 44 | In this case, the prefix registration command and repo's publication data are signed using different keys. 45 | 46 | - In this case, the Repo is supposed obtain the trust schema when it is bootstrapped into the application's namespace. 47 | If it is unable to obtain the trust schema, it should maintain a user list to verify the clients. 48 | The client should do similar things. 49 | -------------------------------------------------------------------------------- /docs/src/specification/specification.rst: -------------------------------------------------------------------------------- 1 | Specification 2 | ============= 3 | 4 | .. toctree:: 5 | 6 | Encoding 7 | Insert 8 | Delete 9 | Check 10 | TCP bulk insert 11 | Sync Group Join 12 | Sync Group Leave 13 | -------------------------------------------------------------------------------- /docs/src/specification/sync.rst: -------------------------------------------------------------------------------- 1 | Joining SVS Sync (Draft) 2 | ======================== 3 | 4 | - Joining SVS Sync group is an optional feature. It should only be enabled when it is deployed as a network service. 5 | 6 | - If the Repo is deployed as an application service, 7 | the application deployer should run another process on the same node as an SVS peer and use the Repo only for Data. 8 | 9 | - The Repo should not join an application's SVS sync group as a producer. 10 | (unless the sync group is specifically designed for Repos to backup data) 11 | 12 | - The Repo should learn how to verify the target SVS group's Sync Interest. 13 | 14 | - The Repo should store its received latest SVS notification Interest as is, 15 | and responds with this Interest when it hears some out-of-dated SVS vector. 16 | 17 | - If there are multiple latest SVS state vectors, e.g. ``[A:1, B:2]`` and ``[A:2, B:1]``, 18 | the Repo will not be able to merge them into ``[A:2, B:2]``. 19 | Instead, it should respond with both stored Interests eventually. 20 | Maybe all at once, maybe one at one time. Not decided yet. 21 | -------------------------------------------------------------------------------- /docs/src/specification/sync_join.rst: -------------------------------------------------------------------------------- 1 | Sync Join 2 | ========= 3 | 4 | The sync join protocol is used to command the repo to join a state vector sync group. 5 | 6 | 1. The repo subscribes to the topic ``//sync/join``. 7 | 8 | 2. The client publishes a message to the topic ``//sync/join``. The message payload is of type 9 | ``RepoCommandParam``, containing one or more ``SyncParam`` with the following fields: 10 | 11 | * ``sync_prefix``: The name prefix of the sync group to join. 12 | * ``register_prefix``: (Optional) The name prefix for the repo to register with the forwarder. This prefix must not 13 | be the same as ``sync_prefix``. 14 | * ``data_name_dedupe``: (Optional) If true, the repo will deduplicate data names in the sync group. 15 | * ``reset``: (Optional) If true, rebuild state vectors from the stored state vectors on the repo disk. This is useful 16 | if interests are sent for permanently unavailable data from an old vector. 17 | 18 | 3. The repo joins the sync group, saving sync information to disk. 19 | 20 | * The repo will listen for state vector interests for the sync group. Then, to fetch any data, the repo will send 21 | interests following the SVSPS data naming convention. For more information, see the 22 | `specification page `_ for State Vector 23 | Synchronization. -------------------------------------------------------------------------------- /docs/src/specification/sync_leave.rst: -------------------------------------------------------------------------------- 1 | Sync Leave 2 | ========== 3 | 4 | The sync leave protocol is used to command the repo to leave the sync group. This command also removes any information 5 | about the sync group from repo storage. 6 | 7 | 1. The repo subscribes to the topic ``//sync/leave``. 8 | 9 | 2. The client publishes a message to the topic ``//sync/leave``. The message payload is of type 10 | ``RepoCommandParam``, containing one or more ``SyncParam`` with the following field: 11 | 12 | * ``sync_prefix``: The name prefix of the sync group to leave. 13 | 14 | 3. The repo leaves the sync group, removing sync information from disk. The repo no longer listens to the originally 15 | specified register prefix. 16 | 17 | * Note that any already-stored data packets that were received prior to leaving the sync group are *not* deleted. -------------------------------------------------------------------------------- /docs/src/specification/tcp_bulk.rst: -------------------------------------------------------------------------------- 1 | TCP bulk insert 2 | =============== -------------------------------------------------------------------------------- /examples/command_checker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | NDN Repo command checker example. 4 | 5 | @Author jonnykong@cs.ucla.edu 6 | """ 7 | 8 | import argparse 9 | import logging 10 | from ndn.app import NDNApp 11 | from ndn.encoding import Name 12 | from ndn_python_repo.clients import CommandChecker 13 | 14 | 15 | async def run_check(app: NDNApp, **kwargs): 16 | """ 17 | Async helper function to run the CommandChecker. 18 | This function is necessary because it's responsible for calling app.shutdown(). 19 | """ 20 | client = CommandChecker(app) 21 | response = await client.check_insert(kwargs['repo_name'], kwargs['process_id']) 22 | if response: 23 | status_code = response.status_code 24 | print('Status Code: {}'.format(status_code)) 25 | app.shutdown() 26 | 27 | 28 | def main(): 29 | parser = argparse.ArgumentParser(description='segmented insert client') 30 | parser.add_argument('-r', '--repo_name', 31 | required=True, help='Name of repo') 32 | parser.add_argument('-p', '--process_id', 33 | required=True, help='Process ID') 34 | args = parser.parse_args() 35 | 36 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 37 | datefmt='%Y-%m-%d %H:%M:%S', 38 | level=logging.INFO) 39 | 40 | app = NDNApp() 41 | try: 42 | app.run_forever( 43 | after_start=run_check(app, 44 | repo_name=Name.from_str(args.repo_name), 45 | process_id=int(args.process_id))) 46 | except FileNotFoundError: 47 | print('Error: could not connect to NFD.') 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /examples/delfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | NDN Repo delfile example. 4 | 5 | @Author jonnykong@cs.ucla.edu 6 | """ 7 | 8 | import argparse 9 | import logging 10 | from ndn.app import NDNApp 11 | from ndn.encoding import Name 12 | from ndn_python_repo.clients import DeleteClient 13 | 14 | 15 | async def run_delete_client(app: NDNApp, **kwargs): 16 | """ 17 | Async helper function to run the DeleteClient. 18 | This function is necessary because it's responsible for calling app.shutdown(). 19 | """ 20 | client = DeleteClient(app=app, 21 | prefix=kwargs['client_prefix'], 22 | repo_name=kwargs['repo_name']) 23 | 24 | # Set pubsub to register ``check_prefix`` directly, so all prefixes under ``check_prefix`` will 25 | # be handled with interest filters. This reduces the number of registered prefixes at NFD, when 26 | # inserting multiple files with one client 27 | check_prefix = kwargs['client_prefix'] 28 | client.pb.set_base_prefix(check_prefix) 29 | 30 | await client.delete_file(prefix=kwargs['name_at_repo'], 31 | start_block_id=kwargs['start_block_id'], 32 | end_block_id=kwargs['end_block_id'], 33 | register_prefix=kwargs['register_prefix'], 34 | check_prefix=check_prefix) 35 | app.shutdown() 36 | 37 | 38 | def main(): 39 | parser = argparse.ArgumentParser(description='delfile') 40 | parser.add_argument('-r', '--repo_name', 41 | required=True, help='Name of repo') 42 | parser.add_argument('-n', '--name_at_repo', 43 | required=True, help='Name used to store file at Repo') 44 | parser.add_argument('-s', '--start_block_id', 45 | required=False, help='Start Block ID') 46 | parser.add_argument('-e', '--end_block_id', 47 | required=False, help='End Block ID') 48 | parser.add_argument('--client_prefix', 49 | required=False, default='/delfile_client', 50 | help='prefix of this client') 51 | parser.add_argument('--register_prefix', default=None, 52 | help='The prefix repo should register') 53 | args = parser.parse_args() 54 | 55 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 56 | datefmt='%Y-%m-%d %H:%M:%S', 57 | level=logging.INFO) 58 | 59 | # process default values 60 | start_block_id = int(args.start_block_id) if args.start_block_id else None 61 | end_block_id = int(args.end_block_id) if args.end_block_id else None 62 | if args.register_prefix is None: 63 | args.register_prefix = args.name_at_repo 64 | args.register_prefix = Name.from_str(args.register_prefix) 65 | 66 | app = NDNApp() 67 | 68 | try: 69 | app.run_forever( 70 | after_start=run_delete_client(app, 71 | repo_name=Name.from_str(args.repo_name), 72 | name_at_repo=Name.from_str(args.name_at_repo), 73 | start_block_id=start_block_id, 74 | end_block_id=end_block_id, 75 | client_prefix=Name.from_str(args.client_prefix), 76 | register_prefix=args.register_prefix)) 77 | except FileNotFoundError: 78 | print('Error: could not connect to NFD.') 79 | 80 | 81 | if __name__ == '__main__': 82 | main() 83 | -------------------------------------------------------------------------------- /examples/getfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | NDN Repo getfile example. 4 | 5 | @Author jonnykong@cs.ucla.edu 6 | """ 7 | 8 | import argparse 9 | import logging 10 | from ndn.app import NDNApp 11 | from ndn.encoding import Name 12 | from ndn_python_repo.clients import GetfileClient 13 | 14 | 15 | async def run_getfile_client(app: NDNApp, **kwargs): 16 | """ 17 | Async helper function to run the GetfileClient. 18 | This function is necessary because it's responsible for calling app.shutdown(). 19 | """ 20 | client = GetfileClient(app, kwargs['repo_name']) 21 | await client.fetch_file(kwargs['name_at_repo']) 22 | app.shutdown() 23 | 24 | 25 | def main(): 26 | parser = argparse.ArgumentParser(description='getfile') 27 | parser.add_argument('-r', '--repo_name', 28 | required=True, help='Name of repo') 29 | parser.add_argument('-n', '--name_at_repo', 30 | required=True, help='Name used to store file at Repo') 31 | args = parser.parse_args() 32 | 33 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 34 | datefmt='%Y-%m-%d %H:%M:%S', 35 | level=logging.INFO) 36 | 37 | app = NDNApp() 38 | try: 39 | app.run_forever( 40 | after_start=run_getfile_client(app, 41 | repo_name=Name.from_str(args.repo_name), 42 | name_at_repo=Name.from_str(args.name_at_repo))) 43 | except FileNotFoundError: 44 | print('Error: could not connect to NFD.') 45 | 46 | 47 | 48 | if __name__ == '__main__': 49 | main() 50 | -------------------------------------------------------------------------------- /examples/leavesync.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | from ndn.app import NDNApp 4 | from ndn.encoding import Name 5 | from ndn.security import KeychainDigest 6 | from ndn_python_repo.clients import SyncClient 7 | 8 | async def run_leave_sync_client(app: NDNApp, **kwargs): 9 | client = SyncClient(app=app, prefix=kwargs['client_prefix'], repo_name=kwargs['repo_name']) 10 | await client.leave_sync(sync_prefix=kwargs['sync_prefix']) 11 | app.shutdown() 12 | 13 | 14 | def main(): 15 | parser = argparse.ArgumentParser(description='leavesync') 16 | parser.add_argument('-r', '--repo_name', 17 | required=True, help='Name of repo') 18 | parser.add_argument('--client_prefix', required=True, 19 | help='prefix of this client') 20 | parser.add_argument('--sync_prefix', required=True, 21 | help='The sync prefix repo should leave') 22 | args = parser.parse_args() 23 | 24 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 25 | datefmt='%Y-%m-%d %H:%M:%S', 26 | level=logging.DEBUG) 27 | 28 | app = NDNApp(face=None, keychain=KeychainDigest()) 29 | try: 30 | app.run_forever( 31 | after_start=run_leave_sync_client(app, repo_name=Name.from_str(args.repo_name), 32 | client_prefix=Name.from_str(args.client_prefix), 33 | sync_prefix=Name.from_str(args.sync_prefix))) 34 | except FileNotFoundError: 35 | print('Error: could not connect to NFD.') 36 | 37 | 38 | if __name__ == '__main__': 39 | main() 40 | -------------------------------------------------------------------------------- /examples/publisher.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Example publisher for `util/pubsub.py`. 4 | 5 | @Author jonnykong@cs.ucla.edu 6 | @Date 2020-05-10 7 | """ 8 | 9 | import asyncio as aio 10 | import datetime 11 | import logging 12 | from ndn.app import NDNApp 13 | from ndn.encoding import Name, NonStrictName 14 | from ndn_python_repo.utils import PubSub 15 | 16 | 17 | async def run_publisher(app: NDNApp, publisher_prefix: NonStrictName): 18 | pb = PubSub(app, publisher_prefix) 19 | await pb.wait_for_ready() 20 | 21 | topic = Name.from_str('/topic_foo') 22 | msg = f'pubsub message generated at {str(datetime.datetime.now())}'.encode() 23 | await pb.publish(topic, msg) 24 | 25 | # wait for msg to be fetched by subscriber 26 | await aio.sleep(10) 27 | app.shutdown() 28 | 29 | 30 | def main(): 31 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 32 | datefmt='%Y-%m-%d %H:%M:%S', 33 | level=logging.INFO) 34 | 35 | publisher_prefix = Name.from_str('/test_publisher') 36 | app = NDNApp() 37 | 38 | try: 39 | app.run_forever( 40 | after_start=run_publisher(app, publisher_prefix)) 41 | except FileNotFoundError: 42 | logging.warning('Error: could not connect to NFD') 43 | 44 | 45 | if __name__ == '__main__': 46 | main() 47 | -------------------------------------------------------------------------------- /examples/putfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | NDN Repo putfile example. 4 | 5 | @Author jonnykong@cs.ucla.edu 6 | """ 7 | 8 | import argparse 9 | import logging 10 | import multiprocessing 11 | from ndn.app import NDNApp 12 | from ndn.encoding import Name 13 | from ndn.security import KeychainDigest 14 | from ndn_python_repo.clients import PutfileClient 15 | import uuid 16 | 17 | 18 | async def run_putfile_client(app: NDNApp, **kwargs): 19 | """ 20 | Async helper function to run the PutfileClient. 21 | This function is necessary because it's responsible for calling app.shutdown(). 22 | """ 23 | client = PutfileClient(app=app, 24 | prefix=kwargs['client_prefix'], 25 | repo_name=kwargs['repo_name']) 26 | 27 | # Set pubsub to register ``check_prefix`` directly, so all prefixes under ``check_prefix`` will 28 | # be handled with interest filters. This reduces the number of registered prefixes at NFD, when 29 | # inserting multiple files with one client 30 | check_prefix = kwargs['client_prefix'] 31 | 32 | await client.insert_file(file_path=kwargs['file_path'], 33 | name_at_repo=kwargs['name_at_repo'], 34 | segment_size=kwargs['segment_size'], 35 | freshness_period=kwargs['freshness_period'], 36 | cpu_count=kwargs['cpu_count'], 37 | forwarding_hint=kwargs['forwarding_hint'], 38 | register_prefix=kwargs['register_prefix'], 39 | check_prefix=check_prefix) 40 | app.shutdown() 41 | 42 | 43 | def main(): 44 | parser = argparse.ArgumentParser(description='putfile') 45 | parser.add_argument('-r', '--repo_name', 46 | required=True, help='Name of repo') 47 | parser.add_argument('-f', '--file_path', 48 | required=True, help='Path to input file') 49 | parser.add_argument('-n', '--name_at_repo', 50 | required=True, help='Prefix used to store file at Repo') 51 | parser.add_argument('--client_prefix', 52 | required=False, default='/putfile_client' + uuid.uuid4().hex.upper()[0:6], 53 | help='prefix of this client') 54 | parser.add_argument('--segment_size', type=int, 55 | required=False, default=8000, 56 | help='Size of each data packet') 57 | parser.add_argument('--freshness_period', type=int, 58 | required=False, default=0, 59 | help='Data packet\'s freshness period') 60 | parser.add_argument('--cpu_count', type=int, 61 | required=False, default=multiprocessing.cpu_count(), 62 | help='Number of cores to use') 63 | parser.add_argument('--forwarding_hint', default=None, 64 | help='Forwarding hint used by the repo when fetching data') 65 | parser.add_argument('--register_prefix', default=None, 66 | help='The prefix repo should register') 67 | args = parser.parse_args() 68 | 69 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 70 | datefmt='%Y-%m-%d %H:%M:%S', 71 | level=logging.INFO) 72 | 73 | # ``register_prefix`` is by default identical to ``name_at_repo`` 74 | if args.register_prefix is None: 75 | args.register_prefix = args.name_at_repo 76 | args.register_prefix = Name.from_str(args.register_prefix) 77 | if args.forwarding_hint: 78 | args.forwarding_hint = Name.from_str(args.forwarding_hint) 79 | 80 | app = NDNApp(face=None, keychain=KeychainDigest()) 81 | try: 82 | app.run_forever( 83 | after_start=run_putfile_client(app, 84 | repo_name=Name.from_str(args.repo_name), 85 | file_path=args.file_path, 86 | name_at_repo=Name.from_str(args.name_at_repo), 87 | client_prefix=Name.from_str(args.client_prefix), 88 | segment_size=args.segment_size, 89 | freshness_period=args.freshness_period, 90 | cpu_count=args.cpu_count, 91 | forwarding_hint=args.forwarding_hint, 92 | register_prefix=args.register_prefix)) 93 | except FileNotFoundError: 94 | print('Error: could not connect to NFD.') 95 | 96 | 97 | if __name__ == '__main__': 98 | main() 99 | -------------------------------------------------------------------------------- /examples/subscriber.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Example subscriber for `util/pubsub.py`. 4 | 5 | @Author jonnykong@cs.ucla.edu 6 | @Date 2020-05-10 7 | """ 8 | 9 | import logging 10 | from ndn.app import NDNApp 11 | from ndn.encoding import Name, NonStrictName 12 | from ndn_python_repo.utils import PubSub 13 | 14 | 15 | async def run_subscriber(app: NDNApp, subscriber_prefix: NonStrictName): 16 | pb = PubSub(app, subscriber_prefix) 17 | await pb.wait_for_ready() 18 | 19 | topic = Name.from_str('/topic_foo') 20 | pb.subscribe(topic, foo_cb) 21 | 22 | 23 | def foo_cb(msg: bytes): 24 | print(f'topic /topic_foo received msg: {msg.decode()}') 25 | 26 | 27 | def main(): 28 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 29 | datefmt='%Y-%m-%d %H:%M:%S', 30 | level=logging.INFO) 31 | 32 | subscriber_prefix = Name.from_str('/test_subscriber') 33 | app = NDNApp() 34 | 35 | try: 36 | app.run_forever( 37 | after_start=run_subscriber(app, subscriber_prefix)) 38 | except FileNotFoundError: 39 | logging.warning('Error: could not connect to NFD') 40 | 41 | 42 | if __name__ == '__main__': 43 | main() -------------------------------------------------------------------------------- /examples/sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import logging 4 | from ndn.app import NDNApp 5 | from ndn.encoding import Name 6 | from ndn.security import KeychainDigest 7 | from ndn_python_repo.clients import SyncClient 8 | 9 | 10 | async def run_sync_client(app: NDNApp, **kwargs): 11 | client = SyncClient(app=app, prefix=kwargs['client_prefix'], repo_name=kwargs['repo_name']) 12 | await client.join_sync(sync_prefix=kwargs['sync_prefix'], register_prefix=kwargs['register_prefix'], 13 | data_name_dedupe=kwargs['data_name_dedupe'], 14 | reset=kwargs['reset']) 15 | app.shutdown() 16 | 17 | 18 | def main(): 19 | parser = argparse.ArgumentParser(description='sync') 20 | parser.add_argument('-r', '--repo_name', 21 | required=True, help='Name of repo') 22 | parser.add_argument('--client_prefix', required=True, 23 | help='prefix of this client') 24 | parser.add_argument('--sync_prefix', required=True, 25 | help='The sync prefix repo should join') 26 | parser.add_argument('--register_prefix', required=False, 27 | help='The prefix repo should register') 28 | parser.add_argument('--data_name_dedupe', required=False, default=False, 29 | help='whether repo should dedupe the sync group in data naming') 30 | parser.add_argument('--reset', required=False, default=False, 31 | help='whether repo should reset the sync group') 32 | args = parser.parse_args() 33 | 34 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 35 | datefmt='%Y-%m-%d %H:%M:%S', 36 | level=logging.DEBUG) 37 | 38 | app = NDNApp(face=None, keychain=KeychainDigest()) 39 | register_prefix = None 40 | if args.register_prefix: 41 | register_prefix = Name.from_str(args.register_prefix) 42 | try: 43 | app.run_forever( 44 | after_start=run_sync_client(app, repo_name=Name.from_str(args.repo_name), 45 | client_prefix=Name.from_str(args.client_prefix), 46 | sync_prefix=Name.from_str(args.sync_prefix), 47 | register_prefix=register_prefix, 48 | data_name_dedupe=args.data_name_dedupe, 49 | reset=args.reset)) 50 | except FileNotFoundError: 51 | print('Error: could not connect to NFD.') 52 | 53 | 54 | if __name__ == '__main__': 55 | main() 56 | -------------------------------------------------------------------------------- /ndn_python_repo/__init__.py: -------------------------------------------------------------------------------- 1 | from .handle import * 2 | from .storage import * 3 | from .repo import Repo 4 | from .config import get_yaml 5 | from .utils import * 6 | -------------------------------------------------------------------------------- /ndn_python_repo/clients/__init__.py: -------------------------------------------------------------------------------- 1 | from .getfile import GetfileClient 2 | from .putfile import PutfileClient 3 | from .delete import DeleteClient 4 | from .sync import SyncClient 5 | from .command_checker import CommandChecker -------------------------------------------------------------------------------- /ndn_python_repo/clients/command_checker.py: -------------------------------------------------------------------------------- 1 | # ----------------------------------------------------------------------------- 2 | # NDN Repo insert check tester. 3 | # 4 | # @Author jonnykong@cs.ucla.edu 5 | # @Date 2019-09-23 6 | # ----------------------------------------------------------------------------- 7 | 8 | import os 9 | import sys 10 | sys.path.insert(1, os.path.join(sys.path[0], '..')) 11 | 12 | import logging 13 | from typing import Optional 14 | from ndn.app import NDNApp 15 | from ndn.encoding import Name, NonStrictName, DecodeError 16 | from ndn.types import InterestNack, InterestTimeout 17 | from ..command.repo_commands import RepoStatQuery, RepoCommandRes 18 | 19 | 20 | class CommandChecker(object): 21 | def __init__(self, app: NDNApp): 22 | """ 23 | This client sends check interests to the repo. 24 | 25 | :param app: NDNApp. 26 | """ 27 | self.app = app 28 | self.logger = logging.getLogger(__name__) 29 | 30 | async def check_insert(self, repo_name: NonStrictName, request_no: bytes) -> RepoCommandRes: 31 | """ 32 | Check the status of an insert process. 33 | 34 | :param repo_name: NonStrictName. The name of the remote repo. 35 | :param request_no: bytes. The request id of the process to check. 36 | :return: The response from the repo. 37 | """ 38 | return await self._check('insert', repo_name, request_no) 39 | 40 | async def check_delete(self, repo_name, request_no: bytes) -> RepoCommandRes: 41 | """ 42 | Check the status of a delete process. 43 | 44 | :param repo_name: NonStrictName. The name of the remote repo. 45 | :param request_no: bytes. The request id of the process to check. 46 | :return: The response from the repo. 47 | """ 48 | return await self._check('delete', repo_name, request_no) 49 | 50 | async def _check(self, method: str, repo_name: NonStrictName, 51 | request_no: bytes) -> Optional[RepoCommandRes]: 52 | """ 53 | Return parsed insert check response message. 54 | 55 | :param method: str. One of `insert` or `delete`. 56 | :param repo_name: NonStrictName. The name of the remote repo. 57 | :param request_no: bytes. The request id of the process to check. 58 | """ 59 | cmd_param = RepoStatQuery() 60 | cmd_param.request_no = request_no 61 | cmd_param_bytes = cmd_param.encode() 62 | 63 | name = Name.normalize(repo_name) 64 | name += Name.from_str(method + ' check') 65 | 66 | try: 67 | self.logger.info(f'Expressing interest: {Name.to_str(name)}') 68 | data_name, meta_info, content = await self.app.express_interest( 69 | name, cmd_param_bytes, must_be_fresh=True, can_be_prefix=False, lifetime=1000) 70 | self.logger.info(f'Received data name: {Name.to_str(data_name)}') 71 | except InterestNack as e: 72 | self.logger.info(f'Nacked with reason={e.reason}') 73 | return None 74 | except InterestTimeout: 75 | self.logger.info(f'Timeout: {Name.to_str(name)}') 76 | return None 77 | 78 | try: 79 | cmd_response = RepoCommandRes.parse(content) 80 | return cmd_response 81 | except DecodeError as exc: 82 | self.logger.warning(f'Response blob decoding failed for {exc}') 83 | return None 84 | except Exception as e: 85 | self.logger.warning(e) 86 | return None 87 | -------------------------------------------------------------------------------- /ndn_python_repo/clients/delete.py: -------------------------------------------------------------------------------- 1 | # ----------------------------------------------------------------------------- 2 | # NDN Repo delete client. 3 | # 4 | # @Author jonnykong@cs.ucla.edu 5 | # @Date 2019-09-26 6 | # ----------------------------------------------------------------------------- 7 | 8 | import os 9 | import sys 10 | sys.path.insert(1, os.path.join(sys.path[0], '..')) 11 | 12 | import asyncio as aio 13 | from ..command import RepoCommandParam, ObjParam, EmbName, RepoStatCode 14 | from .command_checker import CommandChecker 15 | from ..utils import PubSub 16 | import logging 17 | from ndn.app import NDNApp 18 | from ndn.encoding import Name, NonStrictName 19 | from typing import Optional 20 | from hashlib import sha256 21 | 22 | 23 | class DeleteClient(object): 24 | def __init__(self, app: NDNApp, prefix: NonStrictName, repo_name: NonStrictName): 25 | """ 26 | This client deletes data packets from the remote repo. 27 | 28 | :param app: NDNApp. 29 | :param repo_name: NonStrictName. Routable name to remote repo. 30 | """ 31 | self.app = app 32 | self.prefix = prefix 33 | self.repo_name = Name.normalize(repo_name) 34 | self.pb = PubSub(self.app, self.prefix) 35 | self.logger = logging.getLogger(__name__) 36 | 37 | async def delete_file(self, prefix: NonStrictName, start_block_id: int = 0, 38 | end_block_id: int = None, 39 | register_prefix: Optional[NonStrictName] = None, 40 | check_prefix: Optional[NonStrictName] = None) -> int: 41 | """ 42 | Delete from repo packets between "/" and\ 43 | "/" inclusively. 44 | 45 | :param prefix: NonStrictName. The name of the file stored in the remote repo. 46 | :param start_block_id: int. Default value is 0. 47 | :param end_block_id: int. If not specified, repo will attempt to delete all data packets\ 48 | with segment number starting from `start_block_id` continuously. 49 | :param register_prefix: If repo is configured with ``register_root=False``, it unregisters\ 50 | ``register_prefix`` after receiving the deletion command. 51 | :param check_prefix: NonStrictName. The repo will publish process check messages under\ 52 | ``/check``. It is necessary to specify this value in the param, instead\ 53 | of using a predefined prefix, to make sure the subscriber can register this prefix\ 54 | under the NDN prefix registration security model. If not specified, default value is\ 55 | the client prefix. 56 | :return: Number of deleted packets. 57 | """ 58 | # send command interest 59 | cmd_param = RepoCommandParam() 60 | cmd_obj = ObjParam() 61 | cmd_param.objs = [cmd_obj] 62 | cmd_obj.name = prefix 63 | cmd_obj.start_block_id = start_block_id 64 | cmd_obj.end_block_id = end_block_id 65 | cmd_obj.register_prefix = EmbName() 66 | cmd_obj.register_prefix.name = register_prefix 67 | 68 | cmd_param_bytes = bytes(cmd_param.encode()) 69 | request_no = sha256(cmd_param_bytes).digest() 70 | 71 | # publish msg to repo's delete topic 72 | await self.pb.wait_for_ready() 73 | is_success = await self.pb.publish(self.repo_name + Name.from_str('delete'), cmd_param_bytes) 74 | if is_success: 75 | self.logger.info('Published an delete msg and was acknowledged by a subscriber') 76 | else: 77 | self.logger.info('Published an delete msg but was not acknowledged by a subscriber') 78 | 79 | # wait until repo delete all data 80 | delete_num = 0 81 | if is_success: 82 | delete_num = await self._wait_for_finish(check_prefix, request_no) 83 | return delete_num 84 | 85 | async def _wait_for_finish(self, check_prefix: NonStrictName, request_no: bytes): 86 | """ 87 | Send delete check interest to wait until delete process completes 88 | 89 | :param check_prefix: NonStrictName. The prefix under which the check message will be\ 90 | published. 91 | :param request_no: int. The request number to check for delete process (formerly process id) 92 | :return: Number of deleted packets. 93 | """ 94 | # fixme: why is check_prefix never used? 95 | checker = CommandChecker(self.app) 96 | n_retries = 3 97 | while n_retries > 0: 98 | response = await checker.check_delete(self.repo_name, request_no) 99 | if response is None: 100 | self.logger.info(f'No response') 101 | await aio.sleep(1) 102 | # might receive 404 if repo has not yet processed delete command msg 103 | elif response.status_code == RepoStatCode.NOT_FOUND: 104 | n_retries -= 1 105 | # self.logger.info(f'Deletion {request_no} not handled yet') 106 | await aio.sleep(1) 107 | elif response.status_code == RepoStatCode.IN_PROGRESS: 108 | self.logger.info(f'Deletion {request_no} in progress') 109 | await aio.sleep(1) 110 | elif response.status_code == RepoStatCode.COMPLETED: 111 | delete_num = 0 112 | for obj in response.objs: 113 | delete_num += obj.delete_num 114 | self.logger.info(f'Deletion request {request_no} complete, delete_num: {delete_num}') 115 | return delete_num 116 | elif response.status_code == RepoStatCode.FAILED: 117 | self.logger.info(f'Deletion request {request_no} failed') 118 | else: 119 | # Shouldn't get here 120 | self.logger.error(f'Received unrecognized status code {response.status_code}') 121 | -------------------------------------------------------------------------------- /ndn_python_repo/clients/getfile.py: -------------------------------------------------------------------------------- 1 | # ----------------------------------------------------------------------------- 2 | # NDN Repo getfile client. 3 | # 4 | # @Author jonnykong@cs.ucla.edu 5 | # @Date 2019-10-24 6 | # ----------------------------------------------------------------------------- 7 | 8 | import os 9 | import sys 10 | sys.path.insert(1, os.path.join(sys.path[0], '..')) 11 | 12 | import asyncio as aio 13 | import logging 14 | from ndn.app import NDNApp 15 | from ndn.encoding import Name, NonStrictName 16 | from ..utils.concurrent_fetcher import concurrent_fetcher 17 | 18 | 19 | class GetfileClient(object): 20 | """ 21 | This client fetches a file from the repo, and save it to working directory. 22 | """ 23 | def __init__(self, app: NDNApp, repo_name): 24 | """ 25 | A client to retrieve files from the remote repo. 26 | 27 | :param app: NDNApp. 28 | :param repo_name: NonStrictName. Routable name to remote repo. 29 | """ 30 | self.app = app 31 | self.repo_name = repo_name 32 | self.logger = logging.getLogger(__name__) 33 | 34 | async def fetch_file(self, name_at_repo: NonStrictName, local_filename: str = None, overwrite=False): 35 | """ 36 | Fetch a file from remote repo, and write to the current working directory. 37 | 38 | :param name_at_repo: NonStrictName. The name with which this file is stored in the repo. 39 | :param local_filename: str. The filename of the retrieved file on the local file system. 40 | :param overwrite: If true, existing files are replaced. 41 | """ 42 | 43 | # If no local filename is provided, store file with last name component 44 | # of repo filename 45 | if local_filename is None: 46 | local_filename = Name.to_str(name_at_repo) 47 | local_filename = os.path.basename(local_filename) 48 | 49 | # If the file already exists locally and overwrite=False, retrieving the file makes no 50 | # sense. 51 | if os.path.isfile(local_filename) and not overwrite: 52 | raise FileExistsError("{} already exists".format(local_filename)) 53 | 54 | 55 | semaphore = aio.Semaphore(10) 56 | b_array = bytearray() 57 | async for (_, _, content, _) in concurrent_fetcher(self.app, name_at_repo, 0, None, semaphore): 58 | b_array.extend(content) 59 | 60 | if len(b_array) > 0: 61 | 62 | self.logger.info(f'Fetching completed, writing to file {local_filename}') 63 | 64 | # Create folder hierarchy 65 | local_folder = os.path.dirname(local_filename) 66 | if local_folder: 67 | os.makedirs(local_folder, exist_ok=True) 68 | 69 | # Write retrieved data to file 70 | if os.path.isfile(local_filename) and overwrite: 71 | os.remove(local_filename) 72 | with open(local_filename, 'wb') as f: 73 | f.write(b_array) 74 | -------------------------------------------------------------------------------- /ndn_python_repo/clients/putfile.py: -------------------------------------------------------------------------------- 1 | # ----------------------------------------------------------------------------- 2 | # NDN Repo putfile client. 3 | # 4 | # @Author jonnykong@cs.ucla.edu 5 | # susmit@cs.colostate.edu 6 | # @Date 2019-10-18 7 | # ----------------------------------------------------------------------------- 8 | 9 | import os 10 | import sys 11 | sys.path.insert(1, os.path.join(sys.path[0], '..')) 12 | 13 | import asyncio as aio 14 | from .command_checker import CommandChecker 15 | from ..command import RepoCommandParam, ObjParam, EmbName, RepoStatCode 16 | from ..utils import PubSub 17 | import logging 18 | import multiprocessing 19 | from ndn.app import NDNApp 20 | from ndn.encoding import Name, NonStrictName, Component, Links 21 | import os 22 | import platform 23 | from hashlib import sha256 24 | from typing import Optional 25 | 26 | 27 | if not os.environ.get('READTHEDOCS'): 28 | # I don't think global variable is good design 29 | app_to_create_packet = None # used for _create_packets only 30 | 31 | def _create_packets(name, content, freshness_period, final_block_id): 32 | """ 33 | Worker for parallelize prepare_data(). 34 | This function has to be defined at the top level, so that it can be pickled and used 35 | by multiprocessing. 36 | """ 37 | # The keychain's sqlite3 connection is not thread-safe. Create a new NDNApp instance for 38 | # each process, so that each process gets a separate sqlite3 connection 39 | global app_to_create_packet 40 | if app_to_create_packet is None: 41 | app_to_create_packet = NDNApp() 42 | 43 | packet = app_to_create_packet.prepare_data(name, content, 44 | freshness_period=freshness_period, 45 | final_block_id=final_block_id) 46 | return bytes(packet) 47 | 48 | 49 | class PutfileClient(object): 50 | 51 | def __init__(self, app: NDNApp, prefix: NonStrictName, repo_name: NonStrictName): 52 | """ 53 | A client to insert files into the repo. 54 | 55 | :param app: NDNApp. 56 | :param prefix: NonStrictName. The name of this client 57 | :param repo_name: NonStrictName. Routable name to remote repo. 58 | """ 59 | self.app = app 60 | self.prefix = prefix 61 | self.repo_name = Name.normalize(repo_name) 62 | self.encoded_packets = {} 63 | self.pb = PubSub(self.app, self.prefix) 64 | self.pb.base_prefix = self.prefix 65 | self.logger = logging.getLogger(__name__) 66 | 67 | # https://bugs.python.org/issue35219 68 | if platform.system() == 'Darwin': 69 | os.environ['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'YES' 70 | 71 | def _prepare_data(self, file_path: str, name_at_repo, segment_size: int, freshness_period: int, 72 | cpu_count: int): 73 | """ 74 | Shard file into data packets. 75 | 76 | :param file_path: Local FS path to file to insert 77 | :param name_at_repo: Name used to store file at repo 78 | """ 79 | if not os.path.exists(file_path): 80 | self.logger.error(f'file {file_path} does not exist') 81 | return 0 82 | with open(file_path, 'rb') as binary_file: 83 | b_array = bytearray(binary_file.read()) 84 | if len(b_array) == 0: 85 | self.logger.warning("File is empty") 86 | return 0 87 | 88 | # use multiple threads to speed up creating TLV 89 | seg_cnt = (len(b_array) + segment_size - 1) // segment_size 90 | final_block_id = Component.from_segment(seg_cnt - 1) 91 | packet_params = [[ 92 | name_at_repo + [Component.from_segment(seq)], 93 | b_array[seq * segment_size : (seq + 1) * segment_size], 94 | freshness_period, 95 | final_block_id, 96 | ] for seq in range(seg_cnt)] 97 | 98 | self.encoded_packets[Name.to_str(name_at_repo)] = [] 99 | 100 | with multiprocessing.Pool(processes=cpu_count) as p: 101 | self.encoded_packets[Name.to_str(name_at_repo)] = p.starmap(_create_packets, packet_params) 102 | self.logger.info("Prepared {} data for {}".format(seg_cnt, Name.to_str(name_at_repo))) 103 | 104 | def _on_interest(self, int_name, _int_param, _app_param): 105 | # use segment number to index into the encoded packets array 106 | self.logger.info(f'On interest: {Name.to_str(int_name)}') 107 | seq = Component.to_number(int_name[-1]) 108 | name_wo_seq = Name.to_str(int_name[:-1]) 109 | if name_wo_seq in self.encoded_packets and 0 <= seq < len(self.encoded_packets[name_wo_seq]): 110 | encoded_packets = self.encoded_packets[name_wo_seq] 111 | self.app.put_raw_packet(encoded_packets[seq]) 112 | self.logger.info(f'Serve data: {Name.to_str(int_name)}') 113 | else: 114 | self.logger.info(f'Data does not exist: {Name.to_str(int_name)}') 115 | 116 | async def insert_file(self, file_path: str, name_at_repo: NonStrictName, segment_size: int, 117 | freshness_period: int, cpu_count: int, 118 | forwarding_hint: Optional[NonStrictName]=None, 119 | register_prefix: Optional[NonStrictName]=None, 120 | check_prefix: Optional[NonStrictName]=None) -> int: 121 | """ 122 | Insert a file to remote repo. 123 | 124 | :param file_path: Local FS path to file to insert. 125 | :param name_at_repo: NonStrictName. Name used to store file at repo. 126 | :param segment_size: Max size of data packets. 127 | :param freshness_period: Freshness of data packets. 128 | :param cpu_count: Cores used for converting file to TLV format. 129 | :param forwarding_hint: NonStrictName. The forwarding hint the repo uses when fetching data. 130 | :param register_prefix: NonStrictName. If repo is configured with ``register_root=False``,\ 131 | it registers ``register_prefix`` after receiving the insertion command. 132 | :param check_prefix: NonStrictName. The repo will publish process check messages under\ 133 | ``/check``. It is necessary to specify this value in the param, instead\ 134 | of using a predefined prefix, to make sure the subscriber can register this prefix\ 135 | under the NDN prefix registration security model. If not specified, default value is\ 136 | the client prefix. 137 | :return: Number of packets inserted. 138 | """ 139 | self._prepare_data(file_path, name_at_repo, segment_size, freshness_period, cpu_count) 140 | num_packets = len(self.encoded_packets[Name.to_str(name_at_repo)]) 141 | if num_packets == 0: 142 | return 0 143 | 144 | # If the uploaded file has the client's name as prefix, set an interest filter 145 | # for handling corresponding Interests from the repo 146 | if Name.is_prefix(self.prefix, name_at_repo): 147 | self.app.set_interest_filter(name_at_repo, self._on_interest) 148 | else: 149 | # Otherwise, register the file name as prefix for responding interests from the repo 150 | self.logger.info(f'Register prefix for file upload: {Name.to_str(name_at_repo)}') 151 | await self.app.register(name_at_repo, self._on_interest) 152 | 153 | # construct insert cmd msg 154 | cmd_param = RepoCommandParam() 155 | cmd_obj = ObjParam() 156 | cmd_param.objs = [cmd_obj] 157 | cmd_obj.name = name_at_repo 158 | if forwarding_hint is not None: 159 | cmd_obj.forwarding_hint = Links() 160 | cmd_obj.forwarding_hint.names = [forwarding_hint] 161 | else: 162 | cmd_obj.forwarding_hint = None 163 | cmd_obj.start_block_id = 0 164 | cmd_obj.end_block_id = num_packets - 1 165 | cmd_obj.register_prefix = EmbName() 166 | cmd_obj.register_prefix.name = register_prefix 167 | 168 | cmd_param_bytes = bytes(cmd_param.encode()) 169 | request_no = sha256(cmd_param_bytes).digest() 170 | 171 | # publish msg to repo's insert topic 172 | await self.pb.wait_for_ready() 173 | is_success = await self.pb.publish(self.repo_name + Name.from_str('insert'), cmd_param_bytes) 174 | if is_success: 175 | self.logger.info('Published an insert msg and was acknowledged by a subscriber') 176 | else: 177 | self.logger.info('Published an insert msg but was not acknowledged by a subscriber') 178 | 179 | # wait until finish so that repo can finish fetching the data 180 | insert_num = 0 181 | if is_success: 182 | insert_num = await self._wait_for_finish(check_prefix, request_no) 183 | return insert_num 184 | 185 | async def _wait_for_finish(self, check_prefix: NonStrictName, request_no: bytes) -> int: 186 | """ 187 | Wait until process `process_id` completes by sending check interests. 188 | 189 | :param check_prefix: NonStrictName. The prefix under which the check message will be\ 190 | published. 191 | :param request_no: bytes. The request number to check. 192 | :return: int number of inserted packets. 193 | """ 194 | # fixme: why is check_prefix not used? 195 | checker = CommandChecker(self.app) 196 | n_retries = 5 197 | while n_retries > 0: 198 | response = await checker.check_insert(self.repo_name, request_no) 199 | if response is None: 200 | self.logger.info(f'No response') 201 | n_retries -= 1 202 | await aio.sleep(1) 203 | # might receive 404 if repo has not yet processed insert command msg 204 | elif response.status_code == RepoStatCode.NOT_FOUND: 205 | n_retries -= 1 206 | await aio.sleep(1) 207 | elif response.status_code == RepoStatCode.IN_PROGRESS: 208 | self.logger.info(f'Insertion {request_no} in progress') 209 | await aio.sleep(1) 210 | elif response.status_code == RepoStatCode.COMPLETED: 211 | insert_num = 0 212 | for obj in response.objs: 213 | insert_num += obj.insert_num 214 | self.logger.info(f'Deletion request {request_no} complete, insert_num: {insert_num}') 215 | return insert_num 216 | elif response.status_code == RepoStatCode.FAILED: 217 | self.logger.info(f'Deletion request {request_no} failed') 218 | else: 219 | # Shouldn't get here 220 | self.logger.error(f'Received unrecognized status code {response.status_code}') 221 | -------------------------------------------------------------------------------- /ndn_python_repo/clients/sync.py: -------------------------------------------------------------------------------- 1 | # ----------------------------------------------------------------------------- 2 | # NDN Repo putfile client. 3 | # 4 | # @Author jonnykong@cs.ucla.edu 5 | # susmit@cs.colostate.edu 6 | # @Date 2019-10-18 7 | # ----------------------------------------------------------------------------- 8 | 9 | import os 10 | import sys 11 | sys.path.insert(1, os.path.join(sys.path[0], '..')) 12 | 13 | from ..command import RepoCommandParam, SyncParam, EmbName 14 | from ..utils import PubSub 15 | import logging 16 | from ndn.app import NDNApp 17 | from ndn.encoding import Name, NonStrictName 18 | import os 19 | import platform 20 | from hashlib import sha256 21 | 22 | class SyncClient(object): 23 | 24 | def __init__(self, app: NDNApp, prefix: NonStrictName, repo_name: NonStrictName): 25 | """ 26 | A client to sync the repo. 27 | 28 | :param app: NDNApp. 29 | :param prefix: NonStrictName. The name of this client 30 | :param repo_name: NonStrictName. Routable name to remote repo. 31 | """ 32 | self.app = app 33 | self.prefix = prefix 34 | self.repo_name = Name.normalize(repo_name) 35 | self.encoded_packets = {} 36 | self.pb = PubSub(self.app, self.prefix) 37 | self.pb.base_prefix = self.prefix 38 | 39 | # https://bugs.python.org/issue35219 40 | if platform.system() == 'Darwin': 41 | os.environ['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'YES' 42 | 43 | async def join_sync(self, sync_prefix: NonStrictName, register_prefix: NonStrictName = None, 44 | data_name_dedupe: bool = False, reset: bool = False) -> bytes: 45 | 46 | # construct insert cmd msg 47 | cmd_param = RepoCommandParam() 48 | cmd_sync = SyncParam() 49 | cmd_sync.sync_prefix = EmbName.from_name(sync_prefix) 50 | cmd_sync.register_prefix = EmbName.from_name(register_prefix) 51 | cmd_sync.data_name_dedupe = data_name_dedupe 52 | cmd_sync.reset = reset 53 | 54 | cmd_param.sync_groups = [cmd_sync] 55 | cmd_param_bytes = bytes(cmd_param.encode()) 56 | 57 | # publish msg to repo's join topic 58 | await self.pb.wait_for_ready() 59 | is_success = await self.pb.publish(self.repo_name + Name.from_str('sync/join'), cmd_param_bytes) 60 | if is_success: 61 | logging.info('Published an join msg and was acknowledged by a subscriber') 62 | else: 63 | logging.info('Published an join msg but was not acknowledged by a subscriber') 64 | return sha256(cmd_param_bytes).digest() 65 | 66 | async def leave_sync(self, sync_prefix: NonStrictName) -> bytes: 67 | # construct insert cmd msg 68 | cmd_param = RepoCommandParam() 69 | cmd_sync = SyncParam() 70 | cmd_sync.sync_prefix = EmbName.from_name(sync_prefix) 71 | cmd_param.sync_groups = [cmd_sync] 72 | cmd_param_bytes = bytes(cmd_param.encode()) 73 | 74 | # publish msg to repo's leave topic 75 | await self.pb.wait_for_ready() 76 | is_success = await self.pb.publish(self.repo_name + Name.from_str('sync/leave'), cmd_param_bytes) 77 | if is_success: 78 | logging.info('Published an leave msg and was acknowledged by a subscriber') 79 | else: 80 | logging.info('Published an leave msg but was not acknowledged by a subscriber') 81 | return sha256(cmd_param_bytes).digest() 82 | -------------------------------------------------------------------------------- /ndn_python_repo/cmd/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/UCLA-IRL/ndn-python-repo/5fdece99dae72540702052aa93cfaa45d68c6073/ndn_python_repo/cmd/__init__.py -------------------------------------------------------------------------------- /ndn_python_repo/cmd/install.py: -------------------------------------------------------------------------------- 1 | import platform 2 | import shutil 3 | import sys 4 | import importlib.resources 5 | 6 | 7 | def install(source, destination): 8 | shutil.copy(source, destination) 9 | print(f'Installed {source} to {destination}') 10 | 11 | def main(): 12 | # systemd for linux 13 | if platform.system() == 'Linux': 14 | resource = importlib.resources.files('ndn_python_repo').joinpath('ndn-python-repo.service') 15 | # source = resource_filename(__name__, '../ndn-python-repo.service') 16 | destination = '/etc/systemd/system/' 17 | with importlib.resources.as_file(resource) as source: 18 | install(source, destination) 19 | 20 | 21 | if __name__ == "__main__": 22 | sys.exit(main()) -------------------------------------------------------------------------------- /ndn_python_repo/cmd/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import importlib.metadata 4 | import sys 5 | from ndn.app import NDNApp 6 | from ndn_python_repo import * 7 | 8 | 9 | def process_cmd_opts(): 10 | """ 11 | Parse, process, and return cmd options. 12 | """ 13 | def print_version(): 14 | pkg_name = 'ndn-python-repo' 15 | # version = pkg_resources.require(pkg_name)[0].version 16 | version = importlib.metadata.version(pkg_name) 17 | print(pkg_name + ' ' + version) 18 | 19 | def parse_cmd_opts(): 20 | parser = argparse.ArgumentParser(description='ndn-python-repo') 21 | parser.add_argument('-v', '--version', 22 | help='print current version and exit', action='store_true') 23 | parser.add_argument('-c', '--config', 24 | help='path to config file') 25 | parser.add_argument('-r', '--repo_name', 26 | help="""repo's routable prefix. If this option is specified, it 27 | overrides the prefix in the config file""") 28 | args = parser.parse_args() 29 | return args 30 | 31 | args = parse_cmd_opts() 32 | if args.version: 33 | print_version() 34 | exit(0) 35 | return args 36 | 37 | 38 | def process_config(cmdline_args): 39 | """ 40 | Read and process config file. Some config options are overridden by cmdline args. 41 | """ 42 | config = get_yaml(cmdline_args.config) 43 | if cmdline_args.repo_name is not None: 44 | config['repo_config']['repo_name'] = cmdline_args.repo_name 45 | return config 46 | 47 | 48 | def config_logging(config: dict): 49 | log_levels = { 50 | 'CRITICAL': logging.CRITICAL, 51 | 'ERROR': logging.ERROR, 52 | 'WARNING': logging.WARNING, 53 | 'INFO': logging.INFO, 54 | 'DEBUG': logging.DEBUG 55 | } 56 | 57 | # default level is INFO 58 | if config['level'] not in log_levels: 59 | log_level = logging.INFO 60 | else: 61 | log_level = log_levels[config['level']] 62 | 63 | # default is stdout 64 | log_file = config['file'] if 'file' in config else None 65 | 66 | if not log_file: 67 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 68 | # datefmt='%Y-%m-%d %H:%M:%S', 69 | level=log_level) 70 | else: 71 | logging.basicConfig(filename=log_file, 72 | format='[%(asctime)s]%(levelname)s:%(message)s', 73 | # datefmt='%Y-%m-%d %H:%M:%S', 74 | level=log_level) 75 | 76 | 77 | async def async_main(app: NDNApp, config): 78 | storage = create_storage(config['db_config']) 79 | 80 | pb = PubSub(app) 81 | read_handle = ReadHandle(app, storage, config) 82 | write_handle = WriteCommandHandle(app, storage, pb, read_handle, config) 83 | sync_handle = SyncCommandHandle(app, storage, pb, read_handle, config) 84 | delete_handle = DeleteCommandHandle(app, storage, pb, read_handle, config) 85 | tcp_bulk_insert_handle = TcpBulkInsertHandle(storage, read_handle, config) 86 | 87 | repo = Repo(app, storage, read_handle, write_handle, delete_handle, sync_handle, tcp_bulk_insert_handle, config) 88 | await repo.listen() 89 | 90 | 91 | def main() -> int: 92 | cmdline_args = process_cmd_opts() 93 | config = process_config(cmdline_args) 94 | print(config) 95 | 96 | config_logging(config['logging_config']) 97 | 98 | app = NDNApp() 99 | try: 100 | app.run_forever(after_start=async_main(app, config)) 101 | except FileNotFoundError: 102 | print('Error: could not connect to NFD.') 103 | return 0 104 | 105 | 106 | if __name__ == "__main__": 107 | sys.exit(main()) 108 | -------------------------------------------------------------------------------- /ndn_python_repo/cmd/port.py: -------------------------------------------------------------------------------- 1 | # noinspection GrazieInspection 2 | """ 3 | This script ports sqlite db file from repo-ng to ndn-python-repo. 4 | It takes as input a repo-ng sqlite database file, traverses the database, and inserts data into 5 | an ndn-python-repo using TCP bulk insertion. 6 | 7 | @Author jonnykong@cs.ucla.edu 8 | @Date 2019-12-26 9 | """ 10 | 11 | import argparse 12 | import asyncio as aio 13 | import os 14 | import sqlite3 15 | import sys 16 | from ndn.encoding import Name, ndn_format_0_3, tlv_var 17 | 18 | 19 | def create_sqlite3_connection(db_file): 20 | conn = None 21 | try: 22 | conn = sqlite3.connect(db_file) 23 | except Exception as e: 24 | print(e) 25 | return conn 26 | 27 | 28 | def convert_name(name: bytes) -> str: 29 | """ 30 | Convert the name to print. 31 | """ 32 | # Remove ImplicitSha256DigestComponent TLV 33 | data_bytes = name[:-34] 34 | 35 | # Prepend TL of Name 36 | type_len = tlv_var.get_tl_num_size(ndn_format_0_3.TypeNumber.DATA) 37 | len_len = tlv_var.get_tl_num_size(len(data_bytes)) 38 | buf = bytearray(type_len + len_len + len(data_bytes)) 39 | offset = 0 40 | offset += tlv_var.write_tl_num(ndn_format_0_3.TypeNumber.NAME, buf, offset) 41 | offset += tlv_var.write_tl_num(len(data_bytes), buf, offset) 42 | buf[offset:] = data_bytes 43 | 44 | # Convert bytes to URI format 45 | name = Name.from_bytes(buf) 46 | return Name.to_str(name) 47 | 48 | 49 | async def port_over_tcp(src_db_file: str, dest_addr: str, dest_port: str): 50 | conn_from = create_sqlite3_connection(src_db_file) 51 | reader, writer = await aio.open_connection(dest_addr, dest_port) 52 | 53 | # Read from source database 54 | cur = conn_from.cursor() 55 | cur.execute('SELECT name, data FROM NDN_REPO_V2') 56 | rows = cur.fetchall() 57 | for row in rows: 58 | print('Porting data:', convert_name(row[0])) 59 | writer.write(row[1]) 60 | 61 | writer.close() 62 | conn_from.close() 63 | 64 | 65 | def main() -> int: 66 | parser = argparse.ArgumentParser(description='port') 67 | parser.add_argument('-d', '--dbfile', 68 | required=True, help='Source database file') 69 | parser.add_argument('-a', '--addr', 70 | required=True, help='IP address of python repo') 71 | parser.add_argument('-p', '--port', 72 | required=True, help='Port of python repo') 73 | args = parser.parse_args() 74 | 75 | if args.addr is None: 76 | args.addr = '127.0.0.1' 77 | if args.port is None: 78 | args.addr = '7376' 79 | 80 | src_db_file = os.path.expanduser(args.dbfile) 81 | aio.get_event_loop().run_until_complete(port_over_tcp(src_db_file, args.addr, args.port)) 82 | return 0 83 | 84 | 85 | if __name__ == '__main__': 86 | sys.exit(main()) -------------------------------------------------------------------------------- /ndn_python_repo/command/__init__.py: -------------------------------------------------------------------------------- 1 | from .repo_commands import * 2 | 3 | __all__ = [] 4 | __all__.extend(repo_commands.__all__) 5 | -------------------------------------------------------------------------------- /ndn_python_repo/command/repo_commands.py: -------------------------------------------------------------------------------- 1 | """ 2 | Repo command encoding. 3 | 4 | @Author jonnykong@cs.ucla.edu 5 | @Date 2019-11-01 6 | """ 7 | 8 | from typing import TypeVar 9 | import ndn.encoding as enc 10 | 11 | __all__ = [ 12 | "RepoTypeNumber", 13 | "EmbName", 14 | "ObjParam", 15 | "SyncParam", 16 | "SyncStatus", 17 | "RepoCommandParam", 18 | "ObjStatus", 19 | "RepoCommandRes", 20 | "RepeatedNames", 21 | "RepoStatCode", 22 | "RepoStatQuery", 23 | ] 24 | 25 | 26 | class RepoTypeNumber: 27 | START_BLOCK_ID = 204 28 | END_BLOCK_ID = 205 29 | REQUEST_NO = 206 30 | STATUS_CODE = 208 31 | INSERT_NUM = 209 32 | DELETE_NUM = 210 33 | FORWARDING_HINT = 211 34 | REGISTER_PREFIX = 212 35 | CHECK_PREFIX = 213 36 | OBJECT_PARAM = 301 37 | OBJECT_RESULT = 302 38 | SYNC_PARAM = 401 39 | SYNC_RESULT = 402 40 | SYNC_DATA_NAME_DEDUPE = 403 41 | SYNC_RESET = 404 42 | SYNC_PREFIX = 405 43 | 44 | 45 | class RepoStatCode: 46 | # 100 has not been used by previous code, but defined and documented. 47 | # The current code use it for acknowledged but not started yet. 48 | ROGER = 100 49 | # All data have been inserted / deleted 50 | COMPLETED = 200 51 | # Work in progress 52 | IN_PROGRESS = 300 53 | # Some data failed to be inserted / deleted 54 | FAILED = 400 55 | # The command or param is malformed 56 | MALFORMED = 403 57 | # The queried operation cannot be found 58 | NOT_FOUND = 404 59 | 60 | 61 | TEmbName = TypeVar('TEmbName', bound='EmbName') 62 | 63 | 64 | class EmbName(enc.TlvModel): 65 | name = enc.NameField() 66 | 67 | @staticmethod 68 | def from_name(name: enc.NonStrictName) -> TEmbName: 69 | ret = EmbName() 70 | ret.name = name 71 | return ret 72 | 73 | 74 | class ObjParam(enc.TlvModel): 75 | name = enc.NameField() 76 | forwarding_hint = enc.ModelField(RepoTypeNumber.FORWARDING_HINT, enc.Links) 77 | start_block_id = enc.UintField(RepoTypeNumber.START_BLOCK_ID) 78 | end_block_id = enc.UintField(RepoTypeNumber.END_BLOCK_ID) 79 | register_prefix = enc.ModelField(RepoTypeNumber.REGISTER_PREFIX, EmbName) 80 | 81 | 82 | class SyncParam(enc.TlvModel): 83 | sync_prefix = enc.ModelField(RepoTypeNumber.SYNC_PREFIX, EmbName) 84 | register_prefix = enc.ModelField(RepoTypeNumber.REGISTER_PREFIX, EmbName) 85 | data_name_dedupe = enc.BoolField(RepoTypeNumber.SYNC_DATA_NAME_DEDUPE) 86 | reset = enc.BoolField(RepoTypeNumber.SYNC_RESET) 87 | # forwarding_hint = enc.ModelField(RepoTypeNumber.FORWARDING_HINT, enc.Links) 88 | # sync_prefix = enc.ModelField(RepoTypeNumber.REGISTER_PREFIX, EmbName) 89 | 90 | 91 | class RepoCommandParam(enc.TlvModel): 92 | objs = enc.RepeatedField(enc.ModelField(RepoTypeNumber.OBJECT_PARAM, ObjParam)) 93 | sync_groups = enc.RepeatedField( 94 | enc.ModelField(RepoTypeNumber.SYNC_PARAM, SyncParam) 95 | ) 96 | 97 | 98 | class RepoStatQuery(enc.TlvModel): 99 | request_no = enc.BytesField(RepoTypeNumber.REQUEST_NO) 100 | 101 | 102 | class ObjStatus(enc.TlvModel): 103 | name = enc.NameField() 104 | status_code = enc.UintField(RepoTypeNumber.STATUS_CODE) 105 | insert_num = enc.UintField(RepoTypeNumber.INSERT_NUM) 106 | delete_num = enc.UintField(RepoTypeNumber.DELETE_NUM) 107 | 108 | 109 | class SyncStatus(enc.TlvModel): 110 | name = enc.NameField() 111 | status_code = enc.UintField(RepoTypeNumber.STATUS_CODE) 112 | 113 | 114 | class RepoCommandRes(enc.TlvModel): 115 | status_code = enc.UintField(RepoTypeNumber.STATUS_CODE) 116 | objs = enc.RepeatedField(enc.ModelField(RepoTypeNumber.OBJECT_RESULT, ObjStatus)) 117 | sync_groups = enc.RepeatedField( 118 | enc.ModelField(RepoTypeNumber.SYNC_RESULT, SyncStatus) 119 | ) 120 | 121 | 122 | class RepeatedNames(enc.TlvModel): 123 | names = enc.RepeatedField(enc.NameField()) 124 | -------------------------------------------------------------------------------- /ndn_python_repo/config.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | # from pkg_resources import resource_filename 3 | import importlib.resources 4 | 5 | 6 | def get_yaml(path=None): 7 | # if fall back to internal config file, so that repo can run without any external configs 8 | 9 | 10 | try: 11 | if path is None: 12 | resource = importlib.resources.files('ndn_python_repo').joinpath('ndn-python-repo.conf.sample') 13 | with resource.open('r', encoding='utf-8') as file: 14 | config = yaml.safe_load(file) 15 | else: 16 | with open(path, 'r', encoding='utf-8') as file: 17 | config = yaml.safe_load(file) 18 | except FileNotFoundError: 19 | raise FileNotFoundError(f'could not find config file: {path}') from None 20 | return config 21 | 22 | 23 | # For testing 24 | if __name__ == "__main__": 25 | print(get_yaml()) 26 | -------------------------------------------------------------------------------- /ndn_python_repo/handle/__init__.py: -------------------------------------------------------------------------------- 1 | from .read_handle import ReadHandle 2 | from .command_handle_base import CommandHandle 3 | from .write_command_handle import WriteCommandHandle 4 | from .delete_command_handle import DeleteCommandHandle 5 | from .sync_command_handle import SyncCommandHandle 6 | from .tcp_bulk_insert_handle import TcpBulkInsertHandle -------------------------------------------------------------------------------- /ndn_python_repo/handle/command_handle_base.py: -------------------------------------------------------------------------------- 1 | import asyncio as aio 2 | import logging 3 | import json 4 | from ndn.app import NDNApp 5 | from ndn.encoding import Name, NonStrictName, FormalName, Component 6 | from ndn.encoding.tlv_model import DecodeError 7 | 8 | from ..command import RepoStatQuery, RepoCommandRes, RepoStatCode, RepeatedNames, RepoCommandParam 9 | from ..storage import Storage 10 | from ..utils import PubSub 11 | 12 | from hashlib import sha256 13 | 14 | class CommandHandle(object): 15 | """ 16 | Interface for command interest handles 17 | """ 18 | def __init__(self, app: NDNApp, storage: Storage, pb: PubSub, _config: dict): 19 | self.app = app 20 | self.storage = storage 21 | self.pb = pb 22 | self.m_processes = dict() 23 | self.logger = logging.getLogger(__name__) 24 | 25 | async def listen(self, prefix: Name): 26 | raise NotImplementedError 27 | 28 | def _on_check_interest(self, int_name, _int_param, app_param): 29 | self.logger.info('on_check_interest(): {}'.format(Name.to_str(int_name))) 30 | 31 | response = None 32 | request_no = None 33 | try: 34 | if not app_param: 35 | raise DecodeError('Missing Parameters') 36 | parameter = RepoStatQuery.parse(app_param) 37 | request_no = parameter.request_no 38 | if request_no is None: 39 | raise DecodeError('Missing Request No.') 40 | except (DecodeError, IndexError, RuntimeError) as exc: 41 | response = RepoCommandRes() 42 | response.status_code = RepoStatCode.MALFORMED 43 | self.logger.warning(f'Command blob decoding failed for exception {exc}') 44 | 45 | if response is None and request_no not in self.m_processes: 46 | response = RepoCommandRes() 47 | response.status_code = RepoStatCode.NOT_FOUND 48 | self.logger.warning(f'Process does not exist for id={request_no}') 49 | 50 | if response is None: 51 | self.reply_with_response(int_name, self.m_processes[request_no]) 52 | else: 53 | self.reply_with_response(int_name, response) 54 | 55 | def reply_with_response(self, int_name, response: RepoCommandRes): 56 | self.logger.info(f'Reply to command: {Name.to_str(int_name)} w/ code={response.status_code}') 57 | response_bytes = response.encode() 58 | self.app.put_data(int_name, response_bytes, freshness_period=1000) 59 | 60 | async def _delete_process_state_after(self, process_id: bytes, delay: int): 61 | """ 62 | Remove process state after some delay. 63 | """ 64 | await aio.sleep(delay) 65 | if process_id in self.m_processes: 66 | del self.m_processes[process_id] 67 | 68 | def parse_msg(self, msg): 69 | try: 70 | cmd_param = RepoCommandParam.parse(msg) 71 | request_no = sha256(bytes(msg)).digest() 72 | if not cmd_param.objs: 73 | raise DecodeError('Missing objects') 74 | for obj in cmd_param.objs: 75 | if obj.name is None: 76 | raise DecodeError('Missing name for one or more objects') 77 | except (DecodeError, IndexError) as exc: 78 | self.logger.warning(f'Parameter interest blob decoding failed w/ exception: {exc}') 79 | return 80 | 81 | return cmd_param, request_no 82 | 83 | @staticmethod 84 | def add_name_to_set_in_storage(set_name: str, storage: Storage, name: NonStrictName) -> bool: 85 | """ 86 | Add ``name`` to set ``set_name`` in the storage. This function implements a set of Names\ 87 | over the key-value storage interface. The set name is stored as the key, and the set\ 88 | elements are serialized and stored as the value. 89 | :param set_name: str 90 | :param storage: Storage 91 | :param name: NonStrictName 92 | :return: Returns true if ``name`` is already in set ``set_name``. 93 | """ 94 | names_msg = RepeatedNames() 95 | ret = storage._get(set_name.encode('utf-8')) 96 | if ret: 97 | names_msg = RepeatedNames.parse(ret) 98 | 99 | name = Name.normalize(name) 100 | if name in names_msg.names: 101 | return True 102 | else: 103 | names_msg.names.append(name) 104 | names_msg_bytes = names_msg.encode() 105 | storage._put(set_name.encode('utf-8'), bytes(names_msg_bytes)) 106 | return False 107 | 108 | @staticmethod 109 | def get_name_from_set_in_storage(set_name: str, storage: Storage) -> list[FormalName]: 110 | """ 111 | Get all names from set ``set_name`` in the storage. 112 | :param set_name: str 113 | :param storage: Storage 114 | :return: A list of ``FormalName`` 115 | """ 116 | ret = storage._get(set_name.encode('utf-8')) 117 | if ret: 118 | names_msg = RepeatedNames.parse(ret) 119 | return names_msg.names 120 | else: 121 | return [] 122 | 123 | @staticmethod 124 | def remove_name_from_set_in_storage(set_name: str, storage: Storage, name: NonStrictName) -> bool: 125 | """ 126 | Remove ``name`` from set ``set_name`` in the storage. 127 | :param set_name: str 128 | :param storage: Storage 129 | :param name: NonStrictName 130 | :return: Returns true if ``name`` exists in set ``set_name`` and is being successfully\ 131 | removed. 132 | """ 133 | names_msg = RepeatedNames() 134 | ret = storage._get(set_name.encode('utf-8')) 135 | if ret: 136 | names_msg = RepeatedNames.parse(ret) 137 | 138 | name = Name.normalize(name) 139 | if name in names_msg.names: 140 | names_msg.names.remove(Name.normalize(name)) 141 | names_msg_bytes = names_msg.encode() 142 | storage._put(set_name.encode('utf-8'), bytes(names_msg_bytes)) 143 | return True 144 | else: 145 | return False 146 | 147 | # this will overwrite 148 | @staticmethod 149 | def add_dict_in_storage(dict_name: str, storage: Storage, s_dict: dict) -> bool: 150 | ret = storage._get(dict_name.encode('utf-8')) 151 | dict_bytes = json.dumps(s_dict).encode('utf-8') 152 | storage._put(dict_name.encode('utf-8'), dict_bytes) 153 | return ret is not None 154 | 155 | @staticmethod 156 | def get_dict_in_storage(dict_name: str, storage: Storage) -> dict: 157 | res_bytes = storage._get(dict_name.encode('utf-8')) 158 | return json.loads(res_bytes.decode('utf-8')) 159 | 160 | @staticmethod 161 | def remove_dict_in_storage(dict_name: str, storage: Storage) -> bool: 162 | return storage._remove(dict_name.encode('utf-8')) 163 | 164 | # Wrapper for registered prefixes 165 | @staticmethod 166 | def add_registered_prefix_in_storage(storage: Storage, prefix): 167 | ret = CommandHandle.add_name_to_set_in_storage('prefixes', storage, prefix) 168 | if not ret: 169 | logging.getLogger(__name__).info(f'Added new registered prefix to storage: {Name.to_str(prefix)}') 170 | return ret 171 | 172 | @staticmethod 173 | def get_registered_prefix_in_storage(storage: Storage): 174 | return CommandHandle.get_name_from_set_in_storage('prefixes', storage) 175 | 176 | @staticmethod 177 | def remove_registered_prefix_in_storage(storage: Storage, prefix): 178 | ret = CommandHandle.remove_name_from_set_in_storage('prefixes', storage, prefix) 179 | if ret: 180 | logging.getLogger(__name__).info(f'Removed existing registered prefix from storage: {Name.to_str(prefix)}') 181 | return ret 182 | 183 | @staticmethod 184 | def add_sync_states_in_storage(storage: Storage, sync_group: FormalName, states: dict): 185 | store_key = [Component.from_str('sync_states')] + sync_group 186 | logging.info(f'Added new sync states to storage: {Name.to_str(sync_group)}') 187 | return CommandHandle.add_dict_in_storage(Name.to_str(store_key), storage, states) 188 | 189 | @staticmethod 190 | def get_sync_states_in_storage(storage: Storage, sync_group: FormalName): 191 | store_key = [Component.from_str('sync_states')] + sync_group 192 | return CommandHandle.get_dict_in_storage(Name.to_str(store_key), storage) 193 | 194 | @staticmethod 195 | def remove_sync_states_in_storage(storage: Storage, sync_group: FormalName): 196 | store_key = [Component.from_str('sync_states')] + sync_group 197 | logging.info(f'Removed new sync states to storage: {Name.to_str(sync_group)}') 198 | return CommandHandle.remove_dict_in_storage(Name.to_str(store_key), storage) 199 | 200 | @staticmethod 201 | def add_sync_group_in_storage(storage: Storage, sync_group: FormalName): 202 | ret = CommandHandle.add_name_to_set_in_storage('sync_groups', storage, sync_group) 203 | if not ret: 204 | logging.info(f'Added new sync group to storage: {Name.to_str(sync_group)}') 205 | return ret 206 | 207 | @staticmethod 208 | def get_sync_groups_in_storage(storage: Storage): 209 | return CommandHandle.get_name_from_set_in_storage('sync_groups', storage) 210 | 211 | @staticmethod 212 | def remove_sync_group_in_storage(storage: Storage, sync_group: FormalName): 213 | ret = CommandHandle.remove_name_from_set_in_storage('sync_groups', storage, sync_group) 214 | if ret: 215 | logging.info(f'Removed existing sync_group from storage: {Name.to_str(sync_group)}') 216 | return ret -------------------------------------------------------------------------------- /ndn_python_repo/handle/delete_command_handle.py: -------------------------------------------------------------------------------- 1 | import asyncio as aio 2 | import logging 3 | from ndn.app import NDNApp 4 | from ndn.encoding import Name, NonStrictName, Component 5 | from typing import Optional 6 | from . import ReadHandle, CommandHandle 7 | from ..command import RepoCommandRes, RepoCommandParam, ObjParam, ObjStatus, RepoStatCode 8 | from ..storage import Storage 9 | from ..utils import PubSub 10 | from .utils import normalize_block_ids 11 | 12 | 13 | class DeleteCommandHandle(CommandHandle): 14 | """ 15 | DeleteCommandHandle processes delete command handles, and deletes corresponding data stored 16 | in the database. 17 | TODO: Add validator 18 | """ 19 | def __init__(self, app: NDNApp, storage: Storage, pb: PubSub, read_handle: ReadHandle, 20 | config: dict): 21 | """ 22 | Read handle need to keep a reference to write handle to register new prefixes. 23 | 24 | :param app: NDNApp. 25 | :param storage: Storage. 26 | :param read_handle: ReadHandle. This param is necessary because DeleteCommandHandle need to 27 | unregister prefixes. 28 | """ 29 | super(DeleteCommandHandle, self).__init__(app, storage, pb, config) 30 | self.m_read_handle = read_handle 31 | self.prefix = None 32 | self.register_root = config['repo_config']['register_root'] 33 | self.logger = logging.getLogger(__name__) 34 | 35 | async def listen(self, prefix: NonStrictName): 36 | """ 37 | Register routes for command interests. 38 | This function needs to be called explicitly after initialization. 39 | 40 | :param prefix: NonStrictName. The name prefix to listen on. 41 | """ 42 | self.prefix = Name.normalize(prefix) 43 | 44 | # subscribe to delete messages 45 | self.pb.subscribe(self.prefix + Name.from_str('delete'), self._on_delete_msg) 46 | 47 | # listen on delete check interests 48 | self.app.set_interest_filter(self.prefix + Name.from_str('delete check'), self._on_check_interest) 49 | 50 | def _on_delete_msg(self, msg): 51 | cmd_param, request_no = self.parse_msg(msg) 52 | aio.create_task(self._process_delete(cmd_param, request_no)) 53 | 54 | async def _process_delete(self, cmd_param: RepoCommandParam, request_no: bytes): 55 | """ 56 | Process delete command. 57 | """ 58 | objs = cmd_param.objs 59 | self.logger.info(f'Received delete command: {request_no.hex()}') 60 | 61 | # Note that this function still has chance to switch coroutine in _perform_storage_delete. 62 | # So status is required to be defined before actual deletion 63 | def _init_obj_stat(obj: ObjParam) -> ObjStatus: 64 | ret = ObjStatus() 65 | ret.name = obj.name 66 | ret.status_code = RepoStatCode.ROGER 67 | ret.insert_num = None 68 | ret.delete_num = 0 69 | return ret 70 | 71 | # Note: stat is hold by reference 72 | stat = RepoCommandRes() 73 | stat.status_code = RepoStatCode.IN_PROGRESS 74 | stat.objs = [_init_obj_stat(obj) for obj in objs] 75 | self.m_processes[request_no] = stat 76 | 77 | global_deleted = 0 78 | global_succeeded = True 79 | for i, obj in enumerate(objs): 80 | name = obj.name 81 | valid, start_id, end_id = normalize_block_ids(obj) 82 | if not valid: 83 | self.logger.warning('Delete command malformed') 84 | stat.objs[i].status_code = RepoStatCode.MALFORMED 85 | global_succeeded = False 86 | continue 87 | 88 | self.logger.debug(f'Proc del cmd {request_no.hex()} w/' 89 | f'name={Name.to_str(name)}, start={obj.start_block_id}, end={obj.end_block_id}') 90 | 91 | # Start data deleting process 92 | stat.objs[i].status_code = RepoStatCode.IN_PROGRESS 93 | 94 | # If repo does not register root prefix, the client tells repo what to unregister 95 | # TODO: It is probably improper to let the client remember which prefix is registered. 96 | # When register_prefix differs from the insertion command, unexpected result may arise. 97 | if obj.register_prefix: 98 | register_prefix = obj.register_prefix.name 99 | else: 100 | register_prefix = None 101 | if register_prefix: 102 | is_existing = CommandHandle.remove_registered_prefix_in_storage(self.storage, register_prefix) 103 | if not self.register_root and is_existing: 104 | self.m_read_handle.unlisten(register_prefix) 105 | 106 | # Remember what files are removed 107 | # TODO: Warning: this code comes from previous impl. 108 | # When start_id and end_id differs from the insertion command, unexpected result may arise. 109 | # Please do not let such case happen until we fix this problem. 110 | # CommandHandle.remove_inserted_filename_in_storage(self.storage, name) 111 | 112 | # Perform delete 113 | if start_id is not None: 114 | delete_num = await self._perform_storage_delete(name, start_id, end_id) 115 | else: 116 | delete_num = await self._delete_single_data(name) 117 | self.logger.info(f'Deletion {request_no.hex()} name={Name.to_str(name)} finish:' 118 | f'{delete_num} deleted') 119 | 120 | # Delete complete, update process state 121 | stat.objs[i].status_code = RepoStatCode.COMPLETED 122 | stat.objs[i].delete_num = delete_num 123 | global_deleted += delete_num 124 | 125 | # All fetches finished 126 | self.logger.info(f'Deletion {request_no.hex()} done, total {global_deleted} deleted.') 127 | if global_succeeded: 128 | stat.status_code = RepoStatCode.COMPLETED 129 | else: 130 | stat.status_code = RepoStatCode.FAILED 131 | 132 | # Remove process state after some time 133 | await self._delete_process_state_after(request_no, 60) 134 | 135 | async def _perform_storage_delete(self, prefix, start_block_id: int, end_block_id: Optional[int]) -> int: 136 | """ 137 | Delete data packets between [start_block_id, end_block_id]. If end_block_id is None, delete 138 | all continuous data packets from start_block_id. 139 | :param prefix: NonStrictName. 140 | :param start_block_id: int. 141 | :param end_block_id: int. 142 | :return: The number of data items deleted. 143 | """ 144 | delete_num = 0 145 | if end_block_id is None: 146 | end_block_id = 2 ** 30 # TODO: For temp use; Should discover 147 | for idx in range(start_block_id, end_block_id + 1): 148 | key = prefix + [Component.from_segment(idx)] 149 | if self.storage.get_data_packet(key) is not None: 150 | self.logger.debug(f'Data for key {Name.to_str(key)} to be deleted.') 151 | self.storage.remove_data_packet(key) 152 | delete_num += 1 153 | else: 154 | # assume sequence numbers are continuous 155 | self.logger.debug(f'Data for key {Name.to_str(key)} not found, break.') 156 | break 157 | # Temporarily release control to make the process non-blocking 158 | await aio.sleep(0) 159 | return delete_num 160 | 161 | # TODO: previous version only uses _perform_storage_delete 162 | # I doubt if it properly worked. So need test for the current change. 163 | # NOTE: this test cannot done by a client because 164 | # 1) the prefix has been unregistered, so undeleted data become ghost. 165 | # 2) the current client always uses segmented insertion/deletion 166 | async def _delete_single_data(self, name) -> int: 167 | """ 168 | Delete data packets between [start_block_id, end_block_id]. If end_block_id is None, delete 169 | all continuous data packets from start_block_id. 170 | :param name: The name of data to be deleted. 171 | :return: The number of data items deleted. 172 | """ 173 | if self.storage.get_data_packet(name) is not None: 174 | self.storage.remove_data_packet(name) 175 | await aio.sleep(0) 176 | return 1 177 | else: 178 | return 0 179 | -------------------------------------------------------------------------------- /ndn_python_repo/handle/read_handle.py: -------------------------------------------------------------------------------- 1 | import asyncio as aio 2 | import logging 3 | from ndn.app import NDNApp 4 | from ndn.encoding import Name 5 | from ..storage import Storage 6 | 7 | 8 | class ReadHandle(object): 9 | """ 10 | ReadCommandHandle processes ordinary interests, and return corresponding data if exists. 11 | """ 12 | def __init__(self, app: NDNApp, storage: Storage, config: dict): 13 | """ 14 | :param app: NDNApp. 15 | :param storage: Storage. 16 | TODO: determine which prefix to listen on. 17 | """ 18 | self.app = app 19 | self.storage = storage 20 | self.register_root = config['repo_config']['register_root'] 21 | self.logger = logging.getLogger(__name__) 22 | if self.register_root: 23 | self.listen(Name.from_str('/')) 24 | 25 | def listen(self, prefix): 26 | """ 27 | This function needs to be called for prefix of all data stored. 28 | :param prefix: NonStrictName. 29 | """ 30 | self.app.route(prefix)(self._on_interest) 31 | self.logger.info(f'Read handle: listening to {Name.to_str(prefix)}') 32 | 33 | def unlisten(self, prefix): 34 | """ 35 | :param prefix: NonStrictName. 36 | """ 37 | aio.ensure_future(self.app.unregister(prefix)) 38 | self.logger.info(f'Read handle: stop listening to {Name.to_str(prefix)}') 39 | 40 | def _on_interest(self, int_name, int_param, _app_param): 41 | """ 42 | Repo responds to Interests with mustBeFresh flag, following the same logic as the Content Store in NFD 43 | """ 44 | logging.debug(f'Repo got Interest with{"out" if not int_param.must_be_fresh else ""} ' 45 | f'MustBeFresh flag set for name {Name.to_str(int_name)}') 46 | data_bytes = self.storage.get_data_packet(int_name, int_param.can_be_prefix, int_param.must_be_fresh) 47 | if data_bytes is None: 48 | return 49 | self.app.put_raw_packet(data_bytes) 50 | self.logger.info(f'Read handle: serve data {Name.to_str(int_name)}') 51 | -------------------------------------------------------------------------------- /ndn_python_repo/handle/tcp_bulk_insert_handle.py: -------------------------------------------------------------------------------- 1 | import asyncio as aio 2 | import io 3 | import logging 4 | import sys 5 | from . import ReadHandle, CommandHandle 6 | from ..storage import * 7 | from ndn.encoding import Name, read_tl_num_from_stream, parse_data 8 | from ndn.encoding import TypeNumber, FormalName 9 | 10 | 11 | class TcpBulkInsertHandle(object): 12 | 13 | class TcpBulkInsertClient(object): 14 | """ 15 | An instance of this nested class will be created for every new connection. 16 | """ 17 | def __init__(self, reader, writer, storage: Storage, read_handle: ReadHandle, config: dict): 18 | """ 19 | TCP Bulk insertion client need to keep a reference to ReadHandle to register new prefixes. 20 | """ 21 | self.logger = logging.getLogger(__name__) 22 | self.reader = reader 23 | self.writer = writer 24 | self.storage = storage 25 | self.read_handle = read_handle 26 | self.config = config 27 | self.m_inputBufferSize = 0 28 | prefix_strs = self.config['tcp_bulk_insert'].get('prefixes', []) 29 | self.reg_root = self.config['repo_config']['register_root'] 30 | self.reg_prefix = self.config['tcp_bulk_insert']['register_prefix'] 31 | self.prefixes = [Name.from_str(s) for s in prefix_strs] 32 | self.logger.info("New connection") 33 | 34 | async def handle_receive(self): 35 | """ 36 | Handle one incoming TCP connection. 37 | Multiple data packets may be transferred over a single connection. 38 | """ 39 | while True: 40 | try: 41 | bio = io.BytesIO() 42 | ret = await read_tl_num_from_stream(self.reader, bio) 43 | # only accept data packets 44 | if ret != TypeNumber.DATA: 45 | self.logger.fatal('TCP handle received non-data type, closing connection ...') 46 | self.writer.close() 47 | return 48 | siz = await read_tl_num_from_stream(self.reader, bio) 49 | bio.write(await self.reader.readexactly(siz)) 50 | data_bytes = bio.getvalue() 51 | except aio.IncompleteReadError: 52 | self.writer.close() 53 | self.logger.info('Closed TCP connection') 54 | return 55 | except Exception as exc: 56 | print(exc) 57 | return 58 | # Parse data again to obtain the name 59 | data_name, _, _, _ = parse_data(data_bytes, with_tl=True) 60 | self.storage.put_data_packet(data_name, data_bytes) 61 | self.logger.info(f'Inserted data: {Name.to_str(data_name)}') 62 | 63 | # Register prefix 64 | if not self.reg_root and self.reg_prefix: 65 | prefix = self.check_prefix(data_name) 66 | self.logger.info(f'Try to register prefix: {Name.to_str(prefix)}') 67 | is_existing = CommandHandle.add_registered_prefix_in_storage(self.storage, prefix) 68 | if not is_existing: 69 | self.logger.info(f'Registered prefix: {Name.to_str(prefix)}') 70 | self.read_handle.listen(prefix) 71 | 72 | await aio.sleep(0) 73 | 74 | def check_prefix(self, data_name: FormalName) -> FormalName: 75 | for prefix in self.prefixes: 76 | if Name.is_prefix(prefix, data_name): 77 | return prefix 78 | return data_name 79 | 80 | def __init__(self, storage: Storage, read_handle: ReadHandle, config: dict): 81 | """ 82 | TCP bulk insertion handle need to keep a reference to ReadHandle to register new prefixes. 83 | """ 84 | self.logger = logging.getLogger(__name__) 85 | 86 | async def run(): 87 | self.server = await aio.start_server(self.start_receive, server_addr, server_port) 88 | addr = self.server.sockets[0].getsockname() 89 | self.logger.info(f'TCP insertion handle serving on {addr}') 90 | async with self.server: 91 | await self.server.serve_forever() 92 | 93 | self.storage = storage 94 | self.read_handle = read_handle 95 | self.config = config 96 | 97 | server_addr = self.config['tcp_bulk_insert']['addr'] 98 | server_port = self.config['tcp_bulk_insert']['port'] 99 | event_loop = aio.get_event_loop() 100 | 101 | if sys.version_info.minor >= 7: 102 | # python 3.7+ 103 | event_loop.create_task(run()) 104 | else: 105 | coro = aio.start_server(self.start_receive, server_addr, server_port, loop=event_loop) 106 | server = event_loop.run_until_complete(coro) 107 | self.logger.info('TCP insertion handle serving on {}'.format(server.sockets[0].getsockname())) 108 | 109 | async def start_receive(self, reader, writer): 110 | """ 111 | Create a new client for every new connection. 112 | """ 113 | self.logger.info("Accepted new TCP connection") 114 | client = TcpBulkInsertHandle.TcpBulkInsertClient(reader, writer, self.storage, self.read_handle, self.config) 115 | event_loop = aio.get_event_loop() 116 | event_loop.create_task(client.handle_receive()) 117 | 118 | 119 | if __name__ == "__main__": 120 | logging.basicConfig(format='[%(asctime)s]%(levelname)s:%(message)s', 121 | datefmt='%Y-%m-%d %H:%M:%S', 122 | level=logging.INFO) 123 | 124 | storage = LevelDBStorage() # fixme: this should have a parameter for the location of the db 125 | handle = TcpBulkInsertHandle(storage) # fixme: read_handle and config parameters are unfilled 126 | 127 | event_loop = aio.get_event_loop() 128 | event_loop.run_forever() 129 | -------------------------------------------------------------------------------- /ndn_python_repo/handle/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from ..command import ObjParam 3 | 4 | 5 | def normalize_block_ids(obj: ObjParam) -> tuple[bool, Optional[int], Optional[int]]: 6 | """ 7 | Normalize insert parameter, or reject the param if it's invalid. 8 | :param obj: The object to fetch. 9 | :return: Returns (true, start id, end id) if cmd_param is valid. 10 | """ 11 | start_id = obj.start_block_id 12 | end_id = obj.end_block_id 13 | 14 | # Valid if neither start_block_id nor end_block_id is given, fetch single data without seg number 15 | if start_id is None and end_id is None: 16 | return True, None, None 17 | 18 | # If start_block_id is not given, it is set to 0 19 | if start_id is None: 20 | start_id = 0 21 | 22 | # Valid if end_block_id is not given, attempt to fetch all segments until receiving timeout 23 | # Valid if end_block_id is given, and larger than or equal to start_block_id 24 | if end_id is None or end_id >= start_id: 25 | return True, start_id, end_id 26 | 27 | return False, None, None 28 | -------------------------------------------------------------------------------- /ndn_python_repo/handle/write_command_handle.py: -------------------------------------------------------------------------------- 1 | import asyncio as aio 2 | import logging 3 | from ndn.app import NDNApp 4 | from ndn.encoding import Name, NonStrictName 5 | from ndn.types import InterestNack, InterestTimeout 6 | from . import ReadHandle, CommandHandle 7 | from ..command import RepoCommandRes, RepoCommandParam, ObjParam, ObjStatus, RepoStatCode 8 | from ..utils import concurrent_fetcher, PubSub 9 | from ..storage import Storage 10 | from typing import Optional 11 | from .utils import normalize_block_ids 12 | 13 | 14 | class WriteCommandHandle(CommandHandle): 15 | """ 16 | WriteCommandHandle processes insert command interests, and fetches corresponding data to 17 | store them into the database. 18 | TODO: Add validator 19 | """ 20 | def __init__(self, app: NDNApp, storage: Storage, pb: PubSub, read_handle: ReadHandle, 21 | config: dict): 22 | """ 23 | Write handle need to keep a reference to write handle to register new prefixes. 24 | 25 | :param app: NDNApp. 26 | :param storage: Storage. 27 | :param read_handle: ReadHandle. This param is necessary, because WriteCommandHandle need to 28 | call ReadHandle.listen() to register new prefixes. 29 | """ 30 | super(WriteCommandHandle, self).__init__(app, storage, pb, config) 31 | self.m_read_handle = read_handle 32 | self.prefix = None 33 | self.register_root = config['repo_config']['register_root'] 34 | self.logger = logging.getLogger(__name__) 35 | 36 | async def listen(self, prefix: NonStrictName): 37 | """ 38 | Register routes for command interests. 39 | This function needs to be called explicitly after initialization. 40 | 41 | :param prefix: NonStrictName. The name prefix to listen on. 42 | """ 43 | self.prefix = Name.normalize(prefix) 44 | 45 | # subscribe to insert messages 46 | self.pb.subscribe(self.prefix + Name.from_str('insert'), self._on_insert_msg) 47 | 48 | # listen on insert check interests 49 | self.app.set_interest_filter(self.prefix + Name.from_str('insert check'), self._on_check_interest) 50 | 51 | def _on_insert_msg(self, msg): 52 | cmd_param, request_no = self.parse_msg(msg) 53 | aio.create_task(self._process_insert(cmd_param, request_no)) 54 | 55 | async def _process_insert(self, cmd_param: RepoCommandParam, request_no: bytes): 56 | """ 57 | Process segmented insertion command. 58 | Return to client with status code 100 immediately, and then start data fetching process. 59 | """ 60 | objs = cmd_param.objs 61 | self.logger.info(f'Recved insert command: {request_no.hex()}') 62 | 63 | # Cached status response 64 | # Note: no coroutine switching here, so no multithread conflicts 65 | def _init_obj_stat(obj: ObjParam) -> ObjStatus: 66 | ret = ObjStatus() 67 | ret.name = obj.name 68 | ret.status_code = RepoStatCode.ROGER 69 | ret.insert_num = 0 70 | ret.delete_num = None 71 | return ret 72 | 73 | # Note: stat is hold by reference 74 | stat = RepoCommandRes() 75 | stat.status_code = RepoStatCode.IN_PROGRESS 76 | stat.objs = [_init_obj_stat(obj) for obj in objs] 77 | self.m_processes[request_no] = stat 78 | 79 | # Start fetching 80 | global_inserted = 0 81 | global_succeeded = True 82 | for i, obj in enumerate(objs): 83 | name = obj.name 84 | if obj.register_prefix and obj.register_prefix.name: 85 | register_prefix = obj.register_prefix.name 86 | else: 87 | register_prefix = None 88 | if obj.forwarding_hint and obj.forwarding_hint.names: 89 | forwarding_hint = obj.forwarding_hint.names 90 | else: 91 | forwarding_hint = None 92 | 93 | self.logger.debug(f'Proc ins cmd {request_no.hex()} w/' 94 | f'name={Name.to_str(name)}, start={obj.start_block_id}, end={obj.end_block_id}') 95 | 96 | # rejects any data that overlaps with repo's own namespace 97 | if Name.is_prefix(self.prefix, name) or Name.is_prefix(name, self.prefix): 98 | self.logger.warning('Inserted data name overlaps with repo prefix, rejected') 99 | stat.objs[i].status_code = RepoStatCode.MALFORMED 100 | global_succeeded = False 101 | continue 102 | valid, start_block_id, end_block_id = normalize_block_ids(obj) 103 | if not valid: 104 | self.logger.warning('Insert command malformed') 105 | stat.objs[i].status_code = RepoStatCode.MALFORMED 106 | global_succeeded = False 107 | continue 108 | 109 | # Remember the prefixes to register 110 | if register_prefix: 111 | is_existing = CommandHandle.add_registered_prefix_in_storage(self.storage, register_prefix) 112 | # If repo does not register root prefix, the client tells repo what to register 113 | if not self.register_root and not is_existing: 114 | self.m_read_handle.listen(register_prefix) 115 | 116 | # Remember the files inserted, this is useful for enumerating all inserted files 117 | # CommandHandle.add_inserted_filename_in_storage(self.storage, name) 118 | 119 | # Start data fetching process 120 | stat.objs[i].status_code = RepoStatCode.IN_PROGRESS 121 | 122 | if start_block_id is not None: 123 | # Fetch data packets with block ids appended to the end 124 | insert_num = await self.fetch_segmented_data(name, start_block_id, end_block_id, forwarding_hint) 125 | is_success = end_block_id is None or start_block_id + insert_num - 1 == end_block_id 126 | else: 127 | # Both start_block_id and end_block_id are None, fetch a single data packet 128 | insert_num = await self.fetch_single_data(name, forwarding_hint) 129 | is_success = insert_num == 1 130 | 131 | if is_success: 132 | stat.objs[i].status_code = RepoStatCode.COMPLETED 133 | self.logger.info(f'Insertion {request_no.hex()} name={Name.to_str(name)} finish:' 134 | f'{insert_num} inserted') 135 | else: 136 | global_succeeded = False 137 | stat.objs[i].status_code = RepoStatCode.FAILED 138 | self.logger.info(f'Insertion {request_no.hex()} name={Name.to_str(name)} fail:' 139 | f'{insert_num} inserted') 140 | stat.objs[i].insert_num = insert_num 141 | global_inserted += insert_num 142 | 143 | # All fetches finished 144 | self.logger.info(f'Insertion {request_no.hex()} done, total {global_inserted} inserted.') 145 | if global_succeeded: 146 | stat.status_code = RepoStatCode.COMPLETED 147 | else: 148 | stat.status_code = RepoStatCode.FAILED 149 | 150 | # Delete process state after some time 151 | await self._delete_process_state_after(request_no, 60) 152 | 153 | async def fetch_single_data(self, name: NonStrictName, forwarding_hint: Optional[list[NonStrictName]]): 154 | """ 155 | Fetch one Data packet. 156 | :param name: NonStrictName. 157 | :param forwarding_hint: Optional[list[NonStrictName]] 158 | :return: Number of data packets fetched. 159 | """ 160 | try: 161 | data_name, _, _, data_bytes = await self.app.express_interest( 162 | name, need_raw_packet=True, can_be_prefix=False, lifetime=1000, 163 | forwarding_hint=forwarding_hint) 164 | except InterestNack as e: 165 | self.logger.info(f'Nacked with reason={e.reason}') 166 | return 0 167 | except InterestTimeout: 168 | self.logger.info(f'Timeout') 169 | return 0 170 | self.storage.put_data_packet(data_name, data_bytes) 171 | return 1 172 | 173 | async def fetch_segmented_data(self, name, start_block_id: int, end_block_id: Optional[int], 174 | forwarding_hint: Optional[list[NonStrictName]]): 175 | """ 176 | Fetch segmented Data packets. 177 | :param name: NonStrictName. 178 | :param start_block_id: int 179 | :param end_block_id: Optional[int] 180 | :param forwarding_hint: Optional[list[NonStrictName]] 181 | :return: Number of data packets fetched. 182 | """ 183 | semaphore = aio.Semaphore(10) 184 | block_id = start_block_id 185 | async for (data_name, _, _, data_bytes) in ( 186 | concurrent_fetcher(self.app, name, start_block_id, end_block_id, 187 | semaphore, forwarding_hint=forwarding_hint)): 188 | self.storage.put_data_packet(data_name, data_bytes) 189 | block_id += 1 190 | insert_num = block_id - start_block_id 191 | return insert_num 192 | -------------------------------------------------------------------------------- /ndn_python_repo/ndn-python-repo.conf.sample: -------------------------------------------------------------------------------- 1 | --- 2 | repo_config: 3 | # the repo's routable prefix 4 | repo_name: 'testrepo' 5 | # if true, the repo registers the root prefix. If false, client needs to tell repo 6 | # which prefix to register/unregister 7 | register_root: False 8 | 9 | db_config: 10 | # choose one among sqlite3, leveldb, and mongodb 11 | db_type: 'sqlite3' 12 | 13 | # only the chosen db's config will be read 14 | sqlite3: 15 | 'path': '~/.ndn/ndn-python-repo/sqlite3.db' # filepath to sqlite3 database file 16 | leveldb: 17 | 'dir': '~/.ndn/ndn-python-repo/leveldb/' # directory to leveldb database files 18 | mongodb: 19 | 'uri': 'mongodb://127.0.0.1:27017/' 20 | 'db': 'repo' 21 | 'collection': 'data' 22 | 23 | 24 | tcp_bulk_insert: 25 | addr: '0.0.0.0' 26 | port: '7376' 27 | # when register_root is False, whether packets inserted via TCP triggers prefix registration 28 | register_prefix: True 29 | # One prefix in the list is used for registration if: 30 | # 1. register_root is False, and 31 | # 2. register_prefix is True, and 32 | # 3. the prefix matches the name of the sent data 33 | # If 1 and 2 but not 3, the full data name is used to register. 34 | prefixes: 35 | - '/test' 36 | 37 | 38 | logging_config: 39 | # one of 'CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG' 40 | level: 'INFO' 41 | # absolute path to log file. If not given, logs to stdout 42 | # file: 'repo.log' 43 | -------------------------------------------------------------------------------- /ndn_python_repo/ndn-python-repo.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=NDN Repo Service 3 | 4 | [Service] 5 | Type=simple 6 | ExecStart=/usr/local/bin/ndn-python-repo 7 | 8 | [Install] 9 | WantedBy=multi-user.target 10 | -------------------------------------------------------------------------------- /ndn_python_repo/repo.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from ndn.app import NDNApp 3 | from ndn.encoding import Name 4 | 5 | from .storage import * 6 | from .handle import * 7 | 8 | 9 | class Repo(object): 10 | def __init__(self, app: NDNApp, storage: Storage, read_handle: ReadHandle, 11 | write_handle: WriteCommandHandle, delete_handle: DeleteCommandHandle, 12 | sync_handle: SyncCommandHandle, tcp_bulk_insert_handle: TcpBulkInsertHandle, config: dict): 13 | """ 14 | An NDN repo instance. 15 | """ 16 | self.prefix = Name.from_str(config['repo_config']['repo_name']) 17 | self.app = app 18 | self.storage = storage 19 | self.write_handle = write_handle 20 | self.read_handle = read_handle 21 | self.delete_handle = delete_handle 22 | self.sync_handle = sync_handle 23 | self.tcp_bulk_insert_handle = tcp_bulk_insert_handle 24 | 25 | self.running = True 26 | self.register_root = config['repo_config']['register_root'] 27 | self.logger = logging.getLogger(__name__) 28 | 29 | async def listen(self): 30 | """ 31 | Configure pubsub to listen on prefix. The handles share the same pb, so only need to be 32 | done once. 33 | 34 | This method need to be called to make repo working. 35 | """ 36 | # Recover registered prefix to enable hot restart 37 | if not self.register_root: 38 | self.recover_registered_prefixes() 39 | self.recover_sync_states() 40 | 41 | # Init PubSub 42 | self.write_handle.pb.set_publisher_prefix(self.prefix) 43 | self.write_handle.pb.set_base_prefix(self.prefix) 44 | self.delete_handle.pb.set_base_prefix(self.prefix) 45 | self.sync_handle.pb.set_base_prefix(self.prefix) 46 | await self.write_handle.pb.wait_for_ready() 47 | 48 | await self.write_handle.listen(self.prefix) 49 | await self.delete_handle.listen(self.prefix) 50 | await self.sync_handle.listen(self.prefix) 51 | 52 | def recover_registered_prefixes(self): 53 | prefixes = self.write_handle.get_registered_prefix_in_storage(self.storage) 54 | for prefix in prefixes: 55 | self.logger.info(f'Existing Prefix Found: {Name.to_str(prefix)}') 56 | self.read_handle.listen(prefix) 57 | 58 | def recover_sync_states(self): 59 | states = {} 60 | groups = self.sync_handle.get_sync_groups_in_storage(self.storage) 61 | for group in groups: 62 | group_states = self.sync_handle.get_sync_states_in_storage(self.storage, group) 63 | states[Name.to_str(group)] = group_states 64 | self.sync_handle.recover_from_states(states) -------------------------------------------------------------------------------- /ndn_python_repo/storage/__init__.py: -------------------------------------------------------------------------------- 1 | from .storage_base import Storage 2 | from .storage_factory import create_storage 3 | from .sqlite import SqliteStorage 4 | 5 | # import only supported storage backends 6 | try: 7 | from .leveldb import LevelDBStorage 8 | except ImportError as exc: 9 | pass 10 | 11 | try: 12 | from .mongodb import MongoDBStorage 13 | except ImportError as exc: 14 | pass -------------------------------------------------------------------------------- /ndn_python_repo/storage/leveldb.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import plyvel # fixme: my ide tells me this doesn't exist 4 | from .storage_base import Storage 5 | from typing import Optional 6 | 7 | 8 | class LevelDBStorage(Storage): 9 | 10 | def __init__(self, level_dir: str): 11 | """ 12 | Creates a LevelDB storage instance at disk location ``str``. 13 | 14 | :param level_dir: str. The disk location of the database directory. 15 | """ 16 | super().__init__() 17 | db_dir = os.path.expanduser(level_dir) 18 | if not os.path.exists(db_dir): 19 | try: 20 | os.makedirs(db_dir) 21 | except PermissionError: 22 | raise PermissionError(f'Could not create database directory: {db_dir}') from None 23 | self.db = plyvel.DB(db_dir, create_if_missing=True) 24 | 25 | def _put(self, key: bytes, value: bytes, expire_time_ms: int=None): 26 | """ 27 | Insert value and its expiration time into levelDB, overwrite if already exists. 28 | 29 | :param key: bytes. 30 | :param value: bytes. 31 | :param expire_time_ms: Optional[int]. This data is marked unfresh after ``expire_time_ms``\ 32 | milliseconds. 33 | """ 34 | self.db.put(key, pickle.dumps((value, expire_time_ms))) 35 | 36 | def _put_batch(self, keys: list[bytes], values: list[bytes], expire_time_mss:list[Optional[int]]): 37 | """ 38 | Batch insert. 39 | 40 | :param keys: list[bytes]. 41 | :param values: list[bytes]. 42 | :param expire_time_mss: list[Optional[int]]. The expiration time for each data in ``value``. 43 | """ 44 | with self.db.write_batch() as b: 45 | for key, value, expire_time_ms in zip(keys, values, expire_time_mss): 46 | b.put(key, pickle.dumps((value, expire_time_ms))) 47 | 48 | def _get(self, key: bytes, can_be_prefix=False, must_be_fresh=False) -> bytes | None: 49 | """ 50 | Get value from levelDB. 51 | 52 | :param key: bytes. 53 | :param can_be_prefix: bool. If true, use prefix match instead of exact match. 54 | :param must_be_fresh: bool. If true, ignore expired data. 55 | :return: bytes. The value of the data packet in bytes or None if it can't be found or is not fresh when must_be_fresh is True. 56 | """ 57 | if not can_be_prefix: 58 | record = self.db.get(key) 59 | if record is None: 60 | return None 61 | value, expire_time_ms = pickle.loads(record) 62 | if not must_be_fresh or expire_time_ms is not None and expire_time_ms > self._time_ms(): 63 | return value 64 | else: 65 | return None 66 | else: 67 | for _, v_e in self.db.iterator(prefix=key): 68 | value, expire_time_ms = pickle.loads(v_e) 69 | if not must_be_fresh or expire_time_ms is not None and expire_time_ms > self._time_ms(): 70 | return value 71 | return None 72 | 73 | def _remove(self, key: bytes) -> bool: 74 | """ 75 | Remove value from levelDB. Return whether removal is successful. 76 | 77 | :param key: bytes. 78 | :return: True if a data packet is being removed. 79 | """ 80 | if self._get(key) is not None: 81 | self.db.delete(key) 82 | return True 83 | else: 84 | return False -------------------------------------------------------------------------------- /ndn_python_repo/storage/mongodb.py: -------------------------------------------------------------------------------- 1 | import base64 2 | from pymongo import MongoClient, ReplaceOne 3 | from .storage_base import Storage 4 | from typing import Optional 5 | 6 | 7 | class MongoDBStorage(Storage): 8 | 9 | def __init__(self, db: str, collection: str, uri: str = 'mongodb://127.0.0.1:27017/'): 10 | """ 11 | Init a MongoDB storage with unique index on key. 12 | 13 | :param db: str. Database name. 14 | :param collection: str. Collection name. 15 | """ 16 | super().__init__() 17 | self._db = db 18 | self._collection = collection 19 | self._uri = uri 20 | self.client = MongoClient(self._uri) 21 | self.c_db = self.client[self._db] 22 | self.c_collection = self.c_db[self._collection] 23 | 24 | client = MongoClient(self._uri) 25 | client.server_info() # will throw an exception if not connected 26 | c_db = client[self._db] 27 | c_collection = c_db[self._collection] 28 | c_collection.create_index('key', unique=True) 29 | 30 | def _put(self, key: bytes, value: bytes, expire_time_ms: int=None): 31 | """ 32 | Insert document into MongoDB, overwrite if already exists. MongoDB supports prefix search\ 33 | only on strings, so keys are stored in base16 format. 34 | Base32 and base64 don't work here, because they don't preserve prefix search semantics. 35 | 36 | :param key: bytes. 37 | :param value: bytes. 38 | :param expire_time_ms: Optional[int]. This data is marked unfresh after ``expire_time_ms``\ 39 | milliseconds. 40 | """ 41 | key = base64.b16encode(key).decode() 42 | replace = { 43 | 'key': key, 44 | 'value': value, 45 | 'expire_time_ms': expire_time_ms, 46 | } 47 | self.c_collection.replace_one({'key': key}, replace, upsert=True) 48 | 49 | def _put_batch(self, keys: list[bytes], values: list[bytes], expire_time_mss:list[Optional[int]]): 50 | """ 51 | Batch insert. 52 | 53 | :param keys: list[bytes]. 54 | :param values: list[bytes]. 55 | :param expire_time_mss: list[Optional[int]]. The expiration time for each data in ``value``. 56 | """ 57 | keys = [base64.b16encode(key).decode() for key in keys] 58 | replaces = [] 59 | for key, value, expire_time_ms in zip(keys, values, expire_time_mss): 60 | replaces.append(ReplaceOne({'key': key}, { 61 | 'key': key, 62 | 'value': value, 63 | 'expire_time_ms': expire_time_ms, 64 | }, upsert=True)) 65 | self.c_collection.bulk_write(replaces, ordered=False) 66 | 67 | def _get(self, key: bytes, can_be_prefix=False, must_be_fresh=False) -> Optional[bytes]: 68 | """ 69 | Get document from MongoDB. 70 | 71 | :param key: bytes. 72 | :param can_be_prefix: bool. If true, use prefix match instead of exact match. 73 | :param must_be_fresh: bool. If true, ignore expired data. 74 | :return: The value of the data packet. 75 | """ 76 | key = base64.b16encode(key).decode() 77 | query = dict() 78 | if not can_be_prefix: 79 | query.update({'key': key}) 80 | else: 81 | query.update({'key': {'$regex': '^' + key}}) 82 | if must_be_fresh: 83 | query.update({'expire_time_ms': {'$gt': self._time_ms()}}) 84 | ret = self.c_collection.find_one(query) 85 | if ret: 86 | return ret['value'] 87 | else: 88 | return None 89 | 90 | def _remove(self, key: bytes) -> bool: 91 | """ 92 | Remove value from MongoDB, return whether removal is successful. 93 | 94 | :param key: bytes. 95 | :return: True if a data packet is being removed. 96 | """ 97 | key = base64.b16encode(key).decode() 98 | return self.c_collection.delete_one({"key": key}).deleted_count > 0 -------------------------------------------------------------------------------- /ndn_python_repo/storage/sqlite.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sqlite3 3 | from typing import Optional 4 | from .storage_base import Storage 5 | 6 | 7 | class SqliteStorage(Storage): 8 | 9 | def __init__(self, db_path: str): 10 | """ 11 | Init table "data" with the attribute ``key`` being the primary key. 12 | 13 | :param db_path: str. Path to database file. 14 | """ 15 | super().__init__() 16 | db_path = os.path.expanduser(db_path) 17 | if len(os.path.dirname(db_path)) > 0 and not os.path.exists(os.path.dirname(db_path)): 18 | try: 19 | os.makedirs(os.path.dirname(db_path)) 20 | except PermissionError: 21 | raise PermissionError(f'Could not create database directory: {db_path}') from None 22 | 23 | self.conn = sqlite3.connect(os.path.expanduser(db_path)) 24 | c = self.conn.cursor() 25 | c.execute(""" 26 | CREATE TABLE IF NOT EXISTS data ( 27 | key BLOB PRIMARY KEY, 28 | value BLOB, 29 | expire_time_ms INTEGER 30 | ) 31 | """) 32 | self.conn.commit() 33 | 34 | def _put(self, key: bytes, value: bytes, expire_time_ms=None): 35 | """ 36 | Insert value and its expiration time into sqlite3, overwrite if already exists. 37 | 38 | :param key: bytes. 39 | :param value: bytes. 40 | :param expire_time_ms: Optional[int]. This data is marked unfresh after ``expire_time_ms``\ 41 | milliseconds. 42 | """ 43 | c = self.conn.cursor() 44 | c.execute('INSERT OR REPLACE INTO data (key, value, expire_time_ms) VALUES (?, ?, ?)', 45 | (key, value, expire_time_ms)) 46 | self.conn.commit() 47 | 48 | def _put_batch(self, keys: list[bytes], values: list[bytes], expire_time_mss:list[Optional[int]]): 49 | """ 50 | Batch insert. 51 | 52 | :param keys: list[bytes]. 53 | :param values: list[bytes]. 54 | :param expire_time_mss: list[Optional[int]]. The expiration time for each data in ``value``. 55 | """ 56 | c = self.conn.cursor() 57 | c.executemany('INSERT OR REPLACE INTO data (key, value, expire_time_ms) VALUES (?, ?, ?)', 58 | zip(keys, values, expire_time_mss)) 59 | self.conn.commit() 60 | 61 | def _get(self, key: bytes, can_be_prefix=False, must_be_fresh=False) -> Optional[bytes]: 62 | """ 63 | Get value from sqlite3. 64 | 65 | :param key: bytes. 66 | :param can_be_prefix: bool. If true, use prefix match instead of exact match. 67 | :param must_be_fresh: bool. If true, ignore expired data. 68 | :return: The value of the data packet. 69 | """ 70 | c = self.conn.cursor() 71 | query = 'SELECT value FROM data WHERE ' 72 | if must_be_fresh: 73 | query += f'(expire_time_ms > {self._time_ms()}) AND ' 74 | if can_be_prefix: 75 | query += 'hex(key) LIKE ?' 76 | c.execute(query, (key.hex() + '%', )) 77 | else: 78 | query += 'key = ?' 79 | c.execute(query, (key, )) 80 | ret = c.fetchone() 81 | return ret[0] if ret else None 82 | 83 | def _remove(self, key: bytes) -> bool: 84 | """ 85 | Remove value from sqlite. Return whether removal is successful. 86 | 87 | :param key: bytes. 88 | :return: True if a data packet is being removed. 89 | """ 90 | c = self.conn.cursor() 91 | n_removed = c.execute('DELETE FROM data WHERE key = ?', (key, )).rowcount 92 | self.conn.commit() 93 | return n_removed > 0 -------------------------------------------------------------------------------- /ndn_python_repo/storage/storage_base.py: -------------------------------------------------------------------------------- 1 | import asyncio as aio 2 | from hashlib import sha256 3 | import logging 4 | from contextlib import suppress 5 | from ndn.encoding.tlv_var import parse_tl_num 6 | from ndn.encoding import Name, Component, parse_data, NonStrictName 7 | from ndn.name_tree import NameTrie 8 | import time 9 | from typing import Optional 10 | 11 | 12 | class Storage: 13 | cache = NameTrie() 14 | 15 | def __init__(self): 16 | """ 17 | Interface for a unified key-value storage API. 18 | """ 19 | self.write_back_task = aio.create_task(self._periodic_write_back()) 20 | self.logger = logging.getLogger(__name__) 21 | 22 | def __del__(self): 23 | self.write_back_task.cancel() 24 | 25 | def _put(self, key: bytes, data: bytes, expire_time_ms: int=None): 26 | raise NotImplementedError 27 | 28 | def _put_batch(self, keys: list[bytes], values: list[bytes], expire_time_mss:list[Optional[int]]): 29 | raise NotImplementedError 30 | 31 | def _get(self, key: bytes, can_be_prefix: bool=False, must_be_fresh: bool=False) -> bytes: 32 | raise NotImplementedError 33 | 34 | def _remove(self, key: bytes) -> bool: 35 | raise NotImplementedError 36 | 37 | 38 | ###### wrappers around key-value store 39 | async def _periodic_write_back(self): 40 | with suppress(aio.CancelledError): 41 | while True: 42 | self._write_back() 43 | await aio.sleep(10) 44 | 45 | @staticmethod 46 | def _get_name_bytes_wo_tl(name: NonStrictName) -> bytes: 47 | # remove name's TL as key to support efficient prefix search 48 | name = Name.to_bytes(name) 49 | offset = 0 50 | offset += parse_tl_num(name, offset)[1] 51 | offset += parse_tl_num(name, offset)[1] 52 | return name[offset:] 53 | 54 | @staticmethod 55 | def _time_ms(): 56 | return int(time.time() * 1000) 57 | 58 | def _write_back(self): 59 | keys = [] 60 | values = [] 61 | expire_time_mss = [] 62 | for name, (data, expire_time_ms) in self.cache.iteritems(prefix=[], shallow=True): 63 | keys.append(self._get_name_bytes_wo_tl(name)) 64 | values.append(data) 65 | expire_time_mss.append(expire_time_ms) 66 | if len(keys) > 0: 67 | self._put_batch(keys, values, expire_time_mss) 68 | self.logger.info(f'Cache write back {len(keys)} items') 69 | self.cache = NameTrie() 70 | 71 | def put_data_packet(self, name: NonStrictName, data: bytes): 72 | """ 73 | Insert a data packet named ``name`` with value ``data``. 74 | This method will parse ``data`` to get its freshnessPeriod, and compute its expiration time\ 75 | by adding the freshnessPeriod to the current time. 76 | 77 | :param name: NonStrictName. The name of the data packet. 78 | :param data: bytes. The value of the data packet. 79 | """ 80 | _, meta_info, _, _ = parse_data(data) 81 | expire_time_ms = self._time_ms() 82 | if meta_info.freshness_period: 83 | expire_time_ms += meta_info.freshness_period 84 | 85 | # write data packet and freshness_period to cache 86 | name = Name.normalize(name) 87 | self.cache[name] = (data, expire_time_ms) 88 | self.logger.info(f'Cache save: {Name.to_str(name)}') 89 | 90 | def get_data_packet(self, name: NonStrictName, can_be_prefix: bool=False, 91 | must_be_fresh: bool=False) -> Optional[bytes]: 92 | """ 93 | Get a data packet named ``name``. 94 | 95 | :param name: NonStrictName. The name of the data packet. 96 | :param can_be_prefix: bool. If true, use prefix match instead of exact match. 97 | :param must_be_fresh: bool. If true, ignore expired data. 98 | :return: The value of the data packet. 99 | """ 100 | # can_be_prefix must be set to False by default because _delete_single_data would not otherwise be specific enough. 101 | # must_be_fresh must be set to False by default because we want the delete commands to find data we want deleted, regardless of whether it is fresh or not. 102 | name = Name.normalize(name) 103 | if Component.get_type(name[-1]) == Component.TYPE_IMPLICIT_SHA256: 104 | data = self.get_data_packet(name[:-1], can_be_prefix, must_be_fresh) 105 | if sha256(data).digest() == Component.get_value(name[-1]): 106 | self.logger.info('Data digest matches the ImplicitSha256Digest') 107 | return data 108 | else: 109 | raise ValueError("Data digest does not match ImplicitSha256Digest") 110 | else: 111 | # cache lookup 112 | try: 113 | if not can_be_prefix: 114 | data, expire_time_ms = self.cache[name] 115 | if not must_be_fresh or expire_time_ms > self._time_ms(): 116 | self.logger.info('get from cache') 117 | return data 118 | else: 119 | it = self.cache.itervalues(prefix=name, shallow=True) 120 | while True: 121 | data, expire_time_ms = next(it) 122 | if not must_be_fresh or expire_time_ms > self._time_ms(): 123 | self.logger.info('get from cache') 124 | return data 125 | # not in cache, lookup in storage 126 | except (KeyError, StopIteration): 127 | key = self._get_name_bytes_wo_tl(name) 128 | return self._get(key, can_be_prefix, must_be_fresh) 129 | 130 | def remove_data_packet(self, name: NonStrictName) -> bool: 131 | """ 132 | Remove a data packet named ``name``. 133 | 134 | :param name: NonStrictName. The name of the data packet. 135 | :return: True if a data packet is being removed. 136 | """ 137 | removed = False 138 | name = Name.normalize(name) 139 | try: 140 | del self.cache[name] 141 | removed = True 142 | except KeyError: 143 | pass 144 | if self._remove(self._get_name_bytes_wo_tl(name)): 145 | removed = True 146 | return removed 147 | -------------------------------------------------------------------------------- /ndn_python_repo/storage/storage_factory.py: -------------------------------------------------------------------------------- 1 | """ 2 | Factory for storage handles. 3 | 4 | @Author jonnykong@cs.ucla.edu 5 | @Date 2020-02-16 6 | """ 7 | 8 | from .sqlite import SqliteStorage 9 | 10 | # import only supported storage backends 11 | try: 12 | from .leveldb import LevelDBStorage 13 | except ImportError as exc: 14 | pass 15 | 16 | try: 17 | from .mongodb import MongoDBStorage 18 | except ImportError as exc: 19 | pass 20 | 21 | 22 | def create_storage(config): 23 | """ 24 | Factory method to create storage handle. 25 | :param config: config object created by parsing yaml 26 | :return: handle 27 | """ 28 | db_type = config['db_type'] 29 | 30 | try: 31 | if db_type == 'sqlite3': 32 | db_path = config[db_type]['path'] 33 | ret = SqliteStorage(db_path) 34 | elif db_type == 'leveldb': 35 | db_dir = config[db_type]['dir'] 36 | ret = LevelDBStorage(db_dir) 37 | elif db_type == 'mongodb': 38 | db_name = config[db_type]['db'] 39 | db_collection = config[db_type]['collection'] 40 | db_uri = config[db_type]['uri'] 41 | ret = MongoDBStorage(db_name, db_collection, db_uri) 42 | else: 43 | raise NameError() 44 | 45 | except NameError: 46 | raise NotImplementedError(f'Unsupported database backend: {db_type}') 47 | 48 | return ret -------------------------------------------------------------------------------- /ndn_python_repo/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .concurrent_fetcher import concurrent_fetcher, IdNamingConv 2 | from .pubsub import PubSub 3 | from .passive_svs import PassiveSvs -------------------------------------------------------------------------------- /ndn_python_repo/utils/concurrent_fetcher.py: -------------------------------------------------------------------------------- 1 | # ----------------------------------------------------------------------------- 2 | # Concurrent segment fetcher. 3 | # 4 | # @Author jonnykong@cs.ucla.edu tianyuan@cs.ucla.edu 5 | # @Date 2024-05-24 6 | # ----------------------------------------------------------------------------- 7 | 8 | import asyncio as aio 9 | import logging 10 | from ndn.app import NDNApp 11 | from ndn.types import InterestNack, InterestTimeout, InterestCanceled 12 | from ndn.encoding import Name, NonStrictName, Component 13 | from typing import Optional 14 | 15 | class IdNamingConv: 16 | SEGMENT = 1 17 | SEQUENCE = 2 18 | NUMBER = 3 19 | 20 | async def concurrent_fetcher(app: NDNApp, name: NonStrictName, start_id: int, 21 | end_id: Optional[int], semaphore: aio.Semaphore, **kwargs): 22 | """ 23 | An async-generator to fetch data packets between "`name`/`start_id`" and "`name`/`end_id`"\ 24 | concurrently. 25 | 26 | :param app: NDNApp. 27 | :param name: NonStrictName. Name prefix of Data. 28 | :param start_id: int. The start number. 29 | :param end_id: Optional[int]. The end segment number. If not specified, continue fetching\ 30 | until an interest receives timeout or nack or 3 times. 31 | :param semaphore: aio.Semaphore. Semaphore used to fetch data. 32 | :return: Yield ``(FormalName, MetaInfo, Content, RawPacket)`` tuples in order. 33 | """ 34 | name_conv = IdNamingConv.SEGMENT 35 | max_retries = 15 36 | if 'name_conv' in kwargs: 37 | name_conv = kwargs['name_conv'] 38 | if 'max_retries' in kwargs: 39 | max_retries = kwargs['max_retries'] 40 | cur_id = start_id 41 | final_id = end_id if end_id is not None else 0x7fffffff 42 | is_failed = False 43 | tasks = [] 44 | recv_window = cur_id - 1 45 | seq_to_data_packet = dict() # Buffer for out-of-order delivery 46 | received_or_fail = aio.Event() 47 | name = Name.normalize(name) 48 | logger = logging.getLogger(__name__) 49 | 50 | async def _retry(seq: int): 51 | """ 52 | Retry 3 times fetching data of the given sequence number or fail. 53 | :param seq: block_id of data 54 | """ 55 | nonlocal app, name, semaphore, is_failed, received_or_fail, final_id 56 | if name_conv == IdNamingConv.SEGMENT: 57 | int_name = name + [Component.from_segment(seq)] 58 | elif name_conv == IdNamingConv.SEQUENCE: 59 | int_name = name + [Component.from_sequence_num(seq)] 60 | elif name_conv == IdNamingConv.NUMBER: 61 | # fixme: .from_number apparently requires a second parameter for "type" 62 | int_name = name + [Component.from_number(seq)] 63 | else: 64 | logging.error('Unrecognized naming convention') 65 | return 66 | trial_times = 0 67 | while True: 68 | trial_times += 1 69 | # always retry when max_retries is -1 70 | if 0 <= max_retries < trial_times: 71 | semaphore.release() 72 | is_failed = True 73 | received_or_fail.set() 74 | return 75 | try: 76 | logger.info('Express Interest: {}'.format(Name.to_str(int_name))) 77 | data_name, meta_info, content, data_bytes = await app.express_interest( 78 | int_name, need_raw_packet=True, can_be_prefix=False, lifetime=1000, **kwargs) 79 | 80 | # Save data and update final_id 81 | logging.info('Received data: {}'.format(Name.to_str(data_name))) 82 | if name_conv == IdNamingConv.SEGMENT and \ 83 | meta_info is not None and \ 84 | meta_info.final_block_id is not None: 85 | # we need to change final block id before yielding packets, 86 | # preventing window moving beyond the final block id 87 | final_id = Component.to_number(meta_info.final_block_id) 88 | 89 | # cancel the Interests for non-existing data 90 | for task in aio.all_tasks(): 91 | task_name = task.get_name() 92 | try: 93 | task_num = int(task_name) 94 | except: 95 | continue 96 | if task_num and task_num > final_id \ 97 | and task in tasks: 98 | tasks.remove(task) 99 | task.cancel() 100 | seq_to_data_packet[seq] = (data_name, meta_info, content, data_bytes) 101 | break 102 | except InterestNack as e: 103 | logging.info(f'Interest {Name.to_str(int_name)} nacked with reason={e.reason}') 104 | except InterestTimeout: 105 | logging.info(f'Interest {Name.to_str(int_name)} timeout') 106 | except InterestCanceled: 107 | logging.info(f'Interest {Name.to_str(int_name)} (might legally) cancelled') 108 | return 109 | semaphore.release() 110 | received_or_fail.set() 111 | 112 | async def _dispatch_tasks(): 113 | """ 114 | Dispatch retry() tasks using semaphore. 115 | """ 116 | nonlocal semaphore, tasks, cur_id, final_id, is_failed 117 | while cur_id <= final_id: 118 | await semaphore.acquire() 119 | # in case final_id has been updated while waiting semaphore 120 | # typically happened after the first round trip when we update 121 | # the actual final_id with the final block id obtained from data. 122 | if cur_id > final_id: 123 | # giving back the semaphore 124 | semaphore.release() 125 | break 126 | if is_failed: 127 | received_or_fail.set() 128 | semaphore.release() 129 | break 130 | task = aio.get_event_loop().create_task(_retry(cur_id)) 131 | task.set_name(cur_id) 132 | tasks.append(task) 133 | cur_id += 1 134 | 135 | aio.get_event_loop().create_task(_dispatch_tasks()) 136 | while True: 137 | await received_or_fail.wait() 138 | received_or_fail.clear() 139 | # Re-assemble bytes in order 140 | while recv_window + 1 in seq_to_data_packet: 141 | yield seq_to_data_packet[recv_window + 1] 142 | del seq_to_data_packet[recv_window + 1] 143 | recv_window += 1 144 | # Return if all data have been fetched, or the fetching process failed 145 | if recv_window == final_id: 146 | await aio.gather(*tasks) 147 | return 148 | elif is_failed: 149 | await aio.gather(*tasks) 150 | # New data may return during gather(), need to check again 151 | # TODO: complete misuse of async for & yield. The generator does not make any sense since 152 | # all data are already fetched. 153 | while recv_window + 1 in seq_to_data_packet: 154 | yield seq_to_data_packet[recv_window + 1] 155 | del seq_to_data_packet[recv_window + 1] 156 | recv_window += 1 157 | return 158 | -------------------------------------------------------------------------------- /ndn_python_repo/utils/passive_svs.py: -------------------------------------------------------------------------------- 1 | # ----------------------------------------------------------------------------- 2 | # Passive SVS Listener. 3 | # 4 | # @Author tianyuan@cs.ucla.edu 5 | # @Date 2024-03-29 6 | # ----------------------------------------------------------------------------- 7 | 8 | import logging 9 | from base64 import b64decode, b64encode 10 | from typing import Callable 11 | from ndn.app import NDNApp 12 | from ndn.app_support.svs import StateVecWrapper, SvsState 13 | from ndn.encoding import Name, NonStrictName, DecodeError, FormalName, BinaryStr, InterestParam, parse_interest, \ 14 | parse_tl_num, UintField 15 | from ndn.encoding.ndn_format_0_3 import TypeNumber 16 | from ndn.utils import gen_nonce 17 | 18 | OnMissingDataFunc = Callable[["PassiveSvs"], None] 19 | r""" 20 | Called when there is a missing event. 21 | MUST BE NON-BLOCKING. Therefore, it is not allowed to fetch the missing data in this callback. 22 | It can start a task or trigger a signal to fetch missing data. 23 | """ 24 | 25 | 26 | class PassiveSvs: 27 | base_prefix: FormalName 28 | on_missing_data: OnMissingDataFunc 29 | 30 | local_sv: dict[bytes, int] 31 | state: SvsState 32 | running: bool 33 | ndn_app: NDNApp | None 34 | 35 | def __init__(self, base_prefix: NonStrictName, 36 | on_missing_data: OnMissingDataFunc): 37 | self.base_prefix = Name.normalize(base_prefix) 38 | self.on_missing_data = on_missing_data 39 | self.local_sv = {} 40 | self.inst_buffer = {} 41 | self.state = SvsState.SyncSteady 42 | self.running = False 43 | self.ndn_app = None 44 | self.logger = logging.getLogger(__name__) 45 | 46 | def encode_into_states(self): 47 | states = {} 48 | inst_buffer_enc = {} 49 | for nid, inst in self.inst_buffer.items(): 50 | inst_buffer_enc[nid] = b64encode(inst).decode('utf-8') 51 | states['local_sv'] = self.local_sv 52 | states['inst_buffer'] = inst_buffer_enc 53 | return states 54 | 55 | def decode_from_states(self, states: dict): 56 | inst_buffer_dec = {} 57 | for nid, inst in states['inst_buffer'].items(): 58 | inst_buffer_dec[nid] = b64decode(inst) 59 | self.local_sv = states['local_sv'] 60 | self.inst_buffer = inst_buffer_dec 61 | 62 | def send_interest(self, interest_wire): 63 | final_name, interest_param, _, _ = parse_interest(interest_wire) 64 | interest_param.nonce = gen_nonce() 65 | # a bit hack, refresh the nonce and re-encode 66 | wire_ptr = 0 67 | while wire_ptr + 5 < len(interest_wire): 68 | typ, typ_len = parse_tl_num(interest_wire[wire_ptr:], 0) 69 | size, siz_len = parse_tl_num(interest_wire[wire_ptr:], typ_len) 70 | if typ != TypeNumber.NONCE or typ_len != 1 or \ 71 | size != 4 or siz_len != 1: 72 | wire_ptr += 1 73 | continue 74 | else: 75 | # that's it! 76 | wire = bytearray(interest_wire) 77 | nonce = UintField(TypeNumber.NONCE, fixed_len=4) 78 | markers = {} 79 | nonce.encoded_length(interest_param.nonce, markers) 80 | nonce.encode_into(interest_param.nonce, markers, wire, wire_ptr) 81 | break 82 | logging.info(f'Sending buffered interest: {Name.to_str(final_name)}') 83 | # do not await for this 84 | self.ndn_app.express_raw_interest(final_name, interest_param, wire) 85 | 86 | def sync_handler(self, name: FormalName, _param: InterestParam, _app_param: BinaryStr | None, 87 | raw_packet: BinaryStr) -> None: 88 | if len(name) != len(self.base_prefix) + 2: 89 | logging.error(f'Received invalid Sync Interest: {Name.to_str(name)}') 90 | return 91 | _, _, _, sig_ptrs = parse_interest(raw_packet) 92 | sig_info = sig_ptrs.signature_info 93 | if sig_info and sig_info.key_locator and sig_info.key_locator.name: 94 | logging.info(f'Received Sync Interest: {Name.to_str(sig_info.key_locator.name)}') 95 | else: 96 | logging.info(f'Drop unsigned or improperly signed Sync Snterests') 97 | return 98 | try: 99 | remote_sv_pkt = StateVecWrapper.parse(name[-2]).val 100 | except (DecodeError, IndexError) as e: 101 | logging.error(f'Unable to decode state vector [{Name.to_str(name)}]: {e}') 102 | return 103 | if remote_sv_pkt is None: 104 | logging.error(f'Sync Interest does not contain state vectors') 105 | return 106 | remote_sv = remote_sv_pkt.entries 107 | 108 | # No lock is needed since we do not await 109 | # Compare state vectors 110 | rsv_dict = {} 111 | for rsv in remote_sv: 112 | if not rsv.node_id: 113 | continue 114 | rsv_id = Name.to_str(rsv.node_id) 115 | rsv_seq = rsv.seq_no 116 | rsv_dict[rsv_id] = rsv_seq 117 | 118 | need_fetch = False 119 | for rsv_id, rsv_seq in rsv_dict.items(): 120 | already_sent = [] 121 | lsv_seq = self.local_sv.get(rsv_id, 0) 122 | if lsv_seq < rsv_seq: 123 | # Remote is latest 124 | need_fetch = True 125 | self.local_sv[rsv_id] = rsv_seq 126 | self.logger.debug(f'Missing data for: [{Name.to_str(rsv_id)}]: {lsv_seq} < {rsv_seq}') 127 | self.inst_buffer[rsv_id] = raw_packet 128 | elif lsv_seq > rsv_seq: 129 | # Local is latest 130 | self.logger.debug(f'Outdated remote on: [{Name.to_str(rsv_id)}]: {rsv_seq} < {lsv_seq}') 131 | if rsv_id in self.inst_buffer: 132 | raw_inst = self.inst_buffer[rsv_id] 133 | else: raw_inst = None 134 | if raw_inst and raw_inst not in already_sent: 135 | already_sent.append(raw_inst) 136 | self.send_interest(raw_inst) 137 | else: pass 138 | 139 | # Notify remote there are missing nodes 140 | diff = self.local_sv.keys() - rsv_dict.keys() 141 | if len(diff) > 0: 142 | self.logger.info(f'Remote missing nodes: {list(diff)}') 143 | # Missing nodes may only exist in other nodes' sync interest, 144 | # therefore we have to send all buffered sync interest out, 145 | # unless they were sent before. 146 | already_sent = [] 147 | for _, raw_inst in self.inst_buffer.items(): 148 | if raw_inst not in already_sent: 149 | already_sent.append(raw_inst) 150 | self.send_interest(raw_inst) 151 | if need_fetch: 152 | self.on_missing_data(self) 153 | 154 | def start(self, ndn_app: NDNApp): 155 | if self.running: 156 | raise RuntimeError(f'Sync is already running @[{Name.to_str(self.base_prefix)}]') 157 | self.running = True 158 | self.ndn_app = ndn_app 159 | self.ndn_app.route(self.base_prefix, need_raw_packet=True)(self.sync_handler) 160 | 161 | async def stop(self): 162 | if not self.running: 163 | return 164 | self.running = False 165 | await self.ndn_app.unregister(self.base_prefix) 166 | self.logger.info("Passive SVS stopped.") 167 | -------------------------------------------------------------------------------- /ndn_python_repo/utils/pubsub.py: -------------------------------------------------------------------------------- 1 | # ----------------------------------------------------------------------------- 2 | # Pub-sub API. 3 | # 4 | # This pub-sub library provides best-effort, at-most-once delivery guarantee. 5 | # If there are no subscribers reachable when a message is published, this 6 | # message will not be re-transmitted. 7 | # If there are multiple subscribers reachable, the nearest subscriber will be 8 | # notified of the published message in an any-cast style. 9 | # 10 | # @Author jonnykong@cs.ucla.edu 11 | # @Date 2020-05-08 12 | # ----------------------------------------------------------------------------- 13 | 14 | 15 | import asyncio as aio 16 | import logging 17 | from ndn.app import NDNApp 18 | from ndn.encoding import TlvModel, ModelField, NameField, BytesField 19 | from ndn.encoding import Name, NonStrictName, Component, InterestParam 20 | from ndn.name_tree import NameTrie 21 | from ndn.types import InterestNack, InterestTimeout 22 | import os 23 | 24 | 25 | class PubSub(object): 26 | # TODO: prefix has wrong type notation 27 | # Probably Optional[FormalName]. Need investigation. 28 | # Please do not call with unnormalized name. 29 | def __init__(self, app: NDNApp, prefix: NonStrictName=None, forwarding_hint: NonStrictName=None): 30 | """ 31 | Initialize a ``PubSub`` instance with identity ``prefix`` and can be reached at \ 32 | ``forwarding_hint``. 33 | TODO: support msg larger than MTU 34 | 35 | :param app: NDNApp. 36 | :param prefix: NonStrictName. The identity of this ``PubSub`` instance. The publisher needs\ 37 | a prefix under which can publish data. Note that you cannot initialize two ``PubSub``\ 38 | instances with the same ``prefix`` on the same node, since it will cause double\ 39 | registration error. 40 | :param forwarding_hint: NonStrictName. When working as publisher, if ``prefix`` is not\ 41 | reachable, the subscriber can use ``forwarding_hint`` to reach the publisher. 42 | """ 43 | self.app = app 44 | self.publisher_prefix = prefix 45 | self.forwarding_hint = forwarding_hint 46 | self.base_prefix = None 47 | self.published_data = NameTrie() # name -> packet 48 | self.topic_to_cb = NameTrie() 49 | self.nonce_processed = set() # used by subscriber to de-duplicate notify interests 50 | self.logger = logging.getLogger(__name__) 51 | 52 | # prefix has wrong type notation. do not call with unnormalized name. 53 | def set_publisher_prefix(self, prefix: NonStrictName): 54 | """ 55 | Set the identity of the publisher after initialization. 56 | Need to be called before ``_wait_for_ready()``. 57 | 58 | :param prefix: NonStrictName. The identity of this ``PubSub`` instance. 59 | """ 60 | self.publisher_prefix = prefix 61 | 62 | # prefix has wrong type notation. do not call with unnormalized name. 63 | def set_base_prefix(self, prefix: NonStrictName): 64 | """ 65 | Avoid registering too many prefixes, by registering ``prefix`` with NFD. All other prefixes\ 66 | under ``prefix`` will be registered with interest filters, and will not have to be\ 67 | registered with NFD. 68 | Need to be called before ``_wait_for_ready()``. 69 | 70 | :param prefix: NonStrictName. The base prefix to register. 71 | """ 72 | self.base_prefix = prefix 73 | 74 | async def wait_for_ready(self): 75 | """ 76 | Need to be called to wait for pub-sub to be ready. 77 | """ 78 | # Wait until app connected, otherwise app.register() throws an NetworkError 79 | while not self.app.face.running: 80 | await aio.sleep(0.1) 81 | 82 | if self.base_prefix is not None: 83 | try: 84 | await self.app.register(self.base_prefix, func=None) 85 | except ValueError: 86 | pass 87 | 88 | try: 89 | if self.base_prefix is not None and Name.is_prefix(self.base_prefix, self.publisher_prefix + ['msg']): 90 | self.app.set_interest_filter(self.publisher_prefix + ['msg'], self._on_msg_interest) 91 | else: 92 | await self.app.register(self.publisher_prefix + ['msg'], self._on_msg_interest) 93 | except ValueError: 94 | # duplicate registration 95 | pass 96 | 97 | 98 | def subscribe(self, topic: NonStrictName, cb: callable): 99 | """ 100 | Subscribe to ``topic`` with ``cb``. 101 | 102 | :param topic: NonStrictName. The topic to subscribe to. 103 | :param cb: callable. A callback that will be called when a message under ``topic`` is\ 104 | received. This function takes one ``bytes`` argument. 105 | """ 106 | aio.ensure_future(self._subscribe_helper(topic, cb)) 107 | 108 | def unsubscribe(self, topic: NonStrictName): 109 | """ 110 | Unsubscribe from ``topic``. 111 | 112 | :param topic: NonStrictName. The topic to unsubscribe from. 113 | """ 114 | self.logger.info(f'unsubscribing topic: {Name.to_str(topic)}') 115 | topic = Name.normalize(topic) 116 | del self.topic_to_cb[topic] 117 | 118 | async def publish(self, topic: NonStrictName, msg: bytes): 119 | """ 120 | Publish ``msg`` to ``topic``. Make several attempts until the subscriber returns a\ 121 | response. 122 | 123 | :param topic: NonStrictName. The topic to publish ``msg`` to. 124 | :param msg: bytes. The message to publish. The pub-sub API does not make any assumptions on\ 125 | the format of this message. 126 | :return: Return true if received response from a subscriber. 127 | """ 128 | self.logger.info(f'publishing a message to topic: {Name.to_str(topic)}') 129 | # generate a nonce for each message. Nonce is a random sequence of bytes 130 | nonce = os.urandom(4) 131 | # wrap msg in a data packet named //msg//nonce 132 | data_name = Name.normalize(self.publisher_prefix + ['msg'] + topic + [Component.from_bytes(nonce)]) 133 | self.published_data[data_name] = self.app.prepare_data(data_name, msg) 134 | 135 | # prepare notify interest 136 | int_name = topic + ['notify'] 137 | app_param = NotifyAppParam() 138 | app_param.publisher_prefix = self.publisher_prefix 139 | app_param.notify_nonce = nonce 140 | if self.forwarding_hint: 141 | app_param.publisher_fwd_hint = ForwardingHint() 142 | app_param.publisher_fwd_hint.name = self.forwarding_hint 143 | 144 | aio.ensure_future(self._erase_publisher_state_after(data_name, 5)) 145 | 146 | # express notify interest 147 | n_retries = 3 148 | is_success = False 149 | while n_retries > 0: 150 | try: 151 | self.logger.debug(f'sending notify interest: {Name.to_str(int_name)}') 152 | _, _, _ = await self.app.express_interest( 153 | int_name, app_param.encode(), must_be_fresh=False, can_be_prefix=False) 154 | is_success = True 155 | break 156 | except InterestNack as e: 157 | self.logger.debug(f'Nacked with reason: {e.reason}') 158 | await aio.sleep(1) 159 | n_retries -= 1 160 | except InterestTimeout: 161 | self.logger.debug(f'Timeout') 162 | n_retries -= 1 163 | 164 | # if receiving notify response, the subscriber has finished fetching msg 165 | if is_success: 166 | self.logger.debug(f'received notify response for: {data_name}') 167 | else: 168 | self.logger.debug(f'did not receive notify response for: {data_name}') 169 | await self._erase_publisher_state_after(data_name, 0) 170 | return is_success 171 | 172 | async def _subscribe_helper(self, topic: NonStrictName, cb: callable): 173 | """ 174 | Async helper for ``subscribe()``. 175 | """ 176 | topic = Name.normalize(topic) 177 | self.topic_to_cb[topic] = cb 178 | to_register = topic + ['notify'] 179 | if self.base_prefix is not None and Name.is_prefix(self.base_prefix, to_register): 180 | self.app.set_interest_filter(to_register, self._on_notify_interest) 181 | self.logger.info(f'Subscribing to topic (with interest filter): {Name.to_str(topic)}') 182 | else: 183 | await self.app.register(to_register, self._on_notify_interest) 184 | self.logger.info(f'Subscribing to topic: {Name.to_str(topic)}') 185 | 186 | def _on_notify_interest(self, int_name, int_param, app_param): 187 | aio.ensure_future(self._process_notify_interest(int_name, int_param, app_param)) 188 | 189 | async def _process_notify_interest(self, int_name, int_param, app_param): 190 | """ 191 | Async helper for ``_on_notify_interest()``. 192 | """ 193 | # fixme: why isn't int_param used? 194 | self.logger.debug(f'received notify interest: {Name.to_str(int_name)}') 195 | topic = int_name[:-2] # remove digest and `notify` 196 | 197 | # parse notify interest 198 | app_param = NotifyAppParam.parse(app_param) 199 | publisher_prefix = app_param.publisher_prefix 200 | notify_nonce = app_param.notify_nonce 201 | publisher_fwd_hint = app_param.publisher_fwd_hint 202 | int_param = InterestParam() 203 | if publisher_fwd_hint: 204 | # support only 1 forwarding hint now 205 | int_param.forwarding_hint = [(0x0, publisher_fwd_hint.name)] 206 | 207 | # send msg interest, retransmit 3 times 208 | msg_int_name = publisher_prefix + ['msg'] + topic + [Component.from_bytes(notify_nonce)] 209 | n_retries = 3 210 | 211 | # de-duplicate notify interests of the same nonce 212 | if notify_nonce in self.nonce_processed: 213 | self.logger.info(f'Received duplicate notify interest for nonce {notify_nonce}') 214 | return 215 | self.nonce_processed.add(notify_nonce) 216 | aio.ensure_future(self._erase_subscriber_state_after(notify_nonce, 60)) 217 | 218 | msg = None 219 | while n_retries > 0: 220 | try: 221 | self.logger.debug(f'sending msg interest: {Name.to_str(msg_int_name)}') 222 | data_name, meta_info, msg = await self.app.express_interest( 223 | msg_int_name, int_param=int_param) 224 | break 225 | except InterestNack as e: 226 | self.logger.debug(f'Nacked with reason: {e.reason}') 227 | await aio.sleep(1) 228 | n_retries -= 1 229 | except InterestTimeout: 230 | self.logger.debug(f'Timeout') 231 | n_retries -= 1 232 | if msg is None: 233 | return 234 | 235 | # pass msg to application 236 | self.logger.info(f'received subscribed msg: {Name.to_str(msg_int_name)}') 237 | self.topic_to_cb[topic](bytes(msg)) 238 | 239 | # acknowledge notify interest with an empty data packet 240 | self.logger.debug(f'acknowledging notify interest {Name.to_str(int_name)}') 241 | self.app.put_data(int_name, None) 242 | 243 | def _on_msg_interest(self, int_name, int_param, app_param): 244 | aio.ensure_future(self._process_msg_interest(int_name, int_param, app_param)) 245 | 246 | async def _process_msg_interest(self, int_name, int_param, app_param): 247 | """ 248 | Async helper for ``_on_msg_interest()``. 249 | The msg interest has the format of ``//msg//``. 250 | """ 251 | # fixme: neither int_param nor app_param are used here 252 | self.logger.debug(f'received msg interest: {Name.to_str(int_name)}') 253 | if int_name in self.published_data: 254 | self.app.put_raw_packet(self.published_data[int_name]) 255 | self.logger.debug(f'reply msg with name {Name.to_str(int_name)}') 256 | else: 257 | self.logger.debug(f'no matching msg with name {Name.to_str(int_name)}') 258 | 259 | async def _erase_publisher_state_after(self, name: NonStrictName, timeout: int): 260 | """ 261 | Erase data with name ``name`` after ``timeout`` from application cache. 262 | """ 263 | await aio.sleep(timeout) 264 | if name in self.published_data: 265 | del self.published_data[name] 266 | self.logger.debug(f'erased state for data {Name.to_str(name)}') 267 | 268 | async def _erase_subscriber_state_after(self, notify_nonce: bytes, timeout: int): 269 | """ 270 | Erase state associated with nonce ``nonce`` after ``timeout``. 271 | """ 272 | await aio.sleep(timeout) 273 | if notify_nonce in self.nonce_processed: 274 | self.nonce_processed.remove(notify_nonce) 275 | 276 | 277 | class ForwardingHint(TlvModel): 278 | name = NameField() 279 | 280 | class NotifyAppParam(TlvModel): 281 | """ 282 | Used to serialize application parameters for PubSub notify interest. 283 | """ 284 | publisher_prefix = NameField() 285 | notify_nonce = BytesField(128) 286 | publisher_fwd_hint = ModelField(211, ForwardingHint) 287 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "ndn-python-repo" 3 | version = "0.4" 4 | description = "An NDN Repo implementation using Python" 5 | authors = ["Zhaoning Kong "] 6 | maintainers = [ 7 | "Xinyu Ma ", 8 | "Tianyuan Yu ", 9 | ] 10 | license = "Apache-2.0" 11 | readme = "README.rst" 12 | homepage = "https://ndn-python-repo.readthedocs.io" 13 | repository = "https://github.com/UCLA-IRL/ndn-python-repo" 14 | documentation = "https://ndn-python-repo.readthedocs.io" 15 | keywords = ["NDN"] 16 | classifiers = [ 17 | "Development Status :: 4 - Beta", 18 | "Topic :: Database", 19 | "Topic :: Internet", 20 | "Topic :: System :: Networking", 21 | "License :: OSI Approved :: Apache Software License", 22 | "Programming Language :: Python :: 3.10", 23 | "Programming Language :: Python :: 3.11", 24 | "Programming Language :: Python :: 3.12", 25 | ] 26 | 27 | packages = [{include = "ndn_python_repo"}] 28 | include = [ 29 | { path = "tests", format = "sdist" }, 30 | { path = "ndn_python_repo/ndn-python-repo.service", format = ["sdist", "wheel"] }, 31 | { path = "ndn_python_repo/ndn-python-repo.conf.sample", format = ["sdist", "wheel"] }, 32 | ] 33 | 34 | [tool.poetry.dependencies] 35 | python = "^3.10" 36 | python-ndn = "^0.4.2" 37 | pyyaml = "^6.0" 38 | setuptools = "^75.0.0" 39 | 40 | # Extra dependencies [dev] 41 | pytest = { version = "^8.0.0", optional = true } 42 | pytest-cov = { version = "^5.0.0", optional = true } 43 | plyvel = { version = "^1.5.0", optional = true } 44 | pymongo = { version = "^4.4.1", optional = true } 45 | 46 | # Extra dependencies [docs] 47 | Sphinx = { version = "^8.0.0", optional = true } 48 | sphinx-rtd-theme = { version = "^3.0.0", optional = true } 49 | sphinx-autodoc-typehints = { version = "^2.5.0", optional = true } 50 | 51 | [tool.poetry.extras] 52 | docs = ["Sphinx", "sphinx-rtd-theme", "sphinx-autodoc-typehints"] 53 | dev = ["pytest", "pytest-cov", "pymongo"] 54 | leveldb = ["plyvel"] 55 | mongodb = ["pymongo"] 56 | 57 | [tool.poetry.scripts] 58 | ndn-python-repo = "ndn_python_repo.cmd.main:main" 59 | ndn-python-repo-install = "ndn_python_repo.cmd.install:main" 60 | ndn-python-repo-port = "ndn_python_repo.cmd.port:main" 61 | 62 | 63 | [build-system] 64 | requires = ["poetry-core"] 65 | build-backend = "poetry.core.masonry.api" 66 | -------------------------------------------------------------------------------- /tests/concurrent_fetcher_test.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import asyncio as aio 3 | from ndn.app import NDNApp 4 | from ndn.encoding import Name 5 | from ndn.transport.dummy_face import DummyFace 6 | from ndn.security import KeychainDigest 7 | from ndn_python_repo.utils.concurrent_fetcher import concurrent_fetcher 8 | 9 | 10 | class ConcurrentFetcherTestSuite(object): 11 | """ 12 | Abstract test fixture to simulate packet send and recv. 13 | """ 14 | def test_main(self): 15 | aio.run(self.comain()) 16 | 17 | async def comain(self): 18 | face = DummyFace(self.face_proc) 19 | keychain = KeychainDigest() 20 | self.app = NDNApp(face, keychain) 21 | face.app = self.app 22 | await self.app.main_loop(after_start=self.app_main()) 23 | 24 | @abc.abstractmethod 25 | async def face_proc(self, face: DummyFace): 26 | pass 27 | 28 | @abc.abstractmethod 29 | async def app_main(self): 30 | pass 31 | 32 | 33 | class TestConcurrentFetcherBasic(ConcurrentFetcherTestSuite): 34 | async def face_proc(self, face: DummyFace): 35 | await face.consume_output(b'\x05"\x07\x1c\x08\x17test_concurrent_fetcher\x32\x01\x00\x0c\x02\x03\xe8', 36 | timeout=1) 37 | await face.input_packet(b"\x06\x9d\x07\x1c\x08\x17test_concurrent_fetcher\x32\x01\x00" 38 | b"\x14\x07\x18\x01\x00\x19\x02'\x10\x15\rHello, world!" 39 | b"\x16\x1c\x1b\x01\x03\x1c\x17" 40 | b"\x07\x15\x08\x04test\x08\x03KEY\x08\x08\xa0\x04\xf7\xe7\xdd\x0f\x17\xbd" 41 | b"\x17G0E\x02!\x00\x8bD\x12\xacOuY[\xab[\xe3\x04\xea\xd7J\x07\xecxa\x14" 42 | b"\x8d\x88\xf0\xa4\xe5\xf0\x96\xaeI\xfd\xe5\x90\x02 W,/\x13\xf7\xec\x90" 43 | b"\xa5*\xdea\x94\xe9\xa6e5\x15\xbd\xc8P\xa5\xbf\xbeu*um\xf2[XI\xc8") 44 | 45 | async def app_main(self): 46 | semaphore = aio.Semaphore(1) 47 | async for (data_name, _, _, _) in concurrent_fetcher(self.app, Name.from_str('/test_concurrent_fetcher'), 0, 0, semaphore, nonce=None): 48 | assert Name.to_str(data_name) == '/test_concurrent_fetcher/seg=0' 49 | -------------------------------------------------------------------------------- /tests/integration/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | log_cli=true 3 | log_level=INFO 4 | -------------------------------------------------------------------------------- /tests/storage_test.py: -------------------------------------------------------------------------------- 1 | import asyncio as aio 2 | from ndn.encoding import Name 3 | from ndn_python_repo.storage import SqliteStorage 4 | import time 5 | 6 | 7 | class StorageTestFixture(object): 8 | """ 9 | Abstract test cases for all storage types. 10 | """ 11 | storage = None 12 | 13 | @staticmethod 14 | def test_main(_tmp_path): 15 | StorageTestFixture._test_put() 16 | StorageTestFixture._test_get() 17 | StorageTestFixture._test_remove() 18 | StorageTestFixture._test_get_data_packet() 19 | StorageTestFixture._test_freshness_period() 20 | StorageTestFixture._test_freshness_period_again() 21 | StorageTestFixture._test_get_prefix() 22 | StorageTestFixture._test_put_batch() 23 | StorageTestFixture._test_write_back() 24 | 25 | @staticmethod 26 | def _test_put(): 27 | StorageTestFixture.storage._put(b'test_key_1', bytes([0x00, 0x01, 0x02, 0x03, 0x04])) 28 | 29 | @staticmethod 30 | def _test_duplicate_put(): 31 | StorageTestFixture.storage._put(b'test_key_1', bytes([0x00, 0x01, 0x02, 0x03, 0x04])) 32 | StorageTestFixture.storage._put(b'test_key_1', bytes([0x05, 0x06, 0x07, 0x08, 0x09])) 33 | b_out = StorageTestFixture.storage._get(b'test_key_1') 34 | assert b_out == bytes([0x05, 0x06, 0x07, 0x08, 0x09]) 35 | 36 | @staticmethod 37 | def _test_get(): 38 | b_in = bytes([0x00, 0x01, 0x02, 0x03, 0x04]) 39 | StorageTestFixture.storage._put(b'test_key_1', b_in) 40 | b_out = StorageTestFixture.storage._get(b'test_key_1') 41 | assert b_in == b_out 42 | 43 | @staticmethod 44 | def _test_remove(): 45 | StorageTestFixture.storage._put(b'test_key_1', bytes([0x00, 0x01, 0x02, 0x03, 0x04])) 46 | assert StorageTestFixture.storage._remove(b'test_key_1') 47 | assert StorageTestFixture.storage._remove(b'test_key_1') is False 48 | 49 | @staticmethod 50 | def _test_get_data_packet(): 51 | # /test_get_data_packet/0, freshnessPeriod = 10000ms 52 | data_bytes_in = b'\x06\xa2\x07#\x08\x14test_get_data_packet\x08\x010$\x08\x00\x00\x01q\x04\xa3%x\x14\x06\x18\x01\x00\x19\x01\x00\x15\rHello, world!\x16\x1c\x1b\x01\x03\x1c\x17\x07\x15\x08\x04test\x08\x03KEY\x08\x08\xa0\x04\xf7\xe7\xdd\x0f\x17\xbd\x17F0D\x02 P\xcc\r)\xa0\x9c\xc8\xf4E\xe9\xed\x83u\xb2\xfe\\ \xb1\x93\xbb\xbeq5\x18\x91\xd8yl\x96p\xc7\xa5\x02 Q\x07\xad!\xd3\xd5\xff\x07\xbewW~`*\xe9oI\xeb\x01\x12\xe7\xd0\xaf\xf3r\x95\x94q\xb5\xee\xdc\xc9' 53 | StorageTestFixture.storage.put_data_packet(Name.from_str('/test_get_data_packet/0'), data_bytes_in) 54 | data_bytes_out = StorageTestFixture.storage.get_data_packet(Name.from_str('/test_get_data_packet/0')) 55 | assert data_bytes_in == data_bytes_out 56 | 57 | @staticmethod 58 | def _test_freshness_period(): 59 | # /test_freshness_period/0, freshnessPeriod = 0ms 60 | data_bytes_in = b'\x06\xa5\x07$\x08\x15test_freshness_period\x08\x010$\x08\x00\x00\x01q\x04\xa1\xd3\x1b\x14\x06\x18\x01\x00\x19\x01\x00\x15\rHello, world!\x16\x1c\x1b\x01\x03\x1c\x17\x07\x15\x08\x04test\x08\x03KEY\x08\x08\xa0\x04\xf7\xe7\xdd\x0f\x17\xbd\x17H0F\x02!\x00\xc2K\xb7\xa3z\xd5\xd6z\xe0RuX\xa8\x967\xca.\x81!\xb1)\x9a\xf1\xd8\xd8\xcd\x95\x16\xd6\xa9\xb7p\x02!\x00\xe1mb/|$\xc3\xbf\xd3\xb1\x8a\x97\xef\x84\xfe\xebI\x1b5e\xf4\x9f/\xd9\x0e\x9ae\xed7b\xdc/' 61 | StorageTestFixture.storage.put_data_packet(Name.from_str('/test_freshness_period/1'), data_bytes_in) 62 | data_bytes_out = StorageTestFixture.storage.get_data_packet(Name.from_str('/test_freshness_period/1'), 63 | must_be_fresh=True) 64 | assert data_bytes_out is None 65 | 66 | @staticmethod 67 | def _test_freshness_period_again(): 68 | # /test_get_data_packet/0, freshnessPeriod = 3000ms 69 | data_bytes_in = b'\x06\xc3\x07\x1d\x08\x15test_freshness_period\x08\x0122\x01\x00\x14\x0c\x18\x01\x00\x19\x02\x0b\xb8\x1a\x032\x01\x00\x15\x0eHello, World!\n\x16;\x1b\x01\x03\x1c6\x074\x08\tlocalhost\x08\x08operator\x08\x03KEY\x08\x08\xd3\xe9x\xe9\x9cDR0\x08\x04self6\x08\x00\x00\x01\x92T\xa3\xa4\x01\x17G0E\x02!\x00\xcbUrj\xea.\x94\x98\xb5\xe5hG\x14\xad{\xdc|\x9c\xeb\xde\x83,YB?\x83\x1efA\xb8.\xde\x02 k\xd5N\xa0\xef\xa3\xcc~\xfe\xcc\x08,\x08]\tL\xeem\xeff\x9a\x9eoB\xbe\xe1\xae\x8c\x95M\xdb\x00' 70 | StorageTestFixture.storage.put_data_packet(Name.from_str('/test_freshness_period/2'), data_bytes_in) 71 | data_bytes_out = StorageTestFixture.storage.get_data_packet(Name.from_str('/test_freshness_period/2'), 72 | must_be_fresh=True) 73 | assert data_bytes_in == data_bytes_out 74 | time.sleep(3) 75 | data_bytes_out = StorageTestFixture.storage.get_data_packet(Name.from_str('/test_freshness_period/2'), 76 | must_be_fresh=True) 77 | assert data_bytes_out is None 78 | 79 | 80 | @staticmethod 81 | def _test_get_prefix(): 82 | # /test_get_prefix/0, freshnessPeriod = 10000ms 83 | data_bytes_in = b'\x06\x9d\x07\x1e\x08\x0ftest_get_prefix\x08\x010$\x08\x00\x00\x01q\x04\xa3u0\x14\x06\x18\x01\x00\x19\x01\x00\x15\rHello, world!\x16\x1c\x1b\x01\x03\x1c\x17\x07\x15\x08\x04test\x08\x03KEY\x08\x08\xa0\x04\xf7\xe7\xdd\x0f\x17\xbd\x17F0D\x02 L\x16\xe1\xb3v\x11r7"\xcaq\x906\xc1X\xb1N\x168uH\xcb\x18r\x8d\xafHi\x1e\x7fZ", 99 | b'\x06\x91\x07\x11\x08\x0ftest_put_batch1\x14\x07\x18\x01\x00\x19\x02\'\x10\x15\rHello, world!\x16\x1c\x1b\x01\x03\x1c\x17\x07\x15\x08\x04test\x08\x03KEY\x08\x08\xa0\x04\xf7\xe7\xdd\x0f\x17\xbd\x17F0D\x02 2xY\xdb\xa5[\t\x1cxS\xdb$