├── test ├── __init__.py ├── test_cli_fail.py ├── test_packs.py ├── test_index.py ├── test_snapshot.py ├── test_keys.py ├── test_cli_restore.py ├── test_cli.py └── params.py ├── pyrrhic ├── cli │ ├── __init__.py │ ├── state.py │ ├── snapshots.py │ ├── util.py │ ├── ls.py │ ├── main.py │ ├── cat.py │ └── restore.py ├── crypto │ ├── __init__.py │ └── keys.py ├── repo │ ├── __init__.py │ ├── snapshot.py │ ├── repository.py │ ├── index.py │ ├── pack.py │ └── tree.py ├── __init__.py └── util.py ├── .envrc ├── .coveragerc ├── .flake8 ├── pycodestyle.cfg ├── screenshots └── screenshot.png ├── restic_test_repositories ├── restic_test_repository │ ├── config │ ├── data │ │ ├── 22 │ │ │ └── 224dfabf27f07d47d5e0015fbb376039c0150e27958c40d4b26f549e5a4645d3 │ │ ├── 46 │ │ │ └── 46771395523ccd6dda16694f0ce775f9508a4c3e4527c385f55d8efafa36807f │ │ ├── 81 │ │ │ └── 81b0c9b17bcfac837980a154df2422bc1cd02425689258f487b395789566322d │ │ ├── 4b │ │ │ └── 4b24375f07a164e995d06303bfc26f79f94127e4e5c6e1c476495bbee0af7ccc │ │ ├── 6c │ │ │ └── 6c17cc51fe575445c4f01e3832ab4af90fe6c8d6f67a57bcc178a735d3350dc7 │ │ ├── b7 │ │ │ └── b7faaa4eb7471dcf2fb6b72d97c641cbac466320271de88366ac7a68b129a01c │ │ └── f7 │ │ │ └── f7eed29474b7d5756089e9308d58ec9333a7941991938268f713404949bb8987 │ ├── index │ │ └── bd10c8e85d2cdc3267faae1748cf7a334385d021766059580976882556097c0d │ ├── snapshots │ │ ├── 7f9faf70a9889f54124f52e42f0d11d3e5eab185fe423cd2c4bb859ef0f71a8b │ │ ├── d2bbc914bf89a24f16bcb70a4ec1d433856e0a8fef5b73c7dd4fa64a95f3977c │ │ ├── dd62b535d10bd8f24440cc300a868d6bf2f472859f1218883b0a6faca364c10c │ │ └── fb56c7b68e95806244b0080bf8bb05d0132df9228390e4b9dc36e2ba6d3bf1a8 │ └── keys │ │ ├── 98f9e68226bf15a8e9616632df7c9df543e255b388bfca1cde0218009b77cdeb │ │ └── 3f42bbe48d5e1ca65a5186dd9fe8352aef9699cd744f4d74ead6be952e5f6b19 ├── restic_v2_test_repository │ ├── config │ ├── index │ │ ├── 09a56462e4b939181088332cb19ae48506c6a63629eb62ec8257b63b85f2c757 │ │ ├── 2cd9a598873e58bcb067b1d3f6db488047a2bc168da7550b8340c319f0693d8a │ │ └── 7933b7fda4df211bb92e9bedf618e70de5a859baab6a72401a65759e64793a60 │ ├── data │ │ ├── 79 │ │ │ └── 7935cee94288b0ac55957bedb9f2ec1d41fe54eaecb30e695271c332ee606be8 │ │ ├── 7c │ │ │ └── 7c42983e65e74be3911ab1c9177e10128bce39d0c29033a2db9de4a6810d0711 │ │ ├── b7 │ │ │ └── b7843e125c139ab3683a50ce38463c5fda4289ef53dc7fb160638447043f8f69 │ │ ├── d0 │ │ │ └── d0cb4d2b5ad69bdd943fd64daf26f4b53b69d017ae8fdef95d40703be18613f9 │ │ └── ed │ │ │ └── ed505dcfdd8b9c7cb912be91bc19b711279253e37358f70af65466c92816febb │ ├── snapshots │ │ ├── 4d9c1f12d69290f6c3cc8984ef08b8e17b233fddc1bb1b0b2f9a669620f17a59 │ │ ├── 78102f7b5d591244599a84b69bca1fa2323dcc9f4c1d9f5a4bb77d38d545c57c │ │ └── 8e2c0780be7d4cbf4f6007bf4237812d9fcc5bd09eb2febbdcb26fc8e4eb7eef │ └── keys │ │ └── 2cababd397a75a8b6f3ff0ef8ebd51293240f27d3e4e5d576e345d26f083ed6b └── restic_broken_test_repository │ ├── config │ └── keys │ └── 98f9e68226bf15a8e9616632df7c9df543e255b388bfca1cde0218009b77cdeb ├── .gitignore ├── .readthedocs.yaml ├── docs ├── index.rst ├── Makefile ├── make.bat └── conf.py ├── pyproject.toml ├── LICENSE ├── .github └── workflows │ ├── main.yml │ └── codeql-analysis.yml ├── README.md └── noxfile.py /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pyrrhic/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pyrrhic/crypto/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pyrrhic/repo/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | source .venv/bin/activate 2 | 3 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [coverage:report] 2 | show_missing = True 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 160 3 | ignore = E203 # https://github.com/psf/black/issues/315 -------------------------------------------------------------------------------- /pyrrhic/cli/state.py: -------------------------------------------------------------------------------- 1 | from pyrrhic.repo.repository import Repository 2 | 3 | repository: Repository 4 | -------------------------------------------------------------------------------- /pycodestyle.cfg: -------------------------------------------------------------------------------- 1 | [pycodestyle] 2 | count = False 3 | ignore = E203 4 | max-line-length = 160 5 | statistics = True -------------------------------------------------------------------------------- /screenshots/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/screenshots/screenshot.png -------------------------------------------------------------------------------- /pyrrhic/__init__.py: -------------------------------------------------------------------------------- 1 | """Restic implementation in Python""" 2 | # Ensure sync with git tags 3 | __version__ = "0.6.3" 4 | -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/config -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/config -------------------------------------------------------------------------------- /restic_test_repositories/restic_broken_test_repository/config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_broken_test_repository/config -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .mypy_cache/ 2 | /.coverage 3 | /.coverage.* 4 | /.nox/ 5 | /.python-version 6 | /.pytype/ 7 | .venv 8 | /dist/ 9 | /docs/_build/ 10 | /src/*.egg-info/ 11 | __pycache__/ 12 | *.el 13 | *~ 14 | *# 15 | coverage.xml 16 | /docs/reference 17 | -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/data/22/224dfabf27f07d47d5e0015fbb376039c0150e27958c40d4b26f549e5a4645d3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/data/22/224dfabf27f07d47d5e0015fbb376039c0150e27958c40d4b26f549e5a4645d3 -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/data/46/46771395523ccd6dda16694f0ce775f9508a4c3e4527c385f55d8efafa36807f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/data/46/46771395523ccd6dda16694f0ce775f9508a4c3e4527c385f55d8efafa36807f -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/data/4b/4b24375f07a164e995d06303bfc26f79f94127e4e5c6e1c476495bbee0af7ccc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/data/4b/4b24375f07a164e995d06303bfc26f79f94127e4e5c6e1c476495bbee0af7ccc -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/data/6c/6c17cc51fe575445c4f01e3832ab4af90fe6c8d6f67a57bcc178a735d3350dc7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/data/6c/6c17cc51fe575445c4f01e3832ab4af90fe6c8d6f67a57bcc178a735d3350dc7 -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/data/81/81b0c9b17bcfac837980a154df2422bc1cd02425689258f487b395789566322d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/data/81/81b0c9b17bcfac837980a154df2422bc1cd02425689258f487b395789566322d -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/data/b7/b7faaa4eb7471dcf2fb6b72d97c641cbac466320271de88366ac7a68b129a01c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/data/b7/b7faaa4eb7471dcf2fb6b72d97c641cbac466320271de88366ac7a68b129a01c -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/data/f7/f7eed29474b7d5756089e9308d58ec9333a7941991938268f713404949bb8987: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/data/f7/f7eed29474b7d5756089e9308d58ec9333a7941991938268f713404949bb8987 -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/index/bd10c8e85d2cdc3267faae1748cf7a334385d021766059580976882556097c0d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/index/bd10c8e85d2cdc3267faae1748cf7a334385d021766059580976882556097c0d -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | # Set the version of Python and other tools you might need 4 | build: 5 | os: ubuntu-20.04 6 | tools: 7 | python: "3.10" 8 | sphinx: 9 | configuration: docs/conf.py 10 | python: 11 | install: 12 | - method: pip 13 | path: . 14 | 15 | -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/snapshots/7f9faf70a9889f54124f52e42f0d11d3e5eab185fe423cd2c4bb859ef0f71a8b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/snapshots/7f9faf70a9889f54124f52e42f0d11d3e5eab185fe423cd2c4bb859ef0f71a8b -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/snapshots/d2bbc914bf89a24f16bcb70a4ec1d433856e0a8fef5b73c7dd4fa64a95f3977c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/snapshots/d2bbc914bf89a24f16bcb70a4ec1d433856e0a8fef5b73c7dd4fa64a95f3977c -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/snapshots/dd62b535d10bd8f24440cc300a868d6bf2f472859f1218883b0a6faca364c10c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/snapshots/dd62b535d10bd8f24440cc300a868d6bf2f472859f1218883b0a6faca364c10c -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/snapshots/fb56c7b68e95806244b0080bf8bb05d0132df9228390e4b9dc36e2ba6d3bf1a8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_test_repository/snapshots/fb56c7b68e95806244b0080bf8bb05d0132df9228390e4b9dc36e2ba6d3bf1a8 -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/index/09a56462e4b939181088332cb19ae48506c6a63629eb62ec8257b63b85f2c757: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/index/09a56462e4b939181088332cb19ae48506c6a63629eb62ec8257b63b85f2c757 -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/index/2cd9a598873e58bcb067b1d3f6db488047a2bc168da7550b8340c319f0693d8a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/index/2cd9a598873e58bcb067b1d3f6db488047a2bc168da7550b8340c319f0693d8a -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/index/7933b7fda4df211bb92e9bedf618e70de5a859baab6a72401a65759e64793a60: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/index/7933b7fda4df211bb92e9bedf618e70de5a859baab6a72401a65759e64793a60 -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/data/79/7935cee94288b0ac55957bedb9f2ec1d41fe54eaecb30e695271c332ee606be8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/data/79/7935cee94288b0ac55957bedb9f2ec1d41fe54eaecb30e695271c332ee606be8 -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/data/7c/7c42983e65e74be3911ab1c9177e10128bce39d0c29033a2db9de4a6810d0711: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/data/7c/7c42983e65e74be3911ab1c9177e10128bce39d0c29033a2db9de4a6810d0711 -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/data/b7/b7843e125c139ab3683a50ce38463c5fda4289ef53dc7fb160638447043f8f69: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/data/b7/b7843e125c139ab3683a50ce38463c5fda4289ef53dc7fb160638447043f8f69 -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/data/d0/d0cb4d2b5ad69bdd943fd64daf26f4b53b69d017ae8fdef95d40703be18613f9: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/data/d0/d0cb4d2b5ad69bdd943fd64daf26f4b53b69d017ae8fdef95d40703be18613f9 -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/data/ed/ed505dcfdd8b9c7cb912be91bc19b711279253e37358f70af65466c92816febb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/data/ed/ed505dcfdd8b9c7cb912be91bc19b711279253e37358f70af65466c92816febb -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/snapshots/4d9c1f12d69290f6c3cc8984ef08b8e17b233fddc1bb1b0b2f9a669620f17a59: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/snapshots/4d9c1f12d69290f6c3cc8984ef08b8e17b233fddc1bb1b0b2f9a669620f17a59 -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/snapshots/78102f7b5d591244599a84b69bca1fa2323dcc9f4c1d9f5a4bb77d38d545c57c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/snapshots/78102f7b5d591244599a84b69bca1fa2323dcc9f4c1d9f5a4bb77d38d545c57c -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/snapshots/8e2c0780be7d4cbf4f6007bf4237812d9fcc5bd09eb2febbdcb26fc8e4eb7eef: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juergenhoetzel/pyrrhic/HEAD/restic_test_repositories/restic_v2_test_repository/snapshots/8e2c0780be7d4cbf4f6007bf4237812d9fcc5bd09eb2febbdcb26fc8e4eb7eef -------------------------------------------------------------------------------- /pyrrhic/cli/snapshots.py: -------------------------------------------------------------------------------- 1 | import operator 2 | 3 | import pyrrhic.cli.state 4 | 5 | from rich import print 6 | from rich.table import Table 7 | 8 | 9 | def snapshots(): 10 | "List all snapshots" 11 | table = Table("ID", "datetime", "hostname", "tags", "paths", highlight=True) 12 | for s in sorted(pyrrhic.cli.state.repository.get_snapshot(), key=operator.attrgetter("time")): 13 | table.add_row(s.id[:6], f"{s.time:%c}", s.hostname, ", ".join(s.tags or []), ", ".join(s.paths or [])) 14 | print(table) 15 | -------------------------------------------------------------------------------- /pyrrhic/cli/util.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | 3 | import typer 4 | 5 | 6 | def catch_exception(which_exception, exit_code=1): 7 | def decorator(func): 8 | @wraps(func) 9 | def wrapper(*args, **kwargs): 10 | try: 11 | return func(*args, **kwargs) 12 | except which_exception as e: 13 | typer.secho(e, err=True, fg=typer.colors.RED) 14 | raise typer.Exit(code=exit_code) 15 | 16 | return wrapper 17 | 18 | return decorator 19 | -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/keys/98f9e68226bf15a8e9616632df7c9df543e255b388bfca1cde0218009b77cdeb: -------------------------------------------------------------------------------- 1 | {"created":"2022-07-06T21:41:47.707688686+02:00","username":"juergen","hostname":"shaun","kdf":"scrypt","N":32768,"r":8,"p":7,"salt":"UsBY2QtwlpmIEcChK0Uh3NTzrxOLZys1iRlP1/z167MxRDE6x0J134pBWM3Ju4fAzCd2p3zlAiK2mDqaVouXHA==","data":"jjDR5AmwXbi8xAcmLFMhXwRAHrxYqvJJ+ZxYZU7xIEqpUljUt9xKR3Aqx24AFUx6Je0hbfOzv0XAVe2WE6FvQWxqdOd9yNroKon6EurZHfVnG6fBioE4I08O5j2knVGyXhTfhUSgSVi3zQFv/n6IQW/HEAchmj8rQyUOzxPts6xALFcYPdlsjDN2qRiYYgbfKaIYCNCGENYLAjVbluHgJw=="} -------------------------------------------------------------------------------- /restic_test_repositories/restic_test_repository/keys/3f42bbe48d5e1ca65a5186dd9fe8352aef9699cd744f4d74ead6be952e5f6b19: -------------------------------------------------------------------------------- 1 | {"created":"2022-07-15T14:47:51.980913607+02:00","username":"juergen","hostname":"herakles","kdf":"scrypt","N":32768,"r":8,"p":6,"salt":"8YvcfsSo5u59IzeooB4LPIoep+fSIaKlMD9iAzR3CRpbYEeEdEQwPP7kBPpGHhQRI1uJYCEHNWUGAStrVNyrWw==","data":"3H/q1DJQ5u8IlQYvh1YwnY54U9/RBq6nvYPBKfk1AMK2ai4LB+1AZJ27bSWsm6uqgMksefBlNn7p6y2yqF5GXZUz0BMN0iVAmMHx+L0OD6GdMrKlU8K2IRqhiV13eleTl1h9lbMWcGn7m/sw9DLJgKQR//AOZwz1C8yPNQkXKvbDyA484VUPBMa4IpI8ajLa02WZxpc+kYF1PnQ4XFDCtg=="} -------------------------------------------------------------------------------- /restic_test_repositories/restic_v2_test_repository/keys/2cababd397a75a8b6f3ff0ef8ebd51293240f27d3e4e5d576e345d26f083ed6b: -------------------------------------------------------------------------------- 1 | {"created":"2022-10-18T06:51:09.421942637+02:00","username":"juergen","hostname":"shaun","kdf":"scrypt","N":32768,"r":8,"p":7,"salt":"PAD9TjWAyIDwzcV2DZLpJv9MyU4nWoAAV5DEBmKOvKBEzULzKhM0U2X3SPxYMJN4rc0u2nEeGanFJKbZHtZHmA==","data":"rgidniJxpi+OS5o+KXSIqGxL4jyvcpsLG5hjHxcAspKYqLioFlk/Q7umPohvcnDJpLB+n7U8y+/oE8Yg/YugBfS26ZFLScfG2E59grUavNtYlO3VdyH8II8HcU18QChFA+ICfe3F8uVP5RpiUgpoNWsz+e3pIC6VBzGcONGFja6C1Ci7Lv1+2K7aD4InlmeqPvTNGwdMXcY3u38viopqaQ=="} -------------------------------------------------------------------------------- /restic_test_repositories/restic_broken_test_repository/keys/98f9e68226bf15a8e9616632df7c9df543e255b388bfca1cde0218009b77cdeb: -------------------------------------------------------------------------------- 1 | {"created":"2022-07-06T21:41:47.707688686+02:00","username":"juergen","hostname":"shaun","kdf":"scrypt","N":32768,"r":8,"p":7,"salt":"UsBY2QtwlpmIEcChK0Uh3NTzrxOLZys1iRlP1/z167MxRDE6x0J134pBWM3Ju4fAzCd2p3zlAiK2mDqaVouXHA==","data":"jjDR5AmwXbi8xAcmLFMhXwRAHrxYqvJJ+ZxYZU7xIEqpUljUt9xKR3Aqx24AFUx6Je0hbfOzv0XAVe2WE6FvQWxqdOd9yNroKon6EurZHfVnG6fBioE4I08O5j2knVGyXhTfhUSgSVi3zQFv/n6IQW/HEAchmj8rQyUOzxPts6xALFcYPdlsjDN2qRiYYgbfKaIYCNCGENYLAjVbluHgJw=="} -------------------------------------------------------------------------------- /pyrrhic/util.py: -------------------------------------------------------------------------------- 1 | import zstandard 2 | 3 | 4 | _zdec = zstandard.ZstdDecompressor() 5 | 6 | 7 | def decompress(b: bytes, uncompressed_size: int) -> bytes: 8 | return _zdec.decompress(b, uncompressed_size) 9 | 10 | 11 | def maybe_decompress(b: bytes) -> bytes: 12 | if b[0] == 2: # compressed v2 format 13 | return _zdec.decompress( 14 | b[1:], max_output_size=2147483648 15 | ) # https://stackoverflow.com/questions/69270987/how-to-resolve-the-error-related-to-frame-used-in-zstandard-which-requires-too-m 16 | return b 17 | -------------------------------------------------------------------------------- /test/test_cli_fail.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pyrrhic.cli.state 4 | from pyrrhic.repo.repository import Repository, get_masterkey 5 | 6 | import pytest 7 | 8 | 9 | def test_cat_masterkey(capfd): 10 | with pytest.raises(ValueError): 11 | pyrrhic.cli.state.repository = Repository( 12 | Path("restic_test_repositories/restic_test_repository"), 13 | get_masterkey(Path("restic_test_repositories/restic_test_repository"), "invalid!"), 14 | ) 15 | assert capfd.readouterr().out == "Invalid Password" 16 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pyrrhic-restic documentation master file, created by 2 | sphinx-quickstart on Sun Jul 24 13:39:36 2022. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pyrrhic-restic's documentation! 7 | ========================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | reference/modules 14 | 15 | Indices and tables 16 | ================== 17 | 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /test/test_packs.py: -------------------------------------------------------------------------------- 1 | from pyrrhic.repo.index import Index 2 | from pyrrhic.repo.pack import Pack 3 | 4 | import pytest 5 | 6 | from .params import params 7 | 8 | 9 | @pytest.mark.parametrize("repo", [(p["repo"]) for p in params]) 10 | def test_index_matches_packs(repo): 11 | index = repo.get_index() 12 | assert type(index) == Index 13 | for blob_id, packref in index.index.items(): 14 | p = Pack(repo.repository, repo.masterkey, packref.id) 15 | pack_blobs = p.get_blob_index() 16 | assert (matching_blob := next((blob for blob in pack_blobs if blob.hash.hex() == blob_id))) 17 | assert matching_blob.offset == packref.blob.offset 18 | assert matching_blob.length == packref.blob.length 19 | assert matching_blob.uncompressed_length == packref.blob.uncompressed_length 20 | -------------------------------------------------------------------------------- /test/test_index.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from pyrrhic.repo.index import Index 4 | from pyrrhic.repo.repository import Repository, get_masterkey 5 | 6 | REPO_BASE = "restic_test_repositories" 7 | INDEX_ID = "0de57faa699ec0450ddbafb789e165b4e1a3dbe3a09b071075f09ebbfbd6f4b2" 8 | 9 | 10 | def test_load_index(): 11 | repo = Repository( 12 | Path(REPO_BASE) / "restic_test_repository", 13 | get_masterkey(Path(REPO_BASE) / "restic_test_repository", "password"), 14 | ) 15 | indexes = repo.get_index(INDEX_ID) 16 | assert type(indexes) == Index 17 | 18 | 19 | def test_load_indexes(): 20 | repo = Repository( 21 | Path(REPO_BASE) / "restic_test_repository", 22 | get_masterkey(Path(REPO_BASE) / "restic_test_repository", "password"), 23 | ) 24 | indexes = repo.get_index() 25 | assert type(indexes) == Index 26 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=3.2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [project] 6 | name = "pyrrhic-restic" 7 | authors = [{name = "Jürgen Hötzel", email = "juergen@hoetzel.info"}] 8 | readme = "README.md" 9 | license = {file = "LICENSE"} 10 | classifiers = ["License :: OSI Approved :: MIT License"] 11 | dynamic = ["version", "description"] 12 | requires-python = ">=3.7" 13 | dependencies = [ 14 | "cryptography >= 37.0.4", 15 | "typer[all] >= 0.3.2", 16 | "zstandard", 17 | "msgspec >= 0.9.1" 18 | ] 19 | 20 | [project.scripts] 21 | pyrrhic = "pyrrhic.cli.main:app" 22 | 23 | [project.optional-dependencies] 24 | test = [ 25 | "pytest", 26 | ] 27 | 28 | [project.urls] 29 | Home = "https://github.com/juergenhoetzel/pyrrhic" 30 | 31 | [tool.flit.module] 32 | name = "pyrrhic" 33 | 34 | [tool.pytest.ini_options] 35 | minversion = "6.0" 36 | addopts = "-ra -q" 37 | testpaths = [ 38 | "test", 39 | ] 40 | 41 | [tool.black] 42 | line-length = 160 43 | 44 | [tool.isort] 45 | profile = "black" 46 | -------------------------------------------------------------------------------- /pyrrhic/repo/snapshot.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from pathlib import Path 3 | from typing import Generator, List, Optional 4 | 5 | import msgspec 6 | 7 | from pyrrhic.crypto.keys import MasterKey, decrypt_mac 8 | from pyrrhic.util import maybe_decompress 9 | 10 | 11 | class Snapshot(msgspec.Struct): 12 | id: Optional[str] = None # filled by pyrrhic 13 | time: datetime 14 | tree: str 15 | paths: List[str] 16 | hostname: str 17 | username: str 18 | uid: Optional[int] = None # Undocumented https://restic.readthedocs.io/en/stable/100_references.html#repository-format 19 | gid: Optional[int] = None 20 | excludes: Optional[List[str]] = None 21 | tags: Optional[List[str]] = None 22 | 23 | 24 | def get_snapshot(key: MasterKey, repo_path: Path, snapshot_prefix: str) -> Generator[Snapshot, None, None]: 25 | dec = msgspec.json.Decoder(Snapshot) 26 | for snapshot_path in (repo_path / "snapshots").glob(f"{snapshot_prefix}*"): 27 | snapshot = dec.decode(maybe_decompress(decrypt_mac(key, snapshot_path.read_bytes()))) 28 | snapshot.id = snapshot_path.name 29 | yield snapshot 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2022 Jürgen Hötzel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | 2 | name: main 3 | 4 | on: [push, pull_request] 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v3 11 | - name: Setup Python 12 | uses: actions/setup-python@v4 13 | with: 14 | python-version: '3.10' 15 | # FIXME: Python matrix 16 | - name: Install required Python dependencies 17 | run: > 18 | pip install nox 19 | pip install flit 20 | - name: Run nox 21 | # Run tox using the version of Python in `PATH` 22 | run: nox 23 | - name: Package artifacts 24 | run: flit build 25 | - uses: actions/upload-artifact@v3 26 | with: 27 | path: ./dist/* 28 | publish: 29 | name: Publish package 30 | if: startsWith(github.event.ref, 'refs/tags/v') 31 | needs: 32 | - build 33 | runs-on: ubuntu-latest 34 | steps: 35 | - uses: actions/download-artifact@v3 36 | with: 37 | name: artifact 38 | path: dist 39 | - uses: pypa/gh-action-pypi-publish@v1.5.0 40 | with: 41 | verbose: true 42 | user: __token__ 43 | password: ${{ secrets.PYPI_API_TOKEN }} 44 | 45 | -------------------------------------------------------------------------------- /test/test_snapshot.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from pyrrhic.crypto.keys import get_masterkey 4 | from pyrrhic.repo.snapshot import Snapshot, get_snapshot 5 | 6 | import pytest 7 | 8 | REPO_BASE = Path("restic_test_repositories") 9 | SNAPSHOT_ID = "dd62b535d10bd8f24440cc300a868d6bf2f472859f1218883b0a6faca364c10c" 10 | KEY_ID = "98f9e68226bf15a8e9616632df7c9df543e255b388bfca1cde0218009b77cdeb" 11 | KEY_PATH = Path(f"{REPO_BASE}/restic_test_repository/keys/{KEY_ID}") 12 | 13 | 14 | @pytest.fixture 15 | def masterkey(): 16 | return get_masterkey(KEY_PATH, b"password") 17 | 18 | 19 | def test_load_snapshot(masterkey): 20 | snapshot = get_snapshot(masterkey, REPO_BASE / "restic_test_repository", SNAPSHOT_ID) 21 | assert type(next(snapshot, None)) == Snapshot 22 | 23 | 24 | def test_load_snapshot_prefix(masterkey): 25 | snapshots = get_snapshot( 26 | masterkey, 27 | REPO_BASE / "restic_test_repository", 28 | SNAPSHOT_ID, 29 | ) 30 | snapshot = next(snapshots, None) 31 | assert snapshot 32 | assert next(snapshots, None) is None 33 | 34 | 35 | def test_load_snapshot_invalid_prefix(masterkey): 36 | snapshot = next(get_snapshot(masterkey, REPO_BASE / "restic_test_repository", "invalid"), None) 37 | assert snapshot is None 38 | -------------------------------------------------------------------------------- /test/test_keys.py: -------------------------------------------------------------------------------- 1 | from base64 import b64decode 2 | from pathlib import Path 3 | 4 | from pyrrhic.crypto.keys import ( 5 | Mac, 6 | MasterKey, 7 | WrappedKey, 8 | load_key, 9 | ) 10 | from pyrrhic.repo.repository import Repository, get_masterkey 11 | 12 | import pytest 13 | 14 | REPO_BASE = "restic_test_repositories" 15 | KEY = "98f9e68226bf15a8e9616632df7c9df543e255b388bfca1cde0218009b77cdeb" 16 | KEYPATH = Path(f"{REPO_BASE}/restic_test_repository/keys/{KEY}") # noqa: E501 17 | REPO = Path(REPO_BASE) / "restic_test_repository" 18 | BROKEN_REPO = Path(REPO_BASE) / "restic_broken_test_repository" 19 | 20 | 21 | def test_load_key(): 22 | key = load_key(KEYPATH) 23 | assert type(key) == WrappedKey 24 | 25 | 26 | def test_get_masterkey(): 27 | masterkey = get_masterkey(REPO, "password") 28 | assert masterkey == MasterKey( 29 | mac=Mac( 30 | k=b64decode("aSbwRFL8rIOOxL4W+mAW1w=="), 31 | r=b64decode("hQYBDSD/JwpU8XMDIJmAAg=="), 32 | ), 33 | encryption=b64decode("Te0IPiu0wvEtr2+J59McgTrjCp/ynVxC/mmM9mX/t+E="), 34 | ) 35 | 36 | 37 | def test_get_masterkey_with_invalid_password(): 38 | with pytest.raises(ValueError): 39 | get_masterkey(REPO, "password2") 40 | 41 | 42 | def test_config_with_invalid_mac(): 43 | masterkey = get_masterkey(REPO, "password") 44 | repo = Repository(BROKEN_REPO, masterkey) 45 | with pytest.raises(ValueError, match="ciphertext verification failed"): 46 | repo.get_config() 47 | -------------------------------------------------------------------------------- /pyrrhic/repo/repository.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the repository abstractions used by pyrrhic. 3 | """ 4 | import json 5 | from dataclasses import dataclass 6 | from pathlib import Path 7 | from typing import Generator 8 | 9 | from pyrrhic.crypto import keys 10 | from pyrrhic.repo import index, pack, snapshot 11 | 12 | 13 | @dataclass(frozen=True) 14 | class Repository: 15 | """Class that is used for high level access of Repository.""" 16 | 17 | repository: Path 18 | masterkey: keys.MasterKey 19 | 20 | def get_index(self, index_prefix: str = "", glob=True) -> index.Index: 21 | return index.Index(self.masterkey, self.repository, index_prefix, glob) 22 | 23 | def get_snapshot(self, snapshot_prefix: str = "") -> Generator[snapshot.Snapshot, None, None]: 24 | return snapshot.get_snapshot(self.masterkey, self.repository, snapshot_prefix) 25 | 26 | def get_config(self) -> dict: 27 | ct = Path(self.repository / "config").read_bytes() 28 | pt = keys.decrypt_mac(self.masterkey, ct) 29 | return json.loads(pt) 30 | 31 | def get_pack(self, pack_id: str) -> pack.Pack: 32 | return pack.Pack(self.repository, self.masterkey, pack_id) 33 | 34 | 35 | def get_masterkey(repository: Path, password: str) -> keys.MasterKey: 36 | if not password: 37 | raise ValueError("Please specify password") 38 | if not repository: 39 | raise ValueError("Please specify repository location") 40 | 41 | keys_path = repository / "keys" 42 | for kf in keys_path.iterdir(): 43 | try: 44 | return keys.get_masterkey(kf, password.encode("utf8")) 45 | except ValueError as err: 46 | last_err = err 47 | raise last_err 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Restic implementation in Python 2 | 3 | [![image](https://github.com/juergenhoetzel/pyrrhic/workflows/main/badge.svg?branch=master)](https://github.com/juergenhoetzel/pyrrhic/actions?workflow=main) 4 | [![image](https://codecov.io/gh/juergenhoetzel/pyrrhic/branch/master/graph/badge.svg)](https://codecov.io/gh/juergenhoetzel/pyrrhic) 5 | 6 | ## Installation 7 | 8 | ### From pip 9 | 10 | ```bash 11 | pipx install pyrrhic-restic 12 | ``` 13 | ## Screenshot 14 | 15 | 16 | 17 | ## Alpha Relase 18 | 19 | All commands are compatible with `restic` implementation: 20 | ```bash 21 | pyrrhic --repo restic_test_repositories/restic_test_repository cat masterkey 22 | 23 | ``` 24 | 25 | ``` 26 | {'encrypt': 'Te0IPiu0wvEtr2+J59McgTrjCp/ynVxC/mmM9mX/t+E=', 27 | 'mac': {'k': 'aSbwRFL8rIOOxL4W+mAW1w==', 'r': 'hQYBDSD/JwpU8XMDIJmAAg=='}} 28 | ``` 29 | 30 | ```bash 31 | pyrrhic -r restic_test_repositories/restic_test_repository -p <(echo password) ls latest 32 | ``` 33 | 34 | ``` 35 | /usr 36 | /usr/share 37 | /usr/share/cracklib 38 | /usr/share/cracklib/cracklib-small 39 | /usr/share/cracklib/cracklib.magic 40 | /usr/share/cracklib/pw_dict.hwm 41 | /usr/share/cracklib/pw_dict.pwd 42 | /usr/share/cracklib/pw_dict.pwi 43 | ``` 44 | 45 | ## Additional features missing in golang restic implementation 46 | 47 | - pretty-print all objects 48 | - `pyrrhic cat pack SNAPSHOT_ID --header` prints parsed header 49 | - Resumable restore 50 | - Progress bar 51 | 52 | ## Why is it called pyrrhic 53 | 54 | Needed a name starting with `py` containing `r` and ending with `ic`: 55 | 56 | ```bash 57 | grep ^py.*r.*ic$ /usr/share/dict/cracklib-small 58 | ``` 59 | 60 | ## Limitations 61 | 62 | - Supports repository format version 2 only (current restic version). 63 | -------------------------------------------------------------------------------- /pyrrhic/repo/index.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from functools import cache 3 | from pathlib import Path 4 | 5 | import msgspec 6 | 7 | from pyrrhic.crypto.keys import MasterKey, decrypt_mac 8 | from pyrrhic.util import maybe_decompress 9 | 10 | from rich.console import Console 11 | from rich.progress import track 12 | 13 | 14 | class Blob(msgspec.Struct): 15 | id: str 16 | type: str 17 | offset: int 18 | length: int 19 | uncompressed_length: int | None = None 20 | 21 | 22 | class BlobList(msgspec.Struct): 23 | id: str 24 | blobs: list[Blob] 25 | 26 | 27 | class RecPackList(msgspec.Struct): 28 | packs: list[BlobList] 29 | 30 | 31 | @dataclass(frozen=True) 32 | class PackRef: 33 | id: str 34 | blob: Blob 35 | 36 | 37 | @cache 38 | def _get_index(key: MasterKey, repo_path: Path, index_prefix: str, glob: bool) -> dict[str, PackRef]: 39 | if glob: 40 | paths = (repo_path / "index").glob(f"{index_prefix}*") 41 | else: 42 | paths = (repo_path / "index" / name for name in [index_prefix]) 43 | if Console().is_terminal: # FIXME: Should be configurable 44 | paths = track(list(paths), "Loading index") 45 | 46 | dec = msgspec.json.Decoder(type=RecPackList) 47 | 48 | d = { 49 | blob.id: PackRef(packs.id, blob) 50 | for index_path in paths 51 | for packs in dec.decode(maybe_decompress(decrypt_mac(key, index_path.read_bytes()))).packs 52 | for blob in packs.blobs 53 | } 54 | return d 55 | 56 | 57 | class Index: 58 | "Internal Index representation" 59 | index: dict[str, PackRef] 60 | 61 | def __init__(self, key: MasterKey, repo_path: Path, index_prefix: str, glob=True): 62 | self.index = _get_index(key, repo_path, index_prefix, glob) 63 | 64 | def get_packref(self, blob_id) -> PackRef: 65 | if packref := self.index.get(blob_id): 66 | return packref 67 | raise ValueError(f"Invalid blob id {blob_id}") 68 | -------------------------------------------------------------------------------- /test/test_cli_restore.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import logging 3 | import os 4 | from pathlib import Path 5 | 6 | import pyrrhic.cli.state 7 | from pyrrhic.cli.restore import restore 8 | 9 | import pytest 10 | 11 | from .params import params 12 | 13 | RESTORE_FILES = [ 14 | (Path("usr/share/cracklib/cracklib.magic"), "c4b2b3034acf5b35b60a8de27c7ac33f54c6b4ea"), 15 | (Path("usr/share/cracklib/cracklib-double"), "5fc0bc9ea625e08be635226518d47d3070accc5b"), 16 | (Path("usr/share/cracklib/cracklib-small"), "5f97502ab12eac2e3aa869d00a13af41a7f585e6"), 17 | (Path("usr/share/cracklib/pw_dict.hwm"), "5dfc5fa9b8fec7eff807ede3561f4b2cdca17277"), 18 | (Path("usr/share/cracklib/pw_dict.pwd"), "20973efecb1e0239800a6e2437fa3390d3fd415c"), 19 | (Path("usr/share/cracklib/pw_dict.pwi"), "398936961dff2e5f710723c36e67edca92943284"), 20 | ] 21 | 22 | 23 | @pytest.mark.parametrize("repository,snapshot_cracklib", [(p["repo"], p["snapshot_cracklib"]) for p in params]) 24 | def test_restore(capfd, tmp_path, caplog, repository, snapshot_cracklib): 25 | pyrrhic.cli.state.repository = repository 26 | restore(snapshot_cracklib, target=tmp_path) 27 | for snapshot_path, sha1sum in RESTORE_FILES: 28 | assert hashlib.sha1((tmp_path / snapshot_path).read_bytes()).hexdigest() == sha1sum 29 | # Resume backup 30 | resume_from = os.stat(tmp_path / "usr/share/cracklib/cracklib-double").st_size - 10 31 | caplog.set_level(logging.DEBUG) 32 | logging.debug(f"Should resume from {resume_from}") 33 | with open(tmp_path / "usr/share/cracklib/cracklib-double", "a") as f: 34 | f.truncate(resume_from) 35 | logging.debug(f"truncated {tmp_path / 'usr/share/cracklib/cracklib-double'}") 36 | 37 | restore(snapshot_cracklib, target=tmp_path, resume=True) 38 | assert len([record.msg for record in caplog.records if "Resuming from" in record.msg]) == 1 39 | 40 | for snapshot_path, sha1sum in RESTORE_FILES: 41 | assert hashlib.sha1((tmp_path / snapshot_path).read_bytes()).hexdigest() == sha1sum 42 | 43 | symlink_path = Path(tmp_path / "usr/share/cracklib/cracklib-dummy") 44 | assert symlink_path.is_symlink() 45 | assert symlink_path.resolve().name == "cracklib-small" 46 | -------------------------------------------------------------------------------- /pyrrhic/cli/ls.py: -------------------------------------------------------------------------------- 1 | import operator 2 | import stat 3 | 4 | import pyrrhic.cli.state as state 5 | from pyrrhic.repo.tree import Node, walk_breadth_first 6 | 7 | from rich import print 8 | from rich.table import Table 9 | 10 | 11 | import typer 12 | 13 | _MODE_STAT = { 14 | "dir": stat.S_IFDIR, 15 | "file": stat.S_IFREG, 16 | "symlink": stat.S_IFLNK, 17 | "dev": stat.S_IFBLK, 18 | "chardev": stat.S_IFCHR, 19 | "socket": stat.S_IFSOCK, 20 | "fifo": stat.S_IFIFO, 21 | } 22 | 23 | 24 | def _format_name(node: Node) -> str: 25 | if node.type == "symlink": 26 | return f"{node.name} -> {node.linktarget}" 27 | return node.name 28 | 29 | 30 | def _print_long(node: Node, path: str, table: Table) -> None: 31 | name = _format_name(node) 32 | mtime_str = f"{node.mtime:%Y-%m-%d %H:%M:%S}" # rich compatible datetime str 33 | mode = stat.filemode(stat.S_IMODE(node.mode) | _MODE_STAT.get(node.type, 0)) 34 | table.add_row(f"{mode}", f"{node.uid}", f"{node.gid}", f"{node.size}", f"{mtime_str}", f"{path}/{name}") 35 | 36 | 37 | def ls(snapshot_prefix: str, long: bool = typer.Option(False, "--long", "-l", help="Use a long listing (Unix ls -l)")): 38 | "List files in a snapshot" 39 | state.repository.get_snapshot(snapshot_prefix) 40 | if snapshot_prefix == "latest": 41 | snapshots = iter(sorted(state.repository.get_snapshot(), key=operator.attrgetter("time"), reverse=True)[:1]) 42 | else: 43 | snapshots = state.repository.get_snapshot(snapshot_prefix) 44 | snapshot = next(snapshots, None) 45 | if not snapshot: 46 | raise ValueError(f"Index: {snapshot_prefix} not found") 47 | if next(snapshots, None): 48 | raise ValueError(f"Prefix {snapshot_prefix} matches multiple snapshots") 49 | if long: 50 | table = Table("mode", "user", "group", "size", "date", "filename", highlight=True) 51 | for pleaf in walk_breadth_first(state.repository, snapshot.tree): 52 | _print_long(pleaf.node, pleaf.path, table) 53 | print(table) 54 | else: 55 | for pleaf in walk_breadth_first(state.repository, snapshot.tree): 56 | name = _format_name(pleaf.node) 57 | print(f"{pleaf.path}/{name}") 58 | -------------------------------------------------------------------------------- /pyrrhic/cli/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | from enum import Enum 4 | from pathlib import Path 5 | 6 | import pyrrhic 7 | import pyrrhic.cli.cat as cat 8 | import pyrrhic.cli.snapshots as snapshots 9 | import pyrrhic.cli.state 10 | from pyrrhic.cli.ls import ls 11 | from pyrrhic.cli.restore import restore 12 | from pyrrhic.cli.util import catch_exception 13 | from pyrrhic.repo.repository import Repository, get_masterkey 14 | 15 | from rich.logging import RichHandler 16 | 17 | import typer 18 | 19 | 20 | app: typer.Typer = typer.Typer(add_completion=False) 21 | app.add_typer(cat.app, name="cat", help="🐈 Print internal objects to stdout") 22 | app.command()(snapshots.snapshots) 23 | app.command()(ls) 24 | app.command()(restore) 25 | 26 | 27 | class LogLevel(str, Enum): 28 | debug = "DEBUG" 29 | info = "INFO" 30 | warn = "WARN" 31 | error = "ERROR" 32 | 33 | def __str__(self): 34 | return self.value 35 | 36 | 37 | @app.command() 38 | def version(): 39 | """Return version of pyrric application""" 40 | print(pyrrhic.__version__) 41 | 42 | 43 | @app.callback() 44 | @catch_exception(OSError, exit_code=2) 45 | @catch_exception(ValueError) 46 | def global_options( 47 | loglevel: LogLevel = typer.Option(LogLevel.error, case_sensitive=False), 48 | repo: Path = typer.Option(None, "--repo", "-r", help="repository for subcommands ", envvar="RESTIC_REPOSITORY"), 49 | password: str = typer.Option( 50 | None, 51 | help="repository password", 52 | envvar="RESTIC_PASSWORD", 53 | ), 54 | password_file: Path = typer.Option(None, "--password-file", "-p", help="file to read the repository password from", envvar="RESTIC_PASSWORD_FILE"), 55 | ): 56 | logging.basicConfig(level=str(loglevel), format="%(name)s: %(message)s", datefmt="[%X]", handlers=[RichHandler()]) 57 | if password_file: 58 | if password: 59 | print("password and password-file are mutually exclusive", file=sys.stderr) 60 | raise typer.Exit(code=1) 61 | password = Path(password_file).read_text().strip() 62 | if not password: 63 | password = typer.prompt("repository password", hide_input=True) 64 | 65 | masterkey = get_masterkey(repo, password) 66 | pyrrhic.cli.state.repository = Repository(repo, masterkey) 67 | 68 | 69 | if __name__ == "__main__": 70 | app() 71 | -------------------------------------------------------------------------------- /pyrrhic/cli/cat.py: -------------------------------------------------------------------------------- 1 | import operator 2 | import os 3 | import shutil 4 | import sys 5 | 6 | import msgspec 7 | 8 | import pyrrhic.cli.state 9 | from pyrrhic.cli.util import catch_exception 10 | from pyrrhic.repo.pack import blob_to_dict 11 | 12 | from rich import print, print_json 13 | 14 | import typer 15 | 16 | 17 | app: typer.Typer = typer.Typer(add_completion=False) 18 | 19 | 20 | @app.command() 21 | @catch_exception(ValueError) 22 | @catch_exception(FileNotFoundError, exit_code=2) 23 | def masterkey(): 24 | """Return masterkey JSON to stdout""" 25 | state = pyrrhic.cli.state 26 | print(state.repository.masterkey.restic_json()) 27 | 28 | 29 | @app.command() 30 | @catch_exception(ValueError) 31 | @catch_exception(FileNotFoundError, exit_code=2) 32 | def config(): 33 | """Return config JSON to stdout""" 34 | state = pyrrhic.cli.state 35 | config = state.repository.get_config() 36 | print(config) 37 | 38 | 39 | @app.command() 40 | @catch_exception(ValueError) 41 | @catch_exception(FileNotFoundError, exit_code=2) 42 | def index(index_id: str): 43 | """Return index JSON to stdout""" 44 | state = pyrrhic.cli.state 45 | index = state.repository.get_index(index_id) 46 | print(index.index) 47 | 48 | 49 | @app.command() 50 | @catch_exception(ValueError) 51 | @catch_exception(FileNotFoundError, exit_code=2) 52 | def snapshot(snapshot_id: str): 53 | """Return snapshot JSON to stdout""" 54 | state = pyrrhic.cli.state 55 | if snapshot_id == "latest": 56 | snapshots = iter(sorted(pyrrhic.cli.state.repository.get_snapshot(), key=operator.attrgetter("time"), reverse=True)[:1]) 57 | else: 58 | snapshots = state.repository.get_snapshot(snapshot_id) 59 | if (snapshot := next(snapshots, None)) and next(snapshots, None) is None: 60 | print_json(msgspec.json.encode(snapshot).decode("utf-8")) 61 | else: 62 | raise ValueError(f"Invalid Index: {snapshot_id}") 63 | 64 | 65 | @app.command() 66 | @catch_exception(ValueError) 67 | @catch_exception(FileNotFoundError, exit_code=2) 68 | def pack(pack_id: str, header: bool = typer.Option(False, "--header", help="Output parsed pack header")): 69 | """Return pack to stdout""" 70 | state = pyrrhic.cli.state 71 | pack = state.repository.get_pack(pack_id) 72 | 73 | if header: 74 | print([blob_to_dict(blob) for blob in pack.get_blob_index()]) 75 | else: 76 | with os.fdopen(sys.stdout.fileno(), "wb", closefd=False) as stdout, open(pack.path, "rb") as pack_fd: 77 | shutil.copyfileobj(pack_fd, stdout) 78 | -------------------------------------------------------------------------------- /pyrrhic/repo/pack.py: -------------------------------------------------------------------------------- 1 | import os 2 | from dataclasses import dataclass 3 | from pathlib import Path 4 | from struct import unpack 5 | from typing import Optional 6 | 7 | from pyrrhic.crypto.keys import MasterKey, decrypt_mac 8 | 9 | 10 | @dataclass(frozen=True) 11 | class Blob: 12 | type: str # FIXME: Use enum 13 | pack_id: str 14 | offset: int 15 | length: int 16 | hash: bytes 17 | uncompressed_length: Optional[int] = None 18 | 19 | 20 | class Pack: 21 | def __init__(self, repo_path: Path, key: MasterKey, pack_id: str): 22 | "Load Pack at path." 23 | self.pack_id = pack_id 24 | self.path = repo_path / "data" / pack_id[:2] / pack_id 25 | with open(self.path, "rb") as f: 26 | f.seek(-4, os.SEEK_END) 27 | buffer = f.read(4) 28 | header_length = unpack(" list[Blob]: 59 | "Return list of blobs" 60 | return self.get_blobs() 61 | 62 | 63 | def blob_to_dict(blob: Blob) -> dict: 64 | d = {"id": blob.hash.hex(), "offset": blob.offset, "length": blob.length} 65 | if blob.uncompressed_length: 66 | return d | {"length_uncompressed": blob.uncompressed_length} 67 | return d 68 | -------------------------------------------------------------------------------- /test/test_cli.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import json 3 | from ast import literal_eval 4 | from pathlib import Path 5 | 6 | import pyrrhic.cli.state 7 | from pyrrhic.cli.cat import config, index, masterkey, pack, snapshot 8 | from pyrrhic.cli.ls import ls 9 | 10 | import pytest 11 | 12 | from .params import params 13 | 14 | 15 | @pytest.mark.parametrize("repository,expected", [(p["repo"], p["masterkey"]) for p in params]) 16 | def test_cat_masterkey(capfd, repository, expected): 17 | pyrrhic.cli.state.repository = repository 18 | masterkey() 19 | assert literal_eval(capfd.readouterr().out) == expected 20 | 21 | 22 | @pytest.mark.parametrize("repository,config_dict", [(p["repo"], p["config"]) for p in params]) 23 | def test_cat_config(capfd, repository, config_dict): 24 | pyrrhic.cli.state.repository = repository 25 | config() 26 | assert literal_eval(capfd.readouterr().out) == config_dict 27 | 28 | 29 | @pytest.mark.parametrize("repository,snapshot_id,snapshot_json", [(p["repo"], p["snapshot_id"], p["snapshot_json"]) for p in params]) 30 | def test_cat_snapshot(capfd, repository, snapshot_id, snapshot_json): 31 | pyrrhic.cli.state.repository = repository 32 | snapshot(snapshot_id) 33 | out = capfd.readouterr().out 34 | assert json.loads(out) == snapshot_json 35 | 36 | 37 | @pytest.mark.parametrize("repository,index_id,index_substr", [(p["repo"], p["index_id"], p["index_substr"]) for p in params]) 38 | def test_cat_index(capfd, repository, index_id, index_substr): 39 | pyrrhic.cli.state.repository = repository 40 | index(index_id) 41 | out = capfd.readouterr().out 42 | assert "PackRef" in out 43 | assert index_substr in out 44 | 45 | 46 | @pytest.mark.parametrize("repository,pack_id", [(p["repo"], p["pack_id"]) for p in params]) 47 | def test_cat_pack(capfdbinary, repository, pack_id): 48 | pyrrhic.cli.state.repository = repository 49 | sha = pack_id 50 | pack(sha, False) 51 | assert hashlib.sha256(capfdbinary.readouterr().out).hexdigest() == sha 52 | 53 | 54 | @pytest.mark.parametrize("repository,pack_id,pack_blobs", [(p["repo"], p["pack_id"], p["pack_blobs"]) for p in params]) 55 | def test_cat_pack_header(capfd, repository, pack_id, pack_blobs): 56 | pyrrhic.cli.state.repository = repository 57 | sha = pack_id 58 | pack(sha, True) 59 | assert literal_eval(capfd.readouterr().out) == pack_blobs 60 | 61 | 62 | @pytest.mark.parametrize("repository", [(p["repo"]) for p in params]) 63 | def test_ls(capfd, repository): 64 | "Returns a list of paths" 65 | pyrrhic.cli.state.repository = repository 66 | ls("latest", False) 67 | lines = capfd.readouterr().out.splitlines() 68 | for s in lines: 69 | path = Path(s) 70 | assert path.is_absolute() 71 | with pytest.raises(ValueError, match="Index: invalid not found"): 72 | ls("invalid", False) 73 | with pytest.raises(ValueError, match="Prefix matches multiple snapshots"): 74 | ls("", False) 75 | -------------------------------------------------------------------------------- /noxfile.py: -------------------------------------------------------------------------------- 1 | """ 2 | Nox automation tasks for pyrrhic 3 | """ 4 | 5 | import os 6 | from pathlib import Path 7 | 8 | import nox 9 | 10 | # GitHub Actions 11 | ON_CI = bool(os.getenv("CI")) 12 | 13 | # Git info 14 | DEFAULT_BRANCH = "master" 15 | 16 | # Python to use for non-test sessions 17 | DEFAULT_PYTHON: str = "3.10" 18 | 19 | # Global project stuff 20 | PROJECT_ROOT = Path(__file__).parent.resolve() 21 | 22 | SOURCE_FILES = ( 23 | "noxfile.py", 24 | "pyrrhic", 25 | "test", 26 | ) 27 | 28 | 29 | @nox.session(python=DEFAULT_PYTHON) 30 | def dev(session: nox.Session) -> None: 31 | """ 32 | Sets up a python dev environment for the project if one doesn't already exist. 33 | """ 34 | session.run("python", "-m", "venv", os.path.join(PROJECT_ROOT, ".venv")) 35 | session.run( 36 | "flit", 37 | "install", 38 | "--symlink", 39 | "--python", 40 | os.path.join(PROJECT_ROOT, ".venv", "bin", "python"), 41 | external=True, 42 | silent=True, 43 | ) 44 | 45 | 46 | @nox.session(python=["3.10"]) 47 | def test(session: nox.Session) -> None: 48 | """ 49 | Runs the test suite. 50 | """ 51 | session.install( 52 | ".", 53 | "pytest", 54 | "pytest-cov", 55 | "flake8", 56 | ) 57 | session.run("flake8", *SOURCE_FILES) 58 | session.run("pytest", "-v", "-v", "--cov=pyrrhic", "--cov-branch") 59 | 60 | 61 | @nox.session(python=DEFAULT_PYTHON) 62 | def coverage(session: nox.Session): 63 | """Upload coverage data.""" 64 | session.install("coverage[toml]", "codecov") 65 | session.run("coverage", "xml", "--fail-under=0") 66 | session.run("codecov", *session.posargs) 67 | 68 | 69 | @nox.session(python=DEFAULT_PYTHON) 70 | def mypy(session: nox.Session) -> None: 71 | """Type-check using mypy.""" 72 | args = session.posargs or ["pyrrhic", "test"] 73 | session.install("mypy") 74 | session.run("mypy", "--ignore-missing-imports", "--show-error-codes", *args) 75 | 76 | 77 | @nox.session(python=DEFAULT_PYTHON) 78 | def lint(session: nox.Session) -> None: 79 | """Lint using flake8.""" 80 | args = session.posargs or ["pyrrhic", "test"] 81 | session.install( 82 | "flake8", 83 | "flake8-black", 84 | "flake8-import-order", 85 | ) 86 | session.run("flake8", *args) 87 | 88 | 89 | @nox.session(python=DEFAULT_PYTHON) 90 | def sphinx(session: nox.Session) -> None: 91 | """Generate Sphinx documentation from source files.""" 92 | session.install(".", "sphinx") 93 | session.run( 94 | "sphinx-apidoc", 95 | "--force", 96 | "--implicit-namespaces", 97 | "--module-first", 98 | "--separate", 99 | "-o", 100 | "docs/reference/", 101 | "pyrrhic", 102 | ) 103 | session.run("sphinx-build", "-W", "--keep-going", "-b", "html", "docs/", "docs/_build/") 104 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.path.abspath("..")) 17 | from pyrrhic import __version__ 18 | 19 | version = release = __version__ 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = "pyrrhic-restic" 24 | copyright = "2022, Jürgen Hötzel" 25 | author = "Jürgen Hötzel" 26 | 27 | 28 | # -- General configuration --------------------------------------------------- 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [ 34 | "sphinx.ext.autodoc", 35 | "sphinx.ext.autodoc.typehints", 36 | ] 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ["_templates"] 40 | 41 | # List of patterns, relative to source directory, that match files and 42 | # directories to ignore when looking for source files. 43 | # This pattern also affects html_static_path and html_extra_path. 44 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 45 | 46 | 47 | # -- Options for HTML output ------------------------------------------------- 48 | 49 | # The theme to use for HTML and HTML Help pages. See the documentation for 50 | # a list of builtin themes. 51 | # 52 | html_theme = "alabaster" 53 | 54 | # Add any paths that contain custom static files (such as style sheets) here, 55 | # relative to this directory. They are copied after the builtin static files, 56 | # so a file named "default.css" will overwrite the builtin "default.css". 57 | html_static_path = [] 58 | 59 | nitpick_ignore = [("py:class", "type")] 60 | 61 | # Readthedocs config 62 | if os.environ.get("READTHEDOCS") == "True": 63 | from pathlib import Path 64 | 65 | PROJECT_ROOT = Path(__file__).parent.parent 66 | PACKAGE_ROOT = PROJECT_ROOT / "pyrrhic" 67 | 68 | def run_apidoc(_): 69 | from sphinx.ext import apidoc 70 | 71 | apidoc.main( 72 | [ 73 | "--force", 74 | "--implicit-namespaces", 75 | "--module-first", 76 | "--separate", 77 | "-o", 78 | str(PROJECT_ROOT / "docs" / "reference"), 79 | str(PACKAGE_ROOT), 80 | ] 81 | ) 82 | 83 | def setup(app): 84 | app.connect("builder-inited", run_apidoc) 85 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '23 11 * * 3' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /pyrrhic/repo/tree.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import io 3 | from collections import OrderedDict 4 | from copy import copy 5 | from dataclasses import dataclass 6 | from datetime import datetime 7 | from logging import info 8 | from pathlib import Path 9 | from typing import Iterator, Optional 10 | 11 | import msgspec 12 | 13 | from pyrrhic.crypto.keys import decrypt_mac 14 | from pyrrhic.repo.repository import Repository 15 | from pyrrhic.util import decompress 16 | 17 | 18 | class Node(msgspec.Struct): 19 | name: str 20 | type: str 21 | mode: int 22 | mtime: datetime 23 | atime: datetime 24 | ctime: datetime 25 | uid: int 26 | gid: int 27 | size: int = 0 28 | user: str 29 | inode: int 30 | content: Optional[list[str]] = None 31 | linktarget: Optional[str] = None 32 | subtree: Optional[str] = None 33 | 34 | 35 | class Tree(msgspec.Struct): 36 | nodes: list[Node] 37 | 38 | 39 | class ReaderCache: 40 | def __init__(self, capacity: int): 41 | self.cache: OrderedDict[Path, io.BufferedReader] = OrderedDict() 42 | self.misses = 0 # stats 43 | self.requests = 0 44 | self.capacity = capacity 45 | 46 | def get(self, key: Path) -> io.BufferedReader: 47 | self.requests += 1 48 | if key not in self.cache: 49 | self.cache[key] = open(key, "rb") 50 | self.misses += 1 51 | if len(self.cache) > self.capacity: 52 | self.cache.popitem(last=False) 53 | self.cache.move_to_end(key) 54 | return self.cache[key] 55 | 56 | def flush(self): 57 | for item in self.cache.values(): 58 | item.close() 59 | self.cache = OrderedDict() 60 | misses = self.misses / self.requests 61 | info(f"Cache misses: {misses:.2%}") 62 | 63 | 64 | def get_node_blob(repo: Repository, rcache: ReaderCache, blob_id: str) -> bytes: 65 | index = repo.get_index() 66 | packref = index.get_packref(blob_id) 67 | blob = packref.blob 68 | f = rcache.get(repo.repository / "data" / packref.id[:2] / packref.id) 69 | f.seek(blob.offset) 70 | buffer = f.read(blob.length) 71 | plaintext = decrypt_mac(repo.masterkey, buffer) 72 | if uncompressed_length := blob.uncompressed_length: 73 | plaintext = decompress(plaintext, uncompressed_length) 74 | if hashlib.sha256(plaintext).hexdigest() != blob.id: 75 | raise ValueError(f"Invalid hash for blob {blob.id}") 76 | return plaintext 77 | 78 | 79 | # FIXME: Use only one msgspec object 80 | def get_tree(repo: Repository, rcache: ReaderCache, tree_id: str) -> Tree: 81 | plaintext_blob = get_node_blob(repo, rcache, tree_id) 82 | return msgspec.json.decode(plaintext_blob, type=Tree) 83 | 84 | 85 | @dataclass(frozen=True) 86 | class PathNode: 87 | "Represent a Node located at path" 88 | path: str # prefix path 89 | node: Node 90 | 91 | 92 | def walk_breadth_first(repository: Repository, tree_id: str, rcache=ReaderCache(64)) -> Iterator[PathNode]: 93 | tree = get_tree(repository, rcache, tree_id) 94 | pathnodes = [PathNode(f"/{node.name}", node) for node in tree.nodes] 95 | while pathnodes: 96 | pleafes = [pnode for pnode in pathnodes if not pnode.node.subtree] 97 | pathnodes = [pnode for pnode in pathnodes if pnode.node.subtree] # FIXME: traverses 2 times 98 | for pleaf in pleafes: 99 | yield pleaf 100 | if pathnodes: 101 | pnode = pathnodes.pop() 102 | node_without_subdir = copy(pnode.node) 103 | node_without_subdir.subtree = None 104 | tree = get_tree(repository, rcache, pnode.node.subtree or "0123") # just make mypy happy? 105 | pathnodes = [PathNode(pnode.path, node_without_subdir), *[PathNode(f"{pnode.path}/{node.name}", node) for node in tree.nodes], *pathnodes] 106 | rcache.flush() 107 | -------------------------------------------------------------------------------- /test/params.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from pyrrhic.repo.repository import Repository, get_masterkey 4 | 5 | params = [ 6 | { 7 | "repo": Repository( 8 | Path("restic_test_repositories/restic_test_repository"), get_masterkey(Path("restic_test_repositories/restic_test_repository"), "password") 9 | ), 10 | "config": {"chunker_polynomial": "3833148ec41f8d", "id": "2ec6792a9c75a017ec4665a5e7733f45f8bffb67c7d0f3c9ec6cc96e58c6386b", "version": 1}, 11 | "masterkey": {"mac": {"r": "hQYBDSD/JwpU8XMDIJmAAg==", "k": "aSbwRFL8rIOOxL4W+mAW1w=="}, "encrypt": "Te0IPiu0wvEtr2+J59McgTrjCp/ynVxC/mmM9mX/t+E="}, 12 | "snapshot_id": "dd62b535d10bd8f24440cc300a868d6bf2f472859f1218883b0a6faca364c10c", 13 | "snapshot_json": { 14 | "time": "2022-07-19T19:52:28.692252Z", 15 | "tree": "a5dbcc77f63f5dd4f4c67c988aba4a19817aaa9d6c34a6021236a5d40ce653e1", 16 | "paths": ["/home/juergen/shared/python/pyrrhic/test"], 17 | "hostname": "shaun", 18 | "username": "juergen", 19 | "id": "dd62b535d10bd8f24440cc300a868d6bf2f472859f1218883b0a6faca364c10c", 20 | "uid": 1000, 21 | "gid": 1000, 22 | "excludes": None, 23 | "tags": None, 24 | }, 25 | "snapshot_cracklib": "d2bbc914", 26 | "index_id": "bd10c8e85d2cdc3267faae1748cf7a334385d021766059580976882556097c0d", 27 | "index_substr": """'511d1c632ccad135d5407157154eccc17fcfaf501ad252b231c7ce41175473b9': PackRef( 28 | id='46771395523ccd6dda16694f0ce775f9508a4c3e4527c385f55d8efafa36807f'""", 29 | "pack_id": "4b24375f07a164e995d06303bfc26f79f94127e4e5c6e1c476495bbee0af7ccc", 30 | "pack_blobs": [ 31 | {"id": "82deec86c5611cb5ae02b967e49d7aeaca50a732432bd1a5923787bd5d0fbf80", "length": 1643, "offset": 0}, 32 | {"id": "e20d6400fbd4e602c79c8bab98a88726865b168838cc8107d560da10f19b2ff8", "length": 1467, "offset": 1643}, 33 | {"id": "a5dbcc77f63f5dd4f4c67c988aba4a19817aaa9d6c34a6021236a5d40ce653e1", "length": 413, "offset": 3110}, 34 | ], 35 | }, 36 | { 37 | "repo": Repository( 38 | Path("restic_test_repositories/restic_v2_test_repository"), get_masterkey(Path("restic_test_repositories/restic_v2_test_repository"), "password") 39 | ), 40 | "config": {"version": 2, "id": "2c5d1276aa6f58c06883c2a7a2aef35c86ec4a0b57a6ded80e08de3fedc108f8", "chunker_polynomial": "232a535caedb65"}, 41 | "masterkey": {"mac": {"r": "2iQMCQDOPw1UtW0N1C4RAA==", "k": "zpL4Veh3FhT7grSgm0BmCQ=="}, "encrypt": "193cMViwx7TJFtykWOZzEKojzp0fjx4VvaiJ6yMgZ3E="}, 42 | "snapshot_id": "4d9c1f", 43 | "snapshot_json": { 44 | "time": "2022-10-18T04:53:22.965183Z", 45 | "tree": "9178aa841c9fd277478925c7864da28e4500ce236ad89a0ff9a261b1dabbda42", 46 | "paths": ["/usr/share/cracklib"], 47 | "hostname": "shaun", 48 | "username": "juergen", 49 | "id": "4d9c1f12d69290f6c3cc8984ef08b8e17b233fddc1bb1b0b2f9a669620f17a59", 50 | "uid": 1000, 51 | "gid": 1000, 52 | "excludes": None, 53 | "tags": None, 54 | }, 55 | "snapshot_cracklib": "78102f", 56 | "index_id": "2cd9a598873e58bcb067b1d3f6db488047a2bc168da7550b8340c319f0693d8a", 57 | "index_substr": """'8696358373607539e46c42a13955e343cd1d66a45b102c3ccf41b5e38d4b1db1': PackRef( 58 | id='7c42983e65e74be3911ab1c9177e10128bce39d0c29033a2db9de4a6810d0711'""", 59 | "pack_id": "ed505dcfdd8b9c7cb912be91bc19b711279253e37358f70af65466c92816febb", 60 | "pack_blobs": [ 61 | {"id": "8072950e2d305ebcfd94912557e06b6a9e1a7ee487470e4d89efd34e749a2505", "offset": 0, "length": 621, "length_uncompressed": 2197}, 62 | {"id": "25fec2e636c582064ea0927886a93acca579ab1436b01311f18c10776aa70fa4", "offset": 621, "length": 256, "length_uncompressed": 373}, 63 | {"id": "b9c28d59a52a1c5406637280aff6f63195fc9dac603d3e070fbd884aaddb6b5f", "offset": 877, "length": 255, "length_uncompressed": 370}, 64 | {"id": "9178aa841c9fd277478925c7864da28e4500ce236ad89a0ff9a261b1dabbda42", "offset": 1132, "length": 253, "length_uncompressed": 368}, 65 | ], 66 | }, 67 | ] 68 | -------------------------------------------------------------------------------- /pyrrhic/crypto/keys.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the crypto primitives used by restic. 3 | """ 4 | 5 | import json 6 | from base64 import b64decode, b64encode 7 | from dataclasses import dataclass 8 | from datetime import datetime 9 | from pathlib import Path 10 | 11 | from cryptography.hazmat.backends import default_backend 12 | from cryptography.hazmat.primitives import poly1305 13 | from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes 14 | from cryptography.hazmat.primitives.kdf.scrypt import Scrypt 15 | 16 | import msgspec 17 | 18 | # mask for key, (cf. http://cr.yp.to/mac/poly1305-20050329.pdf) 19 | _POLY1305KEYMASK = b"\xff\xff\xff\x0f\xfc\xff\xff\x0f\xfc\xff\xff\x0f\xfc\xff\xff\x0f" 20 | 21 | 22 | class WrappedKey(msgspec.Struct): 23 | """Class that contain all data that is needed to derive the 24 | repository's master encryption and message authentication keys 25 | from a user's password.""" 26 | 27 | hostname: str 28 | username: str 29 | kdf: str 30 | N: int 31 | r: int 32 | p: int 33 | created: datetime 34 | data: bytes 35 | salt: bytes 36 | 37 | 38 | @dataclass(frozen=True) 39 | class Mac: 40 | """Class that holdes the Poly1305-AES parameters""" 41 | 42 | k: bytes 43 | r: bytes 44 | 45 | 46 | @dataclass(frozen=True) 47 | class MasterKey: 48 | """Class that holds encryption and message authentication keys for a 49 | repository.""" 50 | 51 | mac: Mac 52 | encryption: bytes 53 | 54 | def restic_json(self): 55 | """Return restic representation of Masterkey""" 56 | return { 57 | "mac": { 58 | "r": b64encode(self.mac.r).decode(), 59 | "k": b64encode(self.mac.k).decode(), 60 | }, 61 | "encrypt": b64encode(self.encryption).decode(), 62 | } 63 | 64 | def __hash__(self): 65 | return hash(self.encryption + self.mac.k + self.mac.r) 66 | 67 | 68 | def load_key(key_path: Path) -> WrappedKey: 69 | return msgspec.json.decode(key_path.read_bytes(), type=WrappedKey) 70 | 71 | 72 | def _poly1305_validate(nonce: bytes, k: bytes, r: bytes, message: bytes, mac: bytes) -> bool: 73 | r = bytes([(r & m) for r, m in zip(r, _POLY1305KEYMASK)]) 74 | cipher = Cipher(algorithms.AES(k), modes.ECB()) 75 | encryptor = cipher.encryptor() 76 | aes_ciphertext = encryptor.update(nonce) + encryptor.finalize() 77 | poly1305_key = r + aes_ciphertext 78 | p = poly1305.Poly1305(poly1305_key) 79 | p.update(message) 80 | return p.finalize() == mac 81 | 82 | 83 | def _decrypt(aes_key: bytes, nonce: bytes, ciphertext: bytes) -> bytes: 84 | cipher = Cipher(algorithms.AES(aes_key), modes.CTR(nonce)) 85 | decryptor = cipher.decryptor() 86 | return decryptor.update(ciphertext) + decryptor.finalize() 87 | 88 | 89 | def decrypt_mac(key: MasterKey, restic_blob: bytes) -> bytes: 90 | """Decrypt 'IV || CIPHERTEXT || MAC' bytes, validate mac and 91 | return plaintext bytes""" 92 | nonce = restic_blob[:16] 93 | mac = restic_blob[-16:] 94 | ciphertext = restic_blob[16:-16] 95 | if not _poly1305_validate(nonce, key.mac.k, key.mac.r, ciphertext, mac): 96 | raise ValueError("ciphertext verification failed") 97 | return _decrypt(key.encryption, nonce, ciphertext) 98 | 99 | 100 | def get_masterkey(path: Path, password: bytes) -> MasterKey: 101 | wrapped_key = load_key(path) 102 | kdf = Scrypt( 103 | wrapped_key.salt, 104 | 64, 105 | wrapped_key.N, 106 | wrapped_key.r, 107 | wrapped_key.p, 108 | backend=default_backend, 109 | ) 110 | derived_key = kdf.derive(password) 111 | poly_k = derived_key[32:48] 112 | poly_r = derived_key[48:] 113 | nonce = wrapped_key.data[0:16] 114 | message = wrapped_key.data[16:-16] 115 | mac = wrapped_key.data[-16:] 116 | if not _poly1305_validate(nonce, poly_k, poly_r, message, mac): 117 | raise ValueError("Invalid Password") 118 | j = json.loads(_decrypt(derived_key[:32], nonce, message)) 119 | encryption = b64decode(j["encrypt"]) 120 | r = b64decode(j["mac"]["r"]) 121 | k = b64decode(j["mac"]["k"]) 122 | return MasterKey(Mac(k=k, r=r), encryption) 123 | -------------------------------------------------------------------------------- /pyrrhic/cli/restore.py: -------------------------------------------------------------------------------- 1 | import operator 2 | import os 3 | import stat 4 | from logging import debug, info, warn 5 | from pathlib import Path 6 | 7 | import pyrrhic.cli.state as state 8 | from pyrrhic.cli.util import catch_exception 9 | from pyrrhic.repo.tree import ReaderCache, get_node_blob, walk_breadth_first 10 | 11 | from rich.progress import track 12 | 13 | import typer 14 | 15 | 16 | def _restore(tree_id: str, target: Path, resume=False): 17 | isroot = os.geteuid() == 0 18 | rcache = ReaderCache(64) 19 | index = state.repository.get_index() 20 | for pnode in walk_breadth_first(state.repository, tree_id, rcache): 21 | node = pnode.node 22 | abs_path = target / Path(pnode.path).relative_to("/") 23 | mode = stat.S_IMODE(node.mode) 24 | match node.type: 25 | case "file": 26 | # FIXME: Create temporary unique file and do atomic rename 27 | if node.content: # possible empty file 28 | resume_from = 0 29 | blobs = [index.get_packref(content).blob for content in node.content] 30 | if abs_path.is_file(): 31 | if not resume: 32 | raise FileExistsError(f"File exists: {abs_path}") 33 | resume_from = os.stat(abs_path).st_size 34 | if resume_from == node.size: 35 | debug(f"Already restored {pnode.path}") 36 | continue 37 | if resume_from > node.size: 38 | raise ValueError(f"Existing file {abs_path} is larger") 39 | with open(abs_path, "ab") as f: 40 | current_pos = 0 41 | for i, blob in enumerate(track(blobs, pnode.path)): 42 | debug(f"Processing Blob {i} of {abs_path}") 43 | blength = blob.uncompressed_length or (blob.length - 32) 44 | if current_pos < resume_from: 45 | if resume_from < (current_pos + blength): 46 | f.truncate(current_pos) 47 | info(f"Resuming from {current_pos}") 48 | else: 49 | debug(f"{abs_path} Ignoring pos {current_pos} in resumed file") 50 | current_pos += blength 51 | continue 52 | bs = get_node_blob(state.repository, rcache, blob.id) 53 | current_pos += len(bs) 54 | f.write(bs) 55 | 56 | abs_path.chmod(mode) 57 | if isroot: # FIXME: chgrp to groups this user is member of 58 | os.chown(abs_path, node.uid, node.gid) 59 | 60 | case "dir": 61 | debug(f"Creating directory {abs_path}") 62 | try: 63 | abs_path.mkdir(mode) 64 | except FileExistsError as e: 65 | if not resume: 66 | raise e 67 | case "symlink": 68 | if abs_path.is_symlink(): 69 | if not resume: 70 | raise FileExistsError(f"Symlink exists: {abs_path}") 71 | debug(f"Symlink exists: {abs_path}") 72 | elif node.linktarget: 73 | debug(f"Creating symlink {abs_path} -> {node.linktarget}") 74 | os.symlink(node.linktarget, abs_path) 75 | if isroot: # FIXME: chgrp to groups this user is member of 76 | os.chown(abs_path, node.uid, node.gid, follow_symlinks=False) 77 | else: 78 | raise ValueError(f"Symlink target of {abs_path} does not exist") 79 | case _: 80 | warn(f"{node.name}: {node.type} not implemented") 81 | 82 | 83 | @catch_exception(OSError, exit_code=2) 84 | def restore( 85 | snapshot_prefix: str, 86 | target: Path, 87 | resume: bool = typer.Option(False, "--resume", "-r", help="resume exiting files in target"), 88 | help="Restore data from a snapshot", 89 | ): 90 | state.repository.get_snapshot(snapshot_prefix) 91 | if snapshot_prefix == "latest": # FIXME: Duplicated code (ls command) 92 | snapshots = iter(sorted(state.repository.get_snapshot(), key=operator.attrgetter("time"), reverse=True)[:1]) 93 | else: 94 | snapshots = state.repository.get_snapshot(snapshot_prefix) 95 | snapshot = next(snapshots, None) 96 | if not snapshot: 97 | raise ValueError(f"Index: {snapshot_prefix} not found") 98 | if next(snapshots, None): 99 | raise ValueError(f"Prefix {snapshot_prefix} matches multiple snapshots") 100 | _restore(snapshot.tree, target, resume) 101 | --------------------------------------------------------------------------------