├── requirements.txt ├── .gitignore ├── .travis.yml ├── opentimestamps ├── tests │ ├── __init__.py │ ├── core │ │ ├── __init__.py │ │ ├── dubious │ │ │ ├── __init__.py │ │ │ └── test_notary.py │ │ ├── test_serialize.py │ │ ├── test_op.py │ │ ├── test_notary.py │ │ ├── test_git.py │ │ ├── test_packetstream.py │ │ └── test_timestamp.py │ ├── test_calendar.py │ └── test_bitcoin.py ├── __init__.py ├── core │ ├── dubious │ │ ├── __init__.py │ │ └── notary.py │ ├── __init__.py │ ├── packetstream.py │ ├── serialize.py │ ├── git.py │ ├── op.py │ ├── notary.py │ └── timestamp.py ├── timestamp.py ├── bitcoin.py └── calendar.py ├── .github └── workflows │ └── python-package.yml ├── README.md ├── release-notes.md ├── setup.py └── LICENSE /requirements.txt: -------------------------------------------------------------------------------- 1 | python-bitcoinlib>=0.9.0,<0.13.0 2 | GitPython>=2.0.8 3 | pycryptodomex>=3.3.1 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | 3 | local*.cfg 4 | 5 | build/ 6 | dist/ 7 | opentimestamps.egg-info/ 8 | 9 | #Idea IDE 10 | .idea/ 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | git: 2 | depth: 9999999 3 | language: python 4 | python: 5 | - "3.4" 6 | - "3.5" 7 | - "3.6" 8 | - "3.7" 9 | - "3.8" 10 | - "3.9" 11 | # command to install dependencies 12 | install: 13 | - pip install -r requirements.txt 14 | # command to run tests 15 | script: python -m unittest discover -v 16 | -------------------------------------------------------------------------------- /opentimestamps/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | -------------------------------------------------------------------------------- /opentimestamps/tests/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | -------------------------------------------------------------------------------- /opentimestamps/tests/core/dubious/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | -------------------------------------------------------------------------------- /opentimestamps/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | __version__ = "0.4.5" 13 | -------------------------------------------------------------------------------- /opentimestamps/core/dubious/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | """Timestamp proofs with dubious security 13 | 14 | By "dubious" we mean techniques whose security model is uncertain, or likely to 15 | unexpectedly weaken, and should be evaluated on a case-by-case basis. Proofs 16 | falling under this category should be avoided for general production usage. 17 | """ 18 | -------------------------------------------------------------------------------- /opentimestamps/core/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | """Consensus-critical code 13 | 14 | Basically, everything under opentimestamps.core has the property that changes 15 | to it may break timestamp validation in non-backwards-compatible ways that are 16 | difficult to recover from. We keep such code separate as a reminder to 17 | ourselves to pay extra attention when making changes. 18 | """ 19 | -------------------------------------------------------------------------------- /opentimestamps/timestamp.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | """Convenience functions for creating timestamps""" 13 | 14 | import os 15 | 16 | from opentimestamps.core.op import OpAppend, OpSHA256 17 | 18 | def nonce_timestamp(private_timestamp, crypt_op=OpSHA256(), length=16): 19 | """Create a nonced version of a timestamp for privacy""" 20 | stamp2 = private_timestamp.ops.add(OpAppend(os.urandom(length))) 21 | return stamp2.ops.add(crypt_op) 22 | -------------------------------------------------------------------------------- /opentimestamps/tests/core/test_serialize.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import unittest 13 | 14 | from opentimestamps.core.serialize import * 15 | 16 | class Test_serialization(unittest.TestCase): 17 | def test_assert_eof(self): 18 | """End-of-file assertions""" 19 | ctx = BytesDeserializationContext(b'') 20 | ctx.assert_eof() 21 | 22 | with self.assertRaises(TrailingGarbageError): 23 | ctx = BytesDeserializationContext(b'b') 24 | ctx.assert_eof() 25 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [ "master" ] 9 | pull_request: 10 | branches: [ "master" ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: ["3.9", "3.10", "3.11"] 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | with: 24 | fetch-depth: 0 25 | - name: Set up Python ${{ matrix.python-version }} 26 | uses: actions/setup-python@v3 27 | with: 28 | python-version: ${{ matrix.python-version }} 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 33 | - name: Test with pytest 34 | run: | 35 | python3 -m unittest discover -v 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-opentimestamps 2 | 3 | Python3 library for creating and verifying OpenTimestamps proofs. 4 | 5 | ## Installation 6 | 7 | From the PyPi repository: 8 | 9 | pip3 install opentimestamps 10 | 11 | ## Structure 12 | 13 | Similar to the author's `python-bitcoinlib`, the codebase is split between the 14 | consensus-critical `opentimestamps.core.*` modules, and the 15 | non-consensus-critical `opentimestamps.*` modules. The distinction between the 16 | two is whether or not changes to that code are likely to lead to permanent 17 | incompatibilities between versions that could lead to timestamp validation 18 | returning inconsistent results between versions. 19 | 20 | ## Unit tests 21 | 22 | python3 -m unittest discover -v 23 | 24 | Additionally Travis is supported. 25 | 26 | ## SSL Root Certificates 27 | 28 | On some MacOS setups SSL certificates may be missing. The following commands 29 | could be of use to resolve this error (the below example assumes a user is 30 | running Python "3.7", and is using Certifi package): 31 | 32 | ``` 33 | cd /Applications/Python\ 3.7 34 | Install\ Certificates.command 35 | ``` 36 | 37 | -------------------------------------------------------------------------------- /release-notes.md: -------------------------------------------------------------------------------- 1 | # python-opentimestamps release notes 2 | 3 | ## v0.4.5 4 | 5 | * Fix broken upload on pypi 6 | 7 | ## v0.4.4 8 | 9 | * Update requirements to mark python-bitcoinlib v0.12.x as compatible. 10 | 11 | v0.12.x has breaking changes. But they don't affect us. 12 | 13 | ## v0.4.3 14 | 15 | * Replaced pysha3 dependency with pycryptodomex. The former was not compatible 16 | with Python 3.11 17 | 18 | ## v0.4.2 19 | 20 | * Latest python-bitcoinlib marked as compatible; no other changes. 21 | 22 | ## v0.4.1 23 | 24 | * Latest python-bitcoinlib marked as compatible; no other changes. 25 | 26 | 27 | ## v0.4.0 28 | 29 | * Breaking change: Timestamp equality comparison now also checks attestations, 30 | not just operations. 31 | * Fixed issues with timestamp less than/greater than comparisons, (e.g. `ts1 < ts2`) 32 | * Fixed `str_tree()` crash 33 | 34 | 35 | ## v0.3.0 36 | 37 | * New calendar server! Thanks to Vincent Cloutier from Catallaxy. 38 | * URL handling in calendar code now handles tailing slashes. 39 | * New attestation: `LitecoinBlockHeaderAttestation`. 40 | 41 | 42 | ## v0.2.1 43 | 44 | Fixed `make_timestamp_from_block()` w/ blocks containing segwit transactions. 45 | 46 | 47 | ## v0.2.0.1 48 | 49 | Actually get that right... 50 | 51 | 52 | ## v0.2.0 53 | 54 | `python-bitcoinlib` version required bumped to 0.9.0 for segwit compatibility. 55 | 56 | 57 | ## v0.1.0 58 | 59 | Initial release. 60 | -------------------------------------------------------------------------------- /opentimestamps/tests/test_calendar.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import unittest 13 | 14 | from opentimestamps.calendar import * 15 | 16 | class Test_UrlWhitelist(unittest.TestCase): 17 | def test_empty(self): 18 | """Empty whitelist""" 19 | wl = UrlWhitelist() 20 | 21 | self.assertNotIn('', wl) 22 | self.assertNotIn('http://example.com', wl) 23 | 24 | def test_exact_match(self): 25 | """Exact match""" 26 | 27 | wl = UrlWhitelist(("https://example.com",)) 28 | self.assertIn("https://example.com", wl) 29 | self.assertNotIn("http://example.com", wl) 30 | self.assertNotIn("http://example.org", wl) 31 | 32 | # I'm happy for this to be strict 33 | self.assertIn("https://example.com", wl) 34 | 35 | def test_add_scheme(self): 36 | """URL scheme added automatically""" 37 | wl = UrlWhitelist(("example.com",)) 38 | self.assertIn("https://example.com", wl) 39 | self.assertIn("http://example.com", wl) 40 | 41 | def test_glob_match(self): 42 | """Glob matching""" 43 | wl = UrlWhitelist(("*.example.com",)) 44 | self.assertIn("https://foo.example.com", wl) 45 | self.assertIn("http://bar.example.com", wl) 46 | self.assertIn("http://foo.bar.example.com", wl) 47 | 48 | self.assertNotIn("http://barexample.com", wl) 49 | -------------------------------------------------------------------------------- /opentimestamps/core/dubious/notary.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | """Dubious Timestamp signature verification""" 13 | 14 | import opentimestamps.core.serialize 15 | import opentimestamps.core.notary as notary 16 | 17 | 18 | class EthereumBlockHeaderAttestation(notary.TimeAttestation): 19 | """Signed by the Ethereum blockchain 20 | 21 | The commitment digest will be the merkleroot of the blockheader. 22 | 23 | Ethereum attestations are in the "dubious" module as what exactly Ethereum 24 | is has changed repeatedly in the past due to consensus failures and forks; 25 | as of writing the Ethereum developers plan to radically change Ethereum's 26 | consensus model to proof-of-stake, whose security model is at best dubious. 27 | """ 28 | 29 | TAG = bytes.fromhex('30fe8087b5c7ead7') 30 | 31 | def __init__(self, height): 32 | self.height = height 33 | 34 | def __eq__(self, other): 35 | if other.__class__ is EthereumBlockHeaderAttestation: 36 | return self.height == other.height 37 | else: 38 | return super().__eq__(other) 39 | 40 | def __lt__(self, other): 41 | if other.__class__ is EthereumBlockHeaderAttestation: 42 | return self.height < other.height 43 | 44 | else: 45 | return super().__lt__(other) 46 | 47 | def __hash__(self): 48 | return hash(self.height) 49 | 50 | def verify_against_blockheader(self, digest, block): 51 | """Verify attestation against a block header 52 | 53 | Returns the block time on success; raises VerificationError on failure. 54 | """ 55 | 56 | if len(digest) != 32: 57 | raise opentimestamps.core.notary.VerificationError("Expected digest with length 32 bytes; got %d bytes" % len(digest)) 58 | elif digest != bytes.fromhex(block['transactionsRoot'][2:]): 59 | raise opentimestamps.core.notary.VerificationError("Digest does not match merkleroot") 60 | 61 | return block['timestamp'] 62 | 63 | def __repr__(self): 64 | return 'EthereumBlockHeaderAttestation(%r)' % self.height 65 | 66 | def _serialize_payload(self, ctx): 67 | ctx.write_varuint(self.height) 68 | 69 | @classmethod 70 | def deserialize(cls, ctx): 71 | height = ctx.read_varuint() 72 | return EthereumBlockHeaderAttestation(height) 73 | -------------------------------------------------------------------------------- /opentimestamps/tests/core/dubious/test_notary.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import unittest 13 | 14 | from opentimestamps.core.serialize import * 15 | from opentimestamps.core.notary import * 16 | from opentimestamps.core.dubious.notary import * 17 | 18 | class Test_EthereumBlockHeaderAttestation(unittest.TestCase): 19 | def test_serialize(self): 20 | attestation = EthereumBlockHeaderAttestation(0) 21 | expected_serialized = bytes.fromhex('30fe8087b5c7ead7' + '0100') 22 | 23 | ctx = BytesSerializationContext() 24 | attestation.serialize(ctx) 25 | self.assertEqual(ctx.getbytes(), expected_serialized) 26 | 27 | ctx = BytesDeserializationContext(expected_serialized) 28 | attestation2 = TimeAttestation.deserialize(ctx) 29 | 30 | self.assertEqual(attestation2.height, 0) 31 | 32 | def test_verify(self): 33 | eth_block_1 = {'uncles': [], 'size': 537, 'hash': '0x88e96d4537bea4d9c05d12549907b32561d3bf31f45aae734cdc119f13406cb6', 'gasLimit': 5000, 'number': 1, 'totalDifficulty': 34351349760, 'stateRoot': '0xd67e4d450343046425ae4271474353857ab860dbc0a1dde64b41b5cd3a532bf3', 'extraData': '0x476574682f76312e302e302f6c696e75782f676f312e342e32', 'sha3Uncles': '0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347', 'mixHash': '0x969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f59', 'transactionsRoot': '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421', 'sealFields': ['0x969b900de27b6ac6a67742365dd65f55a0526c41fd18e1b16f1a1215c2e66f59', '0x539bd4979fef1ec4'], 'transactions': [], 'parentHash': '0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3', 'logsBloom': '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', 'author': '0x05a56e2d52c817161883f50c441c3228cfe54d9f', 'gasUsed': 0, 'timestamp': 1438269988, 'nonce': '0x539bd4979fef1ec4', 'difficulty': 17171480576, 'receiptsRoot': '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421', 'miner': '0x05a56e2d52c817161883f50c441c3228cfe54d9f'} 34 | attestation = EthereumBlockHeaderAttestation(1) 35 | timestamp = attestation.verify_against_blockheader(bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), eth_block_1) 36 | self.assertEqual(1438269988, timestamp) 37 | -------------------------------------------------------------------------------- /opentimestamps/bitcoin.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | from opentimestamps.core.timestamp import Timestamp, cat_sha256d 13 | from opentimestamps.core.op import OpPrepend 14 | from opentimestamps.core.notary import BitcoinBlockHeaderAttestation 15 | 16 | def __make_btc_block_merkle_tree(blk_txids): 17 | assert len(blk_txids) > 0 18 | 19 | digests = blk_txids 20 | while len(digests) > 1: 21 | # The famously broken Satoshi algorithm: if the # of digests at this 22 | # level is odd, double the last one. 23 | if len(digests) % 2: 24 | digests.append(digests[-1].msg) 25 | 26 | next_level = [] 27 | for i in range(0,len(digests), 2): 28 | next_level.append(cat_sha256d(digests[i], digests[i + 1])) 29 | 30 | digests = next_level 31 | 32 | return digests[0] 33 | 34 | 35 | def make_timestamp_from_block(digest, block, blockheight, *, max_tx_size=1000): 36 | """Make a timestamp for a message in a block 37 | 38 | Every transaction within the block is serialized and checked to see if the 39 | raw serialized bytes contain the message. If one or more transactions do, 40 | the smallest transaction is used to create a timestamp proof for that 41 | specific message to the block header. 42 | 43 | To limit the maximum size of proof, transactions larger than `max_tx_size` 44 | are ignored. 45 | 46 | Returns a timestamp for that message on success, None on failure. 47 | """ 48 | 49 | # Note how strategy changes if we add SHA256 midstate support 50 | len_smallest_tx_found = max_tx_size + 1 51 | commitment_tx = None 52 | prefix = None 53 | suffix = None 54 | for tx in block.vtx: 55 | serialized_tx = tx.serialize(params={'include_witness':False}) 56 | 57 | if len(serialized_tx) > len_smallest_tx_found: 58 | continue 59 | 60 | try: 61 | i = serialized_tx.index(digest) 62 | except ValueError: 63 | continue 64 | 65 | # Found it! 66 | commitment_tx = tx 67 | prefix = serialized_tx[0:i] 68 | suffix = serialized_tx[i + len(digest):] 69 | 70 | len_smallest_tx_found = len(serialized_tx) 71 | 72 | if len_smallest_tx_found > max_tx_size: 73 | return None 74 | 75 | digest_timestamp = Timestamp(digest) 76 | 77 | # Add the commitment ops necessary to go from the digest to the txid op 78 | prefix_stamp = digest_timestamp.ops.add(OpPrepend(prefix)) 79 | txid_stamp = cat_sha256d(prefix_stamp, suffix) 80 | 81 | assert commitment_tx.GetTxid() == txid_stamp.msg 82 | 83 | # Create the txid list, with our commitment txid op in the appropriate 84 | # place 85 | block_txid_stamps = [] 86 | for tx in block.vtx: 87 | if tx.GetTxid() != txid_stamp.msg: 88 | block_txid_stamps.append(Timestamp(tx.GetTxid())) 89 | else: 90 | block_txid_stamps.append(txid_stamp) 91 | 92 | # Build the merkle tree 93 | merkleroot_stamp = __make_btc_block_merkle_tree(block_txid_stamps) 94 | assert merkleroot_stamp.msg == block.hashMerkleRoot 95 | 96 | attestation = BitcoinBlockHeaderAttestation(blockheight) 97 | merkleroot_stamp.attestations.add(attestation) 98 | 99 | return digest_timestamp 100 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Always prefer setuptools over distutils 2 | from setuptools import setup, find_packages 3 | # To use a consistent encoding 4 | from codecs import open 5 | from os import path 6 | 7 | from opentimestamps import __version__ 8 | 9 | here = path.abspath(path.dirname(__file__)) 10 | 11 | # Get the long description from the README file 12 | with open(path.join(here, 'README.md'), encoding='utf-8') as f: 13 | long_description = f.read() 14 | 15 | setup( 16 | name='opentimestamps', 17 | 18 | # Versions should comply with PEP440. For a discussion on single-sourcing 19 | # the version across setup.py and the project code, see 20 | # https://packaging.python.org/en/latest/single_source_version.html 21 | version=__version__, 22 | 23 | description='Create and verify OpenTimestamps proofs', 24 | long_description=long_description, 25 | long_description_content_type='text/markdown', 26 | 27 | # The project's main homepage. 28 | url='https://github.com/opentimestamps/python-opentimestamps', 29 | 30 | # Author details 31 | author='Peter Todd', 32 | author_email='pete@petertodd.org', 33 | 34 | # Choose your license 35 | license='LGPL3', 36 | 37 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 38 | classifiers=[ 39 | # How mature is this project? Common values are 40 | # 3 - Alpha 41 | # 4 - Beta 42 | # 5 - Production/Stable 43 | 'Development Status :: 4 - Beta', 44 | 45 | # Indicate who your project is intended for 46 | 'Intended Audience :: Developers', 47 | 'Topic :: Security :: Cryptography', 48 | 49 | # Pick your license as you wish (should match "license" above) 50 | 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)', 51 | 52 | # Specify the Python versions you support here. In particular, ensure 53 | # that you indicate whether you support Python 2, Python 3 or both. 54 | 'Programming Language :: Python :: 3 :: Only', 55 | ], 56 | 57 | # What does your project relate to? 58 | keywords='cryptography timestamping bitcoin', 59 | 60 | # You can just specify the packages manually here if your project is 61 | # simple. Or you can use find_packages(). 62 | packages=find_packages(exclude=['contrib', 'docs', 'tests']), 63 | 64 | # Alternatively, if you want to distribute just a my_module.py, uncomment 65 | # this: 66 | # py_modules=["my_module"], 67 | 68 | # List run-time dependencies here. These will be installed by pip when 69 | # your project is installed. For an analysis of "install_requires" vs pip's 70 | # requirements files see: 71 | # https://packaging.python.org/en/latest/requirements.html 72 | install_requires=['python-bitcoinlib>=0.9.0,<0.13.0', 73 | 'pycryptodomex>=3.3.1'], 74 | 75 | # List additional groups of dependencies here (e.g. development 76 | # dependencies). You can install these using the following syntax, 77 | # for example: 78 | # $ pip install -e .[dev,test] 79 | extras_require={}, 80 | 81 | # If there are data files included in your packages that need to be 82 | # installed, specify them here. If using Python 2.6 or less, then these 83 | # have to be included in MANIFEST.in as well. 84 | package_data={}, 85 | 86 | # Although 'package_data' is the preferred approach, in some case you may 87 | # need to place data files outside of your packages. See: 88 | # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa 89 | # In this case, 'data_file' will be installed into '/my_data' 90 | data_files=[], 91 | 92 | # To provide executable scripts, use entry points in preference to the 93 | # "scripts" keyword. Entry points provide cross-platform support and allow 94 | # pip to create the appropriate form of executable for the target platform. 95 | entry_points={}, 96 | ) 97 | -------------------------------------------------------------------------------- /opentimestamps/tests/core/test_op.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import unittest 13 | 14 | from opentimestamps.core.op import * 15 | 16 | class Test_Op(unittest.TestCase): 17 | def test_append(self): 18 | """Append operation""" 19 | self.assertEqual(OpAppend(b'suffix')(b'msg'), b'msgsuffix') 20 | 21 | def test_append_invalid_arg(self): 22 | """Append op, invalid argument""" 23 | with self.assertRaises(TypeError): 24 | OpAppend('') 25 | with self.assertRaises(OpArgValueError): 26 | OpAppend(b'') 27 | with self.assertRaises(OpArgValueError): 28 | OpAppend(b'.'*4097) 29 | 30 | def test_append_invalid_msg(self): 31 | """Append op, invalid message""" 32 | with self.assertRaises(TypeError): 33 | OpAppend(b'.')(None) 34 | with self.assertRaises(TypeError): 35 | OpAppend(b'.')('') 36 | 37 | OpAppend(b'.')(b'.'*4095) 38 | with self.assertRaises(MsgValueError): 39 | OpAppend(b'.')(b'.'*4096) 40 | 41 | def test_prepend(self): 42 | """Prepend operation""" 43 | self.assertEqual(OpPrepend(b'prefix')(b'msg'), b'prefixmsg') 44 | 45 | def test_prepend_invalid_arg(self): 46 | """Prepend op, invalid argument""" 47 | with self.assertRaises(TypeError): 48 | OpPrepend('') 49 | with self.assertRaises(OpArgValueError): 50 | OpPrepend(b'') 51 | with self.assertRaises(OpArgValueError): 52 | OpPrepend(b'.'*4097) 53 | 54 | def test_prepend_invalid_msg(self): 55 | """Prepend op, invalid message""" 56 | with self.assertRaises(TypeError): 57 | OpPrepend(b'.')(None) 58 | with self.assertRaises(TypeError): 59 | OpPrepend(b'.')('') 60 | 61 | OpPrepend(b'.')(b'.'*4095) 62 | with self.assertRaises(MsgValueError): 63 | OpPrepend(b'.')(b'.'*4096) 64 | 65 | # def test_reverse(self): 66 | # """Reverse operation""" 67 | # self.assertEqual(OpReverse()(b'abcd'), b'dcba') 68 | 69 | def test_hexlify(self): 70 | """Hexlify operation""" 71 | for msg, expected in ((b'\x00', b'00'), 72 | (b'\xde\xad\xbe\xef', b'deadbeef')): 73 | self.assertEqual(OpHexlify()(msg), expected) 74 | 75 | def test_hexlify_msg_length_limits(self): 76 | """Hexlify message length limits""" 77 | OpHexlify()(b'.'*2048) 78 | with self.assertRaises(MsgValueError): 79 | OpHexlify()(b'.'*2049) 80 | with self.assertRaises(MsgValueError): 81 | OpHexlify()(b'') 82 | 83 | def test_sha256(self): 84 | """SHA256 operation""" 85 | self.assertEqual(OpSHA256()(b''), bytes.fromhex('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')) 86 | 87 | def test_ripemd160(self): 88 | """RIPEMD160 operation""" 89 | self.assertEqual(OpRIPEMD160()(b''), bytes.fromhex('9c1185a5c5e9fc54612808977ee8f548b2258d31')) 90 | 91 | def test_equality(self): 92 | """Operation equality""" 93 | self.assertEqual(OpReverse(), OpReverse()) 94 | self.assertNotEqual(OpReverse(), OpSHA1()) 95 | 96 | self.assertEqual(OpAppend(b'foo'), OpAppend(b'foo')) 97 | self.assertNotEqual(OpAppend(b'foo'), OpAppend(b'bar')) 98 | self.assertNotEqual(OpAppend(b'foo'), OpPrepend(b'foo')) 99 | 100 | def test_ordering(self): 101 | """Operation ordering""" 102 | self.assertTrue(OpSHA1() < OpRIPEMD160()) 103 | # FIXME: more tests 104 | 105 | def test_keccak256(self): 106 | """KECCAK256 operation""" 107 | self.assertEqual(OpKECCAK256()(b''), bytes.fromhex('c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470')) 108 | self.assertEqual(OpKECCAK256()(b'\x80'), bytes.fromhex('56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421')) 109 | -------------------------------------------------------------------------------- /opentimestamps/tests/core/test_notary.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import unittest 13 | 14 | from opentimestamps.core.serialize import * 15 | from opentimestamps.core.notary import * 16 | 17 | class Test_UnknownAttestation(unittest.TestCase): 18 | def test_repr(self): 19 | """repr(UnknownAttestation)""" 20 | a = UnknownAttestation(bytes.fromhex('0102030405060708'), b'Hello World!') 21 | self.assertEqual(repr(a), "UnknownAttestation(b'\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08', b'Hello World!')") 22 | 23 | def test_serialization(self): 24 | """"Serialization/deserialization of unknown attestations""" 25 | expected_serialized = bytes.fromhex('0102030405060708') + b'\x0c' + b'Hello World!' 26 | ctx = BytesDeserializationContext(expected_serialized) 27 | a = TimeAttestation.deserialize(ctx) 28 | 29 | self.assertEqual(a.TAG, bytes.fromhex('0102030405060708')) 30 | self.assertEqual(a.payload, b'Hello World!') 31 | 32 | # Test round trip 33 | ctx = BytesSerializationContext() 34 | a.serialize(ctx) 35 | self.assertEqual(expected_serialized, ctx.getbytes()) 36 | 37 | def test_deserialize_too_long(self): 38 | """Deserialization of attestations with oversized payloads""" 39 | ctx = BytesDeserializationContext(bytes.fromhex('0102030405060708') + b'\x81\x40' + b'x'*8193) 40 | with self.assertRaises(DeserializationError): 41 | TimeAttestation.deserialize(ctx) 42 | 43 | # pending attestation 44 | ctx = BytesDeserializationContext(bytes.fromhex('83dfe30d2ef90c8e') + b'\x81\x40' + b'x'*8193) 45 | with self.assertRaises(DeserializationError): 46 | TimeAttestation.deserialize(ctx) 47 | 48 | class Test_PendingAttestation(unittest.TestCase): 49 | def test_serialize(self): 50 | pending_attestation = PendingAttestation('foobar') 51 | expected_serialized = bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobar' 52 | 53 | ctx = BytesSerializationContext() 54 | pending_attestation.serialize(ctx) 55 | self.assertEqual(ctx.getbytes(), expected_serialized) 56 | 57 | ctx = BytesDeserializationContext(expected_serialized) 58 | pending_attestation2 = TimeAttestation.deserialize(ctx) 59 | 60 | self.assertEqual(pending_attestation2.uri, 'foobar') 61 | 62 | def test_deserialize(self): 63 | pending_attestation = PendingAttestation('foobar') 64 | 65 | ctx = BytesSerializationContext() 66 | pending_attestation.serialize(ctx) 67 | 68 | self.assertEqual(ctx.getbytes(), bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobar') 69 | 70 | def test_invalid_uri_deserialization(self): 71 | # illegal character 72 | ctx = BytesDeserializationContext(bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'fo%bar') 73 | with self.assertRaises(DeserializationError): 74 | TimeAttestation.deserialize(ctx) 75 | 76 | # Too long 77 | 78 | # Exactly 1000 bytes is ok 79 | ctx = BytesDeserializationContext(bytes.fromhex('83dfe30d2ef90c8e' + 'ea07' + 'e807') + b'x'*1000) 80 | TimeAttestation.deserialize(ctx) 81 | 82 | # But 1001 isn't 83 | ctx = BytesDeserializationContext(bytes.fromhex('83dfe30d2ef90c8e' + 'eb07' + 'e907') + b'x'*1001) 84 | with self.assertRaises(DeserializationError): 85 | TimeAttestation.deserialize(ctx) 86 | 87 | def test_deserialization_trailing_garbage(self): 88 | ctx = BytesDeserializationContext(bytes.fromhex('83dfe30d2ef90c8e' + '08' + '06') + b'foobarx') 89 | with self.assertRaises(TrailingGarbageError): 90 | TimeAttestation.deserialize(ctx) 91 | 92 | class Test_BitcoinBlockHeaderAttestation(unittest.TestCase): 93 | def test_deserialization_trailing_garbage(self): 94 | ctx = BytesDeserializationContext(bytes.fromhex('0588960d73d71901' + 95 | '02' + # two bytes of payload 96 | '00' + # genesis block! 97 | 'ff')) # one byte of trailing garbage 98 | with self.assertRaises(TrailingGarbageError): 99 | TimeAttestation.deserialize(ctx) 100 | 101 | class Test_AttestationsComparison(unittest.TestCase): 102 | def test_attestation_comparison(self): 103 | """Comparing attestations""" 104 | self.assertTrue(UnknownAttestation(b'unknown1', b'') < UnknownAttestation(b'unknown2', b'')) 105 | self.assertTrue(BitcoinBlockHeaderAttestation(1) < PendingAttestation("")) 106 | -------------------------------------------------------------------------------- /opentimestamps/tests/test_bitcoin.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import unittest 13 | 14 | from bitcoin.core import * 15 | 16 | from opentimestamps.core.timestamp import * 17 | from opentimestamps.bitcoin import * 18 | 19 | class Test_make_timestamp_from_block(unittest.TestCase): 20 | def test(self): 21 | # genesis block! 22 | block = CBlock.deserialize(x('010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe680e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e362990101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a0100000043410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac00000000')) 23 | 24 | # satoshi's pubkey 25 | digest = x('0496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858ee') 26 | root_stamp = make_timestamp_from_block(digest, block, 0) 27 | 28 | (msg, attestation) = tuple(root_stamp.all_attestations())[0] 29 | self.assertEqual(msg, lx('0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098')) # merkleroot 30 | self.assertEqual(attestation.height, 0) 31 | 32 | 33 | # block #586, first block with 3 txs 34 | block = CBlock.deserialize(x('0100000038babc9586a5fcd60713573494f4377e7c401c33aa24729a4f6cff46000000004d5969c0d10dcce60868fee4d4de80ba5ef38abaeed8a75daa63e48c963d7b1950476f49ffff001d2d9791370301000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d025d06ffffffff0100f2052a0100000043410410daf049ef402de0b6adba8b0f7c392bcf9a6385116efc8b4143b8b7a7841e7de73b478ffe13b60c50ea01e24b4b48c24f5e0fbc5d6c8433c7ca7c3ed3ab8173ac0000000001000000050f40f5e65e115eb4bdb3007f0fb8beaa404cf7ae45de16074e8acc9b69bbf0c3000000004847304402201092da40af6dea8abcbeefb8586335b26d39d36be9b6c38d6c9cc18f20dd5886022045964de79a9008f68d53fc9bc58f9e30b224a1b98dbfda5c7b7b860f32c6aef101ffffffff1bb875b247332e558731c2c510f611d3dde991ea9fe69365bf445a0ccd513b190000000049483045022100b0a1d0a00251c56809a5ab5d7ba6cbe68b82c9bf4f806ee39c568ae537572c840220781ce69017ec3b2d6f96ffff4d19c80c224f40c73b8c26cba4b30e7f4171579b01ffffffff2099e1a92d94c35f0645683257c4c255165385f3e9129a85fed5a3f3d867c9b60000000049483045022100c8e980f43c616232e2d59dce08a5edb84aaa0915ea49780a8af367330216084a02203cc2628f16f995c7aaf6104cba64971963a4e084e4fbd0b6bcf825b47a09f8e301ffffffff5fb770c4de700aca7f74f5e6295f248edafa9423e446d76f4650df9b90f939a700000000494830450220745a8d99c51f98f5c93b8d2f5f14a1f2d8cc42ff7329645681bcafe846cbf50d022100b24e31186129f3ae6cc8a226d1eda389373652a9cf2095631fcc4345067c1ff301ffffffff968d4c096ee861307935d21d797a902b647dc970d3c8374cc13551f8397abbd80000000049483045022100ca65b3f290724d6c56fc333570fa342f2477f34b2a6c93c2e2d7216d9fe9088e022077e259a29ed1f988fab2b9f2ce17a4a56a20c188cadc72bca94e06a73826966501ffffffff0100ba1dd20500000043410497304efd3ab14d0dcbf1e901045a25f4b5dbaf576d074506fd8ded4122ba6f6bec0ed4698ce0e7928c0eaf9ddfb5387929b5d697e82e7aabebe04c10e5c87164ac0000000001000000010d26ba57ff82fefcb43826b45019043e2b6ef9aa8118b7f743167584a7f9cae70000000049483045022024fd7345df2b2bd0e6f8416529046b7d52bda5ffdb70146bc6d72b1ba73cabcd022100ff99c03006cc8f28d92e686f0ae640d20395177f329d0a9dbd560fd2a55aeee701ffffffff0100f2052a01000000434104888d890e1bd84c9e2ac363a9774414a081eb805cd2c0d52e49efc7170ebf342f1cdb284a2e2eb754fc8dd4525fe0caa3d3a525214d0b504dd75376b2f63804a8ac00000000')) 35 | 36 | # one of the txids spent 37 | digest = lx('c3f0bb699bcc8a4e0716de45aef74c40aabeb80f7f00b3bdb45e115ee6f5400f') 38 | root_stamp = make_timestamp_from_block(digest, block, 586) 39 | 40 | (msg, attestation) = tuple(root_stamp.all_attestations())[0] 41 | self.assertEqual(msg, lx('197b3d968ce463aa5da7d8eeba8af35eba80ded4e4fe6808e6cc0dd1c069594d')) # merkleroot 42 | self.assertEqual(attestation.height, 586) 43 | 44 | # Check behavior when the digest is not found 45 | root_stamp = make_timestamp_from_block(b'not in the block', block, 586) 46 | self.assertEqual(root_stamp, None) 47 | 48 | # Check that size limit is respected 49 | root_stamp = make_timestamp_from_block(digest, block, 586, max_tx_size=1) 50 | self.assertEqual(root_stamp, None) 51 | 52 | def test_segwit(self): 53 | # regtest block, with a segwit tx 54 | block = CBlock.deserialize(x('0000002060f13fc5bbde8de36f8896a70071d53494f14bcda132968205db08be651f4402fba57b59d485e5446f674cb42917d85f370ee60bbe8a4149ca9d2d236bb2a5286645835affff7f200000000002020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff050281000101ffffffff02b8f2052a01000000232103c94c88a631f2286bf1404f550742cec64df1701e8374a17426d8375f6fbbc188ac0000000000000000266a24aa21a9edb6e18554334fb03b3ca9e17ef71614853731fffee928f9f63be4de7052b9b1c3012000000000000000000000000000000000000000000000000000000000000000000000000001000000000101bb7e38e3e5bf1ef124e96342cfdd6e4ac8e155a19d845bca68af1c2db420e3a5000000001716001457c8e57a3bdfd4e586fcbecabd958e5f7d5bae49fdffffff0290b1a43e1e00000017a914f55b72549c205fdf490ce331ac3f95ad4f7b2a24870000000000000000226a208e16cdc4bc8b6aae0a217d30662bf5bb7b732b0751746f587b411fecbc41574e02483045022100e6cb9807f0125e6c9bda818b280103177ad4956d8efd5d689d005d95e8d096bf0220138fce520f4cfbb255e0f7c56183002a1f489bbed7eebfa75bf7e9995533d3dc012103e7c5963292645605207996004489e09edb79edfd6b7d7e45562acb064ae9628380000000')) 55 | 56 | # satoshi's pubkey 57 | digest = x('8e16cdc4bc8b6aae0a217d30662bf5bb7b732b0751746f587b411fecbc41574e') 58 | root_stamp = make_timestamp_from_block(digest, block, 129) 59 | 60 | (msg, attestation) = tuple(root_stamp.all_attestations())[0] 61 | self.assertEqual(msg, lx('28a5b26b232d9dca49418abe0be60e375fd81729b44c676f44e585d4597ba5fb')) # merkleroot 62 | self.assertEqual(attestation.height, 129) 63 | -------------------------------------------------------------------------------- /opentimestamps/calendar.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import binascii 13 | import urllib.request 14 | import fnmatch 15 | 16 | from urllib.parse import urljoin 17 | from opentimestamps.core.timestamp import Timestamp 18 | from opentimestamps.core.serialize import BytesDeserializationContext 19 | 20 | 21 | def get_sanitised_resp_msg(exp): 22 | """Get the sanitise response messages from a calendar response 23 | 24 | Returns the sanitised message, with any character not in the whitelist replaced by '_' 25 | """ 26 | 27 | # Note how new lines are _not_ allowed: this is important, as otherwise the 28 | # message could include a second line pretending to be something else. 29 | WHITELIST = b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789#-.,; ' 30 | 31 | # Two lines of text 32 | raw_msg = bytearray(exp.read(160)) 33 | 34 | for i in range(len(raw_msg)): 35 | if raw_msg[i] not in WHITELIST: 36 | raw_msg[i] = ord('_') 37 | 38 | return raw_msg.decode() 39 | 40 | 41 | class CommitmentNotFoundError(KeyError): 42 | def __init__(self, reason): 43 | super().__init__(reason) 44 | self.reason = reason 45 | 46 | 47 | class RemoteCalendar: 48 | """Remote calendar server interface""" 49 | 50 | def __init__(self, url, user_agent="python-opentimestamps"): 51 | if not isinstance(url, str): 52 | raise TypeError("URL must be a string") 53 | self.url = url 54 | 55 | self.request_headers = {"Accept": "application/vnd.opentimestamps.v1", 56 | "User-Agent": user_agent} 57 | 58 | def submit(self, digest, timeout=None): 59 | """Submit a digest to the calendar 60 | 61 | Returns a Timestamp committing to that digest 62 | """ 63 | req = urllib.request.Request(urljoin(self.url, 'digest'), data=digest, headers=self.request_headers) 64 | 65 | with urllib.request.urlopen(req, timeout=timeout) as resp: 66 | if resp.status != 200: 67 | raise Exception("Unknown response from calendar: %d" % resp.status) 68 | 69 | # FIXME: Not a particularly nice way of handling this, but it'll do 70 | # the job for now. 71 | resp_bytes = resp.read(10000) 72 | if len(resp_bytes) > 10000: 73 | raise Exception("Calendar response exceeded size limit") 74 | 75 | ctx = BytesDeserializationContext(resp_bytes) 76 | return Timestamp.deserialize(ctx, digest) 77 | 78 | def get_timestamp(self, commitment, timeout=None): 79 | """Get a timestamp for a given commitment 80 | 81 | Raises KeyError if the calendar doesn't have that commitment 82 | """ 83 | req = urllib.request.Request( 84 | urljoin(self.url, 'timestamp/' + binascii.hexlify(commitment).decode('utf8')), 85 | headers=self.request_headers) 86 | try: 87 | with urllib.request.urlopen(req, timeout=timeout) as resp: 88 | if resp.status == 200: 89 | 90 | # FIXME: Not a particularly nice way of handling this, but it'll do 91 | # the job for now. 92 | resp_bytes = resp.read(10000) 93 | if len(resp_bytes) > 10000: 94 | raise Exception("Calendar response exceeded size limit") 95 | 96 | ctx = BytesDeserializationContext(resp_bytes) 97 | return Timestamp.deserialize(ctx, commitment) 98 | 99 | else: 100 | raise Exception("Unknown response from calendar: %d" % resp.status) 101 | except urllib.error.HTTPError as exp: 102 | if exp.code == 404: 103 | raise CommitmentNotFoundError(get_sanitised_resp_msg(exp)) 104 | else: 105 | raise exp 106 | 107 | 108 | class UrlWhitelist(set): 109 | """Glob-matching whitelist for URL's""" 110 | 111 | def __init__(self, urls=()): 112 | for url in urls: 113 | self.add(url) 114 | 115 | def add(self, url): 116 | if not isinstance(url, str): 117 | raise TypeError("URL must be a string") 118 | 119 | if url.startswith('http://') or url.startswith('https://'): 120 | parsed_url = urllib.parse.urlparse(url) 121 | 122 | # FIXME: should have a more friendly error message 123 | assert not parsed_url.params and not parsed_url.query and not parsed_url.fragment 124 | 125 | set.add(self, parsed_url) 126 | 127 | else: 128 | self.add('http://' + url) 129 | self.add('https://' + url) 130 | 131 | def __contains__(self, url): 132 | parsed_url = urllib.parse.urlparse(url) 133 | 134 | # FIXME: probably should tell user why... 135 | if parsed_url.params or parsed_url.query or parsed_url.fragment: 136 | return False 137 | 138 | for pattern in self: 139 | if (parsed_url.scheme == pattern.scheme and 140 | parsed_url.path == pattern.path and 141 | fnmatch.fnmatch(parsed_url.netloc, pattern.netloc)): 142 | return True 143 | 144 | else: 145 | return False 146 | 147 | def __repr__(self): 148 | return 'UrlWhitelist([%s])' % ','.join("'%s'" % url.geturl() for url in self) 149 | 150 | DEFAULT_CALENDAR_WHITELIST = \ 151 | UrlWhitelist(['https://*.calendar.opentimestamps.org', # Run by Peter Todd 152 | 'https://*.calendar.eternitywall.com', # Run by Riccardo Casatta 153 | 'https://*.calendar.catallaxy.com', # Run by Bull Bitcoin 154 | ]) 155 | 156 | DEFAULT_AGGREGATORS = \ 157 | ('https://a.pool.opentimestamps.org', 158 | 'https://b.pool.opentimestamps.org', 159 | 'https://a.pool.eternitywall.com', 160 | 'https://ots.btc.catallaxy.com') 161 | -------------------------------------------------------------------------------- /opentimestamps/tests/core/test_git.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import unittest 13 | 14 | import dbm 15 | import git 16 | import tempfile 17 | 18 | from bitcoin.core import b2x 19 | 20 | from opentimestamps.core.timestamp import * 21 | from opentimestamps.core.op import * 22 | from opentimestamps.core.git import * 23 | 24 | class Test_GitTreeTimestamper(unittest.TestCase): 25 | 26 | def setUp(self): 27 | self.db_dirs = [] 28 | 29 | def tearDown(self): 30 | for d in self.db_dirs: 31 | d.cleanup() 32 | del self.db_dirs 33 | 34 | def make_stamper(self, commit): 35 | # Yes, we're using our own git repo as the test data! 36 | repo = git.Repo(__file__ + '../../../../../') 37 | db_dir = tempfile.TemporaryDirectory() 38 | self.db_dirs.append(db_dir) 39 | db = dbm.open(db_dir.name + '/db', 'c') 40 | tree = repo.commit(commit).tree 41 | return GitTreeTimestamper(tree, db=db) 42 | 43 | def test_blobs(self): 44 | """Git blob hashing""" 45 | 46 | stamper = self.make_stamper("53c68bc976c581636b84c82fe814fab178adf8a6") 47 | 48 | for expected_hexdigest, path in (('9e34b52cfa5724a4d87e9f7f47e2699c14d918285a20bf47f5a2a7345999e543', 'LICENSE'), 49 | ('ef83ecaca007e8afbfcca834b75510a98b6c10036374bb0d9f42a63f69efcd11', 'opentimestamps/__init__.py'), 50 | ('ef83ecaca007e8afbfcca834b75510a98b6c10036374bb0d9f42a63f69efcd11', 'opentimestamps/tests/__init__.py'), 51 | ('745bd9059cf01edabe3a61198fe1147e01ff57eec69e29f2e617b8e376427082', 'opentimestamps/tests/core/test_core.py'), 52 | ('ef83ecaca007e8afbfcca834b75510a98b6c10036374bb0d9f42a63f69efcd11', 'opentimestamps/tests/core/__init__.py'), 53 | ('7cd2b5a8723814be27fe6b224cc76e52275b1ff149de157ce374d290d032e875', 'opentimestamps/core/__init__.py'), 54 | ('d41fb0337e687b26f3f5dd61d10ec5080ff0bdc32f90f2022f7e2d9eeba91442', 'README')): 55 | 56 | stamp = stamper[path] 57 | actual_hexdigest = b2x(stamp.file_digest) 58 | self.assertEqual(expected_hexdigest, actual_hexdigest) 59 | 60 | stamper = self.make_stamper("30f6c357d578e0921dc6fffd67e2af1ce1ca0ff2") 61 | empty_stamp = stamper["empty"] 62 | self.assertEqual(empty_stamp.file_digest, bytes.fromhex("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")) 63 | 64 | def test_empty_tree(self): 65 | """Git tree with a single empty file""" 66 | stamper = self.make_stamper("30f6c357d578e0921dc6fffd67e2af1ce1ca0ff2") 67 | 68 | # There's a single empty file in this directory. Thus the nonce_key is: 69 | nonce_key = OpSHA256()(OpSHA256()(b'') + # one empty file 70 | b'\x01\x89\x08\x0c\xfb\xd0\xe8\x08') # tag 71 | 72 | nonce1 = OpSHA256()(OpSHA256()(b'') + nonce_key) 73 | assert nonce1[0] & 0b1 == 1 74 | nonce2 = OpSHA256()(nonce1) 75 | 76 | self.assertEqual(stamper.timestamp.msg, 77 | OpSHA256()(b'')) 78 | self.assertEqual(stamper.timestamp.msg, b"\xe3\xb0\xc4B\x98\xfc\x1c\x14\x9a\xfb\xf4\xc8\x99o\xb9$'\xaeA\xe4d\x9b\x93L\xa4\x95\x99\x1bxR\xb8U") 79 | 80 | def test_two_file_tree(self): 81 | """Git tree with a two files""" 82 | stamper = self.make_stamper("78eb5cdc1ec638be72d6fb7a38c4d24f2be5d081") 83 | 84 | nonce_key = OpSHA256()(OpSHA256()(b'a\n') + 85 | OpSHA256()(b'b\n') + 86 | b'\x01\x89\x08\x0c\xfb\xd0\xe8\x08') # tag 87 | 88 | n_a_nonce1 = OpSHA256()(OpSHA256()(b'a\n') + nonce_key) 89 | assert n_a_nonce1[0] & 0b1 == 0 90 | n_a_nonce2 = OpSHA256()(n_a_nonce1) 91 | n_a = OpSHA256()(OpSHA256()(b'a\n') + n_a_nonce2) 92 | 93 | n_b_nonce1 = OpSHA256()(OpSHA256()(b'b\n') + nonce_key) 94 | assert n_b_nonce1[0] & 0b1 == 0 95 | n_b_nonce2 = OpSHA256()(n_b_nonce1) 96 | n_b = OpSHA256()(OpSHA256()(b'b\n') + n_b_nonce2) 97 | 98 | self.assertEqual(stamper.timestamp.msg, 99 | OpSHA256()(n_a + n_b)) 100 | self.assertEqual(stamper.timestamp.msg, b's\x0e\xc2h\xd4\xb3\xa5\xd4\xe6\x0e\xe9\xb2t\x89@\x95\xc8c_F3\x81a=\xc2\xd4qy\xaf\x8e\xa0\x87') 101 | 102 | def test_tree_with_children(self): 103 | """Git tree with child trees""" 104 | stamper = self.make_stamper("b22192fffb9aad27eb57986e7fe89f8047340346") 105 | 106 | # These correspond to the final values from the test_empty_tree() and 107 | # test_two_file_tree() test cases above; git git commit we're testing 108 | # has the trees associated with those test cases in the one/ and two/ 109 | # directories respectively. 110 | d_one = b"\xe3\xb0\xc4B\x98\xfc\x1c\x14\x9a\xfb\xf4\xc8\x99o\xb9$'\xaeA\xe4d\x9b\x93L\xa4\x95\x99\x1bxR\xb8U" 111 | d_two = b's\x0e\xc2h\xd4\xb3\xa5\xd4\xe6\x0e\xe9\xb2t\x89@\x95\xc8c_F3\x81a=\xc2\xd4qy\xaf\x8e\xa0\x87' 112 | 113 | nonce_key = OpSHA256()(d_one + d_two + 114 | b'\x01\x89\x08\x0c\xfb\xd0\xe8\x08') # tag 115 | 116 | n_one_nonce1 = OpSHA256()(d_one + nonce_key) 117 | assert n_one_nonce1[0] & 0b1 == 0 118 | n_one_nonce2 = OpSHA256()(n_one_nonce1) 119 | n_one = OpSHA256()(d_one + n_one_nonce2) 120 | 121 | n_two_nonce1 = OpSHA256()(d_two + nonce_key) 122 | assert n_two_nonce1[0] & 0b1 == 0 123 | n_two_nonce2 = OpSHA256()(n_two_nonce1) 124 | n_two = OpSHA256()(d_two + n_two_nonce2) 125 | 126 | self.assertEqual(stamper.timestamp.msg, 127 | OpSHA256()(n_one + n_two)) 128 | 129 | def test_tree_with_prefix_matching_blob(self): 130 | """Git tree with prefix matching blob""" 131 | stamper = self.make_stamper("75736a2524c624c1a08a574938686f83de5a8a86") 132 | 133 | two_a_stamp = stamper['two/a'] 134 | 135 | def test_submodule(self): 136 | """Git tree with submodule""" 137 | stamper = self.make_stamper("a3efe73f270866bc8d8f6ce01d22c02f14b21a1a") 138 | 139 | self.assertEqual(stamper.timestamp.msg, 140 | OpSHA256()(bytes.fromhex('48b96efa66e2958e955a31a7d9b8f2ac8384b8b9'))) 141 | 142 | def test_dangling_symlink(self): 143 | """Git tree with dangling symlink""" 144 | stamper = self.make_stamper("a59620c107a67c4b6323e6e96aed9929d6a89618") 145 | 146 | self.assertEqual(stamper.timestamp.msg, 147 | OpSHA256()(b'does-not-exist')) 148 | 149 | def test_huge_tree(self): 150 | """Really big git tree""" 151 | # would cause the OpSHA256 length limits to be exceeded if it were used 152 | # directly 153 | stamper = self.make_stamper("a52fe6e3d4b15057ff41df0509dd302bc5863c29") 154 | 155 | self.assertEqual(stamper.timestamp.msg, 156 | b'\x1dW\x9c\xea\x94&`\xc2\xfb\xba \x19Q\x0f\xdb\xf0\x7f\x14\xe3\x14zb\t\xdb\xcf\xf93I\xe9h\xb9\x8d') 157 | -------------------------------------------------------------------------------- /opentimestamps/core/packetstream.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import io 13 | import sys 14 | 15 | """Packet-writing support for append-only streams with truncation handling 16 | 17 | Strictly append-only streams, such as files whose append-only attribute has 18 | been set by chattr(1), are a useful way to avoid losing data. But using them 19 | for complex serialized data poses a problem: truncated writes. 20 | 21 | For instance, suppose we try to serialize the string 'Hello World!' to a stream 22 | using var_bytes(). If everything goes correctly, the following will be written 23 | to the stream: 24 | 25 | b'\x0cHello World!' 26 | 27 | However, suppose that there's an IO error after the third byte, leaving the 28 | file truncated: 29 | 30 | b'\x0cHe' 31 | 32 | Since the stream is strictly append-only, if we write anything else to the 33 | stream, the length byte will cause the deserializer to later incorrectly read 34 | part of the next thing we write as though it were part of the original string. 35 | While in theory we could fix this, doing so requires a lot of invasive code 36 | changes to the (de)serialization code. 37 | 38 | This module implements a much simpler solution with variable length packets. 39 | Each packet can be any length, and it's guaranteed that in the event of a 40 | truncated write, at worst the most recently written packet will be corrupted. 41 | Secondly, it's guaranteed that in the event of a corrupt packet, additional 42 | packets can be written succesfully even if the underlying stream is 43 | append-only. 44 | """ 45 | 46 | 47 | class PacketWriter(io.BufferedIOBase): 48 | """Write an individual packet""" 49 | 50 | def __init__(self, fd): 51 | """Create a new packet stream for writing 52 | 53 | fd must be a buffered stream; a io.BufferedIOBase instance. 54 | 55 | FIXME: fd must be blocking; the BlockingIOError exception isn't handled 56 | correctly yet 57 | """ 58 | if not isinstance(fd, io.BufferedIOBase): 59 | raise TypeError('fd must be buffered IO') 60 | 61 | self.raw = fd 62 | self.pending = b'' 63 | 64 | def write(self, buf): 65 | if self.closed: 66 | raise ValueError("write to closed packet") 67 | 68 | pending = self.pending + buf 69 | 70 | # the + 1 handles the case where the length of buf is an exact multiple 71 | # of the max sub-packet size 72 | for i in range(0, len(pending) + 1, 255): 73 | chunk = pending[i:i+255] 74 | if len(chunk) < 255: 75 | assert 0 <= len(pending) - i < 255 76 | self.pending = chunk 77 | break 78 | else: 79 | assert len(chunk) == 255 80 | 81 | try: 82 | l = self.raw.write(b'\xff' + chunk) 83 | assert l == 256 84 | except io.BlockingIOError as exp: 85 | # To support this, we'd need to look at characters_written to 86 | # figure out what data from pending has been written. 87 | raise Exception("non-blocking IO not yet supported: %r" % exp) 88 | else: 89 | assert False 90 | 91 | return len(buf) 92 | 93 | def flush_pending(self): 94 | """Flush pending data to the underlying stream 95 | 96 | All pending data is written to the underlying stream, creating a 97 | partial-length sub-packet if necessary. However the underlying stream 98 | is _not_ flushed. If there is no pending data, this function is a 99 | no-op. 100 | """ 101 | if self.closed: 102 | raise ValueError("flush of closed packet") 103 | 104 | if not self.pending: 105 | return 106 | 107 | assert len(self.pending) < 255 108 | 109 | l = self.raw.write(bytes([len(self.pending)]) + self.pending) 110 | assert l == 1 + len(self.pending) 111 | 112 | self.pending = b'' 113 | 114 | try: 115 | self.raw.flush() 116 | except io.BlockingIOError as exp: 117 | # To support this, we'd need to look at characters_written to 118 | # figure out what data from pending has been written. 119 | raise Exception("non-blocking IO not yet supported: %r" % exp) 120 | 121 | def flush(self): 122 | """Flush the packet to disk 123 | 124 | All pending data is written to the underlying stream with 125 | flush_pending(), and flush() is called on that stream. 126 | """ 127 | self.flush_pending() 128 | 129 | try: 130 | self.raw.flush() 131 | except io.BlockingIOError as exp: 132 | # To support this, we'd need to look at characters_written to 133 | # figure out what data from pending has been written. 134 | raise Exception("non-blocking IO not yet supported: %r" % exp) 135 | 136 | def close(self): 137 | """Close the packet 138 | 139 | All pending data is written to the underlying stream, and the packet is 140 | closed. 141 | """ 142 | self.flush_pending() 143 | self.raw.write(b'\x00') # terminator to close the packet 144 | 145 | # Note how we didn't call flush above; BufferedIOBase.close() calls 146 | # self.flush() for us. 147 | super().close() 148 | 149 | class PacketMissingError(IOError): 150 | """Raised when a packet is completely missing""" 151 | 152 | class PacketReader(io.BufferedIOBase): 153 | """Read an individual packet""" 154 | 155 | def __init__(self, fd): 156 | """Create a new packet stream reader 157 | 158 | The first byte of the packet will be read immediately; if that read() 159 | fails PacketMissingError will be raised. 160 | """ 161 | self.raw = fd 162 | 163 | # Bytes remaining until the end of the current sub-packet 164 | l = fd.read(1) 165 | if not l: 166 | raise PacketMissingError("Packet completely missing") 167 | 168 | self.len_remaining_subpacket = l[0] 169 | 170 | # Whether the end of the entire packet has been reached 171 | self.end_of_packet = False 172 | 173 | # How many bytes are known to have been truncated (None if not known yet) 174 | self.truncated = None 175 | 176 | def read(self, size=-1): 177 | if self.end_of_packet: 178 | return b'' 179 | 180 | r = [] 181 | remaining = size if size >= 0 else sys.maxsize 182 | while remaining and not self.end_of_packet: 183 | if self.len_remaining_subpacket: 184 | # The current subpacket hasn't been completely read. 185 | l = min(remaining, self.len_remaining_subpacket) 186 | b = self.raw.read(l) 187 | 188 | r.append(b) 189 | self.len_remaining_subpacket -= len(b) 190 | remaining -= len(b) 191 | 192 | if len(b) < l: 193 | # read returned less than requested, so the sub-packet must 194 | # be truncated; record how many bytes are missing. Note how 195 | # we add one to that figure to account for the 196 | # end-of-packet marker. 197 | self.truncated = l - len(b) + 1 198 | self.end_of_packet = True 199 | 200 | else: 201 | # All of the current subpacket has been read, so start reading 202 | # the next sub-packet. 203 | 204 | # Get length of next sub-packet 205 | l = self.raw.read(1) 206 | if l == b'': 207 | # We're truncated by exactly one byte, the end-of-packet 208 | # marker. 209 | self.truncated = 1 210 | self.end_of_packet = True 211 | 212 | else: 213 | # Succesfully read the length 214 | self.len_remaining_subpacket = l[0] 215 | 216 | if not self.len_remaining_subpacket: 217 | self.end_of_packet = True 218 | 219 | return b''.join(r) 220 | -------------------------------------------------------------------------------- /opentimestamps/core/serialize.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | """Consensus-critical recursive descent serialization/deserialization""" 13 | 14 | import binascii 15 | import io 16 | 17 | class DeserializationError(Exception): 18 | """Base class for all errors encountered during deserialization""" 19 | 20 | class BadMagicError(DeserializationError): 21 | """A magic number is incorrect 22 | 23 | Raise this when the file format magic number is incorrect. 24 | """ 25 | def __init__(self, expected_magic, actual_magic): 26 | super().__init__('Expected magic bytes 0x%s, but got 0x%s instead' % (binascii.hexlify(expected_magic).decode(), 27 | binascii.hexlify(actual_magic).decode())) 28 | 29 | class UnsupportedMajorVersion(DeserializationError): 30 | """Unsupported major version 31 | 32 | Raise this a major version is unsupported 33 | """ 34 | 35 | class TruncationError(DeserializationError): 36 | """Truncated data encountered while deserializing""" 37 | 38 | class TrailingGarbageError(DeserializationError): 39 | """Trailing garbage found after deserialization finished 40 | 41 | Raised when deserialization otherwise succeeds without errors, but excess 42 | data is present after the data we expected to get. 43 | """ 44 | 45 | class RecursionLimitError(DeserializationError): 46 | """Data is too deeply nested to be deserialized 47 | 48 | Raised when deserializing recursively defined data structures that exceed 49 | the recursion limit for that particular data structure. 50 | """ 51 | 52 | class SerializerTypeError(TypeError): 53 | """Wrong type for specified serializer""" 54 | 55 | class SerializerValueError(ValueError): 56 | """Inappropriate value to be serialized (of correct type)""" 57 | 58 | 59 | class SerializationContext: 60 | """Context for serialization 61 | 62 | Allows multiple serialization targets to share the same codebase, for 63 | instance bytes, memoized serialization, hashing, etc. 64 | """ 65 | 66 | def write_bool(self, value): 67 | """Write a bool""" 68 | raise NotImplementedError 69 | 70 | def write_varuint(self, value): 71 | """Write a variable-length unsigned integer""" 72 | raise NotImplementedError 73 | 74 | def write_bytes(self, value): 75 | """Write fixed-length bytes""" 76 | raise NotImplementedError 77 | 78 | def write_varbytes(self, value): 79 | """Write variable-length bytes""" 80 | raise NotImplementedError 81 | 82 | class DeserializationContext: 83 | """Context for deserialization 84 | 85 | Allows multiple deserialization sources to share the same codebase, for 86 | instance bytes, memoized serialization, hashing, etc. 87 | """ 88 | def read_bool(self): 89 | """Read a bool""" 90 | raise NotImplementedError 91 | 92 | def read_varuint(self, max_int): 93 | """Read a variable-length unsigned integer""" 94 | raise NotImplementedError 95 | 96 | def read_bytes(self, expected_length): 97 | """Read fixed-length bytes""" 98 | raise NotImplementedError 99 | 100 | def read_varbytes(self, value, max_length=None): 101 | """Read variable-length bytes 102 | 103 | No more than max_length bytes will be read. 104 | """ 105 | raise NotImplementedError 106 | 107 | def assert_magic(self, expected_magic): 108 | """Assert the presence of magic bytes 109 | 110 | Raises BadMagicError if the magic bytes don't match, or if the read was 111 | truncated. 112 | 113 | Note that this isn't an assertion in the Python sense: debug/production 114 | does not change the behavior of this function. 115 | """ 116 | raise NotImplementedError 117 | 118 | def assert_eof(self): 119 | """Assert that we have reached the end of the data 120 | 121 | Raises TrailingGarbageError(msg) if the end of file has not been reached. 122 | 123 | Note that this isn't an assertion in the Python sense: debug/production 124 | does not change the behavior of this function. 125 | """ 126 | raise NotImplementedError 127 | 128 | class StreamSerializationContext(SerializationContext): 129 | def __init__(self, fd): 130 | """Serialize to a stream""" 131 | self.fd = fd 132 | 133 | def write_bool(self, value): 134 | if value is True: 135 | self.fd.write(b'\xff') 136 | 137 | elif value is False: 138 | self.fd.write(b'\x00') 139 | 140 | else: 141 | raise TypeError('Expected bool; got %r' % value.__class__) 142 | 143 | def write_varuint(self, value): 144 | # unsigned little-endian base128 format (LEB128) 145 | if value == 0: 146 | self.fd.write(b'\x00') 147 | 148 | else: 149 | while value != 0: 150 | b = value & 0b01111111 151 | if value > 0b01111111: 152 | b |= 0b10000000 153 | self.fd.write(bytes([b])) 154 | if value <= 0b01111111: 155 | break 156 | value >>= 7 157 | 158 | def write_uint8(self, value): 159 | self.fd.write(bytes([value])) 160 | 161 | def write_bytes(self, value): 162 | self.fd.write(value) 163 | 164 | def write_varbytes(self, value): 165 | self.write_varuint(len(value)) 166 | self.fd.write(value) 167 | 168 | class StreamDeserializationContext(DeserializationContext): 169 | def __init__(self, fd): 170 | """Deserialize from a stream""" 171 | self.fd = fd 172 | 173 | def fd_read(self, l): 174 | r = self.fd.read(l) 175 | if len(r) != l: 176 | raise TruncationError('Tried to read %d bytes but got only %d bytes' % \ 177 | (l, len(r))) 178 | return r 179 | 180 | def read_bool(self): 181 | # unsigned little-endian base128 format (LEB128) 182 | b = self.fd_read(1)[0] 183 | if b == 0xff: 184 | return True 185 | 186 | elif b == 0x00: 187 | return False 188 | 189 | else: 190 | raise DeserializationError('read_bool() expected 0xff or 0x00; got %d' % b) 191 | 192 | def read_varuint(self): 193 | value = 0 194 | shift = 0 195 | 196 | while True: 197 | b = self.fd_read(1)[0] 198 | value |= (b & 0b01111111) << shift 199 | if not (b & 0b10000000): 200 | break 201 | shift += 7 202 | 203 | return value 204 | 205 | def read_uint8(self): 206 | return self.fd_read(1)[0] 207 | 208 | def read_bytes(self, expected_length=None): 209 | if expected_length is None: 210 | expected_length = self.read_varuint(None) 211 | return self.fd_read(expected_length) 212 | 213 | def read_varbytes(self, max_len, min_len=0): 214 | l = self.read_varuint() 215 | if l > max_len: 216 | raise DeserializationError('varbytes max length exceeded; %d > %d' % (l, max_len)) 217 | if l < min_len: 218 | raise DeserializationError('varbytes min length not met; %d < %d' % (l, min_len)) 219 | return self.fd_read(l) 220 | 221 | def assert_magic(self, expected_magic): 222 | actual_magic = self.fd.read(len(expected_magic)) 223 | if expected_magic != actual_magic: 224 | raise BadMagicError(expected_magic, actual_magic) 225 | 226 | def assert_eof(self): 227 | excess = self.fd.read(1) 228 | if excess: 229 | raise TrailingGarbageError("Trailing garbage found after end of deserialized data") 230 | 231 | class BytesSerializationContext(StreamSerializationContext): 232 | def __init__(self): 233 | """Serialize to bytes""" 234 | super().__init__(io.BytesIO()) 235 | 236 | def getbytes(self): 237 | """Return the bytes serialized to date""" 238 | return self.fd.getvalue() 239 | 240 | class BytesDeserializationContext(StreamDeserializationContext): 241 | def __init__(self, buf): 242 | """Deserialize from bytes""" 243 | super().__init__(io.BytesIO(buf)) 244 | 245 | # FIXME: need to check that there isn't extra crap at end of object 246 | -------------------------------------------------------------------------------- /opentimestamps/tests/core/test_packetstream.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import contextlib 13 | import io 14 | import os 15 | import tempfile 16 | import unittest 17 | 18 | from opentimestamps.core.packetstream import * 19 | 20 | class Test_PacketWriter(unittest.TestCase): 21 | def test_open_close(self): 22 | """Open followed by close writes a packet""" 23 | with tempfile.NamedTemporaryFile() as tmpfile: 24 | with open(tmpfile.name, 'wb') as fd: 25 | writer = PacketWriter(fd) 26 | self.assertFalse(writer.closed) 27 | writer.close() 28 | self.assertTrue(writer.closed) 29 | 30 | with open(tmpfile.name, 'rb') as fd: 31 | self.assertEqual(fd.read(), b'\x00') 32 | 33 | def test_with(self): 34 | """Using PacketWrite as a context manager""" 35 | with tempfile.NamedTemporaryFile() as tmpfile: 36 | with open(tmpfile.name, 'wb') as fd: 37 | with PacketWriter(fd) as writer: 38 | pass 39 | 40 | with open(tmpfile.name, 'rb') as fd: 41 | self.assertEqual(fd.read(), b'\x00') 42 | 43 | @contextlib.contextmanager 44 | def assert_written(self, expected_contents): 45 | with tempfile.NamedTemporaryFile() as tmpfile: 46 | with open(tmpfile.name, 'wb') as fd: 47 | with PacketWriter(fd) as writer: 48 | yield writer 49 | 50 | with open(tmpfile.name, 'rb') as fd: 51 | actual_contents = fd.read() 52 | self.assertEqual(expected_contents, actual_contents) 53 | 54 | def test_empty_write(self): 55 | """Empty writes are no-ops""" 56 | with self.assert_written(b'\x00') as writer: 57 | writer.write(b'') 58 | with self.assert_written(b'\x00') as writer: 59 | writer.write(b'') 60 | writer.write(b'') 61 | 62 | def test_sub_block_write(self): 63 | """Writing less than one sub-block""" 64 | with self.assert_written(b'\x01a\x00') as writer: 65 | writer.write(b'a') 66 | with self.assert_written(b'\x02ab\x00') as writer: 67 | writer.write(b'a') 68 | writer.write(b'b') 69 | 70 | with self.assert_written(b'\xff' + b'x'*255 + b'\x00') as writer: 71 | writer.write(b'x'*254) 72 | writer.write(b'x'*1) 73 | with self.assert_written(b'\xff' + b'x'*255 + b'\x00') as writer: 74 | writer.write(b'x'*255) 75 | 76 | def test_multi_sub_block_writes(self): 77 | """Writing more than one sub-block""" 78 | with self.assert_written(b'\xff' + b'x'*255 + b'\x01x' + b'\x00') as writer: 79 | writer.write(b'x' * 255) 80 | writer.write(b'x' * 1) 81 | with self.assert_written(b'\xff' + b'x'*255 + b'\x01x' + b'\x00') as writer: 82 | writer.write(b'x' * (255 + 1)) 83 | 84 | with self.assert_written(b'\xff' + b'x'*255 + b'\xfe' + b'x'*254 + b'\x00') as writer: 85 | writer.write(b'x' * 255) 86 | writer.write(b'x' * 254) 87 | with self.assert_written(b'\xff' + b'x'*255 + b'\xfe' + b'x'*254 + b'\x00') as writer: 88 | writer.write(b'x' * (255 + 254)) 89 | 90 | with self.assert_written(b'\xff' + b'x'*255 + b'\xff' + b'x'*255 + b'\x00') as writer: 91 | writer.write(b'x' * 255) 92 | writer.write(b'x' * 255) 93 | with self.assert_written(b'\xff' + b'x'*255 + b'\xff' + b'x'*255 + b'\x00') as writer: 94 | writer.write(b'x' * (255 + 255)) 95 | 96 | with self.assert_written(b'\xff' + b'x'*255 + b'\xff' + b'x'*255 + b'\x01x' + b'\x00') as writer: 97 | writer.write(b'x' * 255) 98 | writer.write(b'x' * 255) 99 | writer.write(b'x' * 1) 100 | with self.assert_written(b'\xff' + b'x'*255 + b'\xff' + b'x'*255 + b'\x01x' + b'\x00') as writer: 101 | writer.write(b'x' * (255 + 255 + 1)) 102 | 103 | def test_flush(self): 104 | with self.assert_written(b'\x05Hello' + b'\x06World!' + b'\x00') as writer: 105 | writer.write(b'Hello') 106 | writer.flush() 107 | writer.write(b'World!') 108 | 109 | def test_del_does_not_close(self): 110 | """Deleting a PacketWriter does not close the underlying stream""" 111 | with io.BytesIO() as fd: 112 | writer = PacketWriter(fd) 113 | del writer 114 | 115 | self.assertFalse(fd.closed) 116 | 117 | class Test_PacketReader(unittest.TestCase): 118 | def test_close_only_packet(self): 119 | """Close does not close underlying stream""" 120 | with io.BytesIO(b'\x00') as fd: 121 | reader = PacketReader(fd) 122 | reader.close() 123 | 124 | self.assertTrue(reader.closed) 125 | self.assertFalse(fd.closed) 126 | 127 | def test_valid_empty_packet(self): 128 | """Empty, but valid, packets""" 129 | with io.BytesIO(b'\x00') as fd: 130 | reader = PacketReader(fd) 131 | self.assertEqual(fd.tell(), 1) 132 | 133 | self.assertFalse(reader.end_of_packet) 134 | 135 | # reading nothing is a no-op 136 | self.assertEqual(reader.read(0), b'') 137 | self.assertFalse(reader.end_of_packet) 138 | self.assertEqual(fd.tell(), 1) 139 | 140 | self.assertEqual(reader.read(1), b'') 141 | self.assertTrue(reader.end_of_packet) 142 | 143 | self.assertEqual(fd.tell(), 1) 144 | 145 | def test_single_sub_packet_read(self): 146 | """Reading less than a single sub-packet""" 147 | with io.BytesIO(b'\x0cHello World!\x00') as fd: 148 | reader = PacketReader(fd) 149 | self.assertEqual(fd.tell(), 1) 150 | 151 | self.assertEqual(reader.read(12), b'Hello World!') 152 | self.assertFalse(reader.end_of_packet) # reader hasn't found out yet 153 | self.assertEqual(fd.tell(), 13) 154 | 155 | self.assertEqual(reader.read(), b'') 156 | self.assertTrue(reader.end_of_packet) 157 | 158 | self.assertEqual(fd.tell(), 14) 159 | 160 | def test_multi_sub_packet_read(self): 161 | """Reads that span multiple sub-packets""" 162 | with io.BytesIO(b'\x01H' + b'\x0bello World!' + b'\x00') as fd: 163 | reader = PacketReader(fd) 164 | self.assertEqual(fd.tell(), 1) 165 | 166 | self.assertEqual(reader.read(12), b'Hello World!') 167 | self.assertFalse(reader.end_of_packet) # reader hasn't found out yet 168 | self.assertEqual(fd.tell(), 14) 169 | 170 | self.assertEqual(reader.read(), b'') 171 | self.assertTrue(reader.end_of_packet) 172 | 173 | self.assertEqual(fd.tell(), 15) 174 | 175 | def test_missing_packet(self): 176 | """Completely missing packet raises PacketMissingError""" 177 | with io.BytesIO(b'') as fd: 178 | with self.assertRaises(PacketMissingError): 179 | PacketReader(fd) 180 | 181 | def test_truncated_packet(self): 182 | """Packet truncated at the first sub-packet""" 183 | 184 | with io.BytesIO(b'\x01') as fd: 185 | reader = PacketReader(fd) 186 | 187 | self.assertEqual(reader.read(), b'') 188 | self.assertTrue(reader.end_of_packet) 189 | self.assertEqual(reader.truncated, 2) # 1 byte of sub-packet, and the end of packet marker missing 190 | 191 | self.assertEqual(fd.tell(), 1) 192 | 193 | with io.BytesIO(b'\x02a') as fd: 194 | reader = PacketReader(fd) 195 | 196 | self.assertEqual(reader.read(), b'a') 197 | self.assertTrue(reader.end_of_packet) 198 | self.assertEqual(reader.truncated, 2) # 1 byte of sub-packet, and the end of packet marker missing 199 | 200 | self.assertEqual(fd.tell(), 2) 201 | 202 | with io.BytesIO(b'\x04ab') as fd: 203 | reader = PacketReader(fd) 204 | 205 | self.assertEqual(reader.read(1), b'a') 206 | self.assertEqual(reader.read(), b'b') 207 | self.assertTrue(reader.end_of_packet) 208 | self.assertEqual(reader.truncated, 3) # 2 bytes of sub-packet, and the end of packet marker missing 209 | 210 | self.assertEqual(fd.tell(), 3) 211 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | python-opentimestamps is free software: you can redistribute it and/or modify 2 | it under the terms of the GNU Lesser General Public License as published by the 3 | Free Software Foundation, either version 3 of the License, or (at your option) 4 | any later version. 5 | 6 | python-opentimestamps is distributed in the hope that it will be useful, but 7 | WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 8 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License 9 | below for more details. 10 | 11 | 12 | 13 | GNU LESSER GENERAL PUBLIC LICENSE 14 | Version 3, 29 June 2007 15 | 16 | Copyright (C) 2007 Free Software Foundation, Inc. 17 | Everyone is permitted to copy and distribute verbatim copies 18 | of this license document, but changing it is not allowed. 19 | 20 | 21 | This version of the GNU Lesser General Public License incorporates 22 | the terms and conditions of version 3 of the GNU General Public 23 | License, supplemented by the additional permissions listed below. 24 | 25 | 0. Additional Definitions. 26 | 27 | As used herein, "this License" refers to version 3 of the GNU Lesser 28 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 29 | General Public License. 30 | 31 | "The Library" refers to a covered work governed by this License, 32 | other than an Application or a Combined Work as defined below. 33 | 34 | An "Application" is any work that makes use of an interface provided 35 | by the Library, but which is not otherwise based on the Library. 36 | Defining a subclass of a class defined by the Library is deemed a mode 37 | of using an interface provided by the Library. 38 | 39 | A "Combined Work" is a work produced by combining or linking an 40 | Application with the Library. The particular version of the Library 41 | with which the Combined Work was made is also called the "Linked 42 | Version". 43 | 44 | The "Minimal Corresponding Source" for a Combined Work means the 45 | Corresponding Source for the Combined Work, excluding any source code 46 | for portions of the Combined Work that, considered in isolation, are 47 | based on the Application, and not on the Linked Version. 48 | 49 | The "Corresponding Application Code" for a Combined Work means the 50 | object code and/or source code for the Application, including any data 51 | and utility programs needed for reproducing the Combined Work from the 52 | Application, but excluding the System Libraries of the Combined Work. 53 | 54 | 1. Exception to Section 3 of the GNU GPL. 55 | 56 | You may convey a covered work under sections 3 and 4 of this License 57 | without being bound by section 3 of the GNU GPL. 58 | 59 | 2. Conveying Modified Versions. 60 | 61 | If you modify a copy of the Library, and, in your modifications, a 62 | facility refers to a function or data to be supplied by an Application 63 | that uses the facility (other than as an argument passed when the 64 | facility is invoked), then you may convey a copy of the modified 65 | version: 66 | 67 | a) under this License, provided that you make a good faith effort to 68 | ensure that, in the event an Application does not supply the 69 | function or data, the facility still operates, and performs 70 | whatever part of its purpose remains meaningful, or 71 | 72 | b) under the GNU GPL, with none of the additional permissions of 73 | this License applicable to that copy. 74 | 75 | 3. Object Code Incorporating Material from Library Header Files. 76 | 77 | The object code form of an Application may incorporate material from 78 | a header file that is part of the Library. You may convey such object 79 | code under terms of your choice, provided that, if the incorporated 80 | material is not limited to numerical parameters, data structure 81 | layouts and accessors, or small macros, inline functions and templates 82 | (ten or fewer lines in length), you do both of the following: 83 | 84 | a) Give prominent notice with each copy of the object code that the 85 | Library is used in it and that the Library and its use are 86 | covered by this License. 87 | 88 | b) Accompany the object code with a copy of the GNU GPL and this license 89 | document. 90 | 91 | 4. Combined Works. 92 | 93 | You may convey a Combined Work under terms of your choice that, 94 | taken together, effectively do not restrict modification of the 95 | portions of the Library contained in the Combined Work and reverse 96 | engineering for debugging such modifications, if you also do each of 97 | the following: 98 | 99 | a) Give prominent notice with each copy of the Combined Work that 100 | the Library is used in it and that the Library and its use are 101 | covered by this License. 102 | 103 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 104 | document. 105 | 106 | c) For a Combined Work that displays copyright notices during 107 | execution, include the copyright notice for the Library among 108 | these notices, as well as a reference directing the user to the 109 | copies of the GNU GPL and this license document. 110 | 111 | d) Do one of the following: 112 | 113 | 0) Convey the Minimal Corresponding Source under the terms of this 114 | License, and the Corresponding Application Code in a form 115 | suitable for, and under terms that permit, the user to 116 | recombine or relink the Application with a modified version of 117 | the Linked Version to produce a modified Combined Work, in the 118 | manner specified by section 6 of the GNU GPL for conveying 119 | Corresponding Source. 120 | 121 | 1) Use a suitable shared library mechanism for linking with the 122 | Library. A suitable mechanism is one that (a) uses at run time 123 | a copy of the Library already present on the user's computer 124 | system, and (b) will operate properly with a modified version 125 | of the Library that is interface-compatible with the Linked 126 | Version. 127 | 128 | e) Provide Installation Information, but only if you would otherwise 129 | be required to provide such information under section 6 of the 130 | GNU GPL, and only to the extent that such information is 131 | necessary to install and execute a modified version of the 132 | Combined Work produced by recombining or relinking the 133 | Application with a modified version of the Linked Version. (If 134 | you use option 4d0, the Installation Information must accompany 135 | the Minimal Corresponding Source and Corresponding Application 136 | Code. If you use option 4d1, you must provide the Installation 137 | Information in the manner specified by section 6 of the GNU GPL 138 | for conveying Corresponding Source.) 139 | 140 | 5. Combined Libraries. 141 | 142 | You may place library facilities that are a work based on the 143 | Library side by side in a single library together with other library 144 | facilities that are not Applications and are not covered by this 145 | License, and convey such a combined library under terms of your 146 | choice, if you do both of the following: 147 | 148 | a) Accompany the combined library with a copy of the same work based 149 | on the Library, uncombined with any other library facilities, 150 | conveyed under the terms of this License. 151 | 152 | b) Give prominent notice with the combined library that part of it 153 | is a work based on the Library, and explaining where to find the 154 | accompanying uncombined form of the same work. 155 | 156 | 6. Revised Versions of the GNU Lesser General Public License. 157 | 158 | The Free Software Foundation may publish revised and/or new versions 159 | of the GNU Lesser General Public License from time to time. Such new 160 | versions will be similar in spirit to the present version, but may 161 | differ in detail to address new problems or concerns. 162 | 163 | Each version is given a distinguishing version number. If the 164 | Library as you received it specifies that a certain numbered version 165 | of the GNU Lesser General Public License "or any later version" 166 | applies to it, you have the option of following the terms and 167 | conditions either of that published version or of any later version 168 | published by the Free Software Foundation. If the Library as you 169 | received it does not specify a version number of the GNU Lesser 170 | General Public License, you may choose any version of the GNU Lesser 171 | General Public License ever published by the Free Software Foundation. 172 | 173 | If the Library as you received it specifies that a proxy can decide 174 | whether future versions of the GNU Lesser General Public License shall 175 | apply, that proxy's public statement of acceptance of any version is 176 | permanent authorization for you to choose that version for the 177 | Library. 178 | -------------------------------------------------------------------------------- /opentimestamps/core/git.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | """Git integration""" 13 | 14 | import dbm 15 | import git 16 | import io 17 | import os 18 | 19 | from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile, make_merkle_tree 20 | from opentimestamps.core.op import OpAppend, OpPrepend, OpSHA256 21 | 22 | class GitTreeTimestamper: 23 | """Efficient, privacy-preserving, git tree timestamping 24 | 25 | Unfortunately, the way git calculates commit hashes is less than ideal for 26 | timestamping. The first problem is of course the fact that they're SHA1 27 | hashes: still good enough for timestamping, but all the same a dubious 28 | choice of hash algorithm. 29 | 30 | The second problem is more subtle: What if I want to extract a timestamp 31 | proof for an individual file in the commit? Since git hashes tree objects 32 | as linear blobs of data, the proof will contain a significant amount of 33 | extraneous metadata about files other than the one you want - inefficient 34 | and a privacy risk. 35 | 36 | This class solves these problems by recursively re-hashing a git tree and all blobs 37 | in it with SHA256, using a cache of previously calculated hashes for 38 | efficiency. Each git tree is hashed as a merkle tree, allowing paths to 39 | individual blobs to be extracted efficiently. 40 | 41 | For privacy, we guarantee that given a timestamp for a single item in a 42 | given tree, an attacker trying to guess the contents of any other item in 43 | the tree can only do so by brute-forcing all other content in the tree 44 | simultaneously. We achieve this by deterministically generating nonces for 45 | each item in the tree based on the item's hash, and the contents of the 46 | rest of the tree. 47 | 48 | However, note that we do _not_ prevent the attacker from learning about the 49 | directly structure of the repository, including the number of items in each 50 | directory. 51 | 52 | """ 53 | 54 | def __init__(self, tree, db=None, file_hash_op=OpSHA256(), tree_hash_op=OpSHA256()): 55 | self.tree = tree 56 | 57 | if db is None: 58 | os.makedirs(tree.repo.git_dir + '/ots', exist_ok=True) 59 | 60 | # WARNING: change the version number if any of the following is 61 | # changed; __init__() is consensus-critical! 62 | db = dbm.open(tree.repo.git_dir + '/ots/tree-hash-cache-v3', 'c') 63 | 64 | self.db = db 65 | self.file_hash_op = file_hash_op 66 | self.tree_hash_op = tree_hash_op 67 | 68 | def do_item(item): 69 | try: 70 | return (item, Timestamp(db[item.hexsha])) 71 | except KeyError: 72 | timestamp = None 73 | if isinstance(item, git.Blob): 74 | timestamp = Timestamp(file_hash_op.hash_fd(item.data_stream[3])) 75 | 76 | elif isinstance(item, git.Tree): 77 | stamper = GitTreeTimestamper(item, db=db, file_hash_op=file_hash_op, tree_hash_op=tree_hash_op) 78 | timestamp = stamper.timestamp 79 | 80 | elif isinstance(item, git.Submodule): 81 | # A submodule is just a git commit hash. 82 | # 83 | # Unfortunately we're not guaranteed to have the repo 84 | # behind it, so all we can do is timestamp that SHA1 hash. 85 | # 86 | # We do run it through the tree_hash_op to make it 87 | # indistinguishable from other things; consider the 88 | # degenerate case where the only thing in a git repo was a 89 | # submodule. 90 | timestamp = Timestamp(tree_hash_op(item.binsha)) 91 | 92 | else: 93 | raise NotImplementedError("Don't know what to do with %r" % item) 94 | 95 | db[item.hexsha] = timestamp.msg 96 | return (item, timestamp) 97 | 98 | self.contents = tuple(do_item(item) for item in self.tree) 99 | 100 | if len(self.contents) > 1: 101 | # Deterministically nonce contents in an all-or-nothing transform. As 102 | # mentioned in the class docstring, we want to ensure that the the 103 | # siblings of any leaf in the merkle tree don't give the attacker any 104 | # information about what else is in the tree, unless the attacker 105 | # already knows (or can brute-force) the entire contents of the tree. 106 | # 107 | # While not perfect - a user-provided persistant key would prevent the 108 | # attacker from being able to brute-force the contents - this option 109 | # has the advantage of being possible to calculate deterministically 110 | # using only the tree itself, removing the need to keep secret keys 111 | # that can easily be lost. 112 | # 113 | # First, calculate a nonce_key that depends on the entire contents of 114 | # the tree. The 8-byte tag ensures the key calculated is unique for 115 | # this purpose. 116 | contents_sum = b''.join(stamp.msg for item, stamp in self.contents) + b'\x01\x89\x08\x0c\xfb\xd0\xe8\x08' 117 | nonce_key = tree_hash_op.hash_fd(io.BytesIO(contents_sum)) 118 | 119 | # Second, calculate per-item nonces deterministically from that key, 120 | # and add those nonces to the timestamps of every item in the tree. 121 | # 122 | # While we usually use 128-bit nonces, here we're using full-length 123 | # nonces. Additionally, we pick append/prepend pseudo-randomly. This 124 | # helps obscure the directory structure, as a commitment for a git tree 125 | # is indistinguishable from a inner node in the per-git-tree merkle 126 | # tree. 127 | def deterministically_nonce_stamp(private_stamp): 128 | nonce1 = tree_hash_op(private_stamp.msg + nonce_key) 129 | nonce2 = tree_hash_op(nonce1) 130 | 131 | side = OpPrepend if nonce1[0] & 0b1 else OpAppend 132 | nonce_added = private_stamp.ops.add(side(nonce2)) 133 | return nonce_added.ops.add(tree_hash_op) 134 | 135 | nonced_contents = (deterministically_nonce_stamp(stamp) for item, stamp in self.contents) 136 | 137 | # Note how the current algorithm, if asked to timestamp a tree 138 | # with a single thing in it, will return the hash of that thing 139 | # directly. From the point of view of just commiting to the data that's 140 | # perfectly fine, and probably (slightly) better as it reveals a little 141 | # less information about directory structure. 142 | self.timestamp = make_merkle_tree(nonced_stamp for nonced_stamp in nonced_contents) 143 | 144 | elif len(self.contents) == 1: 145 | # If there's only one item in the tree, the fancy all-or-nothing 146 | # transform above is just a waste of ops, so use the tree contents 147 | # directly instead. 148 | self.timestamp = tuple(self.contents)[0][1] 149 | 150 | else: 151 | raise AssertionError("Empty git tree") 152 | 153 | def __getitem__(self, path): 154 | """Get a DetachedTimestampFile for blob at path 155 | 156 | The timestamp object returned will refer to self.timestamp, so 157 | modifying self.timestamp will modify the timestamp returned. 158 | 159 | If path does not exist, FileNotFound error will be raised. 160 | If path exists, but is not a blob, ValueError will be raised. 161 | """ 162 | for item, item_stamp in self.contents: 163 | if item.path == path: 164 | if isinstance(item, git.Blob): 165 | return DetachedTimestampFile(self.file_hash_op, item_stamp) 166 | 167 | else: 168 | raise ValueError("Path %r is not a blob" % item.path) 169 | 170 | elif path.startswith(item.path + '/'): 171 | if isinstance(item, git.Tree): 172 | # recurse 173 | tree_stamper = GitTreeTimestamper(item, db=self.db, file_hash_op=self.file_hash_op, tree_hash_op=self.tree_hash_op) 174 | tree_stamper.timestamp.merge(item_stamp) 175 | return tree_stamper[path] 176 | 177 | else: 178 | raise FileNotFoundError("Path %r not found; prefix %r is a blob" % (path, item.path)) 179 | else: 180 | raise FileNotFoundError("Path %r not found" % path) 181 | -------------------------------------------------------------------------------- /opentimestamps/tests/core/test_timestamp.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import unittest 13 | 14 | from opentimestamps.core.notary import * 15 | from opentimestamps.core.serialize import * 16 | from opentimestamps.core.timestamp import * 17 | from opentimestamps.core.op import * 18 | 19 | class Test_Timestamp(unittest.TestCase): 20 | def test_add_op(self): 21 | """Adding operations to timestamps""" 22 | t = Timestamp(b'abcd') 23 | t.ops.add(OpAppend(b'efgh')) 24 | self.assertEqual(t.ops[OpAppend(b'efgh')], Timestamp(b'abcdefgh')) 25 | 26 | # The second add should succeed with the timestamp unchanged 27 | t.ops.add(OpAppend(b'efgh')) 28 | self.assertEqual(t.ops[OpAppend(b'efgh')], Timestamp(b'abcdefgh')) 29 | 30 | def test_set_result_timestamp(self): 31 | """Setting an op result timestamp""" 32 | t1 = Timestamp(b'foo') 33 | t2 = t1.ops.add(OpAppend(b'bar')) 34 | t3 = t2.ops.add(OpAppend(b'baz')) 35 | 36 | self.assertEqual(t1.ops[OpAppend(b'bar')].ops[OpAppend(b'baz')].msg, b'foobarbaz') 37 | 38 | t1.ops[OpAppend(b'bar')] = Timestamp(b'foobar') 39 | 40 | self.assertTrue(OpAppend(b'baz') not in t1.ops[OpAppend(b'bar')].ops) 41 | 42 | def test_set_fail_if_wrong_message(self): 43 | """Setting an op result timestamp fails if the messages don't match""" 44 | t = Timestamp(b'abcd') 45 | t.ops.add(OpSHA256()) 46 | 47 | with self.assertRaises(ValueError): 48 | t.ops[OpSHA256()] = Timestamp(b'wrong') 49 | 50 | def test_merge(self): 51 | """Merging timestamps""" 52 | with self.assertRaises(ValueError): 53 | Timestamp(b'a').merge(Timestamp(b'b')) 54 | 55 | t1 = Timestamp(b'a') 56 | t2 = Timestamp(b'a') 57 | t2.attestations.add(PendingAttestation('foobar')) 58 | 59 | t1.merge(t2) 60 | self.assertEqual(t1, t2) 61 | 62 | # FIXME: more tests 63 | 64 | def test_serialization(self): 65 | """Timestamp serialization/deserialization""" 66 | def T(expected_instance, expected_serialized): 67 | ctx = BytesSerializationContext() 68 | expected_instance.serialize(ctx) 69 | actual_serialized = ctx.getbytes() 70 | 71 | self.assertEqual(expected_serialized, actual_serialized) 72 | 73 | actual_instance = Timestamp.deserialize(BytesDeserializationContext(expected_serialized), expected_instance.msg) 74 | self.assertEqual(expected_instance, actual_instance) 75 | 76 | 77 | stamp = Timestamp(b'foo') 78 | stamp.attestations.add(PendingAttestation('foobar')) 79 | 80 | T(stamp, b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobar') 81 | 82 | stamp.attestations.add(PendingAttestation('barfoo')) 83 | T(stamp, b'\xff' + (b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'barfoo') + \ 84 | (b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobar')) 85 | 86 | 87 | stamp.attestations.add(PendingAttestation('foobaz')) 88 | T(stamp, b'\xff' + (b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'barfoo') + \ 89 | b'\xff' + (b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobar') + \ 90 | (b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobaz')) 91 | 92 | sha256_stamp = stamp.ops.add(OpSHA256()) 93 | 94 | # Should fail - empty timestamps can't be serialized 95 | with self.assertRaises(ValueError): 96 | ctx = BytesSerializationContext() 97 | stamp.serialize(ctx) 98 | 99 | sha256_stamp.attestations.add(PendingAttestation('deeper')) 100 | T(stamp, b'\xff' + (b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'barfoo') + \ 101 | b'\xff' + (b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobar') + \ 102 | b'\xff' + (b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobaz') + \ 103 | b'\x08' + (b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'deeper')) 104 | 105 | def test_deserialization_invalid_op_msg(self): 106 | """Timestamp deserialization when message is invalid for op""" 107 | serialized = (b'\xf0\x01\x00' + # OpAppend(b'\x00') 108 | b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'barfoo') # perfectly valid pending attestation 109 | 110 | # Perfectly ok, results is 4096 bytes long 111 | Timestamp.deserialize(BytesDeserializationContext(serialized), b'.'*4095) 112 | 113 | with self.assertRaises(DeserializationError): 114 | # Not ok, result would be 4097 bytes long 115 | Timestamp.deserialize(BytesDeserializationContext(serialized), b'.'*4096) 116 | 117 | def test_deserialization_invalid_op_msg_2(self): 118 | """Deserialization of a timestamp that exceeds the recursion limit""" 119 | serialized = (b'\x08'*256 + # OpSHA256, 256 times 120 | b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'barfoo') # perfectly valid pending attestation 121 | 122 | with self.assertRaises(RecursionLimitError): 123 | Timestamp.deserialize(BytesDeserializationContext(serialized), b'') 124 | 125 | def test_str_tree(self): 126 | """Converting timestamp to tree""" 127 | t = Timestamp(b'') 128 | t.ops.add(OpAppend(b'\x01')) 129 | t.ops.add(OpSHA256()) 130 | self.assertEqual(t.str_tree(), " -> sha256\n -> append 01\n") 131 | 132 | def test_equality(self): 133 | """Checking timestamp equality""" 134 | t1 = Timestamp(b'') 135 | t1.attestations = {BitcoinBlockHeaderAttestation(1), PendingAttestation("")} 136 | t2 = Timestamp(b'') 137 | self.assertFalse(t1 == t2) 138 | t2.attestations = {PendingAttestation(""), BitcoinBlockHeaderAttestation(1)} 139 | self.assertTrue(t1 == t2) 140 | 141 | class Test_DetachedTimestampFile(unittest.TestCase): 142 | def test_create_from_file(self): 143 | file_stamp = DetachedTimestampFile.from_fd(OpSHA256(), io.BytesIO(b'')) 144 | self.assertEqual(file_stamp.file_digest, bytes.fromhex('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')) 145 | 146 | def test_hash_fd(self): 147 | file_stamp = DetachedTimestampFile.from_fd(OpSHA256(), io.BytesIO(b'')) 148 | 149 | result = file_stamp.file_hash_op.hash_fd(io.BytesIO(b'')) 150 | self.assertEqual(result, bytes.fromhex('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')) 151 | 152 | def test_serialization(self): 153 | def T(expected_instance, expected_serialized): 154 | ctx = BytesSerializationContext() 155 | expected_instance.serialize(ctx) 156 | actual_serialized = ctx.getbytes() 157 | 158 | self.assertEqual(expected_serialized, actual_serialized) 159 | 160 | actual_instance = DetachedTimestampFile.deserialize(BytesDeserializationContext(expected_serialized)) 161 | self.assertEqual(expected_instance, actual_instance) 162 | 163 | file_stamp = DetachedTimestampFile.from_fd(OpSHA256(), io.BytesIO(b'')) 164 | file_stamp.timestamp.attestations.add(PendingAttestation('foobar')) 165 | 166 | T(file_stamp, (b'\x00OpenTimestamps\x00\x00Proof\x00\xbf\x89\xe2\xe8\x84\xe8\x92\x94' + 167 | b'\x01' + # major version 168 | b'\x08' + bytes.fromhex('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') + 169 | b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobar')) 170 | 171 | def test_deserialization_failures(self): 172 | """Deserialization failures""" 173 | 174 | for serialized, expected_error in ((b'', BadMagicError), 175 | (b'\x00Not a OpenTimestamps Proof \x00\xbf\x89\xe2\xe8\x84\xe8\x92\x94\x01', BadMagicError), 176 | (b'\x00OpenTimestamps\x00\x00Proof\x00\xbf\x89\xe2\xe8\x84\xe8\x92\x94\x00', UnsupportedMajorVersion), 177 | (b'\x00OpenTimestamps\x00\x00Proof\x00\xbf\x89\xe2\xe8\x84\xe8\x92\x94\x01' + 178 | b'\x42' + # Not a valid opcode 179 | b'\x00'*32 + 180 | b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobar', DeserializationError), 181 | (b'\x00OpenTimestamps\x00\x00Proof\x00\xbf\x89\xe2\xe8\x84\xe8\x92\x94\x01' + 182 | b'\x08' + bytes.fromhex('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') + 183 | b'\x00' + bytes.fromhex('83dfe30d2ef90c8e' + '07' + '06') + b'foobar' + 184 | b'trailing garbage', TrailingGarbageError)): 185 | 186 | with self.assertRaises(expected_error): 187 | ctx = BytesDeserializationContext(serialized) 188 | DetachedTimestampFile.deserialize(ctx) 189 | 190 | 191 | class Test_cat_sha256(unittest.TestCase): 192 | def test(self): 193 | left = Timestamp(b'foo') 194 | right = Timestamp(b'bar') 195 | 196 | stamp_left_right= cat_sha256(left, right) 197 | self.assertEqual(stamp_left_right.msg, bytes.fromhex('c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2')) 198 | 199 | righter = Timestamp(b'baz') 200 | stamp_righter = cat_sha256(stamp_left_right, righter) 201 | self.assertEqual(stamp_righter.msg, bytes.fromhex('23388b16c66f1fa37ef14af8eb081712d570813e2afb8c8ae86efa726f3b7276')) 202 | 203 | 204 | class Test_make_merkle_tree(unittest.TestCase): 205 | def test(self): 206 | def T(n, expected_merkle_root): 207 | roots = [Timestamp(bytes([i])) for i in range(n)] 208 | tip = make_merkle_tree(roots) 209 | 210 | self.assertEqual(tip.msg, expected_merkle_root) 211 | 212 | for root in roots: 213 | pass # FIXME: check all roots lead to same timestamp 214 | 215 | # Returned unchanged! 216 | T(1, bytes.fromhex('00')) 217 | 218 | # Manually calculated w/ pen-and-paper 219 | T(2, bytes.fromhex('b413f47d13ee2fe6c845b2ee141af81de858df4ec549a58b7970bb96645bc8d2')) 220 | T(3, bytes.fromhex('e6aa639123d8aac95d13d365ec3779dade4b49c083a8fed97d7bfc0d89bb6a5e')) 221 | T(4, bytes.fromhex('7699a4fdd6b8b6908a344f73b8f05c8e1400f7253f544602c442ff5c65504b24')) 222 | T(5, bytes.fromhex('aaa9609d0c949fee22c1c941a4432f32dc1c2de939e4af25207f0dc62df0dbd8')) 223 | T(6, bytes.fromhex('ebdb4245f648b7e77b60f4f8a99a6d0529d1d372f98f35478b3284f16da93c06')) 224 | T(7, bytes.fromhex('ba4603a311279dea32e8958bfb660c86237157bf79e6bfee857803e811d91b8f')) 225 | -------------------------------------------------------------------------------- /opentimestamps/core/op.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import binascii 13 | import Cryptodome.Hash.keccak 14 | import hashlib 15 | import opentimestamps.core.serialize 16 | 17 | class MsgValueError(ValueError): 18 | """Raised when an operation can't be applied to the specified message. 19 | 20 | For example, because OpHexlify doubles the size of it's input, we restrict 21 | the size of the message it can be applied to to avoid running out of 22 | memory; OpHexlify raises this exception when that happens. 23 | """ 24 | 25 | class OpArgValueError(ValueError): 26 | """Raised when an operation argument has an invalid value 27 | 28 | For example, if OpAppend/OpPrepend's argument is too long. 29 | """ 30 | 31 | class Op(tuple): 32 | """Timestamp proof operations 33 | 34 | Operations are the edges in the timestamp tree, with each operation taking 35 | a message and zero or more arguments to produce a result. 36 | """ 37 | SUBCLS_BY_TAG = {} 38 | __slots__ = [] 39 | 40 | MAX_RESULT_LENGTH = 4096 41 | """Maximum length of an Op result 42 | 43 | For a verifier, this limit is what limits the maximum amount of memory you 44 | need at any one time to verify a particular timestamp path; while verifying 45 | a particular commitment operation path previously calculated results can be 46 | discarded. 47 | 48 | Of course, if everything was a merkle tree you never need to append/prepend 49 | anything near 4KiB of data; 64 bytes would be plenty even with SHA512. The 50 | main need for this is compatibility with existing systems like Bitcoin 51 | timestamps and Certificate Transparency servers. While the pathological 52 | limits required by both are quite large - 1MB and 16MiB respectively - 4KiB 53 | is perfectly adequate in both cases for more reasonable usage. 54 | 55 | Op subclasses should set this limit even lower if doing so is appropriate 56 | for them. 57 | """ 58 | 59 | MAX_MSG_LENGTH = 4096 60 | """Maximum length of the message an Op can be applied too 61 | 62 | Similar to the result length limit, this limit gives implementations a sane 63 | constraint to work with; the maximum result-length limit implicitly 64 | constrains maximum message length anyway. 65 | 66 | Op subclasses should set this limit even lower if doing so is appropriate 67 | for them. 68 | """ 69 | 70 | def __eq__(self, other): 71 | if isinstance(other, Op): 72 | return self.TAG == other.TAG and tuple(self) == tuple(other) 73 | else: 74 | return NotImplemented 75 | 76 | def __ne__(self, other): 77 | if isinstance(other, Op): 78 | return self.TAG != other.TAG or tuple(self) != tuple(other) 79 | else: 80 | return NotImplemented 81 | 82 | def __lt__(self, other): 83 | if isinstance(other, Op): 84 | if self.TAG == other.TAG: 85 | return tuple(self) < tuple(other) 86 | else: 87 | return self.TAG < other.TAG 88 | else: 89 | return NotImplemented 90 | 91 | def __le__(self, other): 92 | if isinstance(other, Op): 93 | if self.TAG == other.TAG: 94 | return tuple(self) <= tuple(other) 95 | else: 96 | return self.TAG < other.TAG 97 | else: 98 | return NotImplemented 99 | 100 | def __gt__(self, other): 101 | if isinstance(other, Op): 102 | if self.TAG == other.TAG: 103 | return tuple(self) > tuple(other) 104 | else: 105 | return self.TAG > other.TAG 106 | else: 107 | return NotImplemented 108 | def __ge__(self, other): 109 | if isinstance(other, Op): 110 | if self.TAG == other.TAG: 111 | return tuple(self) >= tuple(other) 112 | else: 113 | return self.TAG > other.TAG 114 | else: 115 | return NotImplemented 116 | 117 | def __hash__(self): 118 | return self.TAG[0] ^ tuple.__hash__(self) 119 | 120 | def _do_op_call(self, msg): 121 | raise NotImplementedError 122 | 123 | def __call__(self, msg): 124 | """Apply the operation to a message 125 | 126 | Raises MsgValueError if the message value is invalid, such as it being 127 | too long, or it causing the result to be too long. 128 | """ 129 | if not isinstance(msg, bytes): 130 | raise TypeError("Expected message to be bytes; got %r" % msg.__class__) 131 | 132 | elif len(msg) > self.MAX_MSG_LENGTH: 133 | raise MsgValueError("Message too long; %d > %d" % (len(msg), self.MAX_MSG_LENGTH)) 134 | 135 | r = self._do_op_call(msg) 136 | 137 | # No operation should allow the result to be empty; that would 138 | # trivially allow the commitment DAG to have a cycle in it. 139 | assert len(r) 140 | 141 | if len(r) > self.MAX_RESULT_LENGTH: 142 | raise MsgValueError("Result too long; %d > %d" % (len(r), self.MAX_RESULT_LENGTH)) 143 | 144 | else: 145 | return r 146 | 147 | def __repr__(self): 148 | return '%s()' % self.__class__.__name__ 149 | 150 | def __str__(self): 151 | return '%s' % self.TAG_NAME 152 | 153 | @classmethod 154 | def _register_op(cls, subcls): 155 | cls.SUBCLS_BY_TAG[subcls.TAG] = subcls 156 | if cls != Op: 157 | cls.__base__._register_op(subcls) 158 | return subcls 159 | 160 | def serialize(self, ctx): 161 | ctx.write_bytes(self.TAG) 162 | 163 | @classmethod 164 | def deserialize_from_tag(cls, ctx, tag): 165 | if tag in cls.SUBCLS_BY_TAG: 166 | return cls.SUBCLS_BY_TAG[tag].deserialize_from_tag(ctx, tag) 167 | else: 168 | raise opentimestamps.core.serialize.DeserializationError("Unknown operation tag 0x%0x" % tag[0]) 169 | 170 | @classmethod 171 | def deserialize(cls, ctx): 172 | tag = ctx.read_bytes(1) 173 | return cls.deserialize_from_tag(ctx, tag) 174 | 175 | class UnaryOp(Op): 176 | """Operations that act on a single message""" 177 | SUBCLS_BY_TAG = {} 178 | 179 | def __new__(cls): 180 | return tuple.__new__(cls) 181 | 182 | def serialize(self, ctx): 183 | super().serialize(ctx) 184 | 185 | @classmethod 186 | def deserialize_from_tag(cls, ctx, tag): 187 | if tag in cls.SUBCLS_BY_TAG: 188 | return cls.SUBCLS_BY_TAG[tag]() 189 | else: 190 | raise opentimestamps.core.serialize.DeserializationError("Unknown unary op tag 0x%0x" % tag[0]) 191 | 192 | class BinaryOp(Op): 193 | """Operations that act on a message and a single argument""" 194 | SUBCLS_BY_TAG = {} 195 | 196 | def __new__(cls, arg): 197 | if not isinstance(arg, bytes): 198 | raise TypeError("arg must be bytes") 199 | elif not len(arg): 200 | raise OpArgValueError("%s arg can't be empty" % cls.__name__) 201 | elif len(arg) > cls.MAX_RESULT_LENGTH: 202 | raise OpArgValueError("%s arg too long: %d > %d" % (cls.__name__, len(arg), cls.MAX_RESULT_LENGTH)) 203 | return tuple.__new__(cls, (arg,)) 204 | 205 | def __repr__(self): 206 | return '%s(%r)' % (self.__class__.__name__, self[0]) 207 | 208 | def __str__(self): 209 | return '%s %s' % (self.TAG_NAME, binascii.hexlify(self[0]).decode('utf8')) 210 | 211 | def serialize(self, ctx): 212 | super().serialize(ctx) 213 | ctx.write_varbytes(self[0]) 214 | 215 | @classmethod 216 | def deserialize_from_tag(cls, ctx, tag): 217 | if tag in cls.SUBCLS_BY_TAG: 218 | arg = ctx.read_varbytes(cls.MAX_RESULT_LENGTH, min_len=1) 219 | return cls.SUBCLS_BY_TAG[tag](arg) 220 | else: 221 | raise opentimestamps.core.serialize.DeserializationError("Unknown binary op tag 0x%0x" % tag[0]) 222 | 223 | 224 | @BinaryOp._register_op 225 | class OpAppend(BinaryOp): 226 | """Append a suffix to a message""" 227 | TAG = b'\xf0' 228 | TAG_NAME = 'append' 229 | 230 | def _do_op_call(self, msg): 231 | return msg + self[0] 232 | 233 | @BinaryOp._register_op 234 | class OpPrepend(BinaryOp): 235 | """Prepend a prefix to a message""" 236 | TAG = b'\xf1' 237 | TAG_NAME = 'prepend' 238 | 239 | def _do_op_call(self, msg): 240 | return self[0] + msg 241 | 242 | 243 | @UnaryOp._register_op 244 | class OpReverse(UnaryOp): 245 | TAG = b'\xf2' 246 | TAG_NAME = 'reverse' 247 | 248 | def _do_op_call(self, msg): 249 | if not len(msg): 250 | raise MsgValueError("Can't reverse an empty message") 251 | 252 | import warnings 253 | warnings.warn("OpReverse may get removed; see https://github.com/opentimestamps/python-opentimestamps/issues/5", PendingDeprecationWarning) 254 | return msg[::-1] 255 | 256 | @UnaryOp._register_op 257 | class OpHexlify(UnaryOp): 258 | """Convert bytes to lower-case hexadecimal representation 259 | 260 | Note that hexlify can only be performed on messages that aren't empty; 261 | hexlify on an empty message would create a cycle in the commitment graph. 262 | """ 263 | TAG = b'\xf3' 264 | TAG_NAME = 'hexlify' 265 | 266 | MAX_MSG_LENGTH = UnaryOp.MAX_RESULT_LENGTH // 2 267 | """Maximum length of message that we'll hexlify 268 | 269 | Every invocation of hexlify doubles the size of its input, this is simply 270 | half the maximum result length. 271 | """ 272 | 273 | def _do_op_call(self, msg): 274 | if not len(msg): 275 | raise MsgValueError("Can't hexlify an empty message") 276 | return binascii.hexlify(msg) 277 | 278 | 279 | class CryptOp(UnaryOp): 280 | """Cryptographic transformations 281 | 282 | These transformations have the unique property that for any length message, 283 | the size of the result they return is fixed. Additionally, they're the only 284 | type of operation that can be applied directly to a stream. 285 | """ 286 | __slots__ = [] 287 | SUBCLS_BY_TAG = {} 288 | 289 | DIGEST_LENGTH = None 290 | 291 | def _do_op_call(self, msg): 292 | r = hashlib.new(self.HASHLIB_NAME, bytes(msg)).digest() 293 | assert len(r) == self.DIGEST_LENGTH 294 | return r 295 | 296 | def hash_fd(self, fd): 297 | hasher = hashlib.new(self.HASHLIB_NAME) 298 | while True: 299 | chunk = fd.read(2**20) # 1MB chunks 300 | if chunk: 301 | hasher.update(chunk) 302 | else: 303 | break 304 | 305 | return hasher.digest() 306 | 307 | # Cryptographic operation tag numbers taken from RFC4880, although it's not 308 | # guaranteed that they'll continue to match that RFC in the future. 309 | 310 | @CryptOp._register_op 311 | class OpSHA1(CryptOp): 312 | # Remember that for timestamping, hash algorithms with collision attacks 313 | # *are* secure! We've still proven that both messages existed prior to some 314 | # point in time - the fact that they both have the same hash digest doesn't 315 | # change that. 316 | # 317 | # Heck, even md5 is still secure enough for timestamping... but that's 318 | # pushing our luck... 319 | TAG = b'\x02' 320 | TAG_NAME = 'sha1' 321 | HASHLIB_NAME = "sha1" 322 | DIGEST_LENGTH = 20 323 | 324 | @CryptOp._register_op 325 | class OpRIPEMD160(CryptOp): 326 | TAG = b'\x03' 327 | TAG_NAME = 'ripemd160' 328 | HASHLIB_NAME = "ripemd160" 329 | DIGEST_LENGTH = 20 330 | 331 | @CryptOp._register_op 332 | class OpSHA256(CryptOp): 333 | TAG = b'\x08' 334 | TAG_NAME = 'sha256' 335 | HASHLIB_NAME = "sha256" 336 | DIGEST_LENGTH = 32 337 | 338 | 339 | @CryptOp._register_op 340 | class OpKECCAK256(UnaryOp): 341 | __slots__ = [] 342 | TAG = b'\x67' 343 | TAG_NAME = 'keccak256' 344 | DIGEST_LENGTH = 32 345 | 346 | def _do_op_call(self, msg): 347 | r = Cryptodome.Hash.keccak.new(digest_bits=256, data=bytes(msg)).digest() 348 | assert len(r) == self.DIGEST_LENGTH 349 | return r 350 | -------------------------------------------------------------------------------- /opentimestamps/core/notary.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | """Timestamp signature verification""" 13 | 14 | import opentimestamps.core.serialize 15 | 16 | class VerificationError(Exception): 17 | """Attestation verification errors""" 18 | 19 | class TimeAttestation: 20 | """Time-attesting signature""" 21 | 22 | TAG = None 23 | TAG_SIZE = 8 24 | 25 | # FIXME: What should this be? 26 | MAX_PAYLOAD_SIZE = 8192 27 | """Maximum size of a attestation payload""" 28 | 29 | def _serialize_payload(self, ctx): 30 | raise NotImplementedError 31 | 32 | def serialize(self, ctx): 33 | ctx.write_bytes(self.TAG) 34 | 35 | payload_ctx = opentimestamps.core.serialize.BytesSerializationContext() 36 | self._serialize_payload(payload_ctx) 37 | 38 | ctx.write_varbytes(payload_ctx.getbytes()) 39 | 40 | def __eq__(self, other): 41 | """Implementation of equality operator 42 | 43 | WARNING: The exact behavior of this isn't yet well-defined enough to be 44 | used for consensus-critical applications. 45 | """ 46 | if isinstance(other, TimeAttestation): 47 | assert self.__class__ is not other.__class__ # should be implemented by subclass 48 | return False 49 | 50 | else: 51 | return NotImplemented 52 | 53 | def __lt__(self, other): 54 | """Implementation of less than operator 55 | 56 | WARNING: The exact behavior of this isn't yet well-defined enough to be 57 | used for consensus-critical applications. 58 | """ 59 | if isinstance(other, TimeAttestation): 60 | assert self.__class__ is not other.__class__ # should be implemented by subclass 61 | return self.TAG < other.TAG 62 | 63 | else: 64 | return NotImplemented 65 | 66 | @classmethod 67 | def deserialize(cls, ctx): 68 | tag = ctx.read_bytes(cls.TAG_SIZE) 69 | 70 | serialized_attestation = ctx.read_varbytes(cls.MAX_PAYLOAD_SIZE) 71 | 72 | import opentimestamps.core.serialize 73 | payload_ctx = opentimestamps.core.serialize.BytesDeserializationContext(serialized_attestation) 74 | 75 | # FIXME: probably a better way to do this... 76 | import opentimestamps.core.dubious.notary 77 | 78 | if tag == PendingAttestation.TAG: 79 | r = PendingAttestation.deserialize(payload_ctx) 80 | elif tag == BitcoinBlockHeaderAttestation.TAG: 81 | r = BitcoinBlockHeaderAttestation.deserialize(payload_ctx) 82 | elif tag == LitecoinBlockHeaderAttestation.TAG: 83 | r = LitecoinBlockHeaderAttestation.deserialize(payload_ctx) 84 | elif tag == opentimestamps.core.dubious.notary.EthereumBlockHeaderAttestation.TAG: 85 | r = opentimestamps.core.dubious.notary.EthereumBlockHeaderAttestation.deserialize(payload_ctx) 86 | else: 87 | return UnknownAttestation(tag, serialized_attestation) 88 | 89 | # If attestations want to have unspecified fields for future 90 | # upgradability they should do so explicitly. 91 | payload_ctx.assert_eof() 92 | return r 93 | 94 | class UnknownAttestation(TimeAttestation): 95 | """Placeholder for attestations that don't support""" 96 | 97 | def __init__(self, tag, payload): 98 | if tag.__class__ != bytes: 99 | raise TypeError("tag must be bytes instance; got %r" % tag.__class__) 100 | elif len(tag) != self.TAG_SIZE: 101 | raise ValueError("tag must be exactly %d bytes long; got %d" % (self.TAG_SIZE, len(tag))) 102 | 103 | if payload.__class__ != bytes: 104 | raise TypeError("payload must be bytes instance; got %r" % tag.__class__) 105 | elif len(payload) > self.MAX_PAYLOAD_SIZE: 106 | raise ValueError("payload must be <= %d bytes long; got %d" % (self.MAX_PAYLOAD_SIZE, len(payload))) 107 | 108 | # FIXME: we should check that tag != one of the tags that we do know 109 | # about; if it does the operators < and =, and hash() will likely act 110 | # strangely 111 | self.TAG = tag 112 | self.payload = payload 113 | 114 | def __repr__(self): 115 | return 'UnknownAttestation(%r, %r)' % (self.TAG, self.payload) 116 | 117 | def __eq__(self, other): 118 | if other.__class__ is UnknownAttestation: 119 | return self.TAG == other.TAG and self.payload == other.payload 120 | else: 121 | return super().__eq__(other) 122 | 123 | def __lt__(self, other): 124 | if other.__class__ is UnknownAttestation: 125 | return (self.TAG, self.payload) < (other.TAG, other.payload) 126 | else: 127 | return super().__lt__(other) 128 | 129 | def __hash__(self): 130 | return hash((self.TAG, self.payload)) 131 | 132 | def _serialize_payload(self, ctx): 133 | # Notice how this is write_bytes, not write_varbytes - the latter would 134 | # incorrectly add a length header to the actual payload. 135 | ctx.write_bytes(self.payload) 136 | 137 | 138 | # Note how neither of these signatures actually has the time... 139 | 140 | class PendingAttestation(TimeAttestation): 141 | """Pending attestation 142 | 143 | Commitment has been recorded in a remote calendar for future attestation, 144 | and we have a URI to find a more complete timestamp in the future. 145 | 146 | Nothing other than the URI is recorded, nor is there provision made to add 147 | extra metadata (other than the URI) in future upgrades. The rational here 148 | is that remote calendars promise to keep commitments indefinitely, so from 149 | the moment they are created it should be possible to find the commitment in 150 | the calendar. Thus if you're not satisfied with the local verifiability of 151 | a timestamp, the correct thing to do is just ask the remote calendar if 152 | additional attestations are available and/or when they'll be available. 153 | 154 | While we could additional metadata like what types of attestations the 155 | remote calendar expects to be able to provide in the future, that metadata 156 | can easily change in the future too. Given that we don't expect timestamps 157 | to normally have more than a small number of remote calendar attestations, 158 | it'd be better to have verifiers get the most recent status of such 159 | information (possibly with appropriate negative response caching). 160 | 161 | """ 162 | 163 | TAG = bytes.fromhex('83dfe30d2ef90c8e') 164 | 165 | MAX_URI_LENGTH = 1000 166 | """Maximum legal URI length, in bytes""" 167 | 168 | ALLOWED_URI_CHARS = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._/:" 169 | """Characters allowed in URI's 170 | 171 | Note how we've left out the characters necessary for parameters, queries, 172 | or fragments, as well as IPv6 [] notation, percent-encoding special 173 | characters, and @ login notation. Hopefully this keeps us out of trouble! 174 | """ 175 | 176 | @classmethod 177 | def check_uri(cls, uri): 178 | """Check URI for validity 179 | 180 | Raises ValueError appropriately 181 | """ 182 | if len(uri) > cls.MAX_URI_LENGTH: 183 | raise ValueError("URI exceeds maximum length") 184 | for char in uri: 185 | if char not in cls.ALLOWED_URI_CHARS: 186 | raise ValueError("URI contains invalid character %r" % bytes([char])) 187 | 188 | def __init__(self, uri): 189 | if not isinstance(uri, str): 190 | raise TypeError("URI must be a string") 191 | self.check_uri(uri.encode()) 192 | self.uri = uri 193 | 194 | def __repr__(self): 195 | return 'PendingAttestation(%r)' % self.uri 196 | 197 | def __eq__(self, other): 198 | if other.__class__ is PendingAttestation: 199 | return self.uri == other.uri 200 | else: 201 | return super().__eq__(other) 202 | 203 | def __lt__(self, other): 204 | if other.__class__ is PendingAttestation: 205 | return self.uri < other.uri 206 | 207 | else: 208 | return super().__lt__(other) 209 | 210 | def __hash__(self): 211 | return hash(self.uri) 212 | 213 | def _serialize_payload(self, ctx): 214 | ctx.write_varbytes(self.uri.encode()) 215 | 216 | @classmethod 217 | def deserialize(cls, ctx): 218 | utf8_uri = ctx.read_varbytes(cls.MAX_URI_LENGTH) 219 | 220 | try: 221 | cls.check_uri(utf8_uri) 222 | except ValueError as exp: 223 | raise opentimestamps.core.serialize.DeserializationError("Invalid URI: %r" % exp) 224 | 225 | return PendingAttestation(utf8_uri.decode()) 226 | 227 | class BitcoinBlockHeaderAttestation(TimeAttestation): 228 | """Signed by the Bitcoin blockchain 229 | 230 | The commitment digest will be the merkleroot of the blockheader. 231 | 232 | The block height is recorded so that looking up the correct block header in 233 | an external block header database doesn't require every header to be stored 234 | locally (33MB and counting). (remember that a memory-constrained local 235 | client can save an MMR that commits to all blocks, and use an external service to fill 236 | in pruned details). 237 | 238 | Otherwise no additional redundant data about the block header is recorded. 239 | This is very intentional: since the attestation contains (nearly) the 240 | absolute bare minimum amount of data, we encourage implementations to do 241 | the correct thing and get the block header from a by-height index, check 242 | that the merkleroots match, and then calculate the time from the header 243 | information. Providing more data would encourage implementations to cheat. 244 | 245 | Remember that the only thing that would invalidate the block height is a 246 | reorg, but in the event of a reorg the merkleroot will be invalid anyway, 247 | so there's no point to recording data in the attestation like the header 248 | itself. At best that would just give us extra confirmation that a reorg 249 | made the attestation invalid; reorgs deep enough to invalidate timestamps are 250 | exceptionally rare events anyway, so better to just tell the user the timestamp 251 | can't be verified rather than add almost-never tested code to handle that case 252 | more gracefully. 253 | 254 | There is no testnet/regtest/signet/etc. version of this because the purpose 255 | of testnets is to test code. The best test uses the exact same code paths. 256 | """ 257 | 258 | TAG = bytes.fromhex('0588960d73d71901') 259 | 260 | def __init__(self, height): 261 | self.height = height 262 | 263 | def __eq__(self, other): 264 | if other.__class__ is BitcoinBlockHeaderAttestation: 265 | return self.height == other.height 266 | else: 267 | return super().__eq__(other) 268 | 269 | def __lt__(self, other): 270 | if other.__class__ is BitcoinBlockHeaderAttestation: 271 | return self.height < other.height 272 | 273 | else: 274 | return super().__lt__(other) 275 | 276 | def __hash__(self): 277 | return hash(self.height) 278 | 279 | def verify_against_blockheader(self, digest, block_header): 280 | """Verify attestation against a block header 281 | 282 | Returns the block time on success; raises VerificationError on failure. 283 | """ 284 | 285 | if len(digest) != 32: 286 | raise VerificationError("Expected digest with length 32 bytes; got %d bytes" % len(digest)) 287 | elif digest != block_header.hashMerkleRoot: 288 | raise VerificationError("Digest does not match merkleroot") 289 | 290 | return block_header.nTime 291 | 292 | def __repr__(self): 293 | return 'BitcoinBlockHeaderAttestation(%r)' % self.height 294 | 295 | def _serialize_payload(self, ctx): 296 | ctx.write_varuint(self.height) 297 | 298 | @classmethod 299 | def deserialize(cls, ctx): 300 | height = ctx.read_varuint() 301 | return BitcoinBlockHeaderAttestation(height) 302 | 303 | class LitecoinBlockHeaderAttestation(TimeAttestation): 304 | """Signed by the Litecoin blockchain 305 | 306 | Identical in design to the BitcoinBlockHeaderAttestation. 307 | """ 308 | 309 | TAG = bytes.fromhex('06869a0d73d71b45') 310 | 311 | def __init__(self, height): 312 | self.height = height 313 | 314 | def __eq__(self, other): 315 | if other.__class__ is LitecoinBlockHeaderAttestation: 316 | return self.height == other.height 317 | else: 318 | return super().__eq__(other) 319 | 320 | def __lt__(self, other): 321 | if other.__class__ is LitecoinBlockHeaderAttestation: 322 | return self.height < other.height 323 | 324 | else: 325 | return super().__lt__(other) 326 | 327 | def __hash__(self): 328 | return hash(self.height) 329 | 330 | def verify_against_blockheader(self, digest, block_header): 331 | """Verify attestation against a block header 332 | 333 | Not implemented here until there is a well-maintained Litecoin 334 | python library 335 | """ 336 | raise NotImplementedError() 337 | 338 | 339 | def __repr__(self): 340 | return 'LitecoinBlockHeaderAttestation(%r)' % self.height 341 | 342 | def _serialize_payload(self, ctx): 343 | ctx.write_varuint(self.height) 344 | 345 | @classmethod 346 | def deserialize(cls, ctx): 347 | height = ctx.read_varuint() 348 | return LitecoinBlockHeaderAttestation(height) 349 | 350 | -------------------------------------------------------------------------------- /opentimestamps/core/timestamp.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 The OpenTimestamps developers 2 | # 3 | # This file is part of python-opentimestamps. 4 | # 5 | # It is subject to the license terms in the LICENSE file found in the top-level 6 | # directory of this distribution. 7 | # 8 | # No part of python-opentimestamps including this file, may be copied, 9 | # modified, propagated, or distributed except according to the terms contained 10 | # in the LICENSE file. 11 | 12 | import binascii 13 | 14 | from bitcoin.core import CTransaction, SerializationError, b2lx, b2x 15 | 16 | from opentimestamps.core.op import Op, CryptOp, OpSHA256, OpAppend, OpPrepend, MsgValueError 17 | from opentimestamps.core.notary import TimeAttestation, BitcoinBlockHeaderAttestation, LitecoinBlockHeaderAttestation 18 | 19 | import opentimestamps.core.serialize 20 | 21 | class OpSet(dict): 22 | """Set of operations""" 23 | __slots__ = ['__make_timestamp'] 24 | def __init__(self, make_timestamp_func): 25 | self.__make_timestamp = make_timestamp_func 26 | 27 | def add(self, key): 28 | """Add key 29 | 30 | Returns the value associated with that key 31 | """ 32 | try: 33 | return self[key] 34 | except KeyError: 35 | value = self.__make_timestamp(key) 36 | self[key] = value 37 | return value 38 | 39 | def __setitem__(self, op, new_timestamp): 40 | try: 41 | existing_timestamp = self[op] 42 | except KeyError: 43 | dict.__setitem__(self, op, new_timestamp) 44 | return 45 | 46 | if existing_timestamp.msg != new_timestamp.msg: 47 | raise ValueError("Can't change existing result timestamp: timestamps are for different messages") 48 | 49 | dict.__setitem__(self, op, new_timestamp) 50 | 51 | class Timestamp: 52 | """Proof that one or more attestations commit to a message 53 | 54 | The proof is in the form of a tree, with each node being a message, and the 55 | edges being operations acting on those messages. The leafs of the tree are 56 | attestations that attest to the time that messages in the tree existed prior. 57 | """ 58 | __slots__ = ['__msg', 'attestations', 'ops'] 59 | 60 | @property 61 | def msg(self): 62 | return self.__msg 63 | 64 | def __init__(self, msg): 65 | if not isinstance(msg, bytes): 66 | raise TypeError("Expected msg to be bytes; got %r" % msg.__class__) 67 | 68 | elif len(msg) > Op.MAX_MSG_LENGTH: 69 | raise ValueError("Message exceeds Op length limit; %d > %d" % (len(msg), Op.MAX_MSG_LENGTH)) 70 | 71 | self.__msg = bytes(msg) 72 | self.attestations = set() 73 | self.ops = OpSet(lambda op: Timestamp(op(msg))) 74 | 75 | def __eq__(self, other): 76 | if isinstance(other, Timestamp): 77 | return self.__msg == other.__msg and self.attestations == other.attestations and self.ops == other.ops 78 | else: 79 | return False 80 | 81 | def __repr__(self): 82 | return 'Timestamp(<%s>)' % binascii.hexlify(self.__msg).decode('utf8') 83 | 84 | def merge(self, other): 85 | """Add all operations and attestations from another timestamp to this one 86 | 87 | Raises ValueError if the other timestamp isn't for the same message 88 | """ 89 | if not isinstance(other, Timestamp): 90 | raise TypeError("Can only merge Timestamps together") 91 | 92 | if self.__msg != other.__msg: 93 | raise ValueError("Can't merge timestamps for different messages together") 94 | 95 | self.attestations.update(other.attestations) 96 | 97 | for other_op, other_op_stamp in other.ops.items(): 98 | our_op_stamp = self.ops.add(other_op) 99 | our_op_stamp.merge(other_op_stamp) 100 | 101 | def serialize(self, ctx): 102 | if not len(self.attestations) and not len(self.ops): 103 | raise ValueError("An empty timestamp can't be serialized") 104 | 105 | sorted_attestations = sorted(self.attestations) 106 | if len(sorted_attestations) > 1: 107 | for attestation in sorted_attestations[0:-1]: 108 | ctx.write_bytes(b'\xff\x00') 109 | attestation.serialize(ctx) 110 | 111 | if len(self.ops) == 0: 112 | ctx.write_bytes(b'\x00') 113 | sorted_attestations[-1].serialize(ctx) 114 | 115 | elif len(self.ops) > 0: 116 | if len(sorted_attestations) > 0: 117 | ctx.write_bytes(b'\xff\x00') 118 | sorted_attestations[-1].serialize(ctx) 119 | 120 | sorted_ops = sorted(self.ops.items(), key=lambda item: item[0]) 121 | for op, stamp in sorted_ops[0:-1]: 122 | ctx.write_bytes(b'\xff') 123 | op.serialize(ctx) 124 | stamp.serialize(ctx) 125 | 126 | last_op, last_stamp = sorted_ops[-1] 127 | last_op.serialize(ctx) 128 | last_stamp.serialize(ctx) 129 | 130 | @classmethod 131 | def deserialize(cls, ctx, initial_msg, _recursion_limit=256): 132 | """Deserialize 133 | 134 | Because the serialization format doesn't include the message that the 135 | timestamp operates on, you have to provide it so that the correct 136 | operation results can be calculated. 137 | 138 | The message you provide is assumed to be correct; if it causes a op to 139 | raise MsgValueError when the results are being calculated (done 140 | immediately, not lazily) DeserializationError is raised instead. 141 | """ 142 | 143 | # FIXME: The recursion limit is arguably a bit of a hack given that 144 | # it's relatively easily avoided with a different implementation. On 145 | # the other hand, it has the advantage of being a very simple 146 | # solution to the problem, and the limit isn't likely to be reached by 147 | # nearly all timestamps anyway. 148 | # 149 | # FIXME: Corresponding code to detect this condition is missing from 150 | # the serialization/__init__() code. 151 | if not _recursion_limit: 152 | raise opentimestamps.core.serialize.RecursionLimitError("Reached timestamp recursion depth limit while deserializing") 153 | 154 | # FIXME: note how a lazy implementation would have different behavior 155 | # with respect to deserialization errors; is this a good design? 156 | 157 | self = cls(initial_msg) 158 | 159 | def do_tag_or_attestation(tag): 160 | if tag == b'\x00': 161 | attestation = TimeAttestation.deserialize(ctx) 162 | self.attestations.add(attestation) 163 | 164 | else: 165 | op = Op.deserialize_from_tag(ctx, tag) 166 | 167 | try: 168 | result = op(initial_msg) 169 | except MsgValueError as exp: 170 | raise opentimestamps.core.serialize.DeserializationError("Invalid timestamp; message invalid for op %r: %r" % (op, exp)) 171 | 172 | stamp = Timestamp.deserialize(ctx, result, _recursion_limit=_recursion_limit-1) 173 | self.ops[op] = stamp 174 | 175 | tag = ctx.read_bytes(1) 176 | while tag == b'\xff': 177 | do_tag_or_attestation(ctx.read_bytes(1)) 178 | 179 | tag = ctx.read_bytes(1) 180 | 181 | do_tag_or_attestation(tag) 182 | 183 | return self 184 | 185 | def all_attestations(self): 186 | """Iterate over all attestations recursively 187 | 188 | Returns iterable of (msg, attestation) 189 | """ 190 | for attestation in self.attestations: 191 | yield (self.msg, attestation) 192 | 193 | for op_stamp in self.ops.values(): 194 | yield from op_stamp.all_attestations() 195 | 196 | def str_tree(self, indent=0, verbosity=0): 197 | """Convert to tree (for debugging)""" 198 | 199 | class bcolors: 200 | HEADER = '\033[95m' 201 | OKBLUE = '\033[94m' 202 | OKGREEN = '\033[92m' 203 | WARNING = '\033[93m' 204 | FAIL = '\033[91m' 205 | ENDC = '\033[0m' 206 | BOLD = '\033[1m' 207 | UNDERLINE = '\033[4m' 208 | 209 | def str_result(verb, parameter, result): 210 | rr = "" 211 | if verb > 0 and result is not None: 212 | rr += " == " 213 | result_hex = b2x(result) 214 | if parameter is not None: 215 | parameter_hex = b2x(parameter) 216 | try: 217 | index = result_hex.index(parameter_hex) 218 | parameter_hex_highlight = bcolors.BOLD + parameter_hex + bcolors.ENDC 219 | if index == 0: 220 | rr += parameter_hex_highlight + result_hex[index+len(parameter_hex):] 221 | else: 222 | rr += result_hex[0:index] + parameter_hex_highlight 223 | except ValueError: 224 | rr += result_hex 225 | else: 226 | rr += result_hex 227 | 228 | return rr 229 | 230 | r = "" 231 | if len(self.attestations) > 0: 232 | for attestation in sorted(self.attestations): 233 | r += " "*indent + "verify %s" % str(attestation) + str_result(verbosity, self.msg, None) + "\n" 234 | if attestation.__class__ == BitcoinBlockHeaderAttestation: 235 | r += " "*indent + "# Bitcoin block merkle root " + b2lx(self.msg) + "\n" 236 | if attestation.__class__ == LitecoinBlockHeaderAttestation: 237 | r += " "*indent + "# Litecoin block merkle root " + b2lx(self.msg) + "\n" 238 | 239 | if len(self.ops) > 1: 240 | for op, timestamp in sorted(self.ops.items()): 241 | try: 242 | CTransaction.deserialize(self.msg) 243 | r += " " * indent + "* Transaction id " + b2lx( 244 | OpSHA256()(OpSHA256()(self.msg))) + "\n" 245 | except SerializationError: 246 | pass 247 | cur_res = op(self.msg) 248 | cur_par = op[0] if len(op) > 0 else None 249 | r += " " * indent + " -> " + "%s" % str(op) + str_result(verbosity, cur_par, cur_res) + "\n" 250 | r += timestamp.str_tree(indent+4, verbosity=verbosity) 251 | elif len(self.ops) > 0: 252 | try: 253 | CTransaction.deserialize(self.msg) 254 | r += " " * indent + "# Transaction id " + \ 255 | b2lx(OpSHA256()(OpSHA256()(self.msg))) + "\n" 256 | except SerializationError: 257 | pass 258 | op = tuple(self.ops.keys())[0] 259 | cur_res = op(self.msg) 260 | cur_par = op[0] if len(op) > 0 else None 261 | r += " " * indent + "%s" % str(op) + str_result(verbosity, cur_par, cur_res) + "\n" 262 | r += tuple(self.ops.values())[0].str_tree(indent, verbosity=verbosity) 263 | 264 | return r 265 | 266 | 267 | class DetachedTimestampFile: 268 | """A file containing a timestamp for another file 269 | 270 | Contains a timestamp, along with a header and the digest of the file. 271 | """ 272 | 273 | HEADER_MAGIC = b'\x00OpenTimestamps\x00\x00Proof\x00\xbf\x89\xe2\xe8\x84\xe8\x92\x94' 274 | """Header magic bytes 275 | 276 | Designed to be give the user some information in a hexdump, while being 277 | identified as 'data' by the file utility. 278 | """ 279 | 280 | MIN_FILE_DIGEST_LENGTH = 20 # 160-bit hash 281 | MAX_FILE_DIGEST_LENGTH = 32 # 256-bit hash 282 | 283 | MAJOR_VERSION = 1 284 | 285 | # While the git commit timestamps have a minor version, probably better to 286 | # leave it out here: unlike Git commits round-tripping is an issue when 287 | # timestamps are upgraded, and we could end up with bugs related to not 288 | # saving/updating minor version numbers correctly. 289 | 290 | @property 291 | def file_digest(self): 292 | """The digest of the file that was timestamped""" 293 | return self.timestamp.msg 294 | 295 | def __init__(self, file_hash_op, timestamp): 296 | self.file_hash_op = file_hash_op 297 | 298 | if len(timestamp.msg) != file_hash_op.DIGEST_LENGTH: 299 | raise ValueError("Timestamp message length and file_hash_op digest length differ") 300 | 301 | self.timestamp = timestamp 302 | 303 | def __repr__(self): 304 | return 'DetachedTimestampFile(<%s:%s>)' % (str(self.file_hash_op), binascii.hexlify(self.file_digest).decode('utf8')) 305 | 306 | def __eq__(self, other): 307 | return (self.__class__ == other.__class__ and 308 | self.file_hash_op == other.file_hash_op and 309 | self.timestamp == other.timestamp) 310 | 311 | @classmethod 312 | def from_fd(cls, file_hash_op, fd): 313 | fd_hash = file_hash_op.hash_fd(fd) 314 | return cls(file_hash_op, Timestamp(fd_hash)) 315 | 316 | def serialize(self, ctx): 317 | ctx.write_bytes(self.HEADER_MAGIC) 318 | 319 | ctx.write_uint8(self.MAJOR_VERSION) 320 | 321 | self.file_hash_op.serialize(ctx) 322 | assert self.file_hash_op.DIGEST_LENGTH == len(self.timestamp.msg) 323 | ctx.write_bytes(self.timestamp.msg) 324 | 325 | self.timestamp.serialize(ctx) 326 | 327 | @classmethod 328 | def deserialize(cls, ctx): 329 | ctx.assert_magic(cls.HEADER_MAGIC) 330 | 331 | major = ctx.read_uint8() 332 | if major != cls.MAJOR_VERSION: 333 | raise opentimestamps.core.serialize.UnsupportedMajorVersion("Version %d detached timestamp files are not supported" % major) 334 | 335 | file_hash_op = CryptOp.deserialize(ctx) 336 | file_hash = ctx.read_bytes(file_hash_op.DIGEST_LENGTH) 337 | timestamp = Timestamp.deserialize(ctx, file_hash) 338 | 339 | ctx.assert_eof() 340 | 341 | return DetachedTimestampFile(file_hash_op, timestamp) 342 | 343 | 344 | def cat_then_unary_op(unary_op_cls, left, right): 345 | """Concatenate left and right, then perform a unary operation on them 346 | 347 | left and right can be either timestamps or bytes. 348 | 349 | Appropriate intermediary append/prepend operations will be created as 350 | needed for left and right. 351 | """ 352 | if not isinstance(left, Timestamp): 353 | left = Timestamp(left) 354 | 355 | if not isinstance(right, Timestamp): 356 | right = Timestamp(right) 357 | 358 | left_append_stamp = left.ops.add(OpAppend(right.msg)) 359 | right_prepend_stamp = right.ops.add(OpPrepend(left.msg)) 360 | 361 | assert(left_append_stamp == right_prepend_stamp) 362 | 363 | # Left and right should produce the same thing, so we can set the timestamp 364 | # of the left to the right. 365 | left.ops[OpAppend(right.msg)] = right_prepend_stamp 366 | 367 | return right_prepend_stamp.ops.add(unary_op_cls()) 368 | 369 | 370 | def cat_sha256(left, right): 371 | return cat_then_unary_op(OpSHA256, left, right) 372 | 373 | 374 | def cat_sha256d(left, right): 375 | sha256_timestamp = cat_sha256(left, right) 376 | return sha256_timestamp.ops.add(OpSHA256()) 377 | 378 | 379 | def make_merkle_tree(timestamps, binop=cat_sha256): 380 | """Merkelize a set of timestamps 381 | 382 | A merkle tree of all the timestamps is built in-place using binop() to 383 | timestamp each pair of timestamps. The exact algorithm used is structurally 384 | identical to a merkle-mountain-range, although leaf sums aren't committed. 385 | As this function is under the consensus-critical core, it's guaranteed that 386 | the algorithm will not be changed in the future. 387 | 388 | Returns the timestamp for the tip of the tree. 389 | """ 390 | 391 | stamps = timestamps 392 | while True: 393 | stamps = iter(stamps) 394 | 395 | try: 396 | prev_stamp = next(stamps) 397 | except StopIteration: 398 | raise ValueError("Need at least one timestamp") 399 | 400 | next_stamps = [] 401 | for stamp in stamps: 402 | if prev_stamp is not None: 403 | next_stamps.append(binop(prev_stamp, stamp)) 404 | prev_stamp = None 405 | else: 406 | prev_stamp = stamp 407 | 408 | if not next_stamps: 409 | return prev_stamp 410 | 411 | if prev_stamp is not None: 412 | next_stamps.append(prev_stamp) 413 | 414 | stamps = next_stamps 415 | --------------------------------------------------------------------------------