├── .github
    └── README.md
├── CNAME
├── LICENSE
├── bscp
└── index.html


/.github/README.md:
--------------------------------------------------------------------------------
1 | # Bscp – Secure and efficient copying of block devices
2 | 
3 | Please find the project website and documentation at
4 | 
5 | * https://bscp.njh.eu
6 | 


--------------------------------------------------------------------------------
/CNAME:
--------------------------------------------------------------------------------
1 | bscp.njh.eu
2 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Permission to use, copy, modify, and/or distribute this software for any
 2 | purpose with or without fee is hereby granted, provided that the above
 3 | copyright notice and this permission notice appear in all copies.
 4 | 
 5 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 6 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 7 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 8 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 9 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
10 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
11 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
12 | 


--------------------------------------------------------------------------------
/bscp:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | # Copyright (C) 2012-2023  The Bscp Authors <https://github.com/bscp-tool/bscp/graphs/contributors>
  4 | #
  5 | # Permission to use, copy, modify, and/or distribute this software for any
  6 | # purpose with or without fee is hereby granted, provided that the above
  7 | # copyright notice and this permission notice appear in all copies.
  8 | #
  9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 10 | # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 11 | # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 12 | # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 13 | # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 14 | # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 15 | # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 16 | 
 17 | import hashlib
 18 | import struct
 19 | import subprocess
 20 | import sys
 21 | 
 22 | if sys.version_info < (3, 0):
 23 |     range = xrange
 24 | 
 25 | remote_script = r'''
 26 | import hashlib
 27 | import os
 28 | import os.path
 29 | import struct
 30 | import sys
 31 | 
 32 | if sys.version_info < (3, 0):
 33 |     stdin_buffer = sys.stdin
 34 |     stdout_buffer = sys.stdout
 35 |     range = xrange
 36 | else:
 37 |     stdin_buffer = sys.stdin.buffer
 38 |     stdout_buffer = sys.stdout.buffer
 39 | 
 40 | (size, blocksize, filename_len, hashname_len) = struct.unpack('<QQQQ', stdin_buffer.read(8+8+8+8))
 41 | filename_bytes = stdin_buffer.read(filename_len)
 42 | hashname_bytes = stdin_buffer.read(hashname_len)
 43 | filename = filename_bytes.decode('utf-8')
 44 | hashname = hashname_bytes.decode('ascii')
 45 | 
 46 | sanity_hash = hashlib.new(hashname, filename_bytes).digest()
 47 | stdout_buffer.write(sanity_hash)
 48 | stdout_buffer.flush()
 49 | if stdin_buffer.read(2) != b'go':
 50 |     sys.exit()
 51 | 
 52 | if not os.path.exists(filename):
 53 |     # Create sparse file
 54 |     with open(filename, 'wb') as f:
 55 |         f.truncate(size)
 56 |     os.chmod(filename, 0o600)
 57 | 
 58 | with open(filename, 'rb+') as f:
 59 |     f.seek(0, 2)
 60 |     stdout_buffer.write(struct.pack('<Q', f.tell()))
 61 |     readremain = size
 62 |     rblocksize = blocksize
 63 |     f.seek(0)
 64 |     while True:
 65 |         if readremain <= blocksize:
 66 |             rblocksize = readremain
 67 |         block = f.read(rblocksize)
 68 |         if len(block) == 0:
 69 |             break
 70 |         digest = hashlib.new(hashname, block).digest()
 71 |         stdout_buffer.write(digest)
 72 |         readremain -= rblocksize
 73 |         if readremain == 0:
 74 |             break
 75 |     stdout_buffer.flush()
 76 |     while True:
 77 |         position_s = stdin_buffer.read(8)
 78 |         if len(position_s) == 0:
 79 |             break
 80 |         (position,) = struct.unpack('<Q', position_s)
 81 |         block = stdin_buffer.read(blocksize)
 82 |         f.seek(position)
 83 |         f.write(block)
 84 |     readremain = size
 85 |     rblocksize = blocksize
 86 |     hash_total = hashlib.new(hashname)
 87 |     f.seek(0)
 88 |     while True:
 89 |         if readremain <= blocksize:
 90 |             rblocksize = readremain
 91 |         block = f.read(rblocksize)
 92 |         if len(block) == 0:
 93 |             break
 94 |         hash_total.update(block)
 95 |         readremain -= rblocksize
 96 |         if readremain == 0:
 97 |             break
 98 | stdout_buffer.write(hash_total.digest())
 99 | '''
100 | 
101 | class IOCounter:
102 |     def __init__(self, in_stream, out_stream):
103 |         self.in_stream = in_stream
104 |         self.out_stream = out_stream
105 |         self.in_total = 0
106 |         self.out_total = 0
107 |     def read(self, size=None):
108 |         if size is None:
109 |             s = self.in_stream.read()
110 |         else:
111 |             s = self.in_stream.read(size)
112 |         self.in_total += len(s)
113 |         return s
114 |     def write(self, s):
115 |         self.out_stream.write(s)
116 |         self.out_total += len(s)
117 |         self.out_stream.flush()
118 | 
119 | def bscp(local_filename, remote_host, remote_filename, blocksize, hashname):
120 |     remote_filename_bytes = remote_filename.encode('utf-8')
121 |     hashname_bytes = hashname.encode('ascii')
122 |     hash_total = hashlib.new(hashname)
123 |     with open(local_filename, 'rb') as f:
124 |         f.seek(0, 2)
125 |         size = f.tell()
126 |         f.seek(0)
127 | 
128 |         # Calculate number of blocks, including the last block which may be smaller
129 |         blockcount = int((size + blocksize - 1) / blocksize)
130 | 
131 |         remote_command = 'python -c "%s"' % (remote_script,)
132 |         command = ('ssh', '--', remote_host, remote_command)
133 |         p = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
134 |         io = IOCounter(p.stdout, p.stdin)
135 | 
136 |         io.write(struct.pack('<QQQQ', size, blocksize, len(remote_filename_bytes), len(hashname_bytes)))
137 |         io.write(remote_filename_bytes)
138 |         io.write(hashname_bytes)
139 | 
140 |         sanity_digest = hashlib.new(hashname, remote_filename_bytes).digest()
141 |         remote_digest = io.read(len(sanity_digest))
142 |         if remote_digest != sanity_digest:
143 |             raise RuntimeError('Remote script failed to execute properly')
144 |         io.write(b'go')
145 | 
146 |         (remote_size,) = struct.unpack('<Q', io.read(8))
147 |         if remote_size < size:
148 |             raise RuntimeError('Remote size less than local (local: %i, remote: %i)' % (size, remote_size))
149 |         remote_digest_list = [io.read(hash_total.digest_size) for i in range(blockcount)]
150 | 
151 |         for remote_digest in remote_digest_list:
152 |             position = f.tell()
153 |             block = f.read(blocksize)
154 |             hash_total.update(block)
155 |             digest = hashlib.new(hashname, block).digest()
156 |             if digest != remote_digest:
157 |                 try:
158 |                     io.write(struct.pack('<Q', position))
159 |                     io.write(block)
160 |                 except IOError:
161 |                     break
162 |         p.stdin.close()
163 | 
164 |         remote_digest_total = io.read()
165 |         p.wait()
166 |         if remote_digest_total != hash_total.digest():
167 |             raise RuntimeError('Checksum mismatch after transfer')
168 |     return (io.in_total, io.out_total, size)
169 | 
170 | if __name__ == '__main__':
171 |     try:
172 |         local_filename = sys.argv[1]
173 |         (remote_host, remote_filename) = sys.argv[2].split(':')
174 |         if len(sys.argv) >= 4:
175 |             blocksize = int(sys.argv[3])
176 |         else:
177 |             blocksize = 64 * 1024
178 |         if len(sys.argv) >= 5:
179 |             hashname = sys.argv[4]
180 |         else:
181 |             hashname = 'sha256'
182 |         assert len(sys.argv) <= 5
183 |     except:
184 |         usage = 'bscp SRC HOST:DEST [BLOCKSIZE] [HASH]'
185 |         sys.stderr.write('Usage:\n\n    %s\n\n' % (usage,))
186 |         sys.exit(1)
187 |     (in_total, out_total, size) = bscp(local_filename, remote_host, remote_filename, blocksize, hashname)
188 |     speedup = size * 1.0 / (in_total + out_total)
189 |     sys.stderr.write('in=%i out=%i size=%i speedup=%.2f\n' % (in_total, out_total, size, speedup))
190 | 


--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <html>
 3 |   <head>
 4 |     <meta http-equiv="Content-Type" content="text/html;charset=utf-8">
 5 |     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"/>
 6 |     <title>Bscp – Secure and efficient copying of block devices</title>
 7 |     <style>
 8 |       body {
 9 |         font-family: sans-serif;
10 |         font-size: 11pt;
11 |         margin-left: auto;
12 |         margin-right: auto;
13 |         max-width: 400pt;
14 |       }
15 |       h1 {
16 |         font-size: 20pt;
17 |       }
18 |       h2 {
19 |         font-size: 16pt;
20 |       }
21 |       a {
22 |         text-decoration: none;
23 |       }
24 |       a:hover {
25 |         text-decoration: underline;
26 |       }
27 |     </style>
28 |   </head>
29 |   <body>
30 |     <h1>Bscp – Secure and efficient copying of block devices</h1>
31 |     <p>
32 |       Bscp copies a single file or block device over an SSH
33 |       connection, transferring only the parts that have changed.
34 |     </p>
35 |     <p>
36 |       In other words, it handles the edge case where <a href="https://rsync.samba.org/">rsync</a> fails.
37 |     </p>
38 |     <h2>Download</h2>
39 |     <pre>git clone https://github.com/bscp-tool/bscp</pre>
40 |     <h2>Usage</h2>
41 |     <pre>bscp SRC HOST:DEST [BLOCKSIZE] [HASH]</pre>
42 |     <p>The default <code>BLOCKSIZE</code> is <code>65536</code> (64 KiB).</p>
43 |     <p>The default <code>HASH</code> algorithm is <code>sha256</code> (SHA-256).</p>
44 |     <h2>Comparison</h2>
45 |     <p>
46 |       Bscp is similar to the classic
47 |       <a href="https://www.bouncybouncy.net/programs/blocksync.py">blocksync.py</a>,
48 |       but provides the following advantages:
49 |     </p>
50 |     <ul>
51 |       <li>
52 |         It doesn't have to be installed on server side.
53 |       </li>
54 |       <li>
55 |         It uses a strong hash algorithm (SHA-256 instead of MD5) by default,
56 |         and can use even stronger hash algorithms as long as they are supported by Python,
57 |         such as SHA-512 or SHA3-512.
58 |       </li>
59 |       <li>
60 |         DEST is allowed to be larger than SRC.
61 |         In that case,
62 |         SRC is copied to the beginning of DEST
63 |         and the end of DEST remains untouched.
64 |       </li>
65 |       <li>
66 |         The client-server protocol is designed to
67 |         communicate in bigger chunks,
68 |         so it is more suitable to
69 |         network connections with bad latency.
70 |       </li>
71 |     </ul>
72 |     <h2>Contact</h2>
73 |     <ul>
74 |       <li><a href="https://github.com/bscp-tool/bscp/issues">Bscp issue tracker</a></li>
75 |     </ul>
76 |     <h2>See also</h2>
77 |     <ul>
78 |       <li><a href="https://vog.github.io/shasplit">Shasplit</a></li>
79 |       <li><a href="https://github.com/systemd/casync">casync</a>
80 |       <li><a href="https://www.bouncybouncy.net/programs/blocksync.py">blocksync.py</a></li>
81 |       <li><a href="https://theshed.hezmatt.org/lvmsync/">lvmsync</a></li>
82 |       <li><a href="https://github.com/vog/scs">Scs</a></li>
83 |       <li><a href="https://rsync.samba.org/">rsync</a></li>
84 |     </ul>
85 |   </body>
86 | </html>
87 | 


--------------------------------------------------------------------------------