├── .gitignore
├── DDRandom.md
├── LICENSE
├── LVMSnapSend.md
├── LVMSpaceCheck.md
├── README.md
├── build.xml
├── nbproject
    ├── .gitignore
    ├── build-impl.xml
    ├── genfiles.properties
    ├── project.properties
    └── project.xml
├── plan.md
├── sample
    └── dd1.xml
├── src
    └── oneit
    │   └── lvmsendrcv
    │       ├── LVM.java
    │       ├── LVMBlockDiff.java
    │       ├── LVMSnapReceive.java
    │       ├── LVMSnapSend.java
    │       ├── dd
    │           ├── DDRandomReceive.java
    │           └── DDRandomSend.java
    │       ├── service
    │           ├── LVMSnapReceiveService.java
    │           ├── LVMSnapSendService.java
    │           └── LVMSpaceCheck.java
    │       └── utils
    │           ├── ExecUtils.java
    │           ├── IOUtils.java
    │           ├── Utils.java
    │           └── XMLUtils.java
└── thin_delta_output.xml


/.gitignore:
--------------------------------------------------------------------------------
1 | /build/
2 | /dist/
3 | 


--------------------------------------------------------------------------------
/DDRandom.md:
--------------------------------------------------------------------------------
 1 | # DDRandomSend and DDRandomReceive
 2 | This is a pair of utilities designed to work similarly to dd over an ssh tunnel, except that only specific blocks are transferred.
 3 | 
 4 | The XML file is read using SAX, so memory overhead is minimised.
 5 | 
 6 | The intent is something like:
 7 | ```
 8 | java oneit.lvmsendrcv.DDRandomSend --bs 1 --blocks 2,5,7 --if /tmp/some.file | java oneit.lvmsendrcv.DDRandomServe -bs 1 -of /tmp/other.file
 9 | ```
10 | Obviously, this can be done through an ssh tunnel (which should use compression due to the base 64 encoding).
11 | 
12 | ## Example of DDRandomSendOutput
13 | Send blocks 2, 5, and 6 of a file in 1kb blocks.
14 | ```
15 | java -cp lvm-thin-sendrcv.jar oneit.lvmsendrcv.DDRandomSend --bs 1 --blocks 2,5,6 --if /tmp/out.dd 
16 | <?xml version="1.0" encoding="UTF-8"?>
17 | 
18 | <dd>
19 | <block offset='2'>MDItMDItMDItMDItMDItMDItMDItMDItMDItMDItMDItMDItMDItMDItAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==</block>
20 | <block offset='5'>MDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==</block>
21 | <block offset='6'>AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==</block>
22 | </dd>
23 | ```
24 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                     GNU GENERAL PUBLIC LICENSE
  2 |                        Version 3, 29 June 2007
  3 | 
  4 |  Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  5 |  Everyone is permitted to copy and distribute verbatim copies
  6 |  of this license document, but changing it is not allowed.
  7 | 
  8 |                             Preamble
  9 | 
 10 |   The GNU General Public License is a free, copyleft license for
 11 | software and other kinds of works.
 12 | 
 13 |   The licenses for most software and other practical works are designed
 14 | to take away your freedom to share and change the works.  By contrast,
 15 | the GNU General Public License is intended to guarantee your freedom to
 16 | share and change all versions of a program--to make sure it remains free
 17 | software for all its users.  We, the Free Software Foundation, use the
 18 | GNU General Public License for most of our software; it applies also to
 19 | any other work released this way by its authors.  You can apply it to
 20 | your programs, too.
 21 | 
 22 |   When we speak of free software, we are referring to freedom, not
 23 | price.  Our General Public Licenses are designed to make sure that you
 24 | have the freedom to distribute copies of free software (and charge for
 25 | them if you wish), that you receive source code or can get it if you
 26 | want it, that you can change the software or use pieces of it in new
 27 | free programs, and that you know you can do these things.
 28 | 
 29 |   To protect your rights, we need to prevent others from denying you
 30 | these rights or asking you to surrender the rights.  Therefore, you have
 31 | certain responsibilities if you distribute copies of the software, or if
 32 | you modify it: responsibilities to respect the freedom of others.
 33 | 
 34 |   For example, if you distribute copies of such a program, whether
 35 | gratis or for a fee, you must pass on to the recipients the same
 36 | freedoms that you received.  You must make sure that they, too, receive
 37 | or can get the source code.  And you must show them these terms so they
 38 | know their rights.
 39 | 
 40 |   Developers that use the GNU GPL protect your rights with two steps:
 41 | (1) assert copyright on the software, and (2) offer you this License
 42 | giving you legal permission to copy, distribute and/or modify it.
 43 | 
 44 |   For the developers' and authors' protection, the GPL clearly explains
 45 | that there is no warranty for this free software.  For both users' and
 46 | authors' sake, the GPL requires that modified versions be marked as
 47 | changed, so that their problems will not be attributed erroneously to
 48 | authors of previous versions.
 49 | 
 50 |   Some devices are designed to deny users access to install or run
 51 | modified versions of the software inside them, although the manufacturer
 52 | can do so.  This is fundamentally incompatible with the aim of
 53 | protecting users' freedom to change the software.  The systematic
 54 | pattern of such abuse occurs in the area of products for individuals to
 55 | use, which is precisely where it is most unacceptable.  Therefore, we
 56 | have designed this version of the GPL to prohibit the practice for those
 57 | products.  If such problems arise substantially in other domains, we
 58 | stand ready to extend this provision to those domains in future versions
 59 | of the GPL, as needed to protect the freedom of users.
 60 | 
 61 |   Finally, every program is threatened constantly by software patents.
 62 | States should not allow patents to restrict development and use of
 63 | software on general-purpose computers, but in those that do, we wish to
 64 | avoid the special danger that patents applied to a free program could
 65 | make it effectively proprietary.  To prevent this, the GPL assures that
 66 | patents cannot be used to render the program non-free.
 67 | 
 68 |   The precise terms and conditions for copying, distribution and
 69 | modification follow.
 70 | 
 71 |                        TERMS AND CONDITIONS
 72 | 
 73 |   0. Definitions.
 74 | 
 75 |   "This License" refers to version 3 of the GNU General Public License.
 76 | 
 77 |   "Copyright" also means copyright-like laws that apply to other kinds of
 78 | works, such as semiconductor masks.
 79 | 
 80 |   "The Program" refers to any copyrightable work licensed under this
 81 | License.  Each licensee is addressed as "you".  "Licensees" and
 82 | "recipients" may be individuals or organizations.
 83 | 
 84 |   To "modify" a work means to copy from or adapt all or part of the work
 85 | in a fashion requiring copyright permission, other than the making of an
 86 | exact copy.  The resulting work is called a "modified version" of the
 87 | earlier work or a work "based on" the earlier work.
 88 | 
 89 |   A "covered work" means either the unmodified Program or a work based
 90 | on the Program.
 91 | 
 92 |   To "propagate" a work means to do anything with it that, without
 93 | permission, would make you directly or secondarily liable for
 94 | infringement under applicable copyright law, except executing it on a
 95 | computer or modifying a private copy.  Propagation includes copying,
 96 | distribution (with or without modification), making available to the
 97 | public, and in some countries other activities as well.
 98 | 
 99 |   To "convey" a work means any kind of propagation that enables other
100 | parties to make or receive copies.  Mere interaction with a user through
101 | a computer network, with no transfer of a copy, is not conveying.
102 | 
103 |   An interactive user interface displays "Appropriate Legal Notices"
104 | to the extent that it includes a convenient and prominently visible
105 | feature that (1) displays an appropriate copyright notice, and (2)
106 | tells the user that there is no warranty for the work (except to the
107 | extent that warranties are provided), that licensees may convey the
108 | work under this License, and how to view a copy of this License.  If
109 | the interface presents a list of user commands or options, such as a
110 | menu, a prominent item in the list meets this criterion.
111 | 
112 |   1. Source Code.
113 | 
114 |   The "source code" for a work means the preferred form of the work
115 | for making modifications to it.  "Object code" means any non-source
116 | form of a work.
117 | 
118 |   A "Standard Interface" means an interface that either is an official
119 | standard defined by a recognized standards body, or, in the case of
120 | interfaces specified for a particular programming language, one that
121 | is widely used among developers working in that language.
122 | 
123 |   The "System Libraries" of an executable work include anything, other
124 | than the work as a whole, that (a) is included in the normal form of
125 | packaging a Major Component, but which is not part of that Major
126 | Component, and (b) serves only to enable use of the work with that
127 | Major Component, or to implement a Standard Interface for which an
128 | implementation is available to the public in source code form.  A
129 | "Major Component", in this context, means a major essential component
130 | (kernel, window system, and so on) of the specific operating system
131 | (if any) on which the executable work runs, or a compiler used to
132 | produce the work, or an object code interpreter used to run it.
133 | 
134 |   The "Corresponding Source" for a work in object code form means all
135 | the source code needed to generate, install, and (for an executable
136 | work) run the object code and to modify the work, including scripts to
137 | control those activities.  However, it does not include the work's
138 | System Libraries, or general-purpose tools or generally available free
139 | programs which are used unmodified in performing those activities but
140 | which are not part of the work.  For example, Corresponding Source
141 | includes interface definition files associated with source files for
142 | the work, and the source code for shared libraries and dynamically
143 | linked subprograms that the work is specifically designed to require,
144 | such as by intimate data communication or control flow between those
145 | subprograms and other parts of the work.
146 | 
147 |   The Corresponding Source need not include anything that users
148 | can regenerate automatically from other parts of the Corresponding
149 | Source.
150 | 
151 |   The Corresponding Source for a work in source code form is that
152 | same work.
153 | 
154 |   2. Basic Permissions.
155 | 
156 |   All rights granted under this License are granted for the term of
157 | copyright on the Program, and are irrevocable provided the stated
158 | conditions are met.  This License explicitly affirms your unlimited
159 | permission to run the unmodified Program.  The output from running a
160 | covered work is covered by this License only if the output, given its
161 | content, constitutes a covered work.  This License acknowledges your
162 | rights of fair use or other equivalent, as provided by copyright law.
163 | 
164 |   You may make, run and propagate covered works that you do not
165 | convey, without conditions so long as your license otherwise remains
166 | in force.  You may convey covered works to others for the sole purpose
167 | of having them make modifications exclusively for you, or provide you
168 | with facilities for running those works, provided that you comply with
169 | the terms of this License in conveying all material for which you do
170 | not control copyright.  Those thus making or running the covered works
171 | for you must do so exclusively on your behalf, under your direction
172 | and control, on terms that prohibit them from making any copies of
173 | your copyrighted material outside their relationship with you.
174 | 
175 |   Conveying under any other circumstances is permitted solely under
176 | the conditions stated below.  Sublicensing is not allowed; section 10
177 | makes it unnecessary.
178 | 
179 |   3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180 | 
181 |   No covered work shall be deemed part of an effective technological
182 | measure under any applicable law fulfilling obligations under article
183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184 | similar laws prohibiting or restricting circumvention of such
185 | measures.
186 | 
187 |   When you convey a covered work, you waive any legal power to forbid
188 | circumvention of technological measures to the extent such circumvention
189 | is effected by exercising rights under this License with respect to
190 | the covered work, and you disclaim any intention to limit operation or
191 | modification of the work as a means of enforcing, against the work's
192 | users, your or third parties' legal rights to forbid circumvention of
193 | technological measures.
194 | 
195 |   4. Conveying Verbatim Copies.
196 | 
197 |   You may convey verbatim copies of the Program's source code as you
198 | receive it, in any medium, provided that you conspicuously and
199 | appropriately publish on each copy an appropriate copyright notice;
200 | keep intact all notices stating that this License and any
201 | non-permissive terms added in accord with section 7 apply to the code;
202 | keep intact all notices of the absence of any warranty; and give all
203 | recipients a copy of this License along with the Program.
204 | 
205 |   You may charge any price or no price for each copy that you convey,
206 | and you may offer support or warranty protection for a fee.
207 | 
208 |   5. Conveying Modified Source Versions.
209 | 
210 |   You may convey a work based on the Program, or the modifications to
211 | produce it from the Program, in the form of source code under the
212 | terms of section 4, provided that you also meet all of these conditions:
213 | 
214 |     a) The work must carry prominent notices stating that you modified
215 |     it, and giving a relevant date.
216 | 
217 |     b) The work must carry prominent notices stating that it is
218 |     released under this License and any conditions added under section
219 |     7.  This requirement modifies the requirement in section 4 to
220 |     "keep intact all notices".
221 | 
222 |     c) You must license the entire work, as a whole, under this
223 |     License to anyone who comes into possession of a copy.  This
224 |     License will therefore apply, along with any applicable section 7
225 |     additional terms, to the whole of the work, and all its parts,
226 |     regardless of how they are packaged.  This License gives no
227 |     permission to license the work in any other way, but it does not
228 |     invalidate such permission if you have separately received it.
229 | 
230 |     d) If the work has interactive user interfaces, each must display
231 |     Appropriate Legal Notices; however, if the Program has interactive
232 |     interfaces that do not display Appropriate Legal Notices, your
233 |     work need not make them do so.
234 | 
235 |   A compilation of a covered work with other separate and independent
236 | works, which are not by their nature extensions of the covered work,
237 | and which are not combined with it such as to form a larger program,
238 | in or on a volume of a storage or distribution medium, is called an
239 | "aggregate" if the compilation and its resulting copyright are not
240 | used to limit the access or legal rights of the compilation's users
241 | beyond what the individual works permit.  Inclusion of a covered work
242 | in an aggregate does not cause this License to apply to the other
243 | parts of the aggregate.
244 | 
245 |   6. Conveying Non-Source Forms.
246 | 
247 |   You may convey a covered work in object code form under the terms
248 | of sections 4 and 5, provided that you also convey the
249 | machine-readable Corresponding Source under the terms of this License,
250 | in one of these ways:
251 | 
252 |     a) Convey the object code in, or embodied in, a physical product
253 |     (including a physical distribution medium), accompanied by the
254 |     Corresponding Source fixed on a durable physical medium
255 |     customarily used for software interchange.
256 | 
257 |     b) Convey the object code in, or embodied in, a physical product
258 |     (including a physical distribution medium), accompanied by a
259 |     written offer, valid for at least three years and valid for as
260 |     long as you offer spare parts or customer support for that product
261 |     model, to give anyone who possesses the object code either (1) a
262 |     copy of the Corresponding Source for all the software in the
263 |     product that is covered by this License, on a durable physical
264 |     medium customarily used for software interchange, for a price no
265 |     more than your reasonable cost of physically performing this
266 |     conveying of source, or (2) access to copy the
267 |     Corresponding Source from a network server at no charge.
268 | 
269 |     c) Convey individual copies of the object code with a copy of the
270 |     written offer to provide the Corresponding Source.  This
271 |     alternative is allowed only occasionally and noncommercially, and
272 |     only if you received the object code with such an offer, in accord
273 |     with subsection 6b.
274 | 
275 |     d) Convey the object code by offering access from a designated
276 |     place (gratis or for a charge), and offer equivalent access to the
277 |     Corresponding Source in the same way through the same place at no
278 |     further charge.  You need not require recipients to copy the
279 |     Corresponding Source along with the object code.  If the place to
280 |     copy the object code is a network server, the Corresponding Source
281 |     may be on a different server (operated by you or a third party)
282 |     that supports equivalent copying facilities, provided you maintain
283 |     clear directions next to the object code saying where to find the
284 |     Corresponding Source.  Regardless of what server hosts the
285 |     Corresponding Source, you remain obligated to ensure that it is
286 |     available for as long as needed to satisfy these requirements.
287 | 
288 |     e) Convey the object code using peer-to-peer transmission, provided
289 |     you inform other peers where the object code and Corresponding
290 |     Source of the work are being offered to the general public at no
291 |     charge under subsection 6d.
292 | 
293 |   A separable portion of the object code, whose source code is excluded
294 | from the Corresponding Source as a System Library, need not be
295 | included in conveying the object code work.
296 | 
297 |   A "User Product" is either (1) a "consumer product", which means any
298 | tangible personal property which is normally used for personal, family,
299 | or household purposes, or (2) anything designed or sold for incorporation
300 | into a dwelling.  In determining whether a product is a consumer product,
301 | doubtful cases shall be resolved in favor of coverage.  For a particular
302 | product received by a particular user, "normally used" refers to a
303 | typical or common use of that class of product, regardless of the status
304 | of the particular user or of the way in which the particular user
305 | actually uses, or expects or is expected to use, the product.  A product
306 | is a consumer product regardless of whether the product has substantial
307 | commercial, industrial or non-consumer uses, unless such uses represent
308 | the only significant mode of use of the product.
309 | 
310 |   "Installation Information" for a User Product means any methods,
311 | procedures, authorization keys, or other information required to install
312 | and execute modified versions of a covered work in that User Product from
313 | a modified version of its Corresponding Source.  The information must
314 | suffice to ensure that the continued functioning of the modified object
315 | code is in no case prevented or interfered with solely because
316 | modification has been made.
317 | 
318 |   If you convey an object code work under this section in, or with, or
319 | specifically for use in, a User Product, and the conveying occurs as
320 | part of a transaction in which the right of possession and use of the
321 | User Product is transferred to the recipient in perpetuity or for a
322 | fixed term (regardless of how the transaction is characterized), the
323 | Corresponding Source conveyed under this section must be accompanied
324 | by the Installation Information.  But this requirement does not apply
325 | if neither you nor any third party retains the ability to install
326 | modified object code on the User Product (for example, the work has
327 | been installed in ROM).
328 | 
329 |   The requirement to provide Installation Information does not include a
330 | requirement to continue to provide support service, warranty, or updates
331 | for a work that has been modified or installed by the recipient, or for
332 | the User Product in which it has been modified or installed.  Access to a
333 | network may be denied when the modification itself materially and
334 | adversely affects the operation of the network or violates the rules and
335 | protocols for communication across the network.
336 | 
337 |   Corresponding Source conveyed, and Installation Information provided,
338 | in accord with this section must be in a format that is publicly
339 | documented (and with an implementation available to the public in
340 | source code form), and must require no special password or key for
341 | unpacking, reading or copying.
342 | 
343 |   7. Additional Terms.
344 | 
345 |   "Additional permissions" are terms that supplement the terms of this
346 | License by making exceptions from one or more of its conditions.
347 | Additional permissions that are applicable to the entire Program shall
348 | be treated as though they were included in this License, to the extent
349 | that they are valid under applicable law.  If additional permissions
350 | apply only to part of the Program, that part may be used separately
351 | under those permissions, but the entire Program remains governed by
352 | this License without regard to the additional permissions.
353 | 
354 |   When you convey a copy of a covered work, you may at your option
355 | remove any additional permissions from that copy, or from any part of
356 | it.  (Additional permissions may be written to require their own
357 | removal in certain cases when you modify the work.)  You may place
358 | additional permissions on material, added by you to a covered work,
359 | for which you have or can give appropriate copyright permission.
360 | 
361 |   Notwithstanding any other provision of this License, for material you
362 | add to a covered work, you may (if authorized by the copyright holders of
363 | that material) supplement the terms of this License with terms:
364 | 
365 |     a) Disclaiming warranty or limiting liability differently from the
366 |     terms of sections 15 and 16 of this License; or
367 | 
368 |     b) Requiring preservation of specified reasonable legal notices or
369 |     author attributions in that material or in the Appropriate Legal
370 |     Notices displayed by works containing it; or
371 | 
372 |     c) Prohibiting misrepresentation of the origin of that material, or
373 |     requiring that modified versions of such material be marked in
374 |     reasonable ways as different from the original version; or
375 | 
376 |     d) Limiting the use for publicity purposes of names of licensors or
377 |     authors of the material; or
378 | 
379 |     e) Declining to grant rights under trademark law for use of some
380 |     trade names, trademarks, or service marks; or
381 | 
382 |     f) Requiring indemnification of licensors and authors of that
383 |     material by anyone who conveys the material (or modified versions of
384 |     it) with contractual assumptions of liability to the recipient, for
385 |     any liability that these contractual assumptions directly impose on
386 |     those licensors and authors.
387 | 
388 |   All other non-permissive additional terms are considered "further
389 | restrictions" within the meaning of section 10.  If the Program as you
390 | received it, or any part of it, contains a notice stating that it is
391 | governed by this License along with a term that is a further
392 | restriction, you may remove that term.  If a license document contains
393 | a further restriction but permits relicensing or conveying under this
394 | License, you may add to a covered work material governed by the terms
395 | of that license document, provided that the further restriction does
396 | not survive such relicensing or conveying.
397 | 
398 |   If you add terms to a covered work in accord with this section, you
399 | must place, in the relevant source files, a statement of the
400 | additional terms that apply to those files, or a notice indicating
401 | where to find the applicable terms.
402 | 
403 |   Additional terms, permissive or non-permissive, may be stated in the
404 | form of a separately written license, or stated as exceptions;
405 | the above requirements apply either way.
406 | 
407 |   8. Termination.
408 | 
409 |   You may not propagate or modify a covered work except as expressly
410 | provided under this License.  Any attempt otherwise to propagate or
411 | modify it is void, and will automatically terminate your rights under
412 | this License (including any patent licenses granted under the third
413 | paragraph of section 11).
414 | 
415 |   However, if you cease all violation of this License, then your
416 | license from a particular copyright holder is reinstated (a)
417 | provisionally, unless and until the copyright holder explicitly and
418 | finally terminates your license, and (b) permanently, if the copyright
419 | holder fails to notify you of the violation by some reasonable means
420 | prior to 60 days after the cessation.
421 | 
422 |   Moreover, your license from a particular copyright holder is
423 | reinstated permanently if the copyright holder notifies you of the
424 | violation by some reasonable means, this is the first time you have
425 | received notice of violation of this License (for any work) from that
426 | copyright holder, and you cure the violation prior to 30 days after
427 | your receipt of the notice.
428 | 
429 |   Termination of your rights under this section does not terminate the
430 | licenses of parties who have received copies or rights from you under
431 | this License.  If your rights have been terminated and not permanently
432 | reinstated, you do not qualify to receive new licenses for the same
433 | material under section 10.
434 | 
435 |   9. Acceptance Not Required for Having Copies.
436 | 
437 |   You are not required to accept this License in order to receive or
438 | run a copy of the Program.  Ancillary propagation of a covered work
439 | occurring solely as a consequence of using peer-to-peer transmission
440 | to receive a copy likewise does not require acceptance.  However,
441 | nothing other than this License grants you permission to propagate or
442 | modify any covered work.  These actions infringe copyright if you do
443 | not accept this License.  Therefore, by modifying or propagating a
444 | covered work, you indicate your acceptance of this License to do so.
445 | 
446 |   10. Automatic Licensing of Downstream Recipients.
447 | 
448 |   Each time you convey a covered work, the recipient automatically
449 | receives a license from the original licensors, to run, modify and
450 | propagate that work, subject to this License.  You are not responsible
451 | for enforcing compliance by third parties with this License.
452 | 
453 |   An "entity transaction" is a transaction transferring control of an
454 | organization, or substantially all assets of one, or subdividing an
455 | organization, or merging organizations.  If propagation of a covered
456 | work results from an entity transaction, each party to that
457 | transaction who receives a copy of the work also receives whatever
458 | licenses to the work the party's predecessor in interest had or could
459 | give under the previous paragraph, plus a right to possession of the
460 | Corresponding Source of the work from the predecessor in interest, if
461 | the predecessor has it or can get it with reasonable efforts.
462 | 
463 |   You may not impose any further restrictions on the exercise of the
464 | rights granted or affirmed under this License.  For example, you may
465 | not impose a license fee, royalty, or other charge for exercise of
466 | rights granted under this License, and you may not initiate litigation
467 | (including a cross-claim or counterclaim in a lawsuit) alleging that
468 | any patent claim is infringed by making, using, selling, offering for
469 | sale, or importing the Program or any portion of it.
470 | 
471 |   11. Patents.
472 | 
473 |   A "contributor" is a copyright holder who authorizes use under this
474 | License of the Program or a work on which the Program is based.  The
475 | work thus licensed is called the contributor's "contributor version".
476 | 
477 |   A contributor's "essential patent claims" are all patent claims
478 | owned or controlled by the contributor, whether already acquired or
479 | hereafter acquired, that would be infringed by some manner, permitted
480 | by this License, of making, using, or selling its contributor version,
481 | but do not include claims that would be infringed only as a
482 | consequence of further modification of the contributor version.  For
483 | purposes of this definition, "control" includes the right to grant
484 | patent sublicenses in a manner consistent with the requirements of
485 | this License.
486 | 
487 |   Each contributor grants you a non-exclusive, worldwide, royalty-free
488 | patent license under the contributor's essential patent claims, to
489 | make, use, sell, offer for sale, import and otherwise run, modify and
490 | propagate the contents of its contributor version.
491 | 
492 |   In the following three paragraphs, a "patent license" is any express
493 | agreement or commitment, however denominated, not to enforce a patent
494 | (such as an express permission to practice a patent or covenant not to
495 | sue for patent infringement).  To "grant" such a patent license to a
496 | party means to make such an agreement or commitment not to enforce a
497 | patent against the party.
498 | 
499 |   If you convey a covered work, knowingly relying on a patent license,
500 | and the Corresponding Source of the work is not available for anyone
501 | to copy, free of charge and under the terms of this License, through a
502 | publicly available network server or other readily accessible means,
503 | then you must either (1) cause the Corresponding Source to be so
504 | available, or (2) arrange to deprive yourself of the benefit of the
505 | patent license for this particular work, or (3) arrange, in a manner
506 | consistent with the requirements of this License, to extend the patent
507 | license to downstream recipients.  "Knowingly relying" means you have
508 | actual knowledge that, but for the patent license, your conveying the
509 | covered work in a country, or your recipient's use of the covered work
510 | in a country, would infringe one or more identifiable patents in that
511 | country that you have reason to believe are valid.
512 | 
513 |   If, pursuant to or in connection with a single transaction or
514 | arrangement, you convey, or propagate by procuring conveyance of, a
515 | covered work, and grant a patent license to some of the parties
516 | receiving the covered work authorizing them to use, propagate, modify
517 | or convey a specific copy of the covered work, then the patent license
518 | you grant is automatically extended to all recipients of the covered
519 | work and works based on it.
520 | 
521 |   A patent license is "discriminatory" if it does not include within
522 | the scope of its coverage, prohibits the exercise of, or is
523 | conditioned on the non-exercise of one or more of the rights that are
524 | specifically granted under this License.  You may not convey a covered
525 | work if you are a party to an arrangement with a third party that is
526 | in the business of distributing software, under which you make payment
527 | to the third party based on the extent of your activity of conveying
528 | the work, and under which the third party grants, to any of the
529 | parties who would receive the covered work from you, a discriminatory
530 | patent license (a) in connection with copies of the covered work
531 | conveyed by you (or copies made from those copies), or (b) primarily
532 | for and in connection with specific products or compilations that
533 | contain the covered work, unless you entered into that arrangement,
534 | or that patent license was granted, prior to 28 March 2007.
535 | 
536 |   Nothing in this License shall be construed as excluding or limiting
537 | any implied license or other defenses to infringement that may
538 | otherwise be available to you under applicable patent law.
539 | 
540 |   12. No Surrender of Others' Freedom.
541 | 
542 |   If conditions are imposed on you (whether by court order, agreement or
543 | otherwise) that contradict the conditions of this License, they do not
544 | excuse you from the conditions of this License.  If you cannot convey a
545 | covered work so as to satisfy simultaneously your obligations under this
546 | License and any other pertinent obligations, then as a consequence you may
547 | not convey it at all.  For example, if you agree to terms that obligate you
548 | to collect a royalty for further conveying from those to whom you convey
549 | the Program, the only way you could satisfy both those terms and this
550 | License would be to refrain entirely from conveying the Program.
551 | 
552 |   13. Use with the GNU Affero General Public License.
553 | 
554 |   Notwithstanding any other provision of this License, you have
555 | permission to link or combine any covered work with a work licensed
556 | under version 3 of the GNU Affero General Public License into a single
557 | combined work, and to convey the resulting work.  The terms of this
558 | License will continue to apply to the part which is the covered work,
559 | but the special requirements of the GNU Affero General Public License,
560 | section 13, concerning interaction through a network will apply to the
561 | combination as such.
562 | 
563 |   14. Revised Versions of this License.
564 | 
565 |   The Free Software Foundation may publish revised and/or new versions of
566 | the GNU General Public License from time to time.  Such new versions will
567 | be similar in spirit to the present version, but may differ in detail to
568 | address new problems or concerns.
569 | 
570 |   Each version is given a distinguishing version number.  If the
571 | Program specifies that a certain numbered version of the GNU General
572 | Public License "or any later version" applies to it, you have the
573 | option of following the terms and conditions either of that numbered
574 | version or of any later version published by the Free Software
575 | Foundation.  If the Program does not specify a version number of the
576 | GNU General Public License, you may choose any version ever published
577 | by the Free Software Foundation.
578 | 
579 |   If the Program specifies that a proxy can decide which future
580 | versions of the GNU General Public License can be used, that proxy's
581 | public statement of acceptance of a version permanently authorizes you
582 | to choose that version for the Program.
583 | 
584 |   Later license versions may give you additional or different
585 | permissions.  However, no additional obligations are imposed on any
586 | author or copyright holder as a result of your choosing to follow a
587 | later version.
588 | 
589 |   15. Disclaimer of Warranty.
590 | 
591 |   THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592 | APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596 | PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597 | IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599 | 
600 |   16. Limitation of Liability.
601 | 
602 |   IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610 | SUCH DAMAGES.
611 | 
612 |   17. Interpretation of Sections 15 and 16.
613 | 
614 |   If the disclaimer of warranty and limitation of liability provided
615 | above cannot be given local legal effect according to their terms,
616 | reviewing courts shall apply local law that most closely approximates
617 | an absolute waiver of all civil liability in connection with the
618 | Program, unless a warranty or assumption of liability accompanies a
619 | copy of the Program in return for a fee.
620 | 
621 |                      END OF TERMS AND CONDITIONS
622 | 
623 |             How to Apply These Terms to Your New Programs
624 | 
625 |   If you develop a new program, and you want it to be of the greatest
626 | possible use to the public, the best way to achieve this is to make it
627 | free software which everyone can redistribute and change under these terms.
628 | 
629 |   To do so, attach the following notices to the program.  It is safest
630 | to attach them to the start of each source file to most effectively
631 | state the exclusion of warranty; and each file should have at least
632 | the "copyright" line and a pointer to where the full notice is found.
633 | 
634 |     <one line to give the program's name and a brief idea of what it does.>
635 |     Copyright (C) <year>  <name of author>
636 | 
637 |     This program is free software: you can redistribute it and/or modify
638 |     it under the terms of the GNU General Public License as published by
639 |     the Free Software Foundation, either version 3 of the License, or
640 |     (at your option) any later version.
641 | 
642 |     This program is distributed in the hope that it will be useful,
643 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
644 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
645 |     GNU General Public License for more details.
646 | 
647 |     You should have received a copy of the GNU General Public License
648 |     along with this program.  If not, see <https://www.gnu.org/licenses/>.
649 | 
650 | Also add information on how to contact you by electronic and paper mail.
651 | 
652 |   If the program does terminal interaction, make it output a short
653 | notice like this when it starts in an interactive mode:
654 | 
655 |     <program>  Copyright (C) <year>  <name of author>
656 |     This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657 |     This is free software, and you are welcome to redistribute it
658 |     under certain conditions; type `show c' for details.
659 | 
660 | The hypothetical commands `show w' and `show c' should show the appropriate
661 | parts of the General Public License.  Of course, your program's commands
662 | might be different; for a GUI interface, you would use an "about box".
663 | 
664 |   You should also get your employer (if you work as a programmer) or school,
665 | if any, to sign a "copyright disclaimer" for the program, if necessary.
666 | For more information on this, and how to apply and follow the GNU GPL, see
667 | <https://www.gnu.org/licenses/>.
668 | 
669 |   The GNU General Public License does not permit incorporating your program
670 | into proprietary programs.  If your program is a subroutine library, you
671 | may consider it more useful to permit linking proprietary applications with
672 | the library.  If this is what you want to do, use the GNU Lesser General
673 | Public License instead of this License.  But first, please read
674 | <https://www.gnu.org/licenses/why-not-lgpl.html>.
675 | 


--------------------------------------------------------------------------------
/LVMSnapSend.md:
--------------------------------------------------------------------------------
  1 | # LVMSnapSend
  2 | Currently in a very raw state, batteries NOT included.
  3 | 
  4 | **You must never write directly to the destination LV**.
  5 | 
  6 | ## Basic Usage ###
  7 | Assuming that 
  8 | 
  9 | 
 10 | ## Example Testing
 11 | This is an example setting up 2 VGs and LVs on the same server for easier testing.
 12 | 
 13 | ### Set up thin LVM - SOURCE
 14 | ```
 15 | apt install -y thin-provisioning-tools
 16 | apt install -y python
 17 | wget https://raw.githubusercontent.com/theraser/blocksync/master/blocksync.py
 18 | 
 19 | LV=thin_volume
 20 | VG=volg
 21 | THINPOOL=${VG}-thinpool
 22 | THINSIZE=7G   # Slightly smaller than the actual device.  Allow for resizing.
 23 | LVSIZE=3G
 24 | DEVICE=xvdf
 25 | 
 26 | pvcreate /dev/${DEVICE}
 27 | vgcreate ${VG} /dev/${DEVICE}
 28 | 
 29 | lvcreate -L ${THINSIZE} --thinpool ${THINPOOL}  ${VG} 
 30 | 
 31 | # Review the volume and metadata
 32 | lvdisplay ${VG}/${THINPOOL}
 33 | 
 34 | lvcreate -V ${LVSIZE} --thin -n ${LV} ${VG}/${THINPOOL}
 35 | ```
 36 | 
 37 | ### Set up thin LVM - REPLICA
 38 | ```
 39 | LV=thinv_replica
 40 | VG=vgreplica
 41 | THINPOOL=${VG}-thinpoolreplica
 42 | THINSIZE=7G   # Slightly smaller than the actual device.  Allow for resizing.
 43 | LVSIZE=3G
 44 | DEVICE=xvdg
 45 | 
 46 | pvcreate /dev/${DEVICE}
 47 | vgcreate ${VG} /dev/${DEVICE}
 48 | 
 49 | lvcreate -L ${THINSIZE} --thinpool ${THINPOOL}  ${VG} 
 50 | 
 51 | # Review the volume and metadata
 52 | lvdisplay ${VG}/${THINPOOL}
 53 | 
 54 | lvcreate -V ${LVSIZE} --thin -n ${LV} ${VG}/${THINPOOL}
 55 | ```
 56 | 
 57 | ## Set up a file system and write to it
 58 | ```
 59 | mkfs.ext4 /dev/${VG}/${LV}
 60 | mkdir /mnt/${VG}-${LV}
 61 | mount /dev/${VG}/${LV} /mnt/${VG}-${LV}/
 62 | rsync -av /etc /mnt/${VG}-${LV}/
 63 | ```
 64 | 
 65 | ## Run the initial synchronisation
 66 | ```
 67 | time java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVMSnapSend --vg volg --lv thin_volume --targetPath /dev/vgreplica/thinv_replica | ssh -C root@localhost "java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVMSnapReceive"
 68 | 
 69 | # There will be an error, because the initial sync must be done manually
 70 | lvcreate -s -n thin_volume_thinsendrcv_20190623_0554 volg/thin_volume
 71 | lvchange -ay -Ky /dev/volg/thin_volume_thinsendrcv_20190623_0554
 72 | python blocksync.py -c aes128-ctr /dev/volg/thin_volume_thinsendrcv_20190623_0554 root@localhost /dev/vgreplica/thinv_replica
 73 | 
 74 | # Check that the snapshots are created
 75 | lvs
 76 | md5sum /dev/volg/thin_volume_thinsendrcv_* /dev/vgreplica/thinv_replica
 77 | ```
 78 | 
 79 | ## Regular Syncronisation
 80 | ```
 81 | rsync -av /usr/bin /mnt/${VG}-${LV}/usr/
 82 | time java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVMSnapSend --vg volg --lv thin_volume --targetPath /dev/vgreplica/thinv_replica | ssh -C root@localhost "java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVMSnapReceive"
 83 | 
 84 | rsync -av /var /mnt/${VG}-${LV}/
 85 | time java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVMSnapSend --vg volg --lv thin_volume --targetPath /dev/vgreplica/thinv_replica | ssh -C root@localhost "java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVMSnapReceive"
 86 | 
 87 | rsync -av /usr/lib /mnt/volg-thin_volume/usr/
 88 | time java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVMSnapSend --vg volg --lv thin_volume --targetPath /dev/vgreplica/thinv_replica | ssh -C root@localhost "java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVMSnapReceive"
 89 | 
 90 | # Verify that it all worked
 91 | lvchange -ay -Ky /dev/volg/thin_volume_thinsendrcv_20190623_0610
 92 | md5sum /dev/volg/thin_volume_thinsendrcv_* /dev/vgreplica/thinv_replica
 93 | ```
 94 | 
 95 | 
 96 | 
 97 | ## Debugging Problems
 98 | ### Test thin lvm is working properly
 99 | ```
100 | dmsetup message /dev/mapper/volg-volg--thinpool-tpool 0 reserve_metadata_snap
101 | thin_delta  -m --snap1 $(lvs --noheadings -o thin_id volg/thin_volume_snap1) --snap2 $(lvs --noheadings -o thin_id volg/thin_volume_snap2) /dev/mapper/volg-volg--thinpool_tmeta
102 | dmsetup message /dev/mapper/volg-volg--thinpool-tpool 0 release_metadata_snap
103 | ```
104 | 
105 | ### Run it in 2 parts
106 | ```
107 | java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVMSnapSend --vg volg --s1 thin_volume_snap1 --s2 thin_volume_snap2 > /tmp/diff.ddxml
108 | 
109 | # Currently split into 2 parts and I had to hardcode the block size at the destination
110 | cat /tmp/diff.ddxml | java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.LVSnapReceive
111 | 
112 | ```
113 | 


--------------------------------------------------------------------------------
/LVMSpaceCheck.md:
--------------------------------------------------------------------------------
 1 | # LVMSpaceCheck
 2 | If you don't want to autoextend your thin partitions e.g. because you want to be emailed then LVMSpaceCheck is for you
 3 | 
 4 | Run the command once and then exit.  Suitable for a cron job.
 5 | ```
 6 | java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.service.LVMSpaceCheck
 7 | ```
 8 | 
 9 | Run the command every few seconds.  Suitable for running as a daemon.  Need a script for restarting ...
10 | ```
11 | java -cp lvm-thin-sendrcv/lvm-thin-sendrcv.jar oneit.lvmsendrcv.service.LVMSpaceCheck --checkInterval 10
12 | ```
13 | ## When Space is Exceeded
14 | When the configured space is exceeded, an alert email is sent on the 1, 10, 100, 1000th failure.
15 | 
16 | When the system returns to normal, the error count is reverted.
17 | 
18 | ### Autoresizing
19 | TODO.  Would be nice to have
20 | 
21 | ## Help
22 | Run with -h for help
23 | 
24 | 
25 | ## Configuration
26 | The utility uses `/etc/lvmsendrcv/lvmsendrcv.conf`
27 | ```
28 | # Both data and meta limit must be set for each thin pool
29 | monitor.[vgname]/[lvname].dataLimit=0..100
30 | monitor.[vgname]/[lvname].metaimit=0..100
31 | 
32 | # 2 Examples below
33 | monitor.volg/volg-thinpool.dataLimit=70
34 | monitor.volg/volg-thinpool.metaLimit=70
35 | 
36 | monitor.vgreplica/vgreplica-thinpoolreplica.dataLimit=50
37 | monitor.vgreplica/vgreplica-thinpoolreplica.metaLimit=50
38 | ```
39 | 
40 | ## Results
41 | The system generates some information to stdout
42 | 
43 | If the system exceeds the limit, then an email will be sent like:
44 | ```
45 | Subject: [lvmsendrcv] DATA Limit exceeded volg-thinpool
46 | Volume:volg/volg-thinpool : DATA
47 | Usage:78.49 > 70.0
48 | Count:10
49 | lvextend -L+1G volg/volg-thinpool
50 | ```
51 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # lvm-thin-sendrcv
 2 | Send and receive incremental / thin LVM snapshots.  Replication / synchronisation of an LVM volume to a remote server by transmitting only the difference between snapshots.
 3 | 
 4 | # Status
 5 | Working in an initial form.  Needs a little setup, documented at [LVMSnapSend.md](LVMSnapSend.md).  Remember, you must **never write to the target volume**.
 6 | 
 7 | Monitor your thin LVs using [LVMSpaceCheck](LVMSpaceCheck.md)
 8 | Also useful is [DDRandom.md](DDRandom.md)
 9 | 
10 | # Purpose 
11 | LVM volumes (LVs) are useful, however synchronising them between machines is generally slow, especially when repeated.  The goals of this project are:
12 | - Synchronise LVs between machines similar to a zfs send / receive
13 | - No need to read the entire LV to run a sync (low IO)
14 | - Does not have a large performance penalty in either reads, writes, or CPU.
15 | - Able to verify and mount the snapshots 
16 | - No need to unmount the volume / shut down the virtual machine
17 | - Filesystem and encryption agnostic.  I should be able to sync ext4, xfs, etc.
18 | 
19 | The use case I am aiming for is to have VM images synchronised across to another server.
20 | 
21 | # Prerequisites / Platform
22 | This is only intended to run on thinly provisioned LVM.
23 | 
24 | I will be testing on Ubuntu, but welcome feedback for other platforms / OSes.
25 | 
26 | # Goals
27 | ~~This is literally the start of the project.  So far there is nothing.  ~~Goals for the project are:
28 | - ~~Write useful libraries to snapshot the LV, snapshot the thin metadata, and extract the changed blocks and block sizes~~
29 | - ~~Basic proof of concept.  Synchronise two manually created snapshots between 2 servers.  Probably using dd over ssh or something else lame.~~
30 | - ~~Extract things like block size and thin volume using lvs and co.~~
31 | - ~~Create and delete the snapshots as well as synchronise~~
32 | - Server mode over ssh
33 | - Run as a daemon so that your LVs are continuously synchronised.  This requires robust error handling
34 | - Alerts / monitoring / emails
35 | - Ensure continuous consistency at the destination e.g. when there is a failure partway through the sync.  Probably by snapshotting at the destination.
36 | - Allowing for synchronising to a file.  This would be VERY useful.
37 | - Verify the snapshot at both ends (probably daily, as otherwise we are reading the LV every time)
38 | - Automate the original send (something like blocksync) and possibly LV or file creation.
39 | - Alerting on thin pool size and metadata usage
40 | - World domination!
41 | 
42 | There is absolutely no intent to support synchronising without a snapshot.
43 | 
44 | # Performance
45 | Running on a single AWS small instance and via SSH, I am getting changes detected, read, transferred, and written at roughly 10MB/s.  This means that for 50MB of writes, it takes ~5 seconds to do a sync.  This is irrespective of the size of the underlying device.  For devices in the 10s or 100s of GB this is a boon.
46 | 
47 | Running rsyncs of large file collections in the 10s and 100s of MB total e.g. `/usr/bin` or `/usr/lib` the size of changed blocks is very close to the size of the files.  In other words, 10MB of files leads to 11 - 12 MB of block changes.
48 | 
49 | This performance makes it seem reasonable that we could potentially be synchronising the device every minute (or more frequently) once a service has been written.
50 | 
51 | # Alternatives
52 | - Blocksync https://github.com/theraser/blocksync is very good but reads the entire LV.  For large LVs this is **slow** and consumes IO on the source and target
53 | - lvsync https://github.com/mpalmer/lvmsync seems to have a lot of features **but** requires the LV to be closed / VM to be shut down.  It also does an initial full sync just like blocksync.
54 | - drdb https://www.linbit.com/ is costly if you want the proxy (and if running over a WAN, you need a proxy) and there is no easy way to verify the replicas as the other end
55 | - sparsebak https://github.com/tasket/sparsebak seems to be aimed at backups rather than replication
56 | 
57 | # Sponsors
58 | This project is sponsored by:
59 | - [OneIT Custom Software](https://www.oneit.com.au)
60 | - [Secure Hosting Australia](https://www.secure-hosting.com.au)
61 | 
62 | # Technical approach
63 | Thin LVM is not the most well documented API.  The high level approach for now is:
64 | 
65 | I will be maintaining at least a basic LVM document to outline how thing LVM snapshotting works as it seems like a useful service.
66 | ```
67 | # Get the chunk size
68 | lvs -o lv_name,chunksize volg/volg-thinpool
69 | 
70 | # Get the device IDs
71 | SNAP1_ID=$(lvs --noheadings -o thin_id volg/thin_volume_snap4)
72 | SNAP2_ID=$(lvs --noheadings -o thin_id volg/thin_volume_snap5)
73 | 
74 | # Reserve the metadata
75 | dmsetup message /dev/mapper/volg-volg--thinpool-tpool 0 reserve_metadata_snap
76 | dmsetup status /dev/mapper/volg-volg--thinpool-tpool  # Get the snapshot ID piping through: cut -f 7 -d " "
77 | 
78 | # Determine the difference between the snapshots
79 | thin_delta  -m --snap1 $SNAP1_ID --snap2 $SNAP2_ID /dev/mapper/volg-volg--thinpool_tmeta
80 | 
81 | # Release the metadata snapshot.  Try to keep this window short.
82 | dmsetup message /dev/mapper/volg-volg--thinpool-tpool 0 release_metadata_snap
83 | 
84 | # Take those blocks and push them to the target device / file.
85 | ```
86 | 
87 | 


--------------------------------------------------------------------------------
/build.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <!-- You may freely edit this file. See commented blocks below for -->
 3 | <!-- some examples of how to customize the build. -->
 4 | <!-- (If you delete it and reopen the project it will be recreated.) -->
 5 | <!-- By default, only the Clean and Build commands use this build script. -->
 6 | <!-- Commands such as Run, Debug, and Test only use this build script if -->
 7 | <!-- the Compile on Save feature is turned off for the project. -->
 8 | <!-- You can turn off the Compile on Save (or Deploy on Save) setting -->
 9 | <!-- in the project's Project Properties dialog box.-->
10 | <project name="lvm-thin-sendrcv" default="default" basedir=".">
11 |     <description>Builds, tests, and runs the project lvm-thin-sendrcv.</description>
12 |     <import file="nbproject/build-impl.xml"/>
13 |     <!--
14 | 
15 |     There exist several targets which are by default empty and which can be 
16 |     used for execution of your tasks. These targets are usually executed 
17 |     before and after some main targets. They are: 
18 | 
19 |       -pre-init:                 called before initialization of project properties
20 |       -post-init:                called after initialization of project properties
21 |       -pre-compile:              called before javac compilation
22 |       -post-compile:             called after javac compilation
23 |       -pre-compile-single:       called before javac compilation of single file
24 |       -post-compile-single:      called after javac compilation of single file
25 |       -pre-compile-test:         called before javac compilation of JUnit tests
26 |       -post-compile-test:        called after javac compilation of JUnit tests
27 |       -pre-compile-test-single:  called before javac compilation of single JUnit test
28 |       -post-compile-test-single: called after javac compilation of single JUunit test
29 |       -pre-jar:                  called before JAR building
30 |       -post-jar:                 called after JAR building
31 |       -post-clean:               called after cleaning build products
32 | 
33 |     (Targets beginning with '-' are not intended to be called on their own.)
34 | 
35 |     Example of inserting an obfuscator after compilation could look like this:
36 | 
37 |         <target name="-post-compile">
38 |             <obfuscate>
39 |                 <fileset dir="${build.classes.dir}"/>
40 |             </obfuscate>
41 |         </target>
42 | 
43 |     For list of available properties check the imported 
44 |     nbproject/build-impl.xml file. 
45 | 
46 | 
47 |     Another way to customize the build is by overriding existing main targets.
48 |     The targets of interest are: 
49 | 
50 |       -init-macrodef-javac:     defines macro for javac compilation
51 |       -init-macrodef-junit:     defines macro for junit execution
52 |       -init-macrodef-debug:     defines macro for class debugging
53 |       -init-macrodef-java:      defines macro for class execution
54 |       -do-jar:                  JAR building
55 |       run:                      execution of project 
56 |       -javadoc-build:           Javadoc generation
57 |       test-report:              JUnit report generation
58 | 
59 |     An example of overriding the target for project execution could look like this:
60 | 
61 |         <target name="run" depends="lvm-thin-sendrcv-impl.jar">
62 |             <exec dir="bin" executable="launcher.exe">
63 |                 <arg file="${dist.jar}"/>
64 |             </exec>
65 |         </target>
66 | 
67 |     Notice that the overridden target depends on the jar target and not only on 
68 |     the compile target as the regular run target does. Again, for a list of available 
69 |     properties which you can use, check the target you are overriding in the
70 |     nbproject/build-impl.xml file. 
71 | 
72 |     -->
73 | </project>
74 | 


--------------------------------------------------------------------------------
/nbproject/.gitignore:
--------------------------------------------------------------------------------
1 | /private/
2 | 


--------------------------------------------------------------------------------
/nbproject/genfiles.properties:
--------------------------------------------------------------------------------
1 | build.xml.data.CRC32=0ae3ebfc
2 | build.xml.script.CRC32=8cb39794
3 | build.xml.stylesheet.CRC32=f85dc8f2@1.90.1.48
4 | # This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
5 | # Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
6 | nbproject/build-impl.xml.data.CRC32=0ae3ebfc
7 | nbproject/build-impl.xml.script.CRC32=1cdee815
8 | nbproject/build-impl.xml.stylesheet.CRC32=3a2fa800@1.90.1.48
9 | 


--------------------------------------------------------------------------------
/nbproject/project.properties:
--------------------------------------------------------------------------------
  1 | annotation.processing.enabled=true
  2 | annotation.processing.enabled.in.editor=false
  3 | annotation.processing.processors.list=
  4 | annotation.processing.run.all.processors=true
  5 | annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output
  6 | application.title=lvm-thin-sendrcv
  7 | application.vendor=david
  8 | build.classes.dir=${build.dir}/classes
  9 | build.classes.excludes=**/*.java,**/*.form
 10 | # This directory is removed when the project is cleaned:
 11 | build.dir=build
 12 | build.generated.dir=${build.dir}/generated
 13 | build.generated.sources.dir=${build.dir}/generated-sources
 14 | # Only compile against the classpath explicitly listed here:
 15 | build.sysclasspath=ignore
 16 | build.test.classes.dir=${build.dir}/test/classes
 17 | build.test.results.dir=${build.dir}/test/results
 18 | # Uncomment to specify the preferred debugger connection transport:
 19 | #debug.transport=dt_socket
 20 | debug.classpath=\
 21 |     ${run.classpath}
 22 | debug.modulepath=\
 23 |     ${run.modulepath}
 24 | debug.test.classpath=\
 25 |     ${run.test.classpath}
 26 | debug.test.modulepath=\
 27 |     ${run.test.modulepath}
 28 | # Files in build.classes.dir which should be excluded from distribution jar
 29 | dist.archive.excludes=
 30 | # This directory is removed when the project is cleaned:
 31 | dist.dir=dist
 32 | dist.jar=${dist.dir}/lvm-thin-sendrcv.jar
 33 | dist.javadoc.dir=${dist.dir}/javadoc
 34 | dist.jlink.dir=${dist.dir}/jlink
 35 | dist.jlink.output=${dist.jlink.dir}/lvm-thin-sendrcv
 36 | endorsed.classpath=
 37 | excludes=
 38 | file.reference.activation.jar=/home/david/Projects/CougarCMS9.0/cmsWebApp/libraries/optional/activation.jar
 39 | file.reference.commons-cli-1.2.jar=/home/david/Projects/CougarCMS9.0/cmsWebApp/libraries/apache-commons/commons-cli-1.2.jar
 40 | file.reference.json.jar=/home/david/Projects/CougarCMS9.0/cmsWebApp/libraries/script/json.jar
 41 | file.reference.mail.jar=/home/david/Projects/CougarCMS9.0/cmsWebApp/libraries/mail.jar
 42 | includes=**
 43 | jar.compress=false
 44 | javac.classpath=\
 45 |     ${file.reference.commons-cli-1.2.jar}:\
 46 |     ${file.reference.json.jar}:\
 47 |     ${file.reference.mail.jar}:\
 48 |     ${file.reference.activation.jar}
 49 | # Space-separated list of extra javac options
 50 | javac.compilerargs=
 51 | javac.deprecation=false
 52 | javac.external.vm=true
 53 | javac.modulepath=
 54 | javac.processormodulepath=
 55 | javac.processorpath=\
 56 |     ${javac.classpath}
 57 | javac.source=1.8
 58 | javac.target=1.8
 59 | javac.test.classpath=\
 60 |     ${javac.classpath}:\
 61 |     ${build.classes.dir}
 62 | javac.test.modulepath=\
 63 |     ${javac.modulepath}
 64 | javac.test.processorpath=\
 65 |     ${javac.test.classpath}
 66 | javadoc.additionalparam=
 67 | javadoc.author=false
 68 | javadoc.encoding=${source.encoding}
 69 | javadoc.html5=false
 70 | javadoc.noindex=false
 71 | javadoc.nonavbar=false
 72 | javadoc.notree=false
 73 | javadoc.private=false
 74 | javadoc.splitindex=true
 75 | javadoc.use=true
 76 | javadoc.version=false
 77 | javadoc.windowtitle=
 78 | # The jlink additional root modules to resolve
 79 | jlink.additionalmodules=
 80 | # The jlink additional command line parameters
 81 | jlink.additionalparam=
 82 | jlink.launcher=true
 83 | jlink.launcher.name=lvm-thin-sendrcv
 84 | main.class=oneit.lvmsendrcv.dd.DDRandomReceive
 85 | manifest.file=manifest.mf
 86 | meta.inf.dir=${src.dir}/META-INF
 87 | mkdist.disabled=false
 88 | platform.active=default_platform
 89 | run.classpath=\
 90 |     ${javac.classpath}:\
 91 |     ${build.classes.dir}
 92 | # Space-separated list of JVM arguments used when running the project.
 93 | # You may also define separate properties like run-sys-prop.name=value instead of -Dname=value.
 94 | # To set system properties for unit tests define test-sys-prop.name=value:
 95 | run.jvmargs=
 96 | run.modulepath=\
 97 |     ${javac.modulepath}
 98 | run.test.classpath=\
 99 |     ${javac.test.classpath}:\
100 |     ${build.test.classes.dir}
101 | run.test.modulepath=\
102 |     ${javac.test.modulepath}
103 | source.encoding=UTF-8
104 | src.dir=src
105 | test.src.dir=test
106 | 


--------------------------------------------------------------------------------
/nbproject/project.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://www.netbeans.org/ns/project/1">
 3 |     <type>org.netbeans.modules.java.j2seproject</type>
 4 |     <configuration>
 5 |         <data xmlns="http://www.netbeans.org/ns/j2se-project/3">
 6 |             <name>lvm-thin-sendrcv</name>
 7 |             <source-roots>
 8 |                 <root id="src.dir"/>
 9 |             </source-roots>
10 |             <test-roots>
11 |                 <root id="test.src.dir"/>
12 |             </test-roots>
13 |         </data>
14 |     </configuration>
15 | </project>
16 | 


--------------------------------------------------------------------------------
/plan.md:
--------------------------------------------------------------------------------
1 | # First stage plan
2 | 1. Ability to transfor random blocks in bulk
3 | 2. Identify chaned blocks in thin snapshots
4 | 3. Glue 1 and 2 together to be able to transfer differences beween snapshots
5 | 4. Create a verification mode to verify the snapshot after transfer
6 | 
7 | 


--------------------------------------------------------------------------------
/sample/dd1.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | 
3 | <dd>
4 |     <block offset="2">MDItMDItMDItMDItMDItMDItMDItMDItMDItMDItMDItMDItMDItMDIt</block>
5 |     <block offset="5">MDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUtMDUt</block>
6 |     <block offset="7">MDctMDctMDctMDctMDctMDctMDctMDctMDctMDctMDctMDctMDctMDctMDct</block>
7 | </dd>
8 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/LVM.java:
--------------------------------------------------------------------------------
  1 | package oneit.lvmsendrcv;
  2 | 
  3 | import oneit.lvmsendrcv.utils.ExecUtils;
  4 | import java.io.*;
  5 | import java.util.*;
  6 | import org.json.*;
  7 | 
  8 | /**
  9 |  *
 10 |  * @author david
 11 |  */
 12 | public class LVM 
 13 | {
 14 |     /**
 15 |      * getVGInfo (["volg"])
 16 |      * @param vgNames
 17 |      * @return
 18 |      */
 19 |     public static VGInfo getVGInfo (String vgName) throws LVMException
 20 |     {
 21 |         List<JSONObject>            vgs = execVGS(VGInfo.VGS_COLUMNS, new String[] { vgName });
 22 | 
 23 |         if (vgs.size() != 1)
 24 |         {
 25 |             throw new LVMException("VGs.length != 1 : " + vgs.size() + " " + vgs);
 26 |         }
 27 |         
 28 |         return new VGInfo(vgs.get(0));
 29 |         
 30 |     }
 31 |     
 32 |     
 33 |     /**
 34 |      * getThinPoolInfo ("volg", "volg-thinpool")
 35 |      * @param vgName
 36 |      * @param thinPool
 37 |      * @return
 38 |      */
 39 |     public static LVMThinPool getThinPoolInfo (String vgName, String thinPool) throws LVMException
 40 |     {
 41 |         // EXEC lvs --reportformat json -o chunksize,lv_dm_path ${thinPool}
 42 |         List<JSONObject>            thinPools = execLVS(LVMThinPool.LVS_COLUMNS, new String[] { vgName + "/" + thinPool });
 43 | 
 44 |         if (thinPools.size() != 1)
 45 |         {
 46 |             throw new LVMException("Thinpools.length != 1 : " + thinPools.size() + " " + thinPools);
 47 |         }
 48 |         
 49 |         return new LVMThinPool(thinPools.get(0));
 50 |     }
 51 |     
 52 |     
 53 |     /**
 54 |      * getSnapshotInfo ("volg/thin_volume_snap2", "volg/thin_volume_snap1")
 55 |      * @param vgName
 56 |      * @param snapshotNames 
 57 |      * @return map of LVMSnapshot keyed by name
 58 |      */
 59 |     public static Map<String, LVMSnapshot> getSnapshotInfo (String[] snapshotNames) throws LVMException
 60 |     {
 61 |         // EXEC lvs --reportformat json -o lv_name,vg_name,pool_lv,thin_id volg/thin_volume_snap2 volg/thin_volume_snap1
 62 |         List<JSONObject>            snapshots = execLVS(LVMSnapshot.LVS_COLUMNS, snapshotNames);
 63 |         Map<String, LVMSnapshot>    result = new HashMap<>();
 64 | 
 65 |         for (JSONObject  lvsnapJSON : snapshots)
 66 |         {
 67 |             LVMSnapshot lvsnap = new LVMSnapshot(lvsnapJSON);
 68 |             
 69 |             result.put(lvsnap.snapshotName, lvsnap);
 70 |         }
 71 |         
 72 |         return result;
 73 |     }
 74 | 
 75 |     
 76 |     public static List<JSONObject> execLVS (String columns, String[] vols) throws LVMException
 77 |     {
 78 |         return execLVM_JSON("/sbin/lvs", "lv", columns, vols);
 79 |     }
 80 | 
 81 |     
 82 |     public static List<JSONObject> execVGS (String columns, String[] vgs) throws LVMException
 83 |     {
 84 |         return execLVM_JSON("/sbin/vgs", "vg", columns, vgs);
 85 |     }
 86 | 
 87 |     
 88 |     public static List<JSONObject> execLVM_JSON (String command, String jsonReport, String columns, String[] args) throws LVMException
 89 |     {
 90 |         List<String>    lvmArgs         = new ArrayList<>();
 91 |         String          lvmOutputStr;
 92 |         
 93 |         lvmArgs.addAll(Arrays.asList(command, "--reportformat", "json", "-o", columns));
 94 |         lvmArgs.addAll(Arrays.asList(args));
 95 |         
 96 |         try
 97 |         {
 98 |             byte[]          lvsOutput           = new ExecUtils.ExecuteProcess ("/tmp/", lvmArgs.toArray(new String[0])).setHideStdOut(true).executeAsBytes();
 99 |             
100 |             lvmOutputStr        = new String (lvsOutput);
101 |         }
102 |         catch (Exception e)
103 |         {
104 |             System.err.println("problem running "+ command + ":" + e.getMessage()); //@todo
105 |             throw new LVMException(e);
106 |         }
107 |         
108 |         try
109 |         {
110 |             JSONObject          lvmOutputJSON   = new JSONObject(lvmOutputStr);
111 |             JSONArray           lvmOutputArray  = lvmOutputJSON.getJSONArray("report").getJSONObject(0).getJSONArray(jsonReport);
112 |             List<JSONObject>    result          = new ArrayList<>();
113 | 
114 |             for (int x = 0 ; x < lvmOutputArray.length() ; ++x)
115 |             {
116 |                 JSONObject  outputRow = lvmOutputArray.getJSONObject(x);
117 |                 
118 |                 result.add (outputRow);
119 |             }
120 | 
121 |             return result;
122 |         }
123 |         catch (Exception e)
124 |         {
125 |             System.err.println("problem parsing " + command + " output:" + e.getMessage() + "\n" + lvmOutputStr); //@todo
126 |             throw new LVMException(e);
127 |         }
128 |     }
129 | 
130 |     
131 |     public static class VGInfo
132 |     {
133 |         static final String VGS_COLUMNS = "vg_name,vg_free";
134 |         
135 |         public final String  vgName;
136 |         public final String  vgFree;
137 | 
138 |         public VGInfo(JSONObject vgsnap) throws LVMException
139 |         {
140 |             try
141 |             {
142 |                 this.vgName         = vgsnap.getString("vg_name");
143 |                 this.vgFree         = vgsnap.getString("vg_free");
144 |             }
145 |             catch (Exception e)
146 |             {
147 |                 System.err.println("problem parsing vgs output:" + e.getMessage() + "\n" + vgsnap); //@todo
148 |                 throw new LVMException(e);
149 |             }
150 |         }
151 | 
152 |         
153 |         @Override
154 |         public String toString() 
155 |         {
156 |             return "VGInfo " + vgName + " has " + vgFree + " free";
157 |         }
158 |     }
159 |     
160 |     
161 |     public static class LVMSnapshot
162 |     {
163 |         static final String LVS_COLUMNS = "lv_name,vg_name,pool_lv,thin_id";
164 |         
165 |         public final String  snapshotName;
166 |         public final String  vgName;
167 |         public final String  poolLV;
168 |         public final Integer thinID;
169 | 
170 |         public LVMSnapshot(JSONObject lvsnap) throws LVMException
171 |         {
172 |             try
173 |             {
174 |                 String              thinIDStr = lvsnap.getString("thin_id");
175 |                 
176 |                 this.snapshotName   = lvsnap.getString("lv_name");
177 |                 this.vgName         = lvsnap.getString("vg_name");
178 |                 this.poolLV         = lvsnap.getString("pool_lv");
179 |                 this.thinID         = thinIDStr.equals("") ? null : Integer.parseInt(thinIDStr);
180 |             }
181 |             catch (Exception e)
182 |             {
183 |                 System.err.println("problem parsing lvs output:" + e.getMessage() + "\n" + lvsnap); //@todo
184 |                 throw new LVMException(e);
185 |             }
186 |         }
187 | 
188 |         
189 |         @Override
190 |         public String toString() 
191 |         {
192 |             return "LVSnapshot " + vgName + "/" + snapshotName + " in " + poolLV + "@" + thinID;
193 |         }
194 |     }
195 |     
196 |     
197 |     public static class LVMThinPool
198 |     {
199 |         static final String LVS_COLUMNS = "lv_name,vg_name,chunksize,lv_dm_path";
200 |         
201 |         public final String  name;
202 |         public final String  vgName;
203 |         public final int     chunkSizeKB;
204 |         public final String  dmPath;
205 | 
206 |         public LVMThinPool(JSONObject thinJSON) throws LVMException
207 |         {
208 |             try
209 |             {
210 |                 String      chunkSizeStr        = thinJSON.getString("chunk_size");
211 | 
212 |                 assert chunkSizeStr.charAt(chunkSizeStr.length() - 1) == 'k' : "Chunk size must be in kB";
213 | 
214 |                 this.name           = thinJSON.getString("lv_name");
215 |                 this.vgName         = thinJSON.getString("vg_name");
216 |                 this.chunkSizeKB    = (int)Float.parseFloat(chunkSizeStr.substring(0, chunkSizeStr.length() - 1));
217 |                 this.dmPath         = thinJSON.getString("lv_dm_path");
218 |             }
219 |             catch (JSONException e)
220 |             {
221 |                 System.err.println("problem parsing lvs output:" + e.getMessage() + "\n" + thinJSON); //@todo
222 |                 throw new LVMException(e);
223 |             }
224 |         }
225 |         
226 |         @Override
227 |         public String toString() 
228 |         {
229 |             return "LVThinPool " + vgName + "/" + name + "  " + chunkSizeKB + " @ " + dmPath;
230 |         }
231 |     }
232 | 
233 |     
234 |     public static class LVMException extends Exception
235 |     {
236 |         public LVMException(Exception cause) 
237 |         {
238 |             super (cause);
239 |         }
240 |         
241 |         public LVMException(String message) 
242 |         {
243 |             super (message);
244 |         }
245 |     }
246 | }
247 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/LVMBlockDiff.java:
--------------------------------------------------------------------------------
  1 | package oneit.lvmsendrcv;
  2 | 
  3 | import oneit.lvmsendrcv.utils.ExecUtils;
  4 | import oneit.lvmsendrcv.utils.XMLUtils;
  5 | import java.io.*;
  6 | import java.util.*;
  7 | import javax.xml.parsers.*;
  8 | import javax.xml.xpath.XPathExpressionException;
  9 | import org.json.*;
 10 | import org.w3c.dom.Document;
 11 | import org.w3c.dom.Node;
 12 | import org.xml.sax.SAXException;
 13 | 
 14 | /**
 15 |  *
 16 |  * @author david
 17 |  */
 18 | public class LVMBlockDiff 
 19 | {
 20 |     public static LVMSnapshotDiff diffLVMSnapshots (String vgName, String snapshot1, String snapshot2) throws LVM.LVMException, InterruptedException, IOException
 21 |     {
 22 |         Map<String, LVM.LVMSnapshot>    snapshotInfo = LVM.getSnapshotInfo(new String[] { vgName + '/' + snapshot1, vgName + '/' + snapshot2 });
 23 |         
 24 |         System.err.println (snapshotInfo); // @todo
 25 |         
 26 |         String          thinPoolName    = snapshotInfo.get(snapshot1).poolLV;
 27 |         LVM.LVMThinPool thinPool        = LVM.getThinPoolInfo(vgName, thinPoolName);
 28 |         String          snap1ID         = String.valueOf(snapshotInfo.get(snapshot1).thinID);
 29 |         String          snap2ID         = String.valueOf(snapshotInfo.get(snapshot2).thinID);
 30 | 
 31 |         System.err.println (thinPool); // @todo
 32 |         
 33 |         new ExecUtils.ExecuteProcess ("/tmp/", "/sbin/dmsetup", "message", thinPool.dmPath + "-tpool", "0", "reserve_metadata_snap").setHideStdOut(true).executeAsBytes();
 34 |         
 35 |         try
 36 |         {
 37 |             byte[]          thinDeltaOutput = new ExecUtils.ExecuteProcess ("/tmp/", "/usr/sbin/thin_delta", "-m", "--snap1", snap1ID, "--snap2", snap2ID,  thinPool.dmPath + "_tmeta").setHideStdOut(true).executeAsBytes();
 38 |             List<Integer>   result          = getBlocksDiffMatching (thinDeltaOutput, false, true, true, true);
 39 |             
 40 |             return new LVMSnapshotDiff(thinPool.chunkSizeKB, result);
 41 |         }
 42 |         finally
 43 |         {       
 44 |             new ExecUtils.ExecuteProcess ("/tmp/", "/sbin/dmsetup", "message", thinPool.dmPath + "-tpool", "0", "release_metadata_snap").setHideStdOut(true).executeAsBytes();
 45 |         }        
 46 |     }
 47 | 
 48 |     
 49 |     private static List<Integer> getBlocksDiffMatching (byte[] thin_delta_output, boolean includeSame, boolean includeLeftOnly, boolean includeRightOnly, boolean includeDifferent) throws LVM.LVMException
 50 |     {
 51 |         try
 52 |         {
 53 |             DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
 54 |             DocumentBuilder        dBuilder = dbFactory.newDocumentBuilder();        
 55 |             Document               thinDeltaXML = dBuilder.parse(new ByteArrayInputStream(thin_delta_output));
 56 |             List<Integer>          result = new ArrayList<>();
 57 | 
 58 |             for (Node blockNode : XMLUtils.getXPathNodes(thinDeltaXML, "superblock/diff/*"))
 59 |             {
 60 |                 String  nodeName = XMLUtils.getXPathVal(blockNode, "name()");
 61 | 
 62 |                 switch(nodeName) 
 63 |                 { 
 64 |                     case "same": 
 65 |                         addBlockRangeIf(includeSame, blockNode, result);
 66 |                         break; 
 67 |                     case "right_only": 
 68 |                         addBlockRangeIf(includeRightOnly, blockNode, result);
 69 |                         break; 
 70 |                     case "left_only": 
 71 |                         addBlockRangeIf(includeLeftOnly, blockNode, result);
 72 |                         break; 
 73 |                     case "different": 
 74 |                         addBlockRangeIf(includeDifferent, blockNode, result);
 75 |                         break; 
 76 |                     default: 
 77 |                         throw new IllegalArgumentException("Invalid block difference result:" + blockNode); 
 78 |                 } 
 79 |             }
 80 | 
 81 |             return result;
 82 |         }
 83 |         catch (ParserConfigurationException | XPathExpressionException e)
 84 |         {
 85 |             System.err.println("Java configuration issue:" + e);
 86 |             throw new RuntimeException(e);
 87 |         }
 88 |         catch (IOException | SAXException e)
 89 |         {
 90 |             System.err.println("Problem parsing thin_delta output:" + e + "\n" + new String(thin_delta_output));
 91 |             throw new LVM.LVMException(e);
 92 |         }
 93 |     }
 94 |     
 95 |     
 96 |     private static void addBlockRangeIf(boolean includeSame, Node blockNode, List<Integer> blocks) throws XPathExpressionException 
 97 |     {
 98 |         if (includeSame)
 99 |         {
100 |             int     begin = Integer.parseInt(XMLUtils.getXPathVal(blockNode, "@begin"));
101 |             int     length = Integer.parseInt(XMLUtils.getXPathVal(blockNode, "@length"));
102 |             
103 |             for (int x = 0 ; x < length ; ++x)
104 |             {
105 |                 blocks.add(begin + x);
106 |             }
107 |         }        
108 |     }
109 | 
110 |     
111 |     public static class LVMSnapshotDiff 
112 |     {
113 |         private int             chunkSizeKB;
114 |         private List<Integer>   differentBlocks;
115 | 
116 |         
117 |         public LVMSnapshotDiff(int chunkSizeKB, List<Integer> differentBlocks) 
118 |         {
119 |             this.chunkSizeKB = chunkSizeKB;
120 |             this.differentBlocks = differentBlocks;
121 |         }
122 | 
123 |         
124 |         public int getChunkSizeKB() 
125 |         {
126 |             return chunkSizeKB;
127 |     }
128 | 
129 |         
130 |         public List<Integer> getDifferentBlocks() 
131 |         {
132 |             return differentBlocks;
133 |         }
134 |     }
135 | }
136 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/LVMSnapReceive.java:
--------------------------------------------------------------------------------
 1 | package oneit.lvmsendrcv;
 2 | 
 3 | import oneit.lvmsendrcv.dd.DDRandomReceive;
 4 | import oneit.lvmsendrcv.utils.*;
 5 | import java.io.*;
 6 | import java.net.*;
 7 | import java.text.*;
 8 | import java.util.*;
 9 | import org.apache.commons.cli.*;
10 | import org.json.JSONObject;
11 | 
12 | /**
13 |  * 
14 |  * @author david
15 |  */
16 | public class LVMSnapReceive
17 | {
18 |     /**
19 |      * @param args the command line arguments
20 |      */
21 |     public static void main(String[] args) 
22 |     {
23 |         try
24 |         {
25 |             receiveSnapshot(System.in, System.out);
26 |         }
27 |         catch (Exception e)
28 |         {
29 |             e.printStackTrace();
30 |             System.exit(1);
31 |         }
32 |     }
33 | 
34 |     
35 |     public static void receiveSnapshot(InputStream in, OutputStream out) 
36 |     {
37 |         try
38 |         {
39 |             String                  targetPath = IOUtils.readToByte(in, (byte)0); // "/dev/vgreplica/thinv_replica"
40 |             int                     blockSizeBytes = Integer.parseInt(IOUtils.readToByte(in, (byte)0));
41 |             // @todo authentication
42 |             
43 |             System.err.println("Connection received for:" + targetPath + ":" + blockSizeBytes);
44 |             
45 |             // @todo Create temp snapshot
46 |             
47 |             DDRandomReceive receiver = new DDRandomReceive(blockSizeBytes, targetPath);
48 |             
49 |             receiver.writeData(in);
50 |             out.write("OK\0".getBytes());
51 |             out.flush();
52 |             
53 |             // @todo Create final snapshot
54 |         }
55 |         catch (Exception e)
56 |         {
57 |             e.printStackTrace();
58 |         }
59 |         finally
60 |         {
61 |             // @todo remove previous final snapshot ... expiry policy?
62 |             // @todo remove temp snapshot
63 |         }
64 |     }
65 | }
66 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/LVMSnapSend.java:
--------------------------------------------------------------------------------
  1 | package oneit.lvmsendrcv;
  2 | 
  3 | import oneit.lvmsendrcv.dd.DDRandomSend;
  4 | import oneit.lvmsendrcv.utils.Utils;
  5 | import oneit.lvmsendrcv.utils.ExecUtils;
  6 | import java.io.*;
  7 | import java.text.DateFormat;
  8 | import java.text.SimpleDateFormat;
  9 | import java.util.*;
 10 | import java.util.function.BooleanSupplier;
 11 | import java.util.function.Predicate;
 12 | import org.apache.commons.cli.*;
 13 | import org.json.JSONObject;
 14 | 
 15 | /**
 16 |  * 
 17 |  * @author david
 18 |  */
 19 | public class LVMSnapSend 
 20 | {
 21 |     public static final DateFormat    DATE_FORMAT = new SimpleDateFormat("yyyyMMdd_HHmmss");
 22 | 
 23 |     private String          vg;
 24 |     private String          lv;
 25 |     private String          targetPath;
 26 |     private PrintStream     out;
 27 | 
 28 |     public LVMSnapSend(String vg, String lv, String targetPath, PrintStream out) 
 29 |     {
 30 |         this.vg = vg;
 31 |         this.lv = lv;
 32 |         this.targetPath = targetPath;
 33 |         this.out = out;
 34 |     }
 35 |     
 36 |     
 37 |     /**
 38 |      * @param args the command line arguments
 39 |      */
 40 |     public static void main(String[] args) 
 41 |     {
 42 |         try
 43 |         {
 44 |             Options             options = new Options();
 45 |             CommandLineParser   parser = new GnuParser();
 46 | 
 47 |             options.addOption("vg", "volGroup", true, "The volume group e.g. volg");
 48 |             options.addOption("lv", "logicalVolume", true, "The logical volume to snapshot and send e.g. thin_volume");
 49 |             options.addOption("to", "targetPath", true, "The full target path to write to at the destination");
 50 |             options.addOption("h", "help", false, "Print usage help.");
 51 | 
 52 |             CommandLine  cmdLine = parser.parse(options, args);
 53 | 
 54 |             if (cmdLine.hasOption("h"))
 55 |             {
 56 |                 HelpFormatter formatter = new HelpFormatter();
 57 |                 formatter.printHelp( "oneit.lvmsendrcv.LVMSnapSend", 
 58 |                                      "Determines the difference between two thin LVM snapshots and sends the differences.", 
 59 |                                      options,
 60 |                                      null);
 61 |                 return;
 62 |             }
 63 |             else
 64 |             {
 65 |                 LVMSnapSend   sender      = new LVMSnapSend(Utils.getMandatoryString("vg", cmdLine, options),
 66 |                                                             Utils.getMandatoryString("lv", cmdLine, options),
 67 |                                                             Utils.getMandatoryString("to", cmdLine, options),
 68 |                                                             System.out);
 69 | 
 70 |                 sender.createSnapshotsAndSend(() -> { return true; });
 71 |             }
 72 |         }
 73 |         catch (Exception e)
 74 |         {
 75 |             e.printStackTrace();
 76 |             System.exit(1);
 77 |         }
 78 |     }   
 79 | 
 80 |     
 81 |     public void createSnapshotsAndSend (BooleanSupplier checkSendSuccessful) throws InterruptedException, IOException, LVM.LVMException
 82 |     {
 83 |         Map<String, LVM.LVMSnapshot>    snapshots   = LVM.getSnapshotInfo(new String[] { vg });
 84 |         SortedSet<String>               sendrcvSnapshotNames = new TreeSet<>();
 85 | 
 86 |         for (String snapshotName : snapshots.keySet())
 87 |         {
 88 |             if (snapshotName.matches(lv + "_thinsendrcv_.*"))
 89 |             {
 90 |                 sendrcvSnapshotNames.add(snapshotName);
 91 |             }
 92 |         }
 93 | 
 94 |         if (sendrcvSnapshotNames.size() == 1)
 95 |         {
 96 |             String  snapshotFrom    = sendrcvSnapshotNames.first();
 97 |             String  snapshotTo      = lv + "_thinsendrcv_" + DATE_FORMAT.format(new Date ());
 98 |             boolean successful      = false;
 99 | 
100 |             createLVMSnapshot(vg, lv, snapshotTo);
101 | 
102 |             try
103 |             {
104 |                 snapSend(snapshotFrom, snapshotTo);
105 |                 
106 |                 if (checkSendSuccessful.getAsBoolean())
107 |                 {
108 |                     removeLVMSnapshot(vg, snapshotFrom);
109 |                     successful = true;
110 |                 }
111 |             }
112 |             finally
113 |             {
114 |                 if (!successful)
115 |                 {
116 |                     removeLVMSnapshot(vg, snapshotTo);
117 |                 }
118 |             }
119 |         }
120 |         else if (sendrcvSnapshotNames.size() == 0)
121 |         {
122 |             String  snapshotTo      = lv + "_thinsendrcv_" + DATE_FORMAT.format(new Date ());
123 | 
124 |             System.err.println("LVMSnapSend not intialised");
125 |             System.err.println("    # First take a baseline snapshot:");
126 |             System.err.println("    lvcreate -s -n " + snapshotTo + " " + vg + '/' + lv);
127 |             System.err.println("    lvchange -ay -Ky " + vg + '/' + lv);
128 |             System.err.println();
129 |             System.err.println("    # Then sent it to the destination:");
130 |             System.err.println("    python blocksync.py -c aes128-ctr /dev/" + vg + "/" + snapshotTo + " root@server /dev/remotevg/remotelv ");
131 | 
132 |             System.exit(1);
133 |         }
134 |         else
135 |         {
136 |             System.err.println("LVMSnapSend Multiple matching snapshots:" + sendrcvSnapshotNames);
137 |             System.exit(1);
138 |         }
139 |     }
140 |     
141 |     
142 |     public void snapSend(String snapshotFrom, String snapshotTo) throws InterruptedException, IOException, LVM.LVMException
143 |     {
144 |         LVMBlockDiff.LVMSnapshotDiff    diff    = LVMBlockDiff.diffLVMSnapshots(vg, snapshotFrom, snapshotTo);
145 |         int[]                           blocks  = new int[diff.getDifferentBlocks().size()];
146 |         
147 |         for (int x = 0 ; x < blocks.length ; ++x)
148 |         {
149 |             blocks[x] = diff.getDifferentBlocks().get(x).intValue();
150 |         }
151 |         
152 |         System.err.println(diff.getDifferentBlocks()); // @todo
153 |         System.err.println("Blocks changed:" + diff.getDifferentBlocks().size()); // @todo
154 |         setLVMActivationStatus(vg, snapshotTo, "y");
155 |         
156 |         try
157 |         {
158 |             DDRandomSend    ddRandomSend = new DDRandomSend(diff.getChunkSizeKB() * 1024, "/dev/" + vg + "/" + snapshotTo, blocks);
159 |             
160 |             out.print(targetPath + '\0');
161 |             out.print(String.valueOf(diff.getChunkSizeKB() * 1024) + '\0');
162 |             
163 |             ddRandomSend.sendChangedBlocks(new PrintStream(out));
164 |         }
165 |         finally
166 |         {
167 |             setLVMActivationStatus(vg, snapshotTo, "n");
168 |         }
169 |     }
170 |     
171 |     
172 |     private static void setLVMActivationStatus (String vg, String lv, String status) throws InterruptedException, IOException
173 |     {
174 |         // lvchange -ay -Ky storage/snap1
175 |         new ExecUtils.ExecuteProcess ("/tmp/", "/sbin/lvchange", "-a" + status, "-Ky", vg + "/" + lv).setHideStdOut(true).executeAsBytes();
176 |     }
177 |     
178 |     
179 |     private static void createLVMSnapshot (String vg, String lv, String snapshot) throws InterruptedException, IOException
180 |     {
181 |         // lvcreate -s -n ${LV}_snap3 ${VG}/${LV}
182 |         new ExecUtils.ExecuteProcess ("/tmp/", "/sbin/lvcreate", "-s", "-n", snapshot, vg + "/" + lv).setHideStdOut(true).executeAsBytes();
183 |     }
184 |     
185 |     
186 |     private static void removeLVMSnapshot (String vg, String snapshot) throws InterruptedException, IOException
187 |     {
188 |         // lvremove -y ${VG}/${SNAPSHOT}
189 |         new ExecUtils.ExecuteProcess ("/tmp/", "/sbin/lvremove", "-y", vg + "/" + snapshot).setHideStdOut(true).executeAsBytes();
190 |     }
191 | }
192 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/dd/DDRandomReceive.java:
--------------------------------------------------------------------------------
  1 | package oneit.lvmsendrcv.dd;
  2 | 
  3 | import java.io.*;
  4 | import java.sql.*;
  5 | import java.util.*;
  6 | import javax.xml.parsers.*;
  7 | import javax.xml.xpath.XPathExpressionException;
  8 | import org.apache.commons.cli.*;
  9 | import org.xml.sax.*;
 10 | import org.xml.sax.helpers.DefaultHandler;
 11 | 
 12 | /**
 13 |  * 
 14 |  * @author david
 15 |  */
 16 | public class DDRandomReceive 
 17 | {
 18 |     public final long   blockSizeBytes;
 19 |     public final String outputFile;
 20 | 
 21 |     
 22 |     public DDRandomReceive(long blockSizeBytes, String outputFile) 
 23 |     {
 24 |         this.blockSizeBytes = blockSizeBytes;
 25 |         this.outputFile = outputFile;
 26 |     }
 27 |     
 28 |     
 29 |     public void writeData (InputStream in) throws IOException, SAXException, ParserConfigurationException, XPathExpressionException
 30 |     {
 31 |         SAXParserFactory        factory = SAXParserFactory.newInstance();
 32 |         SAXParser               saxParser = factory.newSAXParser();
 33 | 
 34 |         try (RandomAccessFile       outfileRW = new RandomAccessFile(outputFile, "rw"))
 35 |         {
 36 |             DDReceiveSAXHandler handler = new DDReceiveSAXHandler(outfileRW);
 37 |             
 38 |             saxParser.parse(new FilterInputStream(in) {
 39 |                 @Override
 40 |                 public void close() throws IOException {
 41 |                     System.err.println("Trying to close, ignoring");
 42 |                 }
 43 |                 
 44 |             }, handler);
 45 |             
 46 |             System.err.println("DDRandomReceive: Written " + handler.blocksWritten + " blocks");
 47 |         }
 48 |     }
 49 |     
 50 | 
 51 |     /**
 52 |      * @param args the command line arguments
 53 |      */
 54 |     public static void main(String[] args) 
 55 |     {
 56 |         try
 57 |         {
 58 |             Options             options = new Options();
 59 |             CommandLineParser   parser = new GnuParser();
 60 | 
 61 |             options.addOption("of", "outputFile", true, "The file or device to write to.");
 62 |             options.addOption("bs", "blockSize", true, "The block size in kB.");
 63 |             options.addOption("v", "verbose", false, "Be verbose.");
 64 |             options.addOption("h", "help", false, "Print usage help.");
 65 | 
 66 |             CommandLine  cmdLine = parser.parse(options, args);
 67 | 
 68 |             if (cmdLine.hasOption("h"))
 69 |             {
 70 |                 HelpFormatter formatter = new HelpFormatter();
 71 |                 formatter.printHelp( "oneit.lvmsendrcv.DDRandomReceive", 
 72 |                                      "Takes an XML sequence of blocks and writes them to a device / file.  Intended to take the output of DDRandomSend.  Buffers all blocks in memory, so be cautious.", 
 73 |                                      options,
 74 |                                      "XML is sent to stdin and is of the format\n<dd>\n  <block offset='5'>base64 encoded bytes</block>");
 75 |                 return;
 76 |             }
 77 |             else
 78 |             {
 79 |                 String          of = cmdLine.getOptionValue("of");
 80 |                 int             bs = 1024 * Integer.parseInt(cmdLine.getOptionValue("bs"));
 81 |                 DDRandomReceive ddRandomReceive = new DDRandomReceive(bs, of);
 82 |                 
 83 |                 ddRandomReceive.writeData(System.in);
 84 |             }
 85 |         }
 86 |         catch (Exception e)
 87 |         {
 88 |             e.printStackTrace();
 89 |             System.exit(1);
 90 |         }
 91 |     }   
 92 |     
 93 |     
 94 |     public class DDReceiveSAXHandler extends DefaultHandler
 95 |     {
 96 |         RandomAccessFile    outfileRW;
 97 |         StringBuilder       blockBuffer = new StringBuilder();
 98 |         long                blockOffset;
 99 |         int                 blocksWritten = 0;
100 |         
101 |         
102 |         private DDReceiveSAXHandler(RandomAccessFile outfileRW) 
103 |         {
104 |             this.outfileRW= outfileRW;
105 |         }
106 | 
107 |         
108 |         @Override
109 |         public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException 
110 |         {
111 |             if (qName.equals("block"))
112 |             {
113 |                 blockBuffer = new StringBuilder();
114 |                 blockOffset = blockSizeBytes * Integer.parseInt(attributes.getValue("offset"));
115 |             }
116 |         }
117 | 
118 |         
119 |         @Override
120 |         public void endElement(String uri, String localName, String qName) throws SAXException 
121 |         {
122 |             if (qName.equals("block"))
123 |             {
124 |                 String  blockContent = blockBuffer.toString();
125 |                 byte[]  blockBytes = Base64.getDecoder().decode(blockContent);
126 | 
127 |                 assert blockBytes.length == blockSizeBytes : "Block size does not match";
128 | 
129 |                 System.err.println("DDRandomReceive: Receive Block " + blockOffset + ":" + blockBytes.length); // @todo use proper logging
130 |                 
131 |                 try
132 |                 {
133 |                     outfileRW.seek(blockOffset);
134 |                     outfileRW.write(blockBytes);
135 |                     blocksWritten++;
136 |                 }
137 |                 catch(IOException e)
138 |                 {
139 |                     System.err.println("Unable to write file:" + e.getMessage());
140 |                     System.exit (1);
141 |                 }
142 |                 
143 |                 blockBuffer = new StringBuilder();
144 |             }
145 |         }
146 | 
147 |         
148 |         @Override
149 |         public void characters(char[] ch, int start, int length) throws SAXException 
150 |         {
151 |             blockBuffer.append(new String(ch, start, length));
152 |         }
153 | 
154 |     }
155 | }
156 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/dd/DDRandomSend.java:
--------------------------------------------------------------------------------
  1 | package oneit.lvmsendrcv.dd;
  2 | 
  3 | import java.io.*;
  4 | import java.sql.*;
  5 | import java.util.*;
  6 | import javax.xml.parsers.*;
  7 | import javax.xml.xpath.XPathExpressionException;
  8 | import org.w3c.dom.*;
  9 | import org.apache.commons.cli.*;
 10 | import org.xml.sax.SAXException;
 11 | 
 12 | /**
 13 |  * 
 14 |  * @author david
 15 |  */
 16 | public class DDRandomSend 
 17 | {
 18 |     public final int    blockSizeBytes;
 19 |     public final String inputFile;
 20 |     public final int[]  blocks;
 21 | 
 22 |     
 23 |     public DDRandomSend(int blockSizeBytes, String inputFile, int[] blocks) 
 24 |     {
 25 |         this.blockSizeBytes = blockSizeBytes;
 26 |         this.inputFile = inputFile;
 27 |         this.blocks = blocks;
 28 |     }
 29 | 
 30 |    
 31 |         
 32 |     public void sendChangedBlocks (PrintStream out) throws IOException
 33 |     {
 34 |         try (RandomAccessFile       infileRO = new RandomAccessFile(inputFile, "r"))
 35 |         {
 36 |             byte[]  blockbuffer = new byte[blockSizeBytes];
 37 | 
 38 |             out.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\n<dd>");
 39 | 
 40 |             for (long block : blocks)
 41 |             {
 42 |                 System.err.println("DDRandomSend: Send block:" + block + " -> " + (block * blockSizeBytes)); // @todo
 43 |                 infileRO.seek(block * blockSizeBytes);
 44 |                 infileRO.readFully(blockbuffer);    // @todo what if there are not enough bytes left?
 45 |                 
 46 |                 out.print("<block offset='" + block + "'>");
 47 |                 out.print(Base64.getEncoder().encodeToString(blockbuffer));
 48 |                 out.print("</block>");
 49 |                 out.println();
 50 |             }
 51 | 
 52 |             out.println("</dd>");
 53 |             
 54 |             System.err.println("DDRandomSend: Written " + blocks.length + " blocks");
 55 |         }
 56 |     }
 57 |     
 58 | 
 59 |     /**
 60 |      * @param args the command line arguments
 61 |      */
 62 |     public static void main(String[] args) 
 63 |     {
 64 |         try
 65 |         {
 66 |             Options             options = new Options();
 67 |             CommandLineParser   parser = new GnuParser();
 68 | 
 69 |             options.addOption("if", "inputFile", true, "The file or device to write to.");
 70 |             options.addOption("bs", "blockSize", true, "The block size in kB.");
 71 |             options.addOption("blocks", true, "Comma separated list of blocks to read e.g. 5,7,31");
 72 |             options.addOption("h", "help", false, "Print usage help.");
 73 | 
 74 |             CommandLine  cmdLine = parser.parse(options, args);
 75 | 
 76 |             if (cmdLine.hasOption("h"))
 77 |             {
 78 |                 HelpFormatter formatter = new HelpFormatter();
 79 |                 formatter.printHelp( "oneit.lvmsendrcv.DDRandomSend", 
 80 |                                      "Writes a random selection of blocks from a device or file to stdout.  Intended to act as the input of DDRandomReceive.", 
 81 |                                      options,
 82 |                                      "XML is sent to stout and is of the format\n<dd>\n  <block offset='5'>base64 encoded bytes</block>");
 83 |                 return;
 84 |             }
 85 |             else
 86 |             {
 87 |                 String          inf = cmdLine.getOptionValue("if");
 88 |                 int             bs = 1024 * Integer.parseInt(cmdLine.getOptionValue("bs"));
 89 |                 String          blocksStr = cmdLine.getOptionValue("blocks");
 90 |                 String[]        blocksStrArr = blocksStr.split(",");
 91 |                 int[]           blocks = new int[blocksStrArr.length];
 92 |                 
 93 |                 for (int x = 0 ; x < blocksStrArr.length ; ++x)
 94 |                 {
 95 |                     blocks[x] = Integer.parseInt(blocksStrArr[x]);
 96 |                 }
 97 |                 
 98 |                 DDRandomSend    ddRandomSend = new DDRandomSend(bs, inf, blocks);
 99 |                 
100 |                 ddRandomSend.sendChangedBlocks(System.out);
101 |             }
102 |         }
103 |         catch (Exception e)
104 |         {
105 |             e.printStackTrace();
106 |             System.exit(1);
107 |         }
108 |     }   
109 | }
110 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/service/LVMSnapReceiveService.java:
--------------------------------------------------------------------------------
 1 | package oneit.lvmsendrcv.service;
 2 | 
 3 | import oneit.lvmsendrcv.*;
 4 | import oneit.lvmsendrcv.dd.DDRandomReceive;
 5 | import oneit.lvmsendrcv.utils.*;
 6 | import java.io.*;
 7 | import java.net.*;
 8 | import java.text.*;
 9 | import java.util.*;
10 | import java.util.logging.Level;
11 | import java.util.logging.Logger;
12 | import org.apache.commons.cli.*;
13 | import org.json.JSONObject;
14 | 
15 | /**
16 |  * 
17 |  * @author david
18 |  */
19 | public class LVMSnapReceiveService extends Thread
20 | {
21 |     private final Socket socket;
22 |     
23 |     /**
24 |      * @param args the command line arguments
25 |      */
26 |     public static void main(String[] args) 
27 |     {
28 |         try
29 |         {
30 |             ServerSocket    svrSocket = new ServerSocket(15432, 10, Inet4Address.getLocalHost());
31 | 
32 |             while (true)
33 |             {
34 |                 Socket  socket = svrSocket.accept();
35 |                 
36 |                 new LVMSnapReceiveService(socket).start();
37 |             }
38 |         }
39 |         catch (Exception e)
40 |         {
41 |             e.printStackTrace();
42 |             System.exit(1);
43 |         }
44 |     }
45 | 
46 |     
47 |     public LVMSnapReceiveService(Socket socket) 
48 |     {
49 |         this.socket = socket;
50 |     }
51 | 
52 |     
53 |     @Override
54 |     public void run() 
55 |     {
56 |         try
57 |         {
58 |             BufferedInputStream     in = new BufferedInputStream(socket.getInputStream());
59 |             OutputStream            out = socket.getOutputStream();
60 | 
61 |             System.err.println("Ready to receive:" + in);
62 |             LVMSnapReceive.receiveSnapshot(in, out);
63 |         }
64 |         catch (Exception e)
65 |         {
66 |             e.printStackTrace();
67 |         }
68 |         finally
69 |         {
70 |             try
71 |             {
72 |                 socket.close();
73 |             } 
74 |             catch (IOException e) 
75 |             {
76 |                 e.printStackTrace();
77 |             }
78 |         }
79 |     }
80 | }
81 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/service/LVMSnapSendService.java:
--------------------------------------------------------------------------------
  1 | package oneit.lvmsendrcv.service;
  2 | 
  3 | import oneit.lvmsendrcv.*;
  4 | import oneit.lvmsendrcv.dd.DDRandomReceive;
  5 | import oneit.lvmsendrcv.utils.*;
  6 | import java.io.*;
  7 | import java.net.*;
  8 | import java.text.*;
  9 | import java.util.*;
 10 | import java.util.logging.Level;
 11 | import java.util.logging.Logger;
 12 | import org.apache.commons.cli.*;
 13 | import org.json.JSONObject;
 14 | 
 15 | /**
 16 |  * 
 17 |  * @author david
 18 |  */
 19 | public class LVMSnapSendService extends Thread
 20 | {
 21 |     InetAddress target;
 22 |     int         port;
 23 |     String      vg;
 24 |     String      lv;
 25 |     String      targetPath;
 26 |     
 27 |     
 28 |     /**
 29 |      * @param args the command line arguments
 30 |      */
 31 |     public static void main(String[] args) 
 32 |     {
 33 |         try
 34 |         {
 35 |             Options             options = new Options();
 36 |             CommandLineParser   parser = new GnuParser();
 37 | 
 38 |             options.addOption("vg", "volGroup", true, "The volume group e.g. volg");
 39 |             options.addOption("lv", "logicalVolume", true, "The logical volume to snapshot and send e.g. thin_volume");
 40 |             options.addOption("to", "targetPath", true, "The full target path to write to at the destination");
 41 |             options.addOption("h", "help", false, "Print usage help.");
 42 | 
 43 |             CommandLine  cmdLine = parser.parse(options, args);
 44 | 
 45 |             if (cmdLine.hasOption("h"))
 46 |             {
 47 |                 HelpFormatter formatter = new HelpFormatter();
 48 |                 formatter.printHelp( "oneit.lvmsendrcv.LVMSnapSend", 
 49 |                                      "Determines the difference between two thin LVM snapshots and sends the differences.", 
 50 |                                      options,
 51 |                                      null);
 52 |                 return;
 53 |             }
 54 |             else
 55 |             {
 56 |                 String                          vg          = Utils.getMandatoryString("vg", cmdLine, options);
 57 |                 String                          lv          = Utils.getMandatoryString("lv", cmdLine, options);
 58 |                 String                          targetPath  = Utils.getMandatoryString("to", cmdLine, options);
 59 |                 LVMSnapSendService              sender      = new LVMSnapSendService(InetAddress.getLocalHost(), 15432, vg, lv, targetPath);
 60 |                 
 61 |                 sender.start();
 62 |             }
 63 |         }
 64 |         catch (Exception e)
 65 |         {
 66 |             e.printStackTrace();
 67 |             System.exit(1);
 68 |         }
 69 |     }
 70 | 
 71 |     
 72 |     public LVMSnapSendService(InetAddress target, int port, String vg, String lv, String targetPath) 
 73 |     {
 74 |         this.target = target;
 75 |         this.port = port;
 76 |         this.vg = vg;
 77 |         this.lv = lv;
 78 |         this.targetPath = targetPath;
 79 |     }
 80 | 
 81 | 
 82 |     
 83 |     @Override
 84 |     public void run() 
 85 |     {
 86 |         while (true)
 87 |         {
 88 |             connectAndSend();
 89 |             
 90 |             try 
 91 |             {
 92 |                 Thread.sleep(10000);
 93 |             } 
 94 |             catch (InterruptedException ex) 
 95 |             {
 96 |                 return;
 97 |             }
 98 |         }
 99 |     }
100 | 
101 |     
102 |     public void connectAndSend () 
103 |     {
104 |         try (final Socket  connectToServer = new Socket(target, port))
105 |         {
106 |             System.err.println("Connecting to server:" + target + ":" + port);
107 |             
108 |             final PrintStream     out = new PrintStream(new BufferedOutputStream(connectToServer.getOutputStream()));
109 |             final InputStream     in = connectToServer.getInputStream();
110 |             LVMSnapSend           sender = new LVMSnapSend(vg, lv, targetPath, out);
111 |             
112 |             sender.createSnapshotsAndSend(() -> {
113 |                     try
114 |                     {
115 |                         out.flush();
116 |                         connectToServer.shutdownOutput();
117 |                         
118 |                         System.err.println("Ready for response:");
119 | 
120 |                         String  response = IOUtils.readToByte(in, (byte)0);
121 | 
122 |                         if (response.trim().equals("OK"))
123 |                         {
124 |                             System.err.println("Read response Good:" + response);
125 |                             return true;
126 |                         }
127 |                         else
128 |                         {
129 |                             System.err.println("Read response BAD:" + response);
130 |                             return false;
131 |                         }
132 |                     }
133 |                     catch (IOException e)
134 |                     {
135 |                         e.printStackTrace();
136 |                         System.err.println("Error reading response");
137 |                         return false;
138 |                     }
139 |                 });
140 |         }
141 |         catch (Exception e)
142 |         {
143 |             e.printStackTrace();
144 |         }
145 |     }
146 | }
147 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/service/LVMSpaceCheck.java:
--------------------------------------------------------------------------------
  1 | package oneit.lvmsendrcv.service;
  2 | 
  3 | import java.util.*;
  4 | import oneit.lvmsendrcv.LVM;
  5 | import oneit.lvmsendrcv.utils.*;
  6 | import org.apache.commons.cli.*;
  7 | import org.json.*;
  8 | 
  9 | 
 10 | /**
 11 |  *
 12 |  * @author david
 13 |  */
 14 | public class LVMSpaceCheck 
 15 | {
 16 |     public static Map<String, Integer>  ERROR_COUNTS = new HashMap<>();
 17 |     
 18 |     
 19 |     public static void main(String[] args) 
 20 |     {
 21 |         try
 22 |         {
 23 |             Options             options = new Options();
 24 |             CommandLineParser   parser = new GnuParser();
 25 | 
 26 |             options.addOption("checkInterval", true, "Loop forever and run ever [n] seconds.");
 27 |             options.addOption("h", "help", false, "Print usage help.");
 28 | 
 29 |             CommandLine  cmdLine = parser.parse(options, args);
 30 | 
 31 |             if (cmdLine.hasOption("h"))
 32 |             {
 33 |                 HelpFormatter formatter = new HelpFormatter();
 34 |                 formatter.printHelp( "oneit.lvmsendrcv.LVMSpaceCheck", 
 35 |                                      "Monitor the LVMs in lvmsendrcv.conf for free space.", 
 36 |                                      options,
 37 |                                      null);
 38 |                 return;
 39 |             }
 40 |             else
 41 |             {
 42 |                 Properties                      montorProperties = Utils.getPropertiesOf(Utils.CONFIG_PROPERTIES, "monitor.");
 43 |                 Set<String>                     thinLVsToMonitor = Utils.getPropertyGroups(montorProperties, '.', false);
 44 |                 Map<String, SpaceCheckLimit>    limitsByLV = new TreeMap<> ();
 45 |                 int                             checkInterval= 0;
 46 |                     
 47 |                 for (String thinLV : thinLVsToMonitor)
 48 |                 {
 49 |                     Properties  lvProperties = Utils.getPropertiesOf(montorProperties, thinLV + ".");
 50 |                     
 51 |                     limitsByLV.put(thinLV, new SpaceCheckLimit(lvProperties));
 52 |                 }
 53 |                 
 54 |                 if (cmdLine.hasOption("checkInterval"))
 55 |                 {
 56 |                     checkInterval = Integer.parseInt(cmdLine.getOptionValue("checkInterval"));
 57 |                 }
 58 |                 
 59 |                 if (checkInterval > 0)
 60 |                 {
 61 |                     while (true)
 62 |                     {
 63 |                         checkSpace(limitsByLV);
 64 |                         Thread.sleep(checkInterval * 1000);
 65 |                     }
 66 |                 }
 67 |                 else
 68 |                 {
 69 |                     checkSpace(limitsByLV);
 70 |                 }
 71 |             }
 72 |         }
 73 |         catch (Exception e)
 74 |         {
 75 |             e.printStackTrace();
 76 |             System.exit(1);
 77 |         }
 78 |     }   
 79 | 
 80 |     
 81 |     public static void checkSpace (Map<String, SpaceCheckLimit> limitsByLV) throws LVM.LVMException
 82 |     {
 83 |         List<JSONObject>            snapshots = LVM.execLVS("lv_name,vg_name,data_percent,metadata_percent,lv_attr", limitsByLV.keySet().toArray(new String[0]));
 84 | 
 85 |         for (JSONObject  snapJSON : snapshots)
 86 |         {
 87 |             try
 88 |             {
 89 |                 String          lv          = snapJSON.getString("lv_name");
 90 |                 String          vg          = snapJSON.getString("vg_name");
 91 |                 double          metaUsage   = snapJSON.getDouble("metadata_percent");
 92 |                 double          dataUsage    = snapJSON.getDouble("data_percent");
 93 |                 SpaceCheckLimit limit       = limitsByLV.get(vg+ "/" + lv);
 94 | 
 95 |                 System.err.println("Checking " + vg + "/" + lv + " meta:" + metaUsage + "/" + limit.metaLimitPercent + " data:" + dataUsage + "/" + limit.dataLimitPercent); // @todo
 96 |                 checkSpace (vg, lv, limit.dataLimitPercent, dataUsage, ThinSpaceType.DATA);
 97 |                 checkSpace (vg, lv, limit.metaLimitPercent, metaUsage, ThinSpaceType.METADATA);
 98 |             }
 99 |             catch (RuntimeException e)
100 |             {
101 |                 System.err.println("Error processing lvs output:" + snapJSON + "  " + limitsByLV);
102 |                 throw e;
103 |             }
104 |             catch (JSONException e)
105 |             {
106 |                 System.err.println("Error processing lvs output:" + snapJSON + " " + e.getMessage());
107 |                 throw new RuntimeException(e);
108 |             }
109 |         }
110 |     }
111 | 
112 |     
113 |     private static void checkSpace(String vg, String lv, double limitPercent, double actualUsage, ThinSpaceType type) throws LVM.LVMException
114 |     {
115 |         String  errorCountKey = vg + "/" + lv + ":" + type;
116 |         
117 |         if (limitPercent < actualUsage)
118 |         {
119 |             int     errorCount = 0;
120 |             
121 |             if (ERROR_COUNTS.containsKey(errorCountKey))
122 |             {
123 |                 errorCount = ERROR_COUNTS.get(errorCountKey);
124 |             }
125 | 
126 |             
127 |             if (errorCount == 0 || errorCount == 10 || errorCount == 100 || errorCount == 1000)
128 |             {
129 |                 String      vgFreeSpace = LVM.getVGInfo (vg).vgFree;
130 |                 String      extendCmd = (type == ThinSpaceType.DATA) ? "lvextend -L+1G " + vg + "/" + lv
131 |                                                                      : "lvextend --poolmetadatasize +10M " + vg + "/" + lv;
132 |                 System.err.println(type + " limit of " + limitPercent + " exceeded by actual " + actualUsage + " Sending email"); // @todo
133 |                 Utils.sendEmail("[lvmsendrcv] " + type + " Limit exceeded " + lv, 
134 |                                  "Volume:       " + vg + "/" + lv + "\n" + 
135 |                                  "Usage:        " + type + " " + actualUsage + " > " + limitPercent + "\n" + 
136 |                                  "Count:        " + errorCount + "\n" +
137 |                                  "VG Free Space:" + vgFreeSpace + "\n\n" +
138 |                                  extendCmd);
139 |             }
140 |             else
141 |             {
142 |                 System.err.println(type + " limit of " + limitPercent + " exceeded by actual " + actualUsage + " Skipping email:" + errorCount); // @todo
143 |             }
144 |             
145 |             ERROR_COUNTS.put(errorCountKey, errorCount + 1);
146 |         }
147 |         else if (ERROR_COUNTS.containsKey(errorCountKey))
148 |         {
149 |             System.err.println("Clearing error:" + errorCountKey); // @todo
150 |             Utils.sendEmail("[lvmsendrcv] " + type + " Limit OK " + lv, 
151 |                                  "Volume:" + vg + "/" + lv + " : " + type + "\n" + 
152 |                                  "Usage:" + actualUsage + " > " + limitPercent);
153 |             ERROR_COUNTS.remove(errorCountKey);
154 |         }
155 |     }
156 |     
157 |     
158 |     public static class SpaceCheckLimit
159 |     {
160 |         public final    double  dataLimitPercent;
161 |         public final    double  metaLimitPercent;
162 | 
163 |         public SpaceCheckLimit (Properties properties) 
164 |         {
165 |             this.dataLimitPercent = Double.parseDouble(properties.getProperty("dataLimit"));
166 |             this.metaLimitPercent = Double.parseDouble(properties.getProperty("metaLimit"));
167 |         }
168 | 
169 |         @Override
170 |         public String toString() 
171 |         {            
172 |             return dataLimitPercent + " " + metaLimitPercent;
173 |         }
174 |         
175 |         
176 |     }
177 |     
178 |     public enum ThinSpaceType 
179 |     {
180 |         DATA, METADATA;
181 |     }    
182 | }
183 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/utils/ExecUtils.java:
--------------------------------------------------------------------------------
  1 | package oneit.lvmsendrcv.utils;
  2 | 
  3 | import java.util.*;
  4 | import java.io.*;
  5 | import javax.xml.xpath.*;
  6 | import org.w3c.dom.*;
  7 | 
  8 | 
  9 | /**
 10 |  *
 11 |  * @author david
 12 |  */
 13 | public class ExecUtils 
 14 | {
 15 |     public static Thread run(Runnable r, int priority, boolean isDaemon)
 16 |     {
 17 |         Thread  newThread = new Thread (r);
 18 | 		
 19 |         newThread.setPriority(priority);
 20 |         newThread.setDaemon(isDaemon);
 21 |         newThread.start();
 22 | 
 23 |         return newThread;
 24 |     }
 25 |     
 26 |     public static class ExecuteProcess 
 27 |     {
 28 |         private String      homeDirectory;
 29 |         private String[]    command;
 30 |         private boolean     hideStdOut = false; // Hides stdout unless there is an error
 31 |         
 32 |         public ExecuteProcess (String homeDirectory, String... command)
 33 |         {
 34 |             this.homeDirectory = homeDirectory;
 35 |             this.command = command;
 36 |         }
 37 | 
 38 |         
 39 |         public ExecuteProcess setHideStdOut(boolean hideStdOut)
 40 |         {
 41 |             this.hideStdOut = hideStdOut;
 42 |             return this;
 43 |         }
 44 |         
 45 |         
 46 |         public List<String> executeAsLines () throws InterruptedException, IOException
 47 |         {
 48 |             String      output = executeAsBytes().toString();
 49 |             String[]    outputLines = output.split("\n");            
 50 | 
 51 |             return Arrays.asList(outputLines);
 52 |         }
 53 |         
 54 |         
 55 |         public byte[] executeAsBytes () throws InterruptedException, IOException
 56 |         {
 57 |             
 58 |             //System.err.print("Executing:" + Arrays.asList(command)); // @todo log
 59 | 
 60 |             ProcessBuilder          pb = new ProcessBuilder(command).directory(new File (homeDirectory)).redirectInput(ProcessBuilder.Redirect.INHERIT);
 61 | 
 62 |             //pb.environment().putAll(EXEC_ENV);
 63 |             
 64 |             Process                 p = pb.start();       
 65 |             ByteArrayOutputStream   stdoutBuffer = new ByteArrayOutputStream();
 66 |             ByteArrayOutputStream   stderrBuffer = new ByteArrayOutputStream();
 67 |             OutputStream            out = hideStdOut ? stdoutBuffer : new TeeOutputStream(stdoutBuffer, System.out);
 68 | 
 69 |             // This writes stdout and stderr from the process to our own stdout and stderr respectively
 70 |             Thread  t1 = ExecUtils.run(new IOUtils.OutputPusher (p.getInputStream(), out), Thread.NORM_PRIORITY, true);
 71 |             Thread  t2 = ExecUtils.run(new IOUtils.OutputPusher (p.getErrorStream(), new TeeOutputStream(stderrBuffer, System.err)), Thread.NORM_PRIORITY, true);
 72 | 
 73 |             int result = p.waitFor();
 74 |             
 75 |             t1.join();
 76 |             t2.join();
 77 | 
 78 |             if (result == 0)
 79 |             {
 80 |                 return stdoutBuffer.toByteArray();
 81 |             }
 82 |             else
 83 |             {
 84 |                 throw new RuntimeException("Eror running:" + Arrays.asList(command));
 85 |             }
 86 |         }        
 87 |     }
 88 |     
 89 |     public static class TeeOutputStream extends OutputStream
 90 |     {
 91 |         OutputStream[]  targets;
 92 | 
 93 |         
 94 |         public TeeOutputStream(OutputStream... targets)
 95 |         {
 96 |             this.targets = targets;
 97 |         }
 98 |         
 99 |         
100 |         @Override
101 |         public void write(int b) throws IOException
102 |         {
103 |             for (OutputStream target : targets)
104 |             {
105 |                 target.write(b);
106 |             }
107 |         }
108 | 
109 |         
110 |         @Override
111 |         public void flush() throws IOException
112 |         {
113 |             for (OutputStream target : targets)
114 |             {
115 |                 target.flush();
116 |             }
117 |         }
118 | 
119 |         
120 |         @Override
121 |         public void close() throws IOException
122 |         {
123 |             for (OutputStream target : targets)
124 |             {
125 |                 target.close();
126 |             }
127 |         }
128 |     }
129 | }
130 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/utils/IOUtils.java:
--------------------------------------------------------------------------------
 1 | package oneit.lvmsendrcv.utils;
 2 | 
 3 | import java.io.*;
 4 | 
 5 | /**
 6 |  *
 7 |  * @author david
 8 |  */
 9 | public class IOUtils 
10 | {
11 |     public static final int             BUFFER_SIZE = 4096;
12 |     
13 |     
14 |     public static String readToByte (InputStream in, byte terminatorByte) throws IOException
15 |     {
16 |         StringBuilder   result = new StringBuilder();
17 |         
18 |         for (int byteRead = in.read() ; byteRead >= 0 && byteRead != terminatorByte ; byteRead = in.read())
19 |         {
20 |             System.err.print ((char)byteRead);
21 |             result.append((char)byteRead);
22 |         }
23 |         
24 |         return result.toString();
25 |     }
26 |     
27 |     
28 |     public static void pump (InputStream in, OutputStream out) throws IOException
29 |     {
30 |         pump(in, out, Long.MAX_VALUE);
31 |     }
32 |     
33 |     
34 |     public static void pump (InputStream in, OutputStream out, long maxBytes) throws IOException
35 |     {
36 |         if (maxBytes <= 0)
37 |         {
38 |             return;
39 |         }
40 |         
41 |         byte[]  buffer = new byte[BUFFER_SIZE];
42 |         int     bytesRead;
43 |         long    totalBytesRead = 0;
44 |         int     maxBytesToRead = (int)Math.min(maxBytes, BUFFER_SIZE);
45 | 
46 |         while (totalBytesRead < maxBytes && (bytesRead = in.read(buffer, 0, maxBytesToRead)) >= 0)
47 |         {
48 |             out.write(buffer, 0, bytesRead);
49 |             totalBytesRead += bytesRead;
50 |             maxBytesToRead = (int)Math.min(maxBytes - totalBytesRead, BUFFER_SIZE);
51 |         }
52 |     }
53 |     
54 |     
55 |     public static class OutputPusher implements Runnable
56 |     {
57 |         private OutputStream o;
58 |         private InputStream i;
59 | 
60 |         public OutputPusher (InputStream i, OutputStream o)
61 |         {
62 |             this.o = o;
63 |             this.i = i;
64 |         }
65 | 
66 | 
67 |         public void run ()
68 |         {
69 |             try
70 |             {
71 |                 IOUtils.pump(i, o);
72 |             }
73 |             catch (IOException e)
74 |             {
75 |                 e.printStackTrace();
76 |             }
77 |         }
78 |     }
79 |     
80 | }
81 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/utils/Utils.java:
--------------------------------------------------------------------------------
  1 | package oneit.lvmsendrcv.utils;
  2 | 
  3 | import java.io.*;
  4 | import java.util.*;
  5 | import javax.mail.*;  
  6 | import javax.mail.internet.*;  
  7 | //import javax.activation.*;  
  8 | import org.apache.commons.cli.*;
  9 | 
 10 | /**
 11 |  *
 12 |  * @author david
 13 |  */
 14 | public class Utils 
 15 | {
 16 |     public static final Properties CONFIG_PROPERTIES = new Properties();
 17 |     public static final Properties JAVA_MAIL_PROPERTIES;
 18 |     
 19 |     static 
 20 |     {
 21 |         File    configFile = new File ("/etc/lvmsendrcv/lvmsendrcv.conf");
 22 |         
 23 |         if (configFile.exists() && configFile.canRead())
 24 |         {
 25 |             try (FileReader configReader = new FileReader(configFile))
 26 |             {
 27 |                 CONFIG_PROPERTIES.load(configReader);
 28 |             }
 29 |             catch (IOException e)
 30 |             {
 31 |                 System.err.println("Cannot load config:" + configFile); // @todo
 32 |             }
 33 |         }
 34 |         
 35 |         JAVA_MAIL_PROPERTIES = getPropertiesOf(CONFIG_PROPERTIES, "email.");
 36 |     }
 37 |     
 38 |     
 39 | 
 40 |     /**
 41 |      * Returns the unique property names up to the delimiting character
 42 |      * a.foo
 43 |      * a.bar
 44 |      * b.baz
 45 |      * In the example above, if the delimiting character is . then a and b would be returned
 46 |      * 
 47 |      * @param source the source properties
 48 |      * @param delimiter the delimiter for this group.  The group is taken to the first occurence of the delimiter.
 49 |      * @param includeNoDelimiter should the result include properties that do not include the delimiter
 50 |      */
 51 |     public static Set<String> getPropertyGroups (Properties source, char delimiter, boolean includeNoDelimiter)
 52 |     {
 53 |         Set<String>     result = new HashSet<>();
 54 |         
 55 |         for (String propertyName : source.stringPropertyNames() )
 56 |         {
 57 |             int delimIndex = propertyName.indexOf(delimiter);
 58 |             
 59 |             if (delimIndex >= 0)
 60 |             {
 61 |                 result.add(propertyName.substring(0, delimIndex));
 62 |             }
 63 |             else if (includeNoDelimiter)
 64 |             {
 65 |                 result.add(propertyName);
 66 |             }
 67 |         }
 68 | 
 69 |         return result;
 70 |         
 71 |     }
 72 |     
 73 |     
 74 |     /**
 75 |      * Returns a Property set consisting of all Properties in source that start with a specified prefix.
 76 |      * The properties returned will have prefix removed from the key.
 77 |      */
 78 |     public static Properties getPropertiesOf (Properties source, String prefix)
 79 |     {
 80 |         Properties result = new Properties ();
 81 | 
 82 |         for (String propertyName : source.stringPropertyNames() )
 83 |         {
 84 |             if (propertyName.startsWith (prefix))
 85 |             {
 86 |                 result.put (propertyName.substring (prefix.length ()), source.getProperty (propertyName));
 87 |             }
 88 |         }
 89 | 
 90 |         return result;
 91 |     }
 92 |     
 93 |     
 94 |     public static String getMandatoryString (String arg, CommandLine  cmdLine, Options options)
 95 |     {
 96 |         String  param = cmdLine.getOptionValue(arg);
 97 |         
 98 |         if (param == null)
 99 |         {
100 |             Option  opt = options.getOption(arg);
101 |             
102 |             throw new RuntimeException("Missing param:" + arg + " " + opt.getOpt() + "/" + opt.getLongOpt() + " :" + opt.getDescription());            
103 |         }
104 |         else
105 |         {
106 |             return param;
107 |         }
108 |     }
109 |     
110 |     
111 |     public static void sendEmail (String subject, String messageText)
112 |     {
113 |         Session session = Session.getDefaultInstance(JAVA_MAIL_PROPERTIES);  
114 |         
115 |         try
116 |         {  
117 |             MimeMessage message = new MimeMessage(session);  
118 |             message.setFrom(new InternetAddress("dave@oneit.com.au"));  
119 |             message.addRecipient(Message.RecipientType.TO,new InternetAddress("dave@oneit.com.au"));  
120 |             message.setSubject(subject);  
121 |             message.setText(messageText);  
122 | 
123 |             // Send message  
124 |             Transport.send(message);  
125 |         }
126 |         catch (MessagingException mex) 
127 |         {
128 |             throw new RuntimeException(mex);
129 |         }  
130 |     }
131 |     
132 |     public static void main (String[] args)
133 |     {
134 |         System.err.println("Java Mail Properties:" + JAVA_MAIL_PROPERTIES); // @todo
135 |         System.err.println("Config Properties:" + CONFIG_PROPERTIES); // @todo
136 |         
137 |         sendEmail(args[0], args[1]);
138 |     }
139 | }
140 | 


--------------------------------------------------------------------------------
/src/oneit/lvmsendrcv/utils/XMLUtils.java:
--------------------------------------------------------------------------------
 1 | package oneit.lvmsendrcv.utils;
 2 | 
 3 | import java.util.*;
 4 | import java.io.*;
 5 | import javax.xml.xpath.*;
 6 | import org.w3c.dom.*;
 7 | 
 8 | 
 9 | /**
10 |  *
11 |  * @author david
12 |  */
13 | public class XMLUtils 
14 | {
15 |     public static final XPathFactory    XPATH_FACTORY = XPathFactory.newInstance();
16 |     
17 |     
18 |     public static String getXPathVal(Node node, String xpath) throws XPathExpressionException
19 |     {
20 |         XPath           xpathObj = XPATH_FACTORY.newXPath();
21 |             
22 |         return xpathObj.evaluate(xpath, node);
23 |     }
24 | 
25 |     
26 |     public static List<Node> getXPathNodes (Node node, String xpath) throws XPathExpressionException
27 |     {
28 |         XPath           xpathObj = XPATH_FACTORY.newXPath();
29 |         NodeList        nodes = (NodeList)xpathObj.evaluate(xpath, node, XPathConstants.NODESET);
30 |         List<Node>      result = new ArrayList<Node>();
31 | 
32 |         if (nodes != null)
33 |         {
34 |             for(int i=0 ; i < nodes.getLength() ; i++)
35 |             {
36 |                 result.add(nodes.item(i));
37 |             }
38 |         }
39 | 
40 |         return result;
41 |     }
42 | }
43 | 


--------------------------------------------------------------------------------
/thin_delta_output.xml:
--------------------------------------------------------------------------------
  1 | <!-- 
  2 |   dmsetup message /dev/mapper/volg-volg--thinpool-tpool 0 reserve_metadata_snap
  3 |   thin_delta  -m --snap1 $(lvs --noheadings -o thin_id volg/thin_volume_snap4) --snap2 $(lvs --noheadings -o thin_id volg/thin_volume_snap5) /dev/mapper/volg-volg--thinpool_tmeta  
  4 |   dmsetup message /dev/mapper/volg-volg--thinpool-tpool 0 release_metadata_snap
  5 | -->
  6 | <superblock uuid="" time="5" transaction="8" data_block_size="128" nr_data_blocks="0">
  7 |   <diff left="5" right="6">
  8 |     <different begin="0" length="1"/>
  9 |     <same begin="1" length="47"/>
 10 |     <different begin="48" length="1"/>
 11 |     <same begin="49" length="1"/>
 12 |     <different begin="50" length="1"/>
 13 |     <same begin="51" length="36"/>
 14 |     <different begin="87" length="10"/>
 15 |     <same begin="97" length="1"/>
 16 |     <different begin="98" length="11"/>
 17 |     <same begin="109" length="2"/>
 18 |     <different begin="111" length="3"/>
 19 |     <same begin="114" length="1"/>
 20 |     <different begin="115" length="14"/>
 21 |     <same begin="129" length="1"/>
 22 |     <different begin="130" length="13"/>
 23 |     <same begin="143" length="1"/>
 24 |     <different begin="144" length="7"/>
 25 |     <same begin="151" length="1"/>
 26 |     <different begin="152" length="1"/>
 27 |     <same begin="153" length="728"/>
 28 |     <same begin="2048" length="1"/>
 29 |     <same begin="2096" length="4049"/>
 30 |     <same begin="6192" length="4049"/>
 31 |     <same begin="10288" length="4049"/>
 32 |     <same begin="14384" length="4049"/>
 33 |     <same begin="18480" length="395"/>
 34 |     <different begin="18875" length="1"/>
 35 |     <right_only begin="18876" length="37"/>
 36 |     <right_only begin="18944" length="330"/>
 37 |     <different begin="32768" length="4"/>
 38 |     <same begin="32772" length="2"/>
 39 |     <different begin="32774" length="1"/>
 40 |     <same begin="32775" length="1"/>
 41 |     <different begin="32776" length="1"/>
 42 |     <same begin="32777" length="2"/>
 43 |     <different begin="32779" length="1"/>
 44 |     <same begin="32780" length="1"/>
 45 |     <different begin="32781" length="20"/>
 46 |     <same begin="32801" length="1"/>
 47 |     <different begin="32802" length="11"/>
 48 |     <same begin="32813" length="1"/>
 49 |     <different begin="32814" length="3"/>
 50 |     <same begin="32817" length="2"/>
 51 |     <different begin="32819" length="1"/>
 52 |     <same begin="32820" length="1"/>
 53 |     <different begin="32821" length="4"/>
 54 |     <same begin="32825" length="1"/>
 55 |     <different begin="32826" length="5"/>
 56 |     <same begin="32831" length="3"/>
 57 |     <different begin="32834" length="1"/>
 58 |     <same begin="32835" length="3"/>
 59 |     <different begin="32838" length="1"/>
 60 |     <same begin="32839" length="2"/>
 61 |     <different begin="32841" length="1"/>
 62 |     <same begin="32842" length="1"/>
 63 |     <different begin="32843" length="1"/>
 64 |     <same begin="32844" length="2"/>
 65 |     <different begin="32846" length="8"/>
 66 |     <same begin="32854" length="3"/>
 67 |     <different begin="32857" length="12"/>
 68 |     <same begin="32869" length="1"/>
 69 |     <different begin="32870" length="1"/>
 70 |     <same begin="32871" length="1"/>
 71 |     <different begin="32872" length="1"/>
 72 |     <same begin="32873" length="1"/>
 73 |     <different begin="32874" length="1"/>
 74 |     <same begin="32875" length="1"/>
 75 |     <different begin="32876" length="1"/>
 76 |     <same begin="32877" length="1"/>
 77 |     <different begin="32878" length="1"/>
 78 |     <same begin="32879" length="1"/>
 79 |     <different begin="32880" length="1"/>
 80 |     <same begin="32881" length="1"/>
 81 |     <different begin="32882" length="5"/>
 82 |     <same begin="32887" length="395"/>
 83 |     <different begin="33282" length="1"/>
 84 |     <same begin="33283" length="292"/>
 85 |     <different begin="33575" length="1"/>
 86 |     <different begin="34816" length="101"/>
 87 |     <same begin="34917" length="8134"/>
 88 |     <different begin="43051" length="1"/>
 89 |     <right_only begin="43052" length="577"/>
 90 |     <right_only begin="44032" length="512"/>
 91 |     <same begin="51200" length="1"/>
 92 |     <same begin="55296" length="1"/>
 93 |     <same begin="65536" length="9"/>
 94 |     <different begin="65545" length="10"/>
 95 |     <same begin="65555" length="3"/>
 96 |     <different begin="65558" length="4"/>
 97 |     <same begin="65562" length="7"/>
 98 |     <different begin="65569" length="3"/>
 99 |     <same begin="65572" length="3"/>
100 |     <different begin="65575" length="2"/>
101 |     <same begin="65577" length="1"/>
102 |     <different begin="65578" length="2"/>
103 |     <same begin="65580" length="8"/>
104 |     <different begin="65588" length="4"/>
105 |     <same begin="65592" length="3"/>
106 |     <different begin="65595" length="14"/>
107 |     <same begin="65609" length="1"/>
108 |     <different begin="65610" length="6"/>
109 |     <same begin="65616" length="1"/>
110 |     <different begin="65617" length="2"/>
111 |     <same begin="65619" length="2"/>
112 |     <different begin="65621" length="1"/>
113 |     <same begin="65622" length="4"/>
114 |     <different begin="65626" length="5"/>
115 |     <same begin="65631" length="1"/>
116 |     <different begin="65632" length="4"/>
117 |     <same begin="65636" length="1"/>
118 |     <different begin="65637" length="3"/>
119 |     <same begin="65640" length="1"/>
120 |     <different begin="65641" length="16"/>
121 |     <same begin="65657" length="1"/>
122 |     <different begin="65658" length="1"/>
123 |     <same begin="65659" length="4"/>
124 |     <different begin="65663" length="1"/>
125 |     <same begin="65664" length="1"/>
126 |     <different begin="65665" length="3"/>
127 |     <same begin="65668" length="1"/>
128 |     <different begin="65669" length="5"/>
129 |     <same begin="65674" length="1"/>
130 |     <different begin="65675" length="8"/>
131 |     <same begin="65683" length="2"/>
132 |     <different begin="65685" length="10"/>
133 |     <same begin="65695" length="1"/>
134 |     <different begin="65696" length="4"/>
135 |     <same begin="65700" length="726"/>
136 |     <same begin="67584" length="4011"/>
137 |     <same begin="98303" length="1"/>
138 |   </diff>
139 | </superblock>
140 | 


--------------------------------------------------------------------------------