├── paper ├── math │ ├── hba.png │ ├── ss1.png │ ├── ss2.png │ ├── ss3.png │ ├── ws1.png │ ├── ws2.png │ └── ws3.png ├── .htaccess ├── Makefile └── compressed-stream.svg ├── website ├── images │ ├── vcss.png │ ├── i26x25ltb.png │ ├── light_button.png │ └── valid-xhtml11.png ├── Makefile ├── style.css ├── website-mk ├── head.gtml ├── foot.gtml ├── header.gtml ├── papers.gtml ├── author.gtml ├── server-issues.gtml ├── downloads.gtml ├── index.gtml └── server.gtml ├── c ├── doc │ ├── Makefile.am │ ├── zsync.1 │ └── zsyncmake.1 ├── libzsync │ ├── Makefile.am │ ├── sha1test.c │ ├── zmap.h │ ├── sha1.h │ ├── zsync.h │ └── sha1.c ├── zlib │ ├── Makefile.am │ ├── adler32.c │ ├── inftrees.h │ ├── compress.c │ ├── README.zsync │ ├── README │ ├── inflate.h │ ├── inffixed.h │ ├── zutil.h │ └── zutil.c ├── check-zsyncmake ├── librcksum │ ├── Makefile.am │ ├── md4test.c │ ├── md4.h │ ├── rsumtest.c │ ├── rcksum.h │ ├── internal.h │ ├── state.c │ ├── range.c │ ├── md4.c │ └── hash.c ├── autotools │ ├── ac_c_compile_flags.m4 │ ├── herrno.m4 │ ├── sockaddrin6.m4 │ ├── in6addr.m4 │ └── sockinttypes.m4 ├── makegz.h ├── format_string.h ├── zsglobal.h ├── url.h ├── Makefile.am ├── http.h ├── progress.h ├── configure.ac ├── base64.c ├── progress.c ├── url.c ├── makegz.c ├── getaddrinfo.h ├── README └── NEWS ├── .bzrignore └── tools ├── show-stats └── run-tests /paper/math/hba.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/paper/math/hba.png -------------------------------------------------------------------------------- /paper/math/ss1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/paper/math/ss1.png -------------------------------------------------------------------------------- /paper/math/ss2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/paper/math/ss2.png -------------------------------------------------------------------------------- /paper/math/ss3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/paper/math/ss3.png -------------------------------------------------------------------------------- /paper/math/ws1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/paper/math/ws1.png -------------------------------------------------------------------------------- /paper/math/ws2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/paper/math/ws2.png -------------------------------------------------------------------------------- /paper/math/ws3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/paper/math/ws3.png -------------------------------------------------------------------------------- /website/images/vcss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/website/images/vcss.png -------------------------------------------------------------------------------- /c/doc/Makefile.am: -------------------------------------------------------------------------------- 1 | manpages=zsync.1 zsyncmake.1 2 | man_MANS=$(manpages) 3 | EXTRA_DIST=$(manpages) 4 | -------------------------------------------------------------------------------- /website/images/i26x25ltb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/website/images/i26x25ltb.png -------------------------------------------------------------------------------- /website/images/light_button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/website/images/light_button.png -------------------------------------------------------------------------------- /website/images/valid-xhtml11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cph6/zsync/HEAD/website/images/valid-xhtml11.png -------------------------------------------------------------------------------- /.bzrignore: -------------------------------------------------------------------------------- 1 | website/*.html 2 | paper/*.html 3 | c/autotools 4 | c/autotools/* 5 | Makefile 6 | Makefile.in 7 | .deps 8 | -------------------------------------------------------------------------------- /website/Makefile: -------------------------------------------------------------------------------- 1 | PAGES:=author server index papers downloads server-issues 2 | OTHER:=style.css paper200501 paper200503 images download 3 | 4 | include ./website-mk 5 | 6 | -------------------------------------------------------------------------------- /paper/.htaccess: -------------------------------------------------------------------------------- 1 | Redirect permanent /paper/ch05.html http://zsync.moria.org.uk/paper/ch02s08.html 2 | Redirect permanent /paper/ch05s02.html http://zsync.moria.org.uk/paper/ch03s03.html 3 | Redirect gone /paper/ch05s03.html 4 | -------------------------------------------------------------------------------- /c/libzsync/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | noinst_LIBRARIES = libzsync.a 3 | 4 | libzsync_a_SOURCES = zmap.h zsync.h sha1.h zsync.c zmap.c sha1.c 5 | 6 | TESTS = sha1test 7 | noinst_PROGRAMS = sha1test 8 | sha1test_SOURCES = sha1.h sha1.c sha1test.c 9 | 10 | -------------------------------------------------------------------------------- /tools/show-stats: -------------------------------------------------------------------------------- 1 | #!/bin/sh -ex 2 | 3 | mkdir -p stats 4 | alltests=$(ls */stats*|cut -d / -f 1|sort -n -u) 5 | for test in ${1:-$alltests} 6 | do 7 | ( cat $test/stats\ *| head -1 \ 8 | && tail -qn 1 $test/stats\ * ) > stats/$test.csv 9 | oocalc stats/$test.csv 10 | done 11 | -------------------------------------------------------------------------------- /c/zlib/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | noinst_LIBRARIES = libinflate.a libdeflate.a 3 | 4 | libinflate_a_SOURCES = zlib.h inflate.c inflate.h inffixed.h adler32.c inftrees.c inftrees.h zutil.c zutil.h crc32.c crc32.h zconf.h 5 | 6 | libdeflate_a_SOURCES = deflate.c deflate.h compress.c trees.c trees.h 7 | 8 | EXTRA_DIST = README.zsync 9 | 10 | -------------------------------------------------------------------------------- /website/style.css: -------------------------------------------------------------------------------- 1 | #content { 2 | margin-right: 160px; 3 | } 4 | #linklist { 5 | width: 150px; 6 | position: absolute; 7 | top: 0px; 8 | right: 0px; 9 | padding: 10px; 10 | border: 2px black; 11 | background: #ffffb0; 12 | } 13 | .inlinetitle { 14 | font-weight: bolder; 15 | } 16 | a img { 17 | border: 0px; 18 | } 19 | -------------------------------------------------------------------------------- /website/website-mk: -------------------------------------------------------------------------------- 1 | TARGETS:=$(addsuffix .html,$(PAGES)) $(OTHER) 2 | 3 | all : $(TARGETS) 4 | 5 | %.html : %.gtml 6 | gtml $< 7 | 8 | %.html.gz : %.html 9 | gzip --best < $< > $@ 10 | 11 | upload : all 12 | rsync -rRvlt $(TARGETS) $(UPHOST):$(UPPATH) 13 | 14 | clean-html : 15 | rm -f $(addsuffix .html,$(PAGES)) 16 | 17 | .PHONY : all upload 18 | 19 | -------------------------------------------------------------------------------- /c/check-zsyncmake: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | ${top_builddir-.}/zsyncmake -u foo -f foo ${srcdir-.}/INSTALL 4 | 5 | # Note TAB required for Solaris cksum 6 | x=`perl -pe 's/^(zsync:).*$/$1 XXX/; s/^(MTime:)(.*)$/my ($x, $y) = ($1, $2); $y =~ s![a-zA-Z0-9]!_!g; "$x$y"/e;' foo.zsync | tee foo.tr.zsync | cksum | sed -e 's/[ ][0-9].*//'` 7 | [ $x = 3456223344 ] 8 | rm foo.zsync 9 | 10 | -------------------------------------------------------------------------------- /c/librcksum/Makefile.am: -------------------------------------------------------------------------------- 1 | AUTOMAKE_OPTIONS = subdir-objects 2 | 3 | noinst_LIBRARIES = librcksum.a 4 | 5 | TESTS = md4test rsumtest 6 | noinst_PROGRAMS = md4test rsumtest 7 | 8 | md4test_SOURCES = md4test.c md4.h md4.c 9 | rsumtest_SOURCES = rsum.c rsumtest.c hash.c range.c state.c md4.c ../progress.c 10 | librcksum_a_SOURCES = internal.h rcksum.h md4.h rsum.c hash.c state.c range.c md4.c 11 | -------------------------------------------------------------------------------- /c/autotools/ac_c_compile_flags.m4: -------------------------------------------------------------------------------- 1 | AC_DEFUN([X_C_COMPILE_FLAGS],[ 2 | NEW_CFLAGS="$CFLAGS" 3 | for ac_flag in $1 4 | do 5 | AC_MSG_CHECKING(whether compiler supports $ac_flag) 6 | CFLAGS="$NEW_CFLAGS $ac_flag" 7 | AC_COMPILE_IFELSE( 8 | [AC_LANG_PROGRAM([],[ 9 | void f() {}; 10 | ])],[ 11 | NEW_CFLAGS="$CFLAGS" 12 | AC_MSG_RESULT(yes) 13 | ],AC_MSG_RESULT(no)) 14 | done 15 | CFLAGS="$NEW_CFLAGS" 16 | ]) 17 | -------------------------------------------------------------------------------- /paper/Makefile: -------------------------------------------------------------------------------- 1 | STSHEET ?= chunk 2 | STSHEETDIR := /usr/share/xml/docbook/stylesheet/nwalsh/html 3 | 4 | all : index.html paper.html 5 | ruby -pi -e '$$_.gsub!("@","@")' *.html 6 | 7 | index.html : paper.xml 8 | xmlto html $< 9 | 10 | paper.html : paper.xml 11 | xmlto html-nochunks $< 12 | 13 | upload : 14 | rsync --exclude .svn -rptve ssh *.html *.png *.svg math www-uploader@moria:/var/www/zsync/paper 15 | 16 | clean : 17 | rm *.html 18 | -------------------------------------------------------------------------------- /website/head.gtml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 15 |
16 | -------------------------------------------------------------------------------- /website/foot.gtml: -------------------------------------------------------------------------------- 1 |
2 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /website/header.gtml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /c/makegz.h: -------------------------------------------------------------------------------- 1 | /* 2 | * zsync - client side rsync over http 3 | * Copyright (C) 2004,2005,2007,2009 Colin Phipps 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the Artistic License v2 (see the accompanying 7 | * file COPYING for the full license terms), or, at your option, any later 8 | * version of the same license. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * COPYING file for details. 14 | */ 15 | 16 | FILE* optimal_gzip(FILE* fin, const char* fout, size_t blocksize); 17 | -------------------------------------------------------------------------------- /c/format_string.h: -------------------------------------------------------------------------------- 1 | /* 2 | * zsync - client side rsync over http 3 | * Copyright (C) 2004,2005 Colin Phipps 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the Artistic License v2 (see the accompanying 7 | * file COPYING for the full license terms), or, at your option, any later 8 | * version of the same license. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * COPYING file for details. 14 | */ 15 | 16 | #include 17 | 18 | #ifdef PRIu32 19 | # define SIZE_T_PF "%zd" 20 | #else 21 | # define SIZE_T_PF "%u" 22 | #endif 23 | 24 | #if SIZEOF_OFF_T == 8 25 | # ifdef PRIu64 26 | # define OFF_T_PF "%" PRIu64 27 | # else 28 | # define OFF_T_PF "%llu" 29 | # endif 30 | #else 31 | # ifdef PRIu32 32 | # define OFF_T_PF "%" PRIu32 33 | # else 34 | # define OFF_T_PF "%lu" 35 | # endif 36 | #endif 37 | 38 | -------------------------------------------------------------------------------- /c/zsglobal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * zsync - client side rsync over http 3 | * Copyright (C) 2004,2005,2007,2009 Colin Phipps 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the Artistic License v2 (see the accompanying 7 | * file COPYING for the full license terms), or, at your option, any later 8 | * version of the same license. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * COPYING file for details. 14 | */ 15 | 16 | #ifndef ZSGLOBAL_H 17 | #define ZSGLOBAL_H 18 | 19 | 20 | #ifdef HAVE_CONFIG_H 21 | # include "config.h" 22 | #endif 23 | 24 | #if defined(__GNUC__) && defined (__OpenBSD__) 25 | # define ZS_DECL_BOUNDED(x,y,z) __attribute__((__bounded__(x,y,z))) 26 | #else 27 | # define ZS_DECL_BOUNDED(x,y,z) 28 | #endif /* ZS_DECL_BOUNDED */ 29 | 30 | static inline unsigned min(unsigned short a, unsigned short b) { 31 | return a > b ? b : a; 32 | } 33 | 34 | static inline unsigned max(unsigned short a, unsigned short b) { 35 | return a > b ? a : b; 36 | } 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /c/url.h: -------------------------------------------------------------------------------- 1 | /* 2 | * zsync - client side rsync over http 3 | * Copyright (C) 2004,2005,2009 Colin Phipps 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the Artistic License v2 (see the accompanying 7 | * file COPYING for the full license terms), or, at your option, any later 8 | * version of the same license. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * COPYING file for details. 14 | */ 15 | 16 | extern const char http_scheme[]; 17 | 18 | /* Given an HTTP URL, return the path path of the URL as the return value, and 19 | * return the hostname in the provided buffer (hostn, length hnlen) 20 | * and, if present, return a (malloced) string buffer containing the port string. 21 | * Or return NULL if not HTTP or other parsing failure. 22 | */ 23 | char* get_http_host_port(const char* url, char* hostn, int hnlen, char** port); 24 | 25 | char* __attribute__((pure)) make_url_absolute(const char* base, const char* url); 26 | 27 | int __attribute__((pure)) is_url_absolute(const char* url); 28 | -------------------------------------------------------------------------------- /c/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | AUTOMAKE_OPTIONS = check-news 3 | ACLOCAL_AMFLAGS = -I autotools 4 | 5 | SUBDIRS = librcksum zlib libzsync doc 6 | 7 | bin_PROGRAMS = zsyncmake zsync 8 | 9 | zsyncmake_SOURCES = make.c makegz.c makegz.h format_string.h progress.c progress.h 10 | zsyncmake_LDADD = libzsync/libzsync.a librcksum/librcksum.a zlib/libinflate.a zlib/libdeflate.a -lm 11 | 12 | zsync_SOURCES = client.c http.c http.h url.c url.h progress.c progress.h base64.c format_string.h zsglobal.h 13 | zsync_LDADD = libzsync/libzsync.a librcksum/librcksum.a zlib/libinflate.a $(LIBOBJS) 14 | 15 | EXTRA_zsync_SOURCES = getaddrinfo.h 16 | 17 | # From "GNU autoconf, automake and libtool" Vaughan, Elliston, 18 | # # Tromey and Taylor, publisher New Riders, p.134 19 | # # They made me do it! :-) 20 | AUX_DIST_GEN = \ 21 | $(ac_aux_dir)/install-sh \ 22 | $(ac_aux_dir)/missing \ 23 | $(ac_aux_dir)/ac_c_compile_flags.m4 \ 24 | $(ac_aux_dir)/herrno.m4 \ 25 | $(ac_aux_dir)/in6addr.m4 \ 26 | $(ac_aux_dir)/sockaddrin6.m4 \ 27 | $(ac_aux_dir)/sockinttypes.m4 28 | 29 | docdir=$(prefix)/share/doc/@PACKAGE@ 30 | doc_DATA = COPYING README 31 | EXTRA_DIST = INSTALL README COPYING $(AUX_DIST_GEN) check-zsyncmake 32 | 33 | TESTS_ENVIRONMENT = top_builddir=$(top_builddir) 34 | TESTS = check-zsyncmake 35 | 36 | -------------------------------------------------------------------------------- /c/librcksum/md4test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * zsync - client side rsync over http 3 | * Copyright (C) 2005 Colin Phipps 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the Artistic License v2 (see the accompanying 7 | * file COPYING for the full license terms), or, at your option, any later 8 | * version of the same license. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * COPYING file for details. 14 | */ 15 | 16 | #include "zsglobal.h" 17 | 18 | #include 19 | #include 20 | #include 21 | #include "md4.h" 22 | 23 | // From RFC1320 24 | const char correct_checksum[MD4_DIGEST_LENGTH] = {0xd7, 0x9e, 0x1c, 0x30, 0x8a, 0xa5, 0xbb, 0xcd, 0xee, 0xa8, 0xed, 0x63, 0xdf, 0x41, 0x2d, 0xa9 }; 25 | 26 | int main(void) 27 | { 28 | MD4_CTX ctx; 29 | 30 | MD4Init(&ctx); 31 | 32 | MD4Update(&ctx,(uint8_t*)"a",1); 33 | MD4Update(&ctx,(uint8_t*)"bcdefghijklmnopqrstuvwxyz",25); 34 | { 35 | uint8_t digest[MD4_DIGEST_LENGTH]; 36 | MD4Final(digest,&ctx); 37 | exit(memcmp(digest,correct_checksum,MD4_DIGEST_LENGTH)); 38 | } 39 | 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /c/libzsync/sha1test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * zsync - client side rsync over http 3 | * Copyright (C) 2005 Colin Phipps 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the Artistic License v2 (see the accompanying 7 | * file COPYING for the full license terms), or, at your option, any later 8 | * version of the same license. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * COPYING file for details. 14 | */ 15 | 16 | #include "zsglobal.h" 17 | 18 | #include 19 | #include 20 | #include 21 | #include "sha1.h" 22 | 23 | // From RFC3174 24 | const char correct_checksum[SHA1_DIGEST_LENGTH] = {0xA9, 0x99, 0x3E, 0x36, 0x47, 0x06, 0x81, 0x6A, 0xBA, 0x3E, 0x25, 0x71, 0x78, 0x50, 0xC2, 0x6C, 0x9C, 0xD0, 0xD8, 0x9D}; 25 | 26 | int main(void) 27 | { 28 | SHA1_CTX ctx; 29 | 30 | SHA1Init(&ctx); 31 | 32 | SHA1Update(&ctx,(uint8_t*)"a",1); 33 | SHA1Update(&ctx,(uint8_t*)"bc",2); 34 | { 35 | uint8_t digest[SHA1_DIGEST_LENGTH]; 36 | SHA1Final(digest,&ctx); 37 | exit(memcmp(digest,correct_checksum,SHA1_DIGEST_LENGTH)); 38 | } 39 | 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /c/http.h: -------------------------------------------------------------------------------- 1 | /* 2 | * http.h - simple HTTP client for zsync 3 | * 4 | * Copyright (C) 2004,2005,2009 Colin Phipps 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the Artistic License v2 (see the accompanying 8 | * file COPYING for the full license terms), or, at your option, any later 9 | * version of the same license. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * COPYING file for details. 15 | */ 16 | 17 | 18 | 19 | extern char *referer; 20 | 21 | int set_proxy_from_string(const char* s); 22 | 23 | FILE* http_get(const char* orig_url, char** track_referer, const char* tfname); 24 | 25 | struct range_fetch; 26 | 27 | struct range_fetch* range_fetch_start(const char* orig_url); 28 | void range_fetch_addranges(struct range_fetch* rf, off_t* ranges, int nranges); 29 | int get_range_block(struct range_fetch* rf, off_t* offset, unsigned char* data, size_t dlen); 30 | off_t range_fetch_bytes_down(const struct range_fetch* rf); 31 | void range_fetch_end(struct range_fetch* rf); 32 | 33 | void add_auth(char* host, char* user, char* pass); 34 | 35 | /* base64.c */ 36 | char* base64(const char*); 37 | 38 | -------------------------------------------------------------------------------- /c/progress.h: -------------------------------------------------------------------------------- 1 | /* 2 | * zsync - client side rsync over http 3 | * Copyright (C) 2004,2005,2009 Colin Phipps 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the Artistic License v2 (see the accompanying 7 | * file COPYING for the full license terms), or, at your option, any later 8 | * version of the same license. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * COPYING file for details. 14 | */ 15 | 16 | struct progress; 17 | extern int no_progress; 18 | 19 | /* struct progress* = start_progress() 20 | * Returns a progress structure. Caller is responsible for calling 21 | * end_progress() on it later (which will free the memory that it uses). 22 | */ 23 | struct progress* start_progress(void) __attribute__((malloc)); 24 | 25 | void do_progress(struct progress* p, float pcnt, long long newdl); 26 | 27 | /* end_progress(struct progress*, done) 28 | * done parameter is 0 for error, 1 for okay-but-incomplete, 2 for completed 29 | * This frees the memory allocated for the progress data; the pointer is no 30 | * longer valid when this function returns. 31 | */ 32 | void end_progress(struct progress* p, int done); 33 | -------------------------------------------------------------------------------- /website/papers.gtml: -------------------------------------------------------------------------------- 1 | #include "header.gtml" 2 | zsync Technical Paper 3 | #include "head.gtml" 4 | 5 |

Technical Paper

6 | 7 |

As I develop, I am updating the original technical paper in which I laid out the algorithms, ideas and design decisions behind zsync. The paper is therefore a work in progress. Some sections may be incomplete, or refer to earlier versions of zsync or of the implementation; I have given zsync version numbers and indicate out-of-date sections wherever possible.

8 | 12 |

March 2005 Paper — zsync — Optimised rsync over HTTP

13 |

I have frozen the version of the technical paper as it stood in March 2005, because this when I stopped working on zsync for a while. This is also the point where I had completed my work on matching the efficiency of rsync, so it is a good moment to take a snapshot of where the work stands.

14 |

This is a frozen, finished version of the technical paper as it stood at zsync-0.4.x, and can be used for citations.

15 |

http://zsync.moria.org.uk/paper200503/

16 |

Jan 2005 Paper — zsync — Principles and Implementation

17 |

I have frozen the version of the technical paper as it stood at the time of zsync-0.1.x, because I think it is an interesting insight into the original justification for the look-inside method, and I have not seen this approach discussed elsewhere.

18 |

In the light of subsequent improvements to zsync these ideas may be overtaken, so I want this present as a permanent reference for the original reasoning. This can also be used as a citation if other people take up the idea. This version of the paper is now fixed and will remain at:

19 |

http://zsync.moria.org.uk/paper200501/

20 | #include "foot.gtml" 21 | -------------------------------------------------------------------------------- /c/librcksum/md4.h: -------------------------------------------------------------------------------- 1 | /* $OpenBSD: md4.h,v 1.15 2004/06/22 01:57:30 jfb Exp $ */ 2 | 3 | /* 4 | * This code implements the MD4 message-digest algorithm. 5 | * The algorithm is due to Ron Rivest. This code was 6 | * written by Colin Plumb in 1993, no copyright is claimed. 7 | * This code is in the public domain; do with it what you wish. 8 | * Todd C. Miller modified the MD5 code to do MD4 based on RFC 1186. 9 | * 10 | * Equivalent code is available from RSA Data Security, Inc. 11 | * This code has been tested against that, and is equivalent, 12 | * except that you don't need to include two pages of legalese 13 | * with every copy. 14 | */ 15 | 16 | #ifndef _MD4_H_ 17 | #define _MD4_H_ 18 | 19 | #include "zsglobal.h" 20 | 21 | #ifdef HAVE_INTTYPES_H 22 | #include 23 | #else 24 | #include 25 | #endif 26 | 27 | #define MD4_BLOCK_LENGTH 64 28 | #define MD4_DIGEST_LENGTH 16 29 | #define MD4_DIGEST_STRING_LENGTH (MD4_DIGEST_LENGTH * 2 + 1) 30 | 31 | typedef struct MD4Context { 32 | uint32_t state[4]; /* state */ 33 | uint64_t count; /* number of bits, mod 2^64 */ 34 | uint8_t buffer[MD4_BLOCK_LENGTH]; /* input buffer */ 35 | } MD4_CTX; 36 | 37 | void MD4Init(MD4_CTX *); 38 | void MD4Update(MD4_CTX *, const uint8_t *, size_t) 39 | ZS_DECL_BOUNDED(__string__,2,3); 40 | void MD4Pad(MD4_CTX *); 41 | void MD4Final(uint8_t [MD4_DIGEST_LENGTH], MD4_CTX *) 42 | ZS_DECL_BOUNDED(__minbytes__,1,MD4_DIGEST_LENGTH); 43 | void MD4Transform(uint32_t [4], const uint8_t [MD4_BLOCK_LENGTH]) 44 | ZS_DECL_BOUNDED(__minbytes__,1,4) 45 | ZS_DECL_BOUNDED(__minbytes__,2,MD4_BLOCK_LENGTH); 46 | char *MD4End(MD4_CTX *, char *) 47 | ZS_DECL_BOUNDED(__minbytes__,2,MD4_DIGEST_STRING_LENGTH); 48 | char *MD4File(const char *, char *) 49 | ZS_DECL_BOUNDED(__minbytes__,2,MD4_DIGEST_STRING_LENGTH); 50 | char *MD4FileChunk(const char *, char *, off_t, off_t) 51 | ZS_DECL_BOUNDED(__minbytes__,2,MD4_DIGEST_STRING_LENGTH); 52 | char *MD4Data(const uint8_t *, size_t, char *) 53 | ZS_DECL_BOUNDED(__string__,1,2) 54 | ZS_DECL_BOUNDED(__minbytes__,3,MD4_DIGEST_STRING_LENGTH); 55 | 56 | #endif /* _MD4_H_ */ 57 | -------------------------------------------------------------------------------- /website/author.gtml: -------------------------------------------------------------------------------- 1 | #include "header.gtml" 2 | zsync — Credits & Contact 3 | #include "head.gtml" 4 | 5 |

The Author

6 | 7 |

zsync is written by Colin Phipps, as is this website. You can contact me at cph@moria.org.uk. I retain the copyright to zsync, and make it available to you under the Artistic License v2 — see the file COPYING supplied with zsync.

8 |

I use IRC for my instant messaging, so you can find me from time to time on irc.freenode.net, #zsync.

9 |

The content of this website is copyright 2005 by Colin Phipps (except for the standard images, obviously). You are free to copy the examples and descriptions from the web pages on site without restriction, except for the technical papers. The technical papers you can redistribute and reproduce, but they must not be modified and you must acknowledge me as the author; you can cite from them freely, provided you provide a reference to the full copy of the papers on this site.

10 | 11 |

Acknowledgements

12 | 13 |

zsync uses the rsync algorithm, from the technical paper by Andrew Tridgell and Paul Mackerras. 14 | It also builds on optimisations of the basic rsync concept, including ideas from papers by Utku Irmak, Svilen Mihaylov, and Torsten Suel

15 | 16 |

zsync also contains some code from zlib, Copyright (C) 1995-2003 Jean-loup Gailly and Mark Adler. I have made only slight alterations to this code, which I have put in the public domain.

17 | 18 |

zsync also includes checksum code taken from OpenBSD. The MD4 code is public 19 | domain, by Colin Plumb and Todd C. Miller. The SHA1 code is also public 20 | domain, by Steve Reid.

21 | 22 |

I would like 23 | to thank the Free Software Foundation and its contributors, for gcc, gdb, 24 | automake, autoconf and emacs, the essential development tools. Also I would 25 | thank Sourceforge for providing the hosting facilities for the downloads and 26 | mailing list, and the compile farm for testing portability. 27 |

28 | 29 | #include "foot.gtml" 30 | -------------------------------------------------------------------------------- /c/libzsync/zmap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * zsync - client side rsync over http 3 | * Copyright (C) 2004,2005,2007,2009 Colin Phipps 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the Artistic License v2 (see the accompanying 7 | * file COPYING for the full license terms), or, at your option, any later 8 | * version of the same license. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * COPYING file for details. 14 | */ 15 | 16 | #include "zsglobal.h" 17 | 18 | struct gzblock { 19 | uint16_t inbitoffset; 20 | uint16_t outbyteoffset; 21 | } __attribute__((packed)); 22 | 23 | #define GZB_NOTBLOCKSTART 0x8000 24 | 25 | struct zmap; 26 | struct z_stream_s; 27 | 28 | struct zmap* zmap_make(const struct gzblock* zb, int n); 29 | void zmap_free(struct zmap*); 30 | 31 | off_t* zmap_to_compressed_ranges(const struct zmap* zm, off_t* byterange, int nrange, int* num); 32 | void configure_zstream_for_zdata(const struct zmap* zm, struct z_stream_s* zs, long zoffset, off_t* poutoffset); 33 | 34 | /* gzip flag byte */ 35 | #define GZ_ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ 36 | #define GZ_HEAD_CRC 0x02 /* bit 1 set: header CRC present */ 37 | #define GZ_EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ 38 | #define GZ_ORIG_NAME 0x08 /* bit 3 set: original file name present */ 39 | #define GZ_COMMENT 0x10 /* bit 4 set: file comment present */ 40 | #define GZ_RESERVED 0xE0 /* bits 5..7: reserved */ 41 | 42 | /* mtime is in bytes 4..7 of the gzip header */ 43 | static inline int zhead_has_mtime(const char* p) { 44 | return !!(p[4] || p[5] || p[6] || p[7]); 45 | } 46 | 47 | static inline int zhead_has_fname(const char* p) { 48 | return !!(p[3] & GZ_ORIG_NAME); 49 | } 50 | 51 | static inline const char* skip_zhead(const char* p) 52 | { 53 | const char* q = p + 10; 54 | if (zhead_has_fname(p)) { 55 | q += strlen(q)+1; 56 | } 57 | 58 | return q; 59 | } 60 | 61 | -------------------------------------------------------------------------------- /c/autotools/herrno.m4: -------------------------------------------------------------------------------- 1 | dnl * 2 | dnl * Copyright (c) 2001 Motoyuki Kasahara 3 | dnl * 4 | dnl * Redistribution and use in source and binary forms, with or without 5 | dnl * modification, are permitted provided that the following conditions 6 | dnl * are met: 7 | dnl * 1. Redistributions of source code must retain the above copyright 8 | dnl * notice, this list of conditions and the following disclaimer. 9 | dnl * 2. Redistributions in binary form must reproduce the above copyright 10 | dnl * notice, this list of conditions and the following disclaimer in the 11 | dnl * documentation and/or other materials provided with the distribution. 12 | dnl * 3. Neither the name of the project nor the names of its contributors 13 | dnl * may be used to endorse or promote products derived from this software 14 | dnl * without specific prior written permission. 15 | dnl * 16 | dnl * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 17 | dnl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | dnl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 | dnl * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORSBE 20 | dnl * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | dnl * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | dnl * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | dnl * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | dnl * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | dnl * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 | dnl * THE POSSIBILITY OF SUCH DAMAGE. 27 | dnl * 28 | 29 | dnl * 30 | dnl * Check for h_errno. 31 | dnl * 32 | AC_DEFUN([X_DECL_H_ERRNO], 33 | [AC_CACHE_CHECK(for h_errno declaration in netdb.h, ac_cv_decl_h_errno, 34 | [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 35 | #include ]], [[ 36 | h_errno = 0; 37 | ]])],[ac_cv_decl_h_errno=yes],[ac_cv_decl_h_errno=no])]) 38 | if test "$ac_cv_decl_h_errno" = yes; then 39 | AC_DEFINE(H_ERRNO_DECLARED, 1, 40 | [Define to 1 if `h_errno' is declared by ]) 41 | fi]) 42 | -------------------------------------------------------------------------------- /c/librcksum/rsumtest.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "rcksum.h" 7 | #include "md4.h" 8 | #include "internal.h" 9 | 10 | static void test_eq(int a, int b) { 11 | if (a != b) { 12 | fprintf(stderr, "%x != %x\n", a, b); 13 | exit(1); 14 | } 15 | } 16 | 17 | void make_0000ff00_data(unsigned char* data, size_t len) { 18 | int i; 19 | for (i = 0; i < len; i++) { 20 | if (i % 4 == 2) { 21 | data[i] = 0xff; 22 | } else data[i] = 0x00; 23 | } 24 | } 25 | 26 | void test_00000000(void) { 27 | unsigned char data[4096]; 28 | 29 | memset(data, 0, sizeof(data)); 30 | 31 | struct rsum r = rcksum_calc_rsum_block(data, sizeof(data)); 32 | test_eq(r.a, 0x0000); 33 | test_eq(r.b, 0x0000); 34 | } 35 | 36 | void test_abcde(void) { 37 | unsigned char data[4096]; 38 | int i; 39 | 40 | for (i = 0; i < sizeof(data); i++) { 41 | data[i] = "abcde"[i % 5]; 42 | } 43 | 44 | struct rsum r = rcksum_calc_rsum_block(data, sizeof(data)); 45 | test_eq(r.a, 0x2ffe); 46 | test_eq(r.b, 0xf800); 47 | } 48 | 49 | void test_fc000000(void) { 50 | unsigned char data[4096]; 51 | 52 | make_0000ff00_data(data, sizeof(data)); 53 | 54 | struct rsum r = rcksum_calc_rsum_block(data, sizeof(data)); 55 | test_eq(r.a, 0xfc00); 56 | test_eq(r.b, 0x0000); 57 | } 58 | 59 | void perf_test_fc000000(int n) { 60 | struct timeval start, end; 61 | unsigned char data[4096]; 62 | int i; 63 | volatile int unused = 0; 64 | 65 | make_0000ff00_data(data, sizeof(data)); 66 | 67 | gettimeofday(&start, NULL); 68 | for (i = 0; i < n; i++) { 69 | struct rsum r = rcksum_calc_rsum_block(data, sizeof(data)); 70 | unused += r.a + r.b; 71 | } 72 | gettimeofday(&end, NULL); 73 | 74 | int took_us = (end.tv_sec - start.tv_sec) * 1000000L + end.tv_usec - start.tv_usec; 75 | printf("%d iterations, took %d.%06ds\n", n, took_us / 1000000, took_us % 1000000); 76 | } 77 | 78 | void main(void) { 79 | test_00000000(); 80 | test_abcde(); 81 | test_fc000000(); 82 | 83 | #if 0 84 | perf_test_fc000000(10000000); 85 | #endif 86 | } 87 | -------------------------------------------------------------------------------- /c/configure.ac: -------------------------------------------------------------------------------- 1 | # Process this file with autoconf to produce a configure script. 2 | 3 | AC_INIT([zsync],[0.6.3],[http://zsync.moria.org.uk/]) 4 | 5 | AC_CONFIG_SRCDIR([client.c]) 6 | AC_CONFIG_AUX_DIR([autotools]) 7 | AC_CONFIG_MACRO_DIR([autotools]) 8 | 9 | AC_CANONICAL_HOST 10 | 11 | AM_INIT_AUTOMAKE([dist-bzip2 no-dist-gzip foreign check-news -Woverride -Wobsolete -Wportability -Wsyntax -Wunsupported]) 12 | AM_MAINTAINER_MODE 13 | 14 | dnl --- Check for programs 15 | AC_PROG_CC 16 | AC_PROG_LN_S 17 | AC_PROG_RANLIB 18 | 19 | AC_ARG_ENABLE([profile], 20 | AS_HELP_STRING([--enable-profile],[Turns on profiling])) 21 | 22 | AC_ARG_ENABLE([debug],AS_HELP_STRING([--enable-debug], [turns on some debugging features]), 23 | AC_DEFINE([DEBUG], [], [Turn on some debugging features])) 24 | 25 | AS_IF([test "x$enable_profile" = "xyes"], [ 26 | ZS_CFLAGS="${ZS_CFLAGS} -pg" ]) 27 | 28 | dnl --- Header files, typedefs, structures, libraries 29 | AC_C_CONST 30 | 31 | # string.h, memory.h, stdlib.h both included in the default header checks 32 | # but we do need to give at least one .h to test, or Solaris sh errors 33 | AC_CHECK_HEADERS([string.h]) 34 | AC_TYPE_SIZE_T 35 | AC_CHECK_FUNCS(memcpy pwrite pread mkstemp) 36 | 37 | X_TYPE_SOCKLEN_T 38 | X_TYPE_IN_PORT_T 39 | X_DECL_H_ERRNO 40 | 41 | dnl Solaris needs -lsocket - and we need this for the getaddrinfo test 42 | AC_CHECK_LIB(socket,socket) 43 | 44 | AC_REPLACE_FUNCS(getaddrinfo) 45 | 46 | dnl - Large file support if available 47 | AC_SYS_LARGEFILE 48 | AC_FUNC_FSEEKO 49 | AC_CHECK_SIZEOF(size_t) 50 | AC_CHECK_SIZEOF(off_t) 51 | 52 | AM_WITH_DMALLOC 53 | 54 | case $host_os in 55 | mingw32) 56 | host_os_mingw32=yes 57 | LIBS="${LIBS} -lwsock32" 58 | ;; 59 | *) 60 | AC_DEFINE([_XOPEN_SOURCE], 600, [Enable POSIX extensions if present]) 61 | AC_DEFINE([_BSD_SOURCE],1, [Enable BSD extensions if present]) 62 | ;; 63 | esac 64 | AM_CONDITIONAL([MINGW32], test "x$host_os_mingw32" = "xyes") 65 | 66 | X_C_COMPILE_FLAGS($ZS_CFLAGS -g -Wall -Wwrite-strings -Winline -Wextra -Winline -Wmissing-noreturn -Wredundant-decls -Wnested-externs -Wundef -Wbad-function-cast -Wcast-align -Wvolatile-register-var -ffast-math) 67 | 68 | dnl --- output 69 | AC_SUBST(ac_aux_dir) 70 | AC_CONFIG_HEADERS([config.h]) 71 | AC_CONFIG_FILES([Makefile librcksum/Makefile zlib/Makefile libzsync/Makefile doc/Makefile]) 72 | AC_OUTPUT 73 | 74 | -------------------------------------------------------------------------------- /c/libzsync/sha1.h: -------------------------------------------------------------------------------- 1 | /* $OpenBSD: sha1.h,v 1.23 2004/06/22 01:57:30 jfb Exp $ */ 2 | 3 | /* 4 | * SHA-1 in C 5 | * By Steve Reid 6 | * 100% Public Domain 7 | */ 8 | 9 | #ifndef _SHA1_H 10 | #define _SHA1_H 11 | 12 | #include "config.h" 13 | 14 | #ifdef HAVE_INTTYPES_H 15 | #include 16 | #else 17 | #include 18 | #endif 19 | 20 | #define SHA1_BLOCK_LENGTH 64 21 | #define SHA1_DIGEST_LENGTH 20 22 | #define SHA1_DIGEST_STRING_LENGTH (SHA1_DIGEST_LENGTH * 2 + 1) 23 | 24 | typedef struct { 25 | uint32_t state[5]; 26 | uint64_t count; 27 | uint8_t buffer[SHA1_BLOCK_LENGTH]; 28 | } SHA1_CTX; 29 | 30 | void SHA1Init(SHA1_CTX *); 31 | void SHA1Pad(SHA1_CTX *); 32 | void SHA1Transform(uint32_t [5], const uint8_t [SHA1_BLOCK_LENGTH]) 33 | ZS_DECL_BOUNDED(__minbytes__,1,5) 34 | ZS_DECL_BOUNDED(__minbytes__,2,SHA1_BLOCK_LENGTH); 35 | void SHA1Update(SHA1_CTX *, const uint8_t *, size_t) 36 | ZS_DECL_BOUNDED(__string__,2,3); 37 | void SHA1Final(uint8_t [SHA1_DIGEST_LENGTH], SHA1_CTX *) 38 | ZS_DECL_BOUNDED(__minbytes__,1,SHA1_DIGEST_LENGTH); 39 | char *SHA1End(SHA1_CTX *, char *) 40 | ZS_DECL_BOUNDED(__minbytes__,2,SHA1_DIGEST_STRING_LENGTH); 41 | char *SHA1File(const char *, char *) 42 | ZS_DECL_BOUNDED(__minbytes__,2,SHA1_DIGEST_STRING_LENGTH); 43 | char *SHA1FileChunk(const char *, char *, off_t, off_t) 44 | ZS_DECL_BOUNDED(__minbytes__,2,SHA1_DIGEST_STRING_LENGTH); 45 | char *SHA1Data(const uint8_t *, size_t, char *) 46 | ZS_DECL_BOUNDED(__string__,1,2) 47 | ZS_DECL_BOUNDED(__minbytes__,3,SHA1_DIGEST_STRING_LENGTH); 48 | 49 | #define HTONDIGEST(x) do { \ 50 | x[0] = htonl(x[0]); \ 51 | x[1] = htonl(x[1]); \ 52 | x[2] = htonl(x[2]); \ 53 | x[3] = htonl(x[3]); \ 54 | x[4] = htonl(x[4]); } while (0) 55 | 56 | #define NTOHDIGEST(x) do { \ 57 | x[0] = ntohl(x[0]); \ 58 | x[1] = ntohl(x[1]); \ 59 | x[2] = ntohl(x[2]); \ 60 | x[3] = ntohl(x[3]); \ 61 | x[4] = ntohl(x[4]); } while (0) 62 | 63 | #endif /* _SHA1_H */ 64 | -------------------------------------------------------------------------------- /c/zlib/adler32.c: -------------------------------------------------------------------------------- 1 | /* adler32.c -- compute the Adler-32 checksum of a data stream 2 | * Copyright (C) 1995-2003 Mark Adler 3 | * For conditions of distribution and use, see copyright notice in zlib.h 4 | */ 5 | 6 | /* @(#) $Id$ */ 7 | 8 | #define ZLIB_INTERNAL 9 | #include "zlib.h" 10 | 11 | #define BASE 65521UL /* largest prime smaller than 65536 */ 12 | #define NMAX 5552 13 | /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ 14 | 15 | #define DO1(buf,i) {s1 += buf[i]; s2 += s1;} 16 | #define DO2(buf,i) DO1(buf,i); DO1(buf,i+1); 17 | #define DO4(buf,i) DO2(buf,i); DO2(buf,i+2); 18 | #define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); 19 | #define DO16(buf) DO8(buf,0); DO8(buf,8); 20 | 21 | #ifdef NO_DIVIDE 22 | # define MOD(a) \ 23 | do { \ 24 | if (a >= (BASE << 16)) a -= (BASE << 16); \ 25 | if (a >= (BASE << 15)) a -= (BASE << 15); \ 26 | if (a >= (BASE << 14)) a -= (BASE << 14); \ 27 | if (a >= (BASE << 13)) a -= (BASE << 13); \ 28 | if (a >= (BASE << 12)) a -= (BASE << 12); \ 29 | if (a >= (BASE << 11)) a -= (BASE << 11); \ 30 | if (a >= (BASE << 10)) a -= (BASE << 10); \ 31 | if (a >= (BASE << 9)) a -= (BASE << 9); \ 32 | if (a >= (BASE << 8)) a -= (BASE << 8); \ 33 | if (a >= (BASE << 7)) a -= (BASE << 7); \ 34 | if (a >= (BASE << 6)) a -= (BASE << 6); \ 35 | if (a >= (BASE << 5)) a -= (BASE << 5); \ 36 | if (a >= (BASE << 4)) a -= (BASE << 4); \ 37 | if (a >= (BASE << 3)) a -= (BASE << 3); \ 38 | if (a >= (BASE << 2)) a -= (BASE << 2); \ 39 | if (a >= (BASE << 1)) a -= (BASE << 1); \ 40 | if (a >= BASE) a -= BASE; \ 41 | } while (0) 42 | #else 43 | # define MOD(a) a %= BASE 44 | #endif 45 | 46 | /* ========================================================================= */ 47 | uLong ZEXPORT adler32(adler, buf, len) 48 | uLong adler; 49 | const Bytef *buf; 50 | uInt len; 51 | { 52 | unsigned long s1 = adler & 0xffff; 53 | unsigned long s2 = (adler >> 16) & 0xffff; 54 | int k; 55 | 56 | if (buf == Z_NULL) return 1L; 57 | 58 | while (len > 0) { 59 | k = len < NMAX ? (int)len : NMAX; 60 | len -= k; 61 | while (k >= 16) { 62 | DO16(buf); 63 | buf += 16; 64 | k -= 16; 65 | } 66 | if (k != 0) do { 67 | s1 += *buf++; 68 | s2 += s1; 69 | } while (--k); 70 | MOD(s1); 71 | MOD(s2); 72 | } 73 | return (s2 << 16) | s1; 74 | } 75 | -------------------------------------------------------------------------------- /c/zlib/inftrees.h: -------------------------------------------------------------------------------- 1 | /* inftrees.h -- header to use inftrees.c 2 | * Copyright (C) 1995-2005 Mark Adler 3 | * For conditions of distribution and use, see copyright notice in zlib.h 4 | */ 5 | 6 | /* WARNING: this file should *not* be used by applications. It is 7 | part of the implementation of the compression library and is 8 | subject to change. Applications should only use zlib.h. 9 | */ 10 | 11 | /* Structure for decoding tables. Each entry provides either the 12 | information needed to do the operation requested by the code that 13 | indexed that table entry, or it provides a pointer to another 14 | table that indexes more bits of the code. op indicates whether 15 | the entry is a pointer to another table, a literal, a length or 16 | distance, an end-of-block, or an invalid code. For a table 17 | pointer, the low four bits of op is the number of index bits of 18 | that table. For a length or distance, the low four bits of op 19 | is the number of extra bits to get after the code. bits is 20 | the number of bits in this code or part of the code to drop off 21 | of the bit buffer. val is the actual byte to output in the case 22 | of a literal, the base length or distance, or the offset from 23 | the current table to the next table. Each entry is four bytes. */ 24 | typedef struct { 25 | unsigned char op; /* operation, extra bits, table bits */ 26 | unsigned char bits; /* bits in this part of the code */ 27 | unsigned short val; /* offset in table or code value */ 28 | } code; 29 | 30 | /* op values as set by inflate_table(): 31 | 00000000 - literal 32 | 0000tttt - table link, tttt != 0 is the number of table index bits 33 | 0001eeee - length or distance, eeee is the number of extra bits 34 | 01100000 - end of block 35 | 01000000 - invalid code 36 | */ 37 | 38 | /* Maximum size of dynamic tree. The maximum found in a long but non- 39 | exhaustive search was 1444 code structures (852 for length/literals 40 | and 592 for distances, the latter actually the result of an 41 | exhaustive search). The true maximum is not known, but the value 42 | below is more than safe. */ 43 | #define ENOUGH 2048 44 | #define MAXD 592 45 | 46 | /* Type of code to build for inftable() */ 47 | typedef enum { 48 | CODES, 49 | LENS, 50 | DISTS 51 | } codetype; 52 | 53 | extern int inflate_table OF((codetype type, unsigned short FAR *lens, 54 | unsigned codes, code FAR * FAR *table, 55 | unsigned FAR *bits, unsigned short FAR *work)); 56 | -------------------------------------------------------------------------------- /paper/compressed-stream.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 20 | 21 | Compressed stream: 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | ... 31 | 32 | he_ 33 | l 34 | a 35 | z 36 | y 37 | _ 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /c/zlib/compress.c: -------------------------------------------------------------------------------- 1 | /* compress.c -- compress a memory buffer 2 | * Copyright (C) 1995-2002 Jean-loup Gailly. 3 | * For conditions of distribution and use, see copyright notice in zlib.h 4 | */ 5 | 6 | /* @(#) $Id$ */ 7 | 8 | #define ZLIB_INTERNAL 9 | #include "zlib.h" 10 | 11 | /* =========================================================================== 12 | Compresses the source buffer into the destination buffer. The level 13 | parameter has the same meaning as in deflateInit. sourceLen is the byte 14 | length of the source buffer. Upon entry, destLen is the total size of the 15 | destination buffer, which must be at least 0.1% larger than sourceLen plus 16 | 12 bytes. Upon exit, destLen is the actual size of the compressed buffer. 17 | 18 | compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough 19 | memory, Z_BUF_ERROR if there was not enough room in the output buffer, 20 | Z_STREAM_ERROR if the level parameter is invalid. 21 | */ 22 | int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) 23 | Bytef *dest; 24 | uLongf *destLen; 25 | const Bytef *source; 26 | uLong sourceLen; 27 | int level; 28 | { 29 | z_stream stream; 30 | int err; 31 | 32 | stream.next_in = (Bytef*)source; 33 | stream.avail_in = (uInt)sourceLen; 34 | #ifdef MAXSEG_64K 35 | /* Check for source > 64K on 16-bit machine: */ 36 | if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; 37 | #endif 38 | stream.next_out = dest; 39 | stream.avail_out = (uInt)*destLen; 40 | if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; 41 | 42 | stream.zalloc = (alloc_func)0; 43 | stream.zfree = (free_func)0; 44 | stream.opaque = (voidpf)0; 45 | 46 | err = deflateInit(&stream, level); 47 | if (err != Z_OK) return err; 48 | 49 | err = deflate(&stream, Z_FINISH); 50 | if (err != Z_STREAM_END) { 51 | deflateEnd(&stream); 52 | return err == Z_OK ? Z_BUF_ERROR : err; 53 | } 54 | *destLen = stream.total_out; 55 | 56 | err = deflateEnd(&stream); 57 | return err; 58 | } 59 | 60 | /* =========================================================================== 61 | */ 62 | int ZEXPORT compress (dest, destLen, source, sourceLen) 63 | Bytef *dest; 64 | uLongf *destLen; 65 | const Bytef *source; 66 | uLong sourceLen; 67 | { 68 | return compress2(dest, destLen, source, sourceLen, Z_DEFAULT_COMPRESSION); 69 | } 70 | 71 | /* =========================================================================== 72 | If the default memLevel or windowBits for deflateInit() is changed, then 73 | this function needs to be updated. 74 | */ 75 | uLong ZEXPORT compressBound (sourceLen) 76 | uLong sourceLen; 77 | { 78 | return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + 11; 79 | } 80 | -------------------------------------------------------------------------------- /c/base64.c: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 1997 Massachusetts Institute of Technology 4 | * 5 | * Permission to use, copy, modify, and distribute this software and 6 | * its documentation for any purpose and without fee is hereby 7 | * granted, provided that both the above copyright notice and this 8 | * permission notice appear in all copies, that both the above 9 | * copyright notice and this permission notice appear in all 10 | * supporting documentation, and that the name of M.I.T. not be used 11 | * in advertising or publicity pertaining to distribution of the 12 | * software without specific, written prior permission. M.I.T. makes 13 | * no representations about the suitability of this software for any 14 | * purpose. It is provided "as is" without express or implied 15 | * warranty. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS 18 | * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, 19 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT 21 | * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF 24 | * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 27 | * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 | * SUCH DAMAGE. 29 | */ 30 | 31 | #include 32 | #include 33 | 34 | /* 35 | * Base64 encoding 36 | */ 37 | char * base64(const char *src) 38 | { 39 | static const char base64[] = 40 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 41 | "abcdefghijklmnopqrstuvwxyz" 42 | "0123456789+/"; 43 | char *str, *dst; 44 | size_t l; 45 | int t, r; 46 | 47 | l = strlen(src); 48 | if ((str = malloc(((l + 2) / 3) * 4 + 1)) == NULL) 49 | return (NULL); 50 | dst = str; 51 | r = 0; 52 | 53 | while (l >= 3) { 54 | t = (src[0] << 16) | (src[1] << 8) | src[2]; 55 | dst[0] = base64[(t >> 18) & 0x3f]; 56 | dst[1] = base64[(t >> 12) & 0x3f]; 57 | dst[2] = base64[(t >> 6) & 0x3f]; 58 | dst[3] = base64[(t >> 0) & 0x3f]; 59 | src += 3; l -= 3; 60 | dst += 4; r += 4; 61 | } 62 | 63 | switch (l) { 64 | case 2: 65 | t = (src[0] << 16) | (src[1] << 8); 66 | dst[0] = base64[(t >> 18) & 0x3f]; 67 | dst[1] = base64[(t >> 12) & 0x3f]; 68 | dst[2] = base64[(t >> 6) & 0x3f]; 69 | dst[3] = '='; 70 | dst += 4; 71 | r += 4; 72 | break; 73 | case 1: 74 | t = src[0] << 16; 75 | dst[0] = base64[(t >> 18) & 0x3f]; 76 | dst[1] = base64[(t >> 12) & 0x3f]; 77 | dst[2] = dst[3] = '='; 78 | dst += 4; 79 | r += 4; 80 | break; 81 | case 0: 82 | break; 83 | } 84 | 85 | *dst = 0; 86 | return (str); 87 | } 88 | 89 | 90 | -------------------------------------------------------------------------------- /c/autotools/sockaddrin6.m4: -------------------------------------------------------------------------------- 1 | dnl * 2 | dnl * Copyright (c) 2001 Motoyuki Kasahara 3 | dnl * 4 | dnl * Redistribution and use in source and binary forms, with or without 5 | dnl * modification, are permitted provided that the following conditions 6 | dnl * are met: 7 | dnl * 1. Redistributions of source code must retain the above copyright 8 | dnl * notice, this list of conditions and the following disclaimer. 9 | dnl * 2. Redistributions in binary form must reproduce the above copyright 10 | dnl * notice, this list of conditions and the following disclaimer in the 11 | dnl * documentation and/or other materials provided with the distribution. 12 | dnl * 3. Neither the name of the project nor the names of its contributors 13 | dnl * may be used to endorse or promote products derived from this software 14 | dnl * without specific prior written permission. 15 | dnl * 16 | dnl * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 17 | dnl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | dnl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 | dnl * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORSBE 20 | dnl * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | dnl * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | dnl * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | dnl * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | dnl * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | dnl * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 | dnl * THE POSSIBILITY OF SUCH DAMAGE. 27 | dnl * 28 | 29 | dnl * 30 | dnl * Check for struct sockaddr_in6 31 | dnl * 32 | AC_DEFUN([X_STRUCT_SOCKADDR_IN6], 33 | [AC_CACHE_CHECK(for struct sockaddr_in6, ac_cv_struct_sockaddr_in6, 34 | [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 35 | #include 36 | #include 37 | #include ]], [[ 38 | struct sockaddr_in6 address; 39 | ]])],[ac_cv_struct_sockaddr_in6=yes],[ac_cv_struct_sockaddr_in6=no])]) 40 | if test "$ac_cv_struct_sockaddr_in6" = yes; then 41 | AC_DEFINE(HAVE_STRUCT_SOCKADDR_IN6, 1, 42 | [Define to 1 if defines `struct sockaddr_in6']) 43 | fi]) 44 | 45 | dnl * 46 | dnl * Check for struct sockaddr_storage 47 | dnl * 48 | AC_DEFUN([X_STRUCT_SOCKADDR_STORAGE], 49 | [AC_CACHE_CHECK(for struct sockaddr_storage, ac_cv_struct_sockaddr_storage, 50 | [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 51 | #include 52 | #include 53 | #include ]], [[ 54 | struct sockaddr_storage address; 55 | ]])],[ac_cv_struct_sockaddr_storage=yes],[ac_cv_struct_sockaddr_storage=no])]) 56 | if test "$ac_cv_struct_sockaddr_storage" = yes; then 57 | AC_DEFINE(HAVE_STRUCT_SOCKADDR_STORAGE, 1, 58 | [Define to 1 if defines `struct sockaddr_storage']) 59 | fi]) 60 | 61 | -------------------------------------------------------------------------------- /website/server-issues.gtml: -------------------------------------------------------------------------------- 1 | #include "header.gtml" 2 | zsync Server Issues 3 | #include "head.gtml" 4 | 5 |

General

6 | 7 |

zsync makes an unusual pattern of requests; because it is retrieving lots of small parts of the target file, it will typically result in more than one hit appearing in the server's log. It could also put slightly more load on the server than a simple download (although it should be much faster and use less bandwidth).

8 |

Of course, if the server operator themselves has created the .zsync file, or has given permission for it, then there is no issue. And in any case, zsync saves bandwidth and doesn't do anything except do selective downloads of bits of files, so it should not cause a problem. Its traffic is not much different to, say, Adobe's PDF viewer that also does partial downloads.

9 |

zsync has been programmed conservatively so that it is unlikely to annoy a webmaster. It even self-imposes a few restrictions to prevent some less-than-ideal traffic patterns that could spook the webmaster. I would just ask that people using zsync exercise the same care in using it.

10 | 11 |

Server Partial Content Support

12 | 13 |

To make use of zsync, there has to be a server holding a full version of the content that supports partial downloading (Range requests). And, reasonably speaking, zsync needs slightly more: support for multiple byte ranges per request; without this, zsync has to make a large number of requests to do any sync where there are a large number of scattered changes. 14 | 15 |

Redirects

16 | 17 |

zsync follows redirects on the .zsync file, and sets suitable referer headers.

18 | 19 |

zsync 0.6 and earlier versions refused to follow redirects on the target file, because, due to the number of requests it makes, it is impractical to revisit the original URL to get another temporary redirect before every request. You need to regenerate the .zsync if the URL of the target changes.

20 | 21 |

zsync 0.6.1 and later do follow redirects on the target file. Note that you can't use temporary redirects to load-balance single sessions of zsync — it treats all redirects as permanent for the duration of the session. It is simply not practical to maintain a pipelined connection to maximise throughput if the individual requests go to different servers (at the request of the server; the client could do it if it was programmed to do so, although it currently does not). zsync won't remember redirects between runs so you can use 302 redirects to load-balance if you have multiple clients (which would be the normal case for zsync's intended use, file distribution).

22 |

Note that you are still recommended to put the correct, direct URL in the .zsync file in most cases; only where you need the redirect for load-balancing is it desirable to put in a URL that is a redirect. But the redirect support is also useful in that it allows servers to be restructured and redirects put in place and existing .zsync files will continue working.

23 | 24 | #include "foot.gtml" 25 | -------------------------------------------------------------------------------- /c/librcksum/rcksum.h: -------------------------------------------------------------------------------- 1 | /* 2 | * rcksum/lib - library for using the rsync algorithm to determine 3 | * which parts of a file you have and which you need. 4 | * Copyright (C) 2004,2005,2009 Colin Phipps 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the Artistic License v2 (see the accompanying 8 | * file COPYING for the full license terms), or, at your option, any later 9 | * version of the same license. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * COPYING file for details. 15 | */ 16 | 17 | /* This is the library interface. Very changeable at this stage. */ 18 | 19 | #include 20 | 21 | struct rcksum_state; 22 | 23 | typedef int zs_blockid; 24 | 25 | struct rsum { 26 | unsigned short a; 27 | unsigned short b; 28 | } __attribute__((packed)); 29 | 30 | #define CHECKSUM_SIZE 16 31 | 32 | struct rcksum_state *rcksum_init(zs_blockid nblocks, size_t blocksize, 33 | int rsum_butes, unsigned int checksum_bytes, 34 | int require_consecutive_matches); 35 | void rcksum_end(struct rcksum_state* z); 36 | 37 | /* These transfer out the filename and handle of the file backing the data retrieved. 38 | * Once you have transferred out the file handle, you can no longer read and write data through librcksum - it has handed it over to you, and can use it no more itself. 39 | * If you transfer out the filename, you are responsible for renaming it to something useful. If you don't transfer out the filename, librcksum will unlink it at rcksum_end. 40 | */ 41 | char* rcksum_filename(struct rcksum_state* z); 42 | int rcksum_filehandle(struct rcksum_state* z); 43 | 44 | void rcksum_add_target_block(struct rcksum_state* z, zs_blockid b, struct rsum r, void* checksum); 45 | 46 | int rcksum_submit_blocks(struct rcksum_state* z, const unsigned char* data, zs_blockid bfrom, zs_blockid bto); 47 | int rcksum_submit_source_data(struct rcksum_state* z, unsigned char* data, size_t len, off_t offset); 48 | int rcksum_submit_source_file(struct rcksum_state* z, FILE* f, int progress); 49 | 50 | /* This reads back in data which is already known. */ 51 | ssize_t rcksum_read_known_data(struct rcksum_state *z, unsigned char *buf, 52 | off_t offset, size_t len); 53 | 54 | /* rcksum_needed_block_ranges tells you what blocks, within the given range, 55 | * are still unknown. It returns a list of block ranges in r[] 56 | * (at most max ranges, so spece for 2*max elements must be there) 57 | * these are half-open ranges, so r[0] <= x < r[1], r[2] <= x < r[3] etc are needed */ 58 | zs_blockid* rcksum_needed_block_ranges(const struct rcksum_state* z, int* num, zs_blockid from, zs_blockid to); 59 | int rcksum_blocks_todo(const struct rcksum_state*); 60 | 61 | /* For preparing rcksum control files - in both cases len is the block size. */ 62 | struct rsum __attribute__((pure)) 63 | rcksum_calc_rsum_block(const unsigned char *data, size_t len); 64 | 65 | void rcksum_calc_checksum(unsigned char *c, const unsigned char* data, size_t len); 66 | 67 | -------------------------------------------------------------------------------- /c/autotools/in6addr.m4: -------------------------------------------------------------------------------- 1 | dnl * 2 | dnl * Copyright (c) 2001 Motoyuki Kasahara 3 | dnl * 4 | dnl * Redistribution and use in source and binary forms, with or without 5 | dnl * modification, are permitted provided that the following conditions 6 | dnl * are met: 7 | dnl * 1. Redistributions of source code must retain the above copyright 8 | dnl * notice, this list of conditions and the following disclaimer. 9 | dnl * 2. Redistributions in binary form must reproduce the above copyright 10 | dnl * notice, this list of conditions and the following disclaimer in the 11 | dnl * documentation and/or other materials provided with the distribution. 12 | dnl * 3. Neither the name of the project nor the names of its contributors 13 | dnl * may be used to endorse or promote products derived from this software 14 | dnl * without specific prior written permission. 15 | dnl * 16 | dnl * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 17 | dnl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | dnl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 | dnl * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORSBE 20 | dnl * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | dnl * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | dnl * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | dnl * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | dnl * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | dnl * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 | dnl * THE POSSIBILITY OF SUCH DAMAGE. 27 | dnl * 28 | 29 | dnl * 30 | dnl * Check for struct in6_addr 31 | dnl * 32 | AC_DEFUN([X_STRUCT_IN6_ADDR], 33 | [AC_CACHE_CHECK(for struct in6_addr, ac_cv_struct_in6_addr, 34 | [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 35 | #include 36 | #include 37 | #include ]], [[ 38 | struct in6_addr address; 39 | ]])],[ac_cv_struct_in6_addr=yes],[ac_cv_struct_in6_addr=no])]) 40 | if test "$ac_cv_struct_in6_addr" = yes; then 41 | AC_DEFINE(HAVE_STRUCT_IN6_ADDR, 1, 42 | [Define to 1 if defines `struct in6_addr']) 43 | fi]) 44 | 45 | dnl * 46 | dnl * Check for in6addr_any. 47 | dnl * 48 | AC_DEFUN([X_DECL_IN6ADDR_ANY], 49 | [AC_REQUIRE([AC_STRUCT_IN6_ADDR]) 50 | if test $ac_cv_struct_in6_addr = no; then 51 | ac_cv_decl_in6addr_any=no 52 | else 53 | AC_CACHE_CHECK(for in6addr_any declaration in netinet/in.h, 54 | ac_cv_decl_in6addr_any, 55 | [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 56 | #include 57 | #include 58 | #include ]], [[ 59 | unsigned char *address; 60 | address = (char *)&in6addr_any; 61 | ]])],[ac_cv_decl_in6addr_any=yes],[ac_cv_decl_in6addr_any=no])]) 62 | if test "$ac_cv_decl_in6addr_any" = yes; then 63 | AC_DEFINE(IN6ADDR_ANY_DECLARED, 1, 64 | [Define to 1 if `in6addr_any' is declared by ]) 65 | fi 66 | fi]) 67 | 68 | dnl * 69 | dnl * Check for in6addr_loopback. 70 | dnl * 71 | AC_DEFUN([X_DECL_IN6ADDR_LOOPBACK], 72 | [AC_REQUIRE([AC_STRUCT_IN6_ADDR]) 73 | if test $ac_cv_struct_in6_addr = no; then 74 | ac_cv_decl_in6addr_loopback=no 75 | else 76 | AC_CACHE_CHECK(for in6addr_loopback declaration in netinet/in.h, 77 | ac_cv_decl_in6addr_loopback, 78 | [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 79 | #include 80 | #include 81 | #include ]], [[ 82 | unsigned char *address; 83 | address = (char *)&in6addr_loopback; 84 | ]])],[ac_cv_decl_in6addr_loopback=yes],[ac_cv_decl_in6addr_loopback=no])]) 85 | if test "$ac_cv_decl_in6addr_loopback" = yes; then 86 | AC_DEFINE(IN6ADDR_LOOPBACK_DECLARED, 1, 87 | [Define to 1 if `in6addr_loopback' is declared by ]) 88 | fi 89 | fi]) 90 | -------------------------------------------------------------------------------- /c/doc/zsync.1: -------------------------------------------------------------------------------- 1 | .TH "zsync" "1" "0.6.3" "Colin Phipps" "File Transfer" 2 | .SH "NAME" 3 | .LP 4 | zsync \- Partial/differential file download client over HTTP 5 | .SH "SYNTAX" 6 | .LP 7 | zsync [ \-u \fIurl\fR ] [ \-i \fIinputfile\fP ] [ \-o \fIoutputfile\fP ] [ { \-s | \-q } ] [ \-k \fIfile\fR.zsync ] [ -A \fIhostname\fP=\fIusername\fR:\fIpassword\fR ] { \fIfilename\fP | \fIurl\fR } 8 | .LP 9 | zsync \-V 10 | .SH "DESCRIPTION" 11 | .LP 12 | Downloads a file over HTTP. zsync uses a control file to determine whether any blocks in the file are already known to the downloader, and only downloads the new blocks. 13 | .LP 14 | Either a filename or a URL can be given on the command line \- this is the path of the \fBcontrol\fR file for the download, which normally has the name of the actual file to downlaod with .zsync appended. (To create this .zsync file you have to have a copy of the target file, so this file should be generated by the person providing the download). 15 | .LP 16 | zsync downloads to your current directory. It looks for any file in the directory of the same name as the file to download. If it finds one, it assumes that this is an earlier or incomplete version of the new file to download, and scans this file for any blocks that it can use to build the target file. (It also looks for a file of the same name with .part appended, so it will automatically find previously interrupted zsync downloads and reuse the data already downloaded. If you know that the local file to use as input has a different name, you must use \fB\-i\fR) 17 | .LP 18 | zsync retrieves the rest of the target file over HTTP. Once the download is finished, the old version (if the new file wants the same name) is moved aside (a .zs\-old extension is appended). The modification time of the file is set to be the same as the remote source file (if specified in the .zsync). 19 | .SH "OPTIONS" 20 | .LP 21 | .TP 22 | \fB\-A\fR \fIhostname\fP=\fIusername\fR:\fIpassword\fR 23 | Specifies a username and password to be used with the given hostname. \fB-A\fR 24 | can be used multiple times (with different hostnames), in cases where e.g. the 25 | .zsync file is on a different server from the download, or there are multiple 26 | download servers (there could be different auth details for different servers - 27 | and zsync never assumes that your password should be sent to a server other 28 | than the one named - otherwise redirects would be dangerous!). 29 | .TP 30 | \fB\-i\fR \fIinputfile\fP 31 | Specifies (extra) input files. \fIinputfile\fP is scanned to identify blocks in common with the target file and zsync uses any blocks found. Can be used multiple times. 32 | .TP 33 | \fB\-k\fR \fIfile\fP.zsync 34 | Indicates that zsync should save the zsync file that it downloads, with the given filename. If that file already exists, then zsync will make a conditional request to the web server, such that it will only download it again if the server's copy is newer. zsync will append .part to the filename for storing it while it is downloading, and will only overwrite the main file once the download is done - and if the download is interrupted, it will resume using the data in the .part file. 35 | .TP 36 | \fB\-o\fR \fIoutputfile\fP 37 | Override the default output file name. 38 | .TP 39 | \fB\-q\fR 40 | Suppress the progress bar, download rate and ETA display. 41 | .TP 42 | \fB\-s\fR 43 | Deprecated synonym for -q. 44 | .TP 45 | \fB\-u\fR \fIurl\fP 46 | This specifies the referring URL. If you have a .zsync file locally (if you 47 | downloaded it separately, with wget, say) and the .zsync file contains a 48 | relative URL, you need to specify where you got the .zsync file from so that 49 | zsync knows which server and path to use for the rest of the download (this is 50 | analogous to adding a to a downloaded web page to make the 51 | links work). 52 | .TP 53 | \fB\-V\fR 54 | Prints the version of zsync. 55 | .SH "FILES" 56 | 57 | .SH "ENVIRONMENT VARIABLES" 58 | .LP 59 | .TP 60 | \fBhttp_proxy\fP 61 | Should be the [http://]hostname:port for your web proxy, if one is required to access the target web server(s). 62 | .SH "EXAMPLES" 63 | .LP 64 | zsync \-i /var/lib/apt/lists/server.debian.org_debian_dists_etch_main_binary-i386_Packages http://zsync.moria.org.uk/s/etch/Packages.zsync 65 | 66 | .SH "AUTHORS" 67 | .LP 68 | Colin Phipps 69 | .SH "SEE ALSO" 70 | .LP 71 | zsyncmake(1) 72 | -------------------------------------------------------------------------------- /c/libzsync/zsync.h: -------------------------------------------------------------------------------- 1 | /* 2 | * zsync - client side rsync over http 3 | * Copyright (C) 2004,2005,2009 Colin Phipps 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the Artistic License v2 (see the accompanying 7 | * file COPYING for the full license terms), or, at your option, any later 8 | * version of the same license. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * COPYING file for details. 14 | */ 15 | 16 | struct zsync_state; 17 | 18 | /* zsync_begin - load a zsync file and return data structure to use for the rest of the process. 19 | */ 20 | struct zsync_state* zsync_begin(FILE* cf); 21 | 22 | /* zsync_hint_decompress - if it returns non-zero, this suggests that 23 | * compressed seed files should be decompressed */ 24 | int zsync_hint_decompress(const struct zsync_state*); 25 | 26 | /* zsync_filename - return the suggested filename from the .zsync file */ 27 | char* zsync_filename(const struct zsync_state*); 28 | /* zsync_mtime - return the suggested mtime from the .zsync file */ 29 | time_t zsync_mtime(const struct zsync_state*); 30 | 31 | /* zsync_rename_file - renames the temporary file used by zsync to the given name. 32 | * You don't "own" the filename until you zsync_end, but you can use this to give zsync a more 33 | * appropriate intermediate filename (in case the user ctrl-c's). 34 | * This is purely a hint; zsync could ignore it. Returns 0 if successful. */ 35 | int zsync_rename_file(struct zsync_state* zs, const char* f); 36 | 37 | /* zsync_status - returns the current state: 38 | * 0 - no relevant local data found yet. 39 | * 1 - some data present 40 | * 2+ - all data downloaded (higher values may be added later to indicate completion 41 | * of checksumming and file handle states) 42 | */ 43 | 44 | int zsync_status(const struct zsync_state* zs); 45 | 46 | /* zsync_progress - returns bytes of the file known so far in *got, 47 | * and the total (roughly, the file length) in *total */ 48 | void zsync_progress(const struct zsync_state* zs, long long* got, long long* total); 49 | 50 | /* zsync_submit_source_file - submit local file data to zsync 51 | */ 52 | int zsync_submit_source_file(struct zsync_state* zs, FILE* f, int progress); 53 | 54 | /* zsync_get_url - returns a URL from which to get needed data. 55 | * Returns NULL on failure, or a array of pointers to URLs. 56 | * Returns the size of the array in *n, 57 | * and the url type (to pass to needed_byte_ranges & begin_receive) 58 | * (the URL pointers are still referenced by the library, and are valid only until zsync_end). 59 | */ 60 | 61 | const char * const * zsync_get_urls(struct zsync_state* zs, int* n, int* t); 62 | 63 | /* zsync_needed_byte_ranges - get the byte ranges needed from a URL. 64 | * Returns the number of ranges in *num, and a malloc'd array (to be freed 65 | * by the caller) of 2*(*num) off_t's which are the starts and ends 66 | * of byte ranges. 67 | */ 68 | 69 | off_t* zsync_needed_byte_ranges(struct zsync_state* zs, int* num, int type); 70 | 71 | /* zsync_complete - set file length and verify checksum if available 72 | * Returns -1 for failure, 1 for success, 0 for unable to verify (e.g. no checksum in the .zsync) */ 73 | int zsync_complete(struct zsync_state* zs); 74 | 75 | /* Clean up and free all resources. The pointer is freed by this call. 76 | * Returns a strdup()d pointer to the name of the file resulting from the process. */ 77 | char* zsync_end(struct zsync_state* zs); 78 | 79 | /* And functions for receiving data on the network */ 80 | struct zsync_receiver; 81 | 82 | /* Begin and end receiving from a particular URL. 83 | * Note that the zsync_receiver stores a reference to the zsync_state, 84 | * and libzsync does not do reference counting, so it is the caller's 85 | * responsibility not to do a zsync_end without doing a zsync_end_receive 86 | * first. 87 | * The url_type is as in the value returned by zsync_get_url. 88 | */ 89 | struct zsync_receiver* zsync_begin_receive(struct zsync_state*zs, int url_type); 90 | void zsync_end_receive(struct zsync_receiver* zr); 91 | 92 | /* Supply data buf of length len received corresponding to offset offset from the URL. 93 | * Returns 0 for success; if not, you should not submit more data. */ 94 | int zsync_receive_data(struct zsync_receiver* zr, const unsigned char* buf, off_t offset, size_t len); 95 | 96 | -------------------------------------------------------------------------------- /website/downloads.gtml: -------------------------------------------------------------------------------- 1 | #include "header.gtml" 2 | zsync Downloads 3 | #include "head.gtml" 4 | 5 |

Download

6 | 7 |

Current Version

8 | 9 |

Download zsync-0.6.3.tar.bz2.

10 | 11 |

To install, it is the usual configure, make, make install sequence — see the INSTALL file for step-by-step instructions. You do not have to install zsync if you are just trying it out, as both zsync and zsyncmake require no data files installed; installation just puts the programs and manual pages into the normal paths on your system.

12 | 13 |

In-development & Source Code History

14 |

zsync history is kept in a git repository. You can get a copy of the history with:

15 | 
git clone https://github.com/cph6/zsync.git
16 | 17 |

Download Details & Old Versions

18 | 19 | 51 | 52 | #include "foot.gtml" 53 | -------------------------------------------------------------------------------- /c/progress.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * zsync - client side rsync over http 4 | * Copyright (C) 2004,2005,2007,2009 Colin Phipps 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the Artistic License v2 (see the accompanying 8 | * file COPYING for the full license terms), or, at your option, any later 9 | * version of the same license. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * COPYING file for details. 15 | */ 16 | 17 | #include "zsglobal.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #ifdef WITH_DMALLOC 25 | # include 26 | #endif 27 | 28 | #include "progress.h" 29 | 30 | #define HISTORY 10 31 | struct progress { 32 | time_t starttime; 33 | struct { 34 | time_t hist_time; 35 | long long dl; 36 | float pcnt; 37 | } history[HISTORY]; 38 | int num_history; 39 | }; 40 | 41 | int no_progress; 42 | 43 | /* progbar(chars, percent) 44 | * (Re)print progress bar with chars out of 20 shown and followed by the given 45 | * percentage done. */ 46 | static void progbar(int j, float pcnt) { 47 | int i; 48 | char buf[21]; 49 | 50 | for (i = 0; i < j && i < 20; i++) 51 | buf[i] = '#'; 52 | for (; i < 20; i++) 53 | buf[i] = '-'; 54 | buf[i] = 0; 55 | printf("\r%s %.1f%%", buf, pcnt); 56 | } 57 | 58 | /* struct progress* = start_progress() 59 | * Returns a progress structure. Caller is responsible for calling 60 | * end_progress() on it later (which will free the memory that it uses). 61 | */ 62 | struct progress* start_progress(void) { 63 | return calloc(1, sizeof(struct progress)); 64 | } 65 | 66 | /* do_progress(progress, percent, total_bytes_retrieved 67 | * Updates the supplied progress structure with the new % done given, and 68 | * recalculates the rolling download rate given the supplied 69 | * total_bytes_retrieved (and the current time) */ 70 | void do_progress(struct progress *p, float pcnt, long long newdl) { 71 | time_t newtime = time(NULL); 72 | if (!p->num_history) 73 | p->starttime = newtime; 74 | else if (p->history[p->num_history-1].hist_time == newtime) 75 | return; 76 | 77 | /* Add to the history, rolling off some old history if needed. */ 78 | if (p->num_history >= HISTORY) { 79 | p->num_history = HISTORY-1; 80 | memmove(p->history, &(p->history[1]), (HISTORY-1)*sizeof(p->history[0])); 81 | } 82 | p->history[p->num_history].hist_time = newtime; 83 | p->history[p->num_history].dl = newdl; 84 | p->history[p->num_history].pcnt = pcnt; 85 | p->num_history++; 86 | 87 | /* Update progress bar displayed */ 88 | progbar(pcnt * (20.0 / 100.0), pcnt); 89 | 90 | /* If we have more than one data point, we can calculate and show rates */ 91 | if (p->num_history > 1) { 92 | int passed = p->history[p->num_history-1].hist_time - p->history[0].hist_time; 93 | float rate = (p->history[p->num_history-1].dl - p->history[0].dl) / (float)passed; 94 | float pcnt_change = (p->history[p->num_history-1].pcnt - p->history[0].pcnt); 95 | int sleft = (100.0f - pcnt) * passed / pcnt_change; 96 | printf(" %.1f kBps ", rate / 1000.0); 97 | if (sleft < 60 * 1000) 98 | printf("%d:%02d ETA ", sleft / 60, sleft % 60); 99 | else 100 | fputs(" ", stdout); 101 | } 102 | fflush(stdout); 103 | } 104 | 105 | /* end_progress(progress, done) 106 | * Do one final update of the progress display (set to 100% if done is set to 107 | * true), and then release the progress data structures. 108 | */ 109 | void end_progress(struct progress *p, int done) { 110 | if (done == 2) 111 | progbar(20, 100.0); 112 | else { 113 | float lastpcnt = p->history[p->num_history-1].pcnt; 114 | progbar(lastpcnt * (20.0 / 100.0), lastpcnt); 115 | } 116 | 117 | { /* For the final display, show the rate for the whole download. */ 118 | float rate = (float)(p->history[p->num_history-1].dl) / 119 | (p->history[p->num_history-1].hist_time - p->starttime + 0.5); 120 | printf(" %.1f kBps ", rate / 1000.0); 121 | } 122 | puts(done == 2 ? "DONE \n" : !done ? "aborted \n" : " \n"); 123 | fflush(stdout); 124 | free(p); 125 | } 126 | -------------------------------------------------------------------------------- /c/zlib/README.zsync: -------------------------------------------------------------------------------- 1 | Local changes to zlib used by zsync 2 | ----------------------------------- 3 | 4 | There are two different modes of operation that zsync supports that 5 | these patches are designed to support: 6 | 7 | Changes to the deflate code: Compressing a file in a way that is optimised for 8 | zsync's block-based rsync algorithm - starting a new zlib block for each 1024 9 | byte (for example) block in the source file. cf 10 | http://zsync.moria.org.uk/paper/ch03s04.html . This is used by makegz.c in the 11 | zsync source. 12 | 13 | Changes to the inflate code: Working with files compressed with the standard 14 | gzip(1). To enable people to get started with zsync, I want it to work with 15 | existing compressed content. To achieve optimal results with standard gzip 16 | files, I made zsync capable of starting decompression in the middle of a block. 17 | In these cases it has to download the block header, then skips forward to the 18 | part of the block that gives it the data that it wants. cf 19 | http://zsync.moria.org.uk/paper/ch03s02.html 20 | 21 | Contrary to some internet discussion, the changes are not related to rsync's 22 | changes nor to rsync compatibility (zsync isn't compatible with rsync - 23 | whatever that would mean - nor do these changes relate in any way to the 24 | rsyncable gzip patch). 25 | 26 | Changes to the deflate code 27 | --------------------------- 28 | 29 | Essentially, I hijacked Z_PARTIAL_FLUSH to mean something new - I want to start 30 | a new zlib block, but unlike Z_PARTIAL_FLUSH I don't need to emit a whole byte 31 | between blocks, so I took that out. Correctly this ought to be implemented by 32 | adding a Z_NEWBLOCKONLY_FLUSH or something like that instead of repurposing an 33 | existing state. 34 | 35 | (If this were the only issue preventing the use of a standard zlib, distros 36 | could change it to use Z_PARTIAL_FLUSH with only a slight loss of compression 37 | efficiency.) 38 | 39 | Changes to the inflate code 40 | --------------------------- 41 | 42 | zsync uses the rsync algorithm to construct a desired file from an 43 | (e.g.) older local version of the file and then downloading any 44 | new/needed blocks from a server; the aim being to minimise the amount of 45 | data downloaded to construct the target file. It supports downloading 46 | those blocks from a gzipped version of the file on the server. If I want 47 | e.g. bytes 4096-8192 of the file from inside the gzipped file, I could 48 | download the whole zlib block (using a map of the compressed file that I 49 | construct beforehand and is downloaded first) containing the range 50 | 4096-8192 (zsync 0.1.0 used this method); but it can do better (fewer 51 | bytes downloaded) than that, by downloading just the block header and 52 | then downloading the bytes within that compressed block that correspond 53 | to bytes 4096-8192 of the contained data. 54 | 55 | To do that, I need: 56 | a) to be able to start inflating at the start of "*any* length/literal/eob code 57 | in any dynamic or fixed block, or at *any* stored byte in a stored block.". 58 | That is what additional function inflate_advance() and the export of 59 | updatewindow() allow me to do. 60 | 61 | b) make a map of the gzip file that lets me know what points I should start 62 | downloading at in order to inflate particular byte ranges of the contained 63 | content. To do this, I can decompress each byte range into a buffer of that 64 | size and then quiz zlib for the position in the stream; but I need to know that 65 | the position in the stream does correspond to the start of a code or the middle 66 | of a stored block (not, e.g., that we have just read a backref and the backref 67 | expands to span the boundary - in that case, I would need to know that position 68 | where the backref started and the lib doesn't give me a way to find that out). 69 | 70 | This is given by inflateSafePoint(), and the implicit guarantee provided by 71 | using my own copy of the library that I know how the library behaves around 72 | internal states and stream position (I need a guarantee that the library won't 73 | read ahead more than it needs to, and I need to access certain member variables 74 | directly to get the bit position in the stream). 75 | 76 | I also removed inflate_fast as I did not want to spend the time working out if 77 | it was compatible with these changes. 78 | 79 | I have started a discussion with one of the zlib maintainers about getting 80 | changes such as I would need put into zlib, but it is too early to know whether 81 | suitable changes will developed or accepted. 82 | 83 | Copyright 84 | --------- 85 | 86 | I place all my modifications to these files in the public domain. 87 | 88 | - Colin Phipps, 2009/12/13 89 | 90 | -------------------------------------------------------------------------------- /c/librcksum/internal.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * rcksum/lib - library for using the rsync algorithm to determine 4 | * which parts of a file you have and which you need. 5 | * Copyright (C) 2004,2005,2009 Colin Phipps 6 | * 7 | * This program is free software; you can redistribute it and/or modify 8 | * it under the terms of the Artistic License v2 (see the accompanying 9 | * file COPYING for the full license terms), or, at your option, any later 10 | * version of the same license. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * COPYING file for details. 16 | */ 17 | 18 | /* Internal data structures to the library. Not to be included by code outside librcksum. */ 19 | 20 | /* Two types of checksum - 21 | * rsum: rolling Adler-style checksum 22 | * checksum: hopefully-collision-resistant MD4 checksum of the block 23 | */ 24 | 25 | struct hash_entry { 26 | struct hash_entry *next; /* next entry with the same rsum */ 27 | struct rsum r; 28 | unsigned char checksum[CHECKSUM_SIZE]; 29 | }; 30 | 31 | /* An rcksum_state contains the set of checksums of the blocks of a target 32 | * file, and is used to apply the rsync algorithm to detect data in common with 33 | * a local file. It essentially contains as rsum and a checksum per block of 34 | * the target file, plus various hash tables to speed up lookups when looping 35 | * over data looking for matching blocks. */ 36 | 37 | struct rcksum_state { 38 | struct rsum r[2]; /* Current rsums */ 39 | 40 | zs_blockid blocks; /* Number of blocks in the target file */ 41 | size_t blocksize; /* And how many bytes per block */ 42 | int blockshift; /* log2(blocksize) */ 43 | unsigned short rsum_a_mask; /* The mask to apply to rsum values before looking up */ 44 | unsigned short rsum_bits; /* # of bits of rsum data in the .zsync for each block */ 45 | unsigned short hash_func_shift; /* Config for the hash function */ 46 | unsigned int checksum_bytes; /* How many bytes of the MD4 checksum are available */ 47 | int seq_matches; 48 | unsigned int context; /* precalculated blocksize * seq_matches */ 49 | 50 | /* These are used by the library. Note, not thread safe. */ 51 | int skip; /* skip forward on next submit_source_data */ 52 | const struct hash_entry *rover; 53 | 54 | /* Internal; hint to rcksum_submit_source_data that it should try matching 55 | * the following block of input data against the block ->next_match. 56 | * next_known is a cached lookup of the id of the next block after that 57 | * that we already have data for. */ 58 | const struct hash_entry *next_match; 59 | zs_blockid next_known; 60 | 61 | /* Hash table for rsync algorithm */ 62 | unsigned int hashmask; 63 | struct hash_entry *blockhashes; 64 | struct hash_entry **rsum_hash; 65 | 66 | /* And a 1-bit per rsum value table to allow fast negative lookups for hash 67 | * values that don't occur in the target file. */ 68 | unsigned char *bithash; 69 | unsigned int bithashmask; 70 | 71 | /* Current state and stats for data collected by algorithm */ 72 | int numranges; 73 | zs_blockid *ranges; 74 | int gotblocks; 75 | struct { 76 | long long hashhit; 77 | int weakhit, stronghit, checksummed; 78 | } stats; 79 | 80 | /* Temp file for output */ 81 | char *filename; 82 | int fd; 83 | }; 84 | 85 | #define BITHASHBITS 3 86 | 87 | /* rcksum_state methods */ 88 | 89 | /* From a hash entry, return the corresponding blockid */ 90 | static inline zs_blockid get_HE_blockid(const struct rcksum_state *z, 91 | const struct hash_entry *e) { 92 | return e - z->blockhashes; 93 | } 94 | 95 | void add_to_ranges(struct rcksum_state *z, zs_blockid n); 96 | int already_got_block(struct rcksum_state *z, zs_blockid n); 97 | zs_blockid next_known_block(struct rcksum_state *rs, zs_blockid x); 98 | 99 | struct hash_entry *calc_hash_entry(void *data, size_t len); 100 | 101 | /* Hash the checksum values for the given hash entry and return the hash value */ 102 | static inline unsigned calc_rhash(const struct rcksum_state *const z, 103 | const struct hash_entry *const e) { 104 | unsigned h = e[0].r.b; 105 | 106 | h ^= ((z->seq_matches > 1) ? e[1].r.b 107 | : e[0].r.a & z->rsum_a_mask) << z->hash_func_shift; 108 | 109 | return h; 110 | } 111 | 112 | int build_hash(struct rcksum_state *z); 113 | void remove_block_from_hash(struct rcksum_state *z, zs_blockid id); 114 | -------------------------------------------------------------------------------- /c/librcksum/state.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * rcksum/lib - library for using the rsync algorithm to determine 4 | * which parts of a file you have and which you need. 5 | * Copyright (C) 2004,2005,2007,2009 Colin Phipps 6 | * 7 | * This program is free software; you can redistribute it and/or modify 8 | * it under the terms of the Artistic License v2 (see the accompanying 9 | * file COPYING for the full license terms), or, at your option, any later 10 | * version of the same license. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * COPYING file for details. 16 | */ 17 | 18 | /* Effectively the constructor and destructor for the rcksum object. 19 | * Also handles the file handles on the temporary store. 20 | */ 21 | 22 | #include "zsglobal.h" 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #ifdef WITH_DMALLOC 29 | # include 30 | #endif 31 | 32 | #include "rcksum.h" 33 | #include "internal.h" 34 | 35 | /* rcksum_init(num_blocks, block_size, rsum_bytes, checksum_bytes, require_consecutive_matches) 36 | * Creates and returns an rcksum_state with the given properties 37 | */ 38 | struct rcksum_state *rcksum_init(zs_blockid nblocks, size_t blocksize, 39 | int rsum_bytes, unsigned int checksum_bytes, 40 | int require_consecutive_matches) { 41 | /* Allocate memory for the object */ 42 | struct rcksum_state *z = malloc(sizeof(struct rcksum_state)); 43 | if (z == NULL) return NULL; 44 | 45 | /* Enter supplied properties. */ 46 | z->blocksize = blocksize; 47 | z->blocks = nblocks; 48 | z->rsum_a_mask = rsum_bytes < 3 ? 0 : rsum_bytes == 3 ? 0xff : 0xffff; 49 | z->rsum_bits = rsum_bytes * 8; 50 | z->checksum_bytes = checksum_bytes; 51 | z->seq_matches = require_consecutive_matches; 52 | 53 | /* require_consecutive_matches is 1 if true; and if true we need 1 block of 54 | * context to do block matching */ 55 | z->context = blocksize * require_consecutive_matches; 56 | 57 | /* Temporary file to hold the target file as we get blocks for it */ 58 | z->filename = strdup("rcksum-XXXXXX"); 59 | 60 | /* Initialise to 0 various state & stats */ 61 | z->gotblocks = 0; 62 | memset(&(z->stats), 0, sizeof(z->stats)); 63 | z->ranges = NULL; 64 | z->numranges = 0; 65 | 66 | /* Hashes for looking up checksums are generated when needed. 67 | * So initially store NULL so we know there's nothing there yet. 68 | */ 69 | z->rsum_hash = NULL; 70 | z->bithash = NULL; 71 | 72 | if (!(z->blocksize & (z->blocksize - 1)) && z->filename != NULL 73 | && z->blocks) { 74 | /* Create temporary file */ 75 | z->fd = mkstemp(z->filename); 76 | if (z->fd == -1) { 77 | perror("open"); 78 | } 79 | else { 80 | { /* Calculate bit-shift for blocksize */ 81 | int i; 82 | for (i = 0; i < 32; i++) 83 | if (z->blocksize == (1u << i)) { 84 | z->blockshift = i; 85 | break; 86 | } 87 | } 88 | 89 | z->blockhashes = 90 | malloc(sizeof(z->blockhashes[0]) * 91 | (z->blocks + z->seq_matches)); 92 | if (z->blockhashes != NULL) 93 | return z; 94 | 95 | /* All below is error handling */ 96 | } 97 | } 98 | free(z->filename); 99 | free(z); 100 | return NULL; 101 | } 102 | 103 | /* rcksum_filename(self) 104 | * Returns temporary filename to caller as malloced string. 105 | * Ownership of the file passes to the caller - the function returns NULL if 106 | * called again, and it is up to the caller to deal with the file. */ 107 | char *rcksum_filename(struct rcksum_state *rs) { 108 | char *p = rs->filename; 109 | rs->filename = NULL; 110 | return p; 111 | } 112 | 113 | /* rcksum_filehandle(self) 114 | * Returns the filehandle for the temporary file. 115 | * Ownership of the handle passes to the caller - the function returns -1 if 116 | * called again, and it is up to the caller to close it. */ 117 | int rcksum_filehandle(struct rcksum_state *rs) { 118 | int h = rs->fd; 119 | rs->fd = -1; 120 | return h; 121 | } 122 | 123 | /* rcksum_end - destructor */ 124 | void rcksum_end(struct rcksum_state *z) { 125 | /* Free temporary file resources */ 126 | if (z->fd != -1) 127 | close(z->fd); 128 | if (z->filename) { 129 | unlink(z->filename); 130 | free(z->filename); 131 | } 132 | 133 | /* Free other allocated memory */ 134 | free(z->rsum_hash); 135 | free(z->blockhashes); 136 | free(z->bithash); 137 | free(z->ranges); // Should be NULL already 138 | #ifdef DEBUG 139 | fprintf(stderr, "hashhit %lld, weakhit %d, checksummed %d, stronghit %d\n", 140 | z->stats.hashhit, z->stats.weakhit, z->stats.checksummed, 141 | z->stats.stronghit); 142 | #endif 143 | free(z); 144 | } 145 | -------------------------------------------------------------------------------- /website/index.gtml: -------------------------------------------------------------------------------- 1 | #include "header.gtml" 2 | zsync 3 | #include "head.gtml" 4 | 5 |

Overview

6 | 7 |

zsync is a file transfer program. It allows you to download a file from a remote server, where you have a copy of an older version of the file on your computer already. zsync downloads only the new parts of the file. It uses the same algorithm as rsync. However, where rsync is designed for synchronising data from one computer to another within an organisation, zsync is designed for file distribution, with one file on a server to be distributed to thousands of downloaders. zsync requires no special server software — just a web server to host the files — and imposes no extra load on the server, making it ideal for large scale file distribution.

8 | 9 |

zsync is open source, distributed under version 2 of the Artistic License. Feedback, bugs reports and patches are welcome.

10 | 11 |

Advantages of zsync

12 | 13 |

zsync fills a gap in the technology available for large-scale file distribution. Three key points explain why zsync provides a genuinely new technique for file distribution:

14 |
    15 |
  • Client-side rsync — zsync uses the rsync algorithm, but runs it on the client side, thus avoiding the high server load associated with rsync.
    • 16 |
  • Rsync over HTTP — zsync provides transfers that are nearly as efficient as rsync -z or cvsup, without the need to run a special server application. All that is needed is an HTTP/1.1-compliant web server. So it works through firewalls and on shared hosting accounts, and gives less security worries.
    • 17 |
  • Handling for compressed files — rsync is ineffective on compressed files, unless they are compressed with a patched version of gzip. zsync has special handling for gzipped files, which enables update transfers of files which are distributed in compressed form.
    • 18 |
19 | 20 |

The special handling of gzip files is, as far as I know, entirely new and unique to zsync.

21 | 22 |

I have put up technical papers. There is a current paper, which reflects zsync's latest development, and which I update with new figures as I run new tests and comparisons; there are also older snapshots of the technical paper, included for reference and citation.

23 | 24 |

Practical Engineering

25 | 26 |

zsync is only useful if people offer zsync downloads. I have made a deliberate effort to make zsync simple and reliable in use, and easy to deploy.

27 | 28 |
    29 |
  • Single meta-file — zsync downloads are offered by building a .zsync file, which contains the meta-data needed by zsync. This file contains the precalculated checksums for the rsync algorithm; it is generated on the server, once, and is then used by any number of downloaders.
    • 30 |
  • Easy file format — .zsync files have a simple text header, so they can be edited with any good editor to change details like the download URL.
    • 31 |
  • Flexible use — The .zsync contains either a relative URL to the full content of the file, so you can either generate and offer it on the same server as hosts the full download (and so it can be distributed like any other file onto mirror sites). Or it can contains one or more absolute URLs (the client will try multiple URLs if necessary to find the content), meaning the .zsync can be generated and offered by a third party, while still pointing to the original distribution site for the content.
    • 32 |
33 | 34 |

35 | zsync can only be used to download files for which a .zsync file has been generated by someone who already has a copy of the file. So it requires that the download distributor, or third party, generates a .zsync file for the download. 36 |

37 | 38 |

If you are a server operator looking to use zsync for file distribution, see the server page, which gives some guidance on how to create and offer .zsync files.

39 | 40 |

Current Status

41 | 42 |

zsync was originally released in 2004, and has seen few changes in the last 15 years. While it never reached a v1.0 release, the program is stable and has been used by many projects for file distribution.

43 | 44 |

I am interested in hearing feedback about zsync. Bug reports are welcome. At this time I am not planning to make any large feature additions to zsync in its current form. Note in particular that:

45 | 46 |
    47 |
  • Multiple file support is not planned to be added. From the number of feature requests that I received for this, it is clear that there is or was a gap in the market for a directory synchronisation tool without the need for specialist server software like rsync; but this is not zsync's target.
    • 48 |
  • HTTPS support. Yes, I know that the internet has moved on and doing stuff over unencrypted HTTP looks old fashioned now. In particular the authentication support in zsync makes little sense without HTTPS support. But zsync's current implementation is not suited to adding HTTPS support. Really the HTTP layer in the program needs to be rewritten from hand-rolled to using a library. So HTTPS support is likely to happen only if and when I feel like rewriting a larger part of the client program.
    • 49 |
50 | 51 | #include "foot.gtml" 52 | -------------------------------------------------------------------------------- /website/server.gtml: -------------------------------------------------------------------------------- 1 | #include "header.gtml" 2 | zsync Server-Side 3 | #include "head.gtml" 4 | 5 |

Making zsync Downloads

6 | 7 |

Note that zsync only (currently) supports single-file downloads. I am currently of the opinion that this is sufficient: it is a file distribution method, and, like .tar.gz, it is up to the downloader to worry about extraction, permisions, etc. zsync is not competing with rsync for synchronising directory trees between machines and over shell accounts.

8 | 9 |

There are three cases:

10 |
    11 |
  1. The file to download is already gzipped, and you don't want to recompress it.
    2. 12 |
  2. The content of the file to download is already compressed or not worth compressing further (e.g. deb/rpm packages, ISOs of Linux/BSD distributions, mpegs etc).
    3. 13 |
  3. The file should be compressed for transfer.
    4. 14 |
15 | 16 |

Already Gzipped

17 |

Simply run zsyncmake on the target file. zsyncmake will automatically look inside the compressed file and produce a zsync for the content (this avoids the problem that rsync has, that it does not normally work with compressed files).

18 |

Use the -C option if you want the client to receive the uncompressed data, or if you want the download to work for versions of zsync before 0.3.1. Without -C, zsyncmake will include information to cause the client to compress the content again after it receives it — if you must have the client receive a .tar.gz identical to the original, specify -e.

19 | 20 |

Not Compressible & Not Gzipped

21 |

If the content of a file is compressed, but not in a simple gzip format, bear in mind that zsync may not be effective. Each compressed stream can typically only be efficiently updated via the rsync method if it is either completely unchanged, or the compression has been made rsync-friendly (with, for example, gzip --rsync).

22 |

So, for example, zsync is useless for individual Debian or RPM package files, and is useless for bzip2 files. But it might still be useful for an ISO containing lots of such files, if, say, some packages have changed but some are completely unchanged.

23 |

Just run zsyncmake over the file. It should choose a suitable block size for you.

24 | 25 |

To Be Compressed

26 |

You could just gzip the file first, but it is more efficient to use zsyncmake -z. This causes zsyncmake to build a gzip file for you; it constructs it in such a way that the zsync client will have less to download. It is this approach which gives transfer efficiencies approaching that of rsync -z. 27 | If you have control over the file being downloaded, it may even be worth decompressing it and recompressing with zsyncmake, to get that extra efficiency.

28 | 29 |

Distribution

30 |

The other consideration is how people will access the zsync download. Remember that people have to have access to both the .zsync meta-file, and the full download that it describes.

31 |

The server hosting the full download must support HTTP Range: requests (almost all do). It should support HTTP multipart/byteranges responses, HTTP/1.1 and pipelining. Currently it must not use Transfer-Encoding: chunked (yes, I know that makes zsync non-compliant), and should not be generating the content dynamically.

32 |

If you are hosting the download yourself, then simply put the .zsync in the same directory on the webserver as the download (this is the setup assumed by default by zsyncmake). zsyncmake inserts a relative URL by default, so this will work even if your download directory is then mirrored to other web servers. Remember that if you use zsyncmake -z, you must put the .gz file made by it on the server as well as the .zsync. If you want the .zsync to be in a different directory, specify the relative URL to the full download (for example, if the .zsync will go in a zsync/ subdirectory, specify -u ../orig_filename.tar.gz).

33 |

If the main download and the .zsync are on different servers, specify the full URL to the main download with -u. If you have a set of mirror sites, if you had only a single absolute URL in the .zsync then you would lose the benefit of the mirrors; so you can specify multiple -u options to provide multiple URLs, and the zsync client will choose between them (currently randomely, but it may change later to choose more intelligently).

34 |

You can even offer a .zsync for a download provided/hosted by someone else. You have to download the file yourself, and generate a .zsync from it. Specify the original URL with -u, and then put the .zsync on your own webserver.

35 | 36 |

Other Considerations

37 | 38 |

Serve .zsync files with a MIME type of application/x-zsync if possible. The zsync client does not need this, though. For example, for Apache, include

39 | 
40 | application/x-zsync             zsync
41 |
42 |

in your mime.types.

43 |

Set suitable content expiry headers on .zsync and other downloads, to help web proxies work out how long they can cache it. zsync itself always goes back to the server, although with the -k option it will use If-Modified-Since headers to avoid redownloading the .zsync unnecessarily.

44 |

Do not point the full download at content which is negotiated: it must always be exactly the same as the file supplied to (or generated by, with -z) zsyncmake.

45 |

See also the page on client/server issues, which contains useful info for webmasters and also explains errors that could be seen by users.

46 | 47 | #include "foot.gtml" 48 | -------------------------------------------------------------------------------- /c/url.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * zsync - client side rsync over http 4 | * Copyright (C) 2004,2005,2009 Colin Phipps 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the Artistic License v2 (see the accompanying 8 | * file COPYING for the full license terms), or, at your option, any later 9 | * version of the same license. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * COPYING file for details. 15 | */ 16 | 17 | /* Very crude URL parsing module */ 18 | 19 | #include "zsglobal.h" 20 | 21 | #include 22 | #include 23 | 24 | #ifdef WITH_DMALLOC 25 | # include 26 | #endif 27 | 28 | #include "url.h" 29 | 30 | const char http_scheme[] = { "http://" }; 31 | 32 | /* path_str = get_http_host_port(url_str, &host, host_len, &port_str) 33 | * For the given url_str, returns: 34 | * the hostname, in host[] array provided by caller (up to length host_len) 35 | * the port (if any) as a malloced string, pointer to this stored in *port_str 36 | * return value is the path part of the URL, a malloced string. 37 | * Or return value NULL on failure (host and port could have been written to). 38 | */ 39 | char *get_http_host_port(const char *url, char *hostn, int hnlen, char **port) { 40 | char *p, *q; 41 | 42 | /* Check it's HTTP */ 43 | if (memcmp(url, http_scheme, strlen(http_scheme))) 44 | return NULL; 45 | 46 | q = url + strlen(http_scheme); 47 | 48 | p = strchr(q, ':'); 49 | if (p) { /* if : is after the first /, we have looked too far ahead */ 50 | char *r = strchr(q, '/'); 51 | if (r && r < p) 52 | p = NULL; 53 | } 54 | if (!p) { 55 | *port = strdup("http"); 56 | p = strchr(q, '/'); 57 | } 58 | 59 | if (!p) 60 | return NULL; 61 | 62 | if (p - q < hnlen - 1) { 63 | memcpy(hostn, q, p - q); 64 | hostn[p - q] = 0; 65 | } 66 | 67 | if (*p == ':') { 68 | size_t l; 69 | q = p; 70 | p = strchr(p, '/'); 71 | l = p ? (size_t) (p - q - 1) : strlen(q) - 1; 72 | *port = malloc(l + 1); 73 | if (!*port) 74 | return NULL; 75 | memcpy(*port, q + 1, l); 76 | (*port)[l] = 0; 77 | if (!p) 78 | p = strdup("/"); 79 | } 80 | return p; 81 | } 82 | 83 | /* abs_url_str = make_url_absolute(base_str, url_str) 84 | * Returns an absolute version of url_str relative to base_str, as a malloced 85 | * string. Or NULL on error. */ 86 | char * __attribute__ ((pure)) make_url_absolute(const char *base, const char *url) { 87 | if (is_url_absolute(url)) 88 | return strdup(url); 89 | 90 | /* Otherwise, we'll need a base URL to get the scheme and host */ 91 | if (!base) 92 | return NULL; 93 | 94 | /* Next, is it a full-path URL? */ 95 | if (*url == '/') { 96 | size_t l; 97 | char *newurl; 98 | 99 | /* Find the end of the scheme of the base URL, then the end of the 100 | * hostname[:port]/ */ 101 | const char *p = strchr(base, ':'); 102 | if (!p || p[1] != '/' || p[2] != '/') 103 | return NULL; 104 | p = strchr(p + 3, '/'); 105 | if (!p) /* We have a http://hostname URL (no trailing / or path) */ 106 | l = strlen(base); 107 | else /* http://hostname/... , we want just http://hostname */ 108 | l = p - base; 109 | 110 | /* assert */ 111 | if (base[l] != '/' && base[l] != '\0') 112 | return NULL; 113 | 114 | /* Append full path to the base http://hostname */ 115 | newurl = malloc(l + strlen(url) + 1); 116 | memcpy(newurl, base, l); 117 | strcpy(newurl + l, url); 118 | return newurl; 119 | } 120 | 121 | /* No leading / or scheme - relative path */ 122 | { 123 | int l; 124 | char *newurl; 125 | 126 | /* Find the end of the path part of the base URL */ 127 | const char *q; 128 | const char *p = strchr(base, '?'); 129 | if (!p) 130 | p = strchr(base, '#'); 131 | if (!p) 132 | p = base + strlen(base); 133 | 134 | /* Find the last / in the path part */ 135 | for (q = p; q > base && *q != '/'; q--); 136 | if (*q != '/') 137 | return NULL; 138 | 139 | /* Take the base URL up to and including the last /, and append the relative URL */ 140 | l = q - base + 1; 141 | newurl = malloc(l + strlen(url) + 1); 142 | memcpy(newurl, base, l); 143 | strcpy(newurl + l, url); 144 | return newurl; 145 | } 146 | } 147 | 148 | /* int n = is_url_absolute(url) 149 | * Returns 0 if the supplied string is not an absolute URL. 150 | * Returns the number of characters in the URL scheme if it is. 151 | */ 152 | static const char special[] = { ":/?" }; 153 | 154 | int __attribute__ ((pure)) is_url_absolute(const char *url) { 155 | /* find end of first no-special-URL-characters part of the string */ 156 | int n = strcspn(url, special); 157 | 158 | /* If the first special character is a :, the start is a URL scheme */ 159 | if (n > 0 && url[n] == ':') 160 | return n; 161 | 162 | /* otherwise, it's a full path or relative path URL, or just a local file 163 | * path (caller knows the context) */ 164 | return 0; 165 | } 166 | -------------------------------------------------------------------------------- /c/makegz.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * zsync - client side rsync over http 4 | * Copyright (C) 2004,2005,2007,2009 Colin Phipps 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the Artistic License v2 (see the accompanying 8 | * file COPYING for the full license terms), or, at your option, any later 9 | * version of the same license. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * COPYING file for details. 15 | */ 16 | 17 | #include "zsglobal.h" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "zlib/zlib.h" 24 | 25 | #include 26 | #include 27 | 28 | #ifdef WITH_DMALLOC 29 | # include 30 | #endif 31 | 32 | /* fputlong(filehandle, long) 33 | * Writes a 32bit int as raw bytes in little-endian to the given filehandle. 34 | * Returns 0 if successful; otherwise ferror(filehandle) to see the error */ 35 | static int fputlong(FILE * f, unsigned long x) { 36 | int n; 37 | for (n = 0; n < 4; n++) { 38 | if (fputc((int)(x & 0xff), f) == EOF) 39 | return -1; 40 | x >>= 8; 41 | } 42 | return 0; 43 | } 44 | 45 | /* time = get_mtime(filehandle) 46 | * Get the mtime of a file from an open filehandle; or 0 if unavailable */ 47 | time_t get_mtime(FILE * f) { 48 | struct stat s; 49 | 50 | if (fstat(fileno(f), &s) == 0) 51 | return s.st_mtime; 52 | else 53 | return 0; 54 | } 55 | 56 | /* filehandle = optimal_gzip(filehandle, out_filename_str, blocksize) 57 | * Constructs a compressed version of the data in the file referenced by the 58 | * supplied filehandle; this is saved in a file with the filename supplied as 59 | * the second parameter. The compressed version is optimised for the zsync 60 | * algorithm with the supplied blocksize. The function returns a handle to the 61 | * compressed file (opened for r+w but rewound to the start of the file, 62 | * ready for reading. 63 | */ 64 | 65 | /* Algorithm: simple really. We construct a gzip (have to write header with 66 | * mtime, footer with length and crc; we do a standard zlib compress on the 67 | * content _except_ that we supply one block (of the size that zsync will be 68 | * using) and a Z_PARTIAL_FLUSH at the end of each one, so zlib breaks the 69 | * compression runs at exactly the places zsync will start/stop retrieving data. 70 | */ 71 | FILE *optimal_gzip(FILE * ffin, const char *fout, size_t blocksize) { 72 | time_t mtime = get_mtime(ffin); 73 | 74 | /* Open output file (for writing, but also reading so we can return the 75 | * handle for reading by the caller. */ 76 | FILE *ffout = fopen(fout, "wb+"); 77 | if (!ffout) { 78 | perror("open"); 79 | return NULL; 80 | } 81 | 82 | /* Write gzip header */ 83 | if (fwrite("\x1f\x8b\x08\x00", 4, 1, ffout) != 1) { 84 | perror("write"); 85 | return NULL; 86 | } 87 | if (fputlong(ffout, mtime) == -1) { 88 | perror("write"); 89 | return NULL; 90 | } 91 | if (fwrite("\x00\x03", 2, 1, ffout) != 1) { 92 | perror("write"); 93 | return NULL; 94 | } 95 | 96 | { /* Now write compressed content */ 97 | z_stream zs; 98 | unsigned char *inbuf = malloc(blocksize); 99 | unsigned char *outbuf = malloc(blocksize + 500); 100 | int err, r; 101 | unsigned long crc = crc32(0L, Z_NULL, 0); 102 | 103 | /* Set up zlib object */ 104 | zs.zalloc = Z_NULL; 105 | zs.zfree = Z_NULL; 106 | zs.opaque = NULL; 107 | zs.total_in = 0; 108 | zs.total_out = 0; 109 | 110 | /* windowBits is passed < 0 to suppress zlib header */ 111 | err = deflateInit2(&zs, 9, 112 | Z_DEFLATED, -MAX_WBITS, 8, Z_DEFAULT_STRATEGY); 113 | 114 | /* Until end of file or error */ 115 | for (r = 1; r > 0;) { 116 | r = fread(inbuf, 1, blocksize, ffin); 117 | if (r < 0) 118 | break; 119 | 120 | /* Maintain running crc (for footer) */ 121 | crc = crc32(crc, inbuf, r); 122 | 123 | /* Set up compressor for this block */ 124 | zs.next_in = inbuf; 125 | zs.avail_in = r; 126 | zs.next_out = outbuf; 127 | zs.avail_out = blocksize + 500; 128 | 129 | /* Compress with partial flush at the end of this block 130 | * unless EOF, in which case finish */ 131 | err = deflate(&zs, r ? Z_PARTIAL_FLUSH : Z_FINISH); 132 | switch (err) { 133 | case Z_STREAM_END: 134 | case Z_OK: 135 | { 136 | size_t w = zs.next_out - outbuf; 137 | 138 | if (w != fwrite(outbuf, 1, w, ffout)) { 139 | perror("write"); 140 | r = -1; 141 | } 142 | } 143 | break; 144 | default: 145 | fprintf(stderr, "zlib error: %s (%d)\n", zs.msg, err); 146 | r = -1; 147 | } 148 | } 149 | 150 | /* Write gzip footer */ 151 | if (fputlong(ffout, crc) == -1) { 152 | perror("write"); 153 | return NULL; 154 | } 155 | if (fputlong(ffout, zs.total_in) == -1) { 156 | perror("write"); 157 | return NULL; 158 | } 159 | 160 | /* Clean up */ 161 | fflush(ffout); 162 | free(outbuf); 163 | free(inbuf); 164 | if (fclose(ffin) != 0 || r != 0) { 165 | fclose(ffout); 166 | return NULL; 167 | } 168 | } 169 | 170 | /* Return rewound handle on the compressed data to the caller */ 171 | rewind(ffout); 172 | return ffout; 173 | } 174 | -------------------------------------------------------------------------------- /c/getaddrinfo.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2001, 02 Motoyuki Kasahara 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 1. Redistributions of source code must retain the above copyright 8 | * notice, this list of conditions and the following disclaimer. 9 | * 2. Redistributions in binary form must reproduce the above copyright 10 | * notice, this list of conditions and the following disclaimer in the 11 | * documentation and/or other materials provided with the distribution. 12 | * 3. Neither the name of the project nor the names of its contributors 13 | * may be used to endorse or promote products derived from this software 14 | * without specific prior written permission. 15 | * 16 | * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 17 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 20 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 | * SUCH DAMAGE. 27 | */ 28 | 29 | #ifndef GETADDRINFO_H 30 | #define GETADDRINFO_H 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | /********************************************************************/ 37 | /* 38 | * Undefine all the macros. 39 | * might defines some of them. 40 | */ 41 | #ifdef EAI_ADDRFAMILY 42 | #undef EAI_ADDRFAMILY 43 | #endif 44 | #ifdef EAI_AGAIN 45 | #undef EAI_AGAIN 46 | #endif 47 | #ifdef EAI_BADFLAGS 48 | #undef EAI_BADFLAGS 49 | #endif 50 | #ifdef EAI_FAIL 51 | #undef EAI_FAIL 52 | #endif 53 | #ifdef EAI_FAMILY 54 | #undef EAI_FAMILY 55 | #endif 56 | #ifdef EAI_MEMORY 57 | #undef EAI_MEMORY 58 | #endif 59 | #ifdef EAI_NONAME 60 | #undef EAI_NONAME 61 | #endif 62 | #ifdef EAI_OVERFLOW 63 | #undef EAI_OVERFLOW 64 | #endif 65 | #ifdef EAI_SERVICE 66 | #undef EAI_SERVICE 67 | #endif 68 | #ifdef EAI_SOCKTYPE 69 | #undef EAI_SOCKTYPE 70 | #endif 71 | #ifdef EAI_SYSTEM 72 | #undef EAI_SYSTEM 73 | #endif 74 | 75 | #ifdef AI_PASSIVE 76 | #undef AI_PASSIVE 77 | #endif 78 | #ifdef AI_CANONNAME 79 | #undef AI_CANONNAME 80 | #endif 81 | #ifdef AI_NUMERICHOST 82 | #undef AI_NUMERICHOST 83 | #endif 84 | #ifdef AI_NUMERICSERV 85 | #undef AI_NUMERICSERV 86 | #endif 87 | #ifdef AI_V4MAPPED 88 | #undef AI_V4MAPPED 89 | #endif 90 | #ifdef AI_ALL 91 | #undef AI_ALL 92 | #endif 93 | #ifdef AI_ADDRCONFIG 94 | #undef AI_ADDRCONFIG 95 | #endif 96 | #ifdef AI_DEFAULT 97 | #undef AI_DEFAULT 98 | #endif 99 | 100 | #ifdef NI_NOFQDN 101 | #undef NI_NOFQDN 102 | #endif 103 | #ifdef NI_NUMERICHOST 104 | #undef NI_NUMERICHOST 105 | #endif 106 | #ifdef NI_NAMEREQD 107 | #undef NI_NAMEREQD 108 | #endif 109 | #ifdef NI_NUMERICSERV 110 | #undef NI_NUMERICSERV 111 | #endif 112 | #ifdef NI_NUMERICSCOPE 113 | #undef NI_NUMERICSCOPE 114 | #endif 115 | 116 | #ifdef NI_DGRAM 117 | #undef NI_DGRAM 118 | #endif 119 | #ifdef NI_MAXHOST 120 | #undef NI_MAXHOST 121 | #endif 122 | #ifdef NI_MAXSERV 123 | #undef NI_MAXSERV 124 | #endif 125 | 126 | /* 127 | * Fake struct and function names. 128 | * might declares all or some of them. 129 | */ 130 | #if defined(HAVE_GETADDRINFO) || defined(HAVE_GETNAMEINFO) 131 | #define addrinfo my_addrinfo 132 | #define gai_strerror my_gai_strerror 133 | #define freeaddrinfo my_freeaddrinfo 134 | #define getaddrinfo my_getaddrinfo 135 | #define getnameinfo my_getnameinfo 136 | #endif 137 | 138 | /********************************************************************/ 139 | /* 140 | * Error codes. 141 | */ 142 | #define EAI_ADDRFAMILY 1 143 | #define EAI_AGAIN 2 144 | #define EAI_BADFLAGS 3 145 | #define EAI_FAIL 4 146 | #define EAI_FAMILY 5 147 | #define EAI_MEMORY 6 148 | #define EAI_NONAME 7 149 | #define EAI_OVERFLOW 8 150 | #define EAI_SERVICE 9 151 | #define EAI_SOCKTYPE 10 152 | #define EAI_SYSTEM 11 153 | 154 | /* 155 | * Flags for getaddrinfo(). 156 | */ 157 | #define AI_ADDRCONFIG 0x0001 158 | #define AI_ALL 0x0002 159 | #define AI_CANONNAME 0x0004 160 | #define AI_NUMERICHOST 0x0008 161 | #define AI_NUMERICSERV 0x0010 162 | #define AI_PASSIVE 0x0020 163 | #define AI_V4MAPPED 0x0040 164 | #define AI_DEFAULT (AI_V4MAPPED | AI_ADDRCONFIG) 165 | 166 | /* 167 | * Flags for getnameinfo(). 168 | */ 169 | #define NI_DGRAM 0x0001 170 | #define NI_NAMEREQD 0x0002 171 | #define NI_NOFQDN 0x0004 172 | #define NI_NUMERICHOST 0x0008 173 | #define NI_NUMERICSCOPE 0x0010 174 | #define NI_NUMERICSERV 0x0020 175 | 176 | /* 177 | * Maximum length of FQDN and servie name for getnameinfo(). 178 | */ 179 | #define NI_MAXHOST 1025 180 | #define NI_MAXSERV 32 181 | 182 | /* 183 | * Address families and Protocol families. 184 | */ 185 | #ifndef AF_UNSPEC 186 | #define AF_UNSPEC AF_INET 187 | #endif 188 | #ifndef PF_UNSPEC 189 | #define PF_UNSPEC PF_INET 190 | #endif 191 | 192 | /* 193 | * struct addrinfo. 194 | */ 195 | struct addrinfo { 196 | int ai_flags; 197 | int ai_family; 198 | int ai_socktype; 199 | int ai_protocol; 200 | socklen_t ai_addrlen; 201 | char *ai_canonname; 202 | struct sockaddr *ai_addr; 203 | struct addrinfo *ai_next; 204 | }; 205 | 206 | /* 207 | * Functions. 208 | */ 209 | #ifdef __STDC__ 210 | const char *gai_strerror(int); 211 | void freeaddrinfo(struct addrinfo *); 212 | int getaddrinfo(const char *, const char *, const struct addrinfo *, 213 | struct addrinfo **); 214 | int getnameinfo(const struct sockaddr *, socklen_t, char *, 215 | socklen_t, char *, socklen_t, int); 216 | #else 217 | const char *gai_strerror(); 218 | void freeaddrinfo(); 219 | int getaddrinfo(); 220 | int getnameinfo(); 221 | #endif 222 | 223 | #endif /* not GETADDRINFO_H */ 224 | -------------------------------------------------------------------------------- /c/zlib/README: -------------------------------------------------------------------------------- 1 | ZLIB DATA COMPRESSION LIBRARY 2 | 3 | zlib 1.2.1.1 is a general purpose data compression library. All the code is 4 | thread safe. The data format used by the zlib library is described by RFCs 5 | (Request for Comments) 1950 to 1952 in the files 6 | http://www.ietf.org/rfc/rfc1950.txt (zlib format), rfc1951.txt (deflate format) 7 | and rfc1952.txt (gzip format). These documents are also available in other 8 | formats from ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html 9 | 10 | All functions of the compression library are documented in the file zlib.h 11 | (volunteer to write man pages welcome, contact zlib@gzip.org). A usage example 12 | of the library is given in the file example.c which also tests that the library 13 | is working correctly. Another example is given in the file minigzip.c. The 14 | compression library itself is composed of all source files except example.c and 15 | minigzip.c. 16 | 17 | To compile all files and run the test program, follow the instructions given at 18 | the top of Makefile. In short "make test; make install" should work for most 19 | machines. For Unix: "./configure; make test; make install" For MSDOS, use one 20 | of the special makefiles such as Makefile.msc. For VMS, use Make_vms.com or 21 | descrip.mms. 22 | 23 | Questions about zlib should be sent to , or to Gilles Vollant 24 | for the Windows DLL version. The zlib home page is 25 | http://www.zlib.org or http://www.gzip.org/zlib/ Before reporting a problem, 26 | please check this site to verify that you have the latest version of zlib; 27 | otherwise get the latest version and check whether the problem still exists or 28 | not. 29 | 30 | PLEASE read the zlib FAQ http://www.gzip.org/zlib/zlib_faq.html before asking 31 | for help. 32 | 33 | Mark Nelson wrote an article about zlib for the Jan. 1997 34 | issue of Dr. Dobb's Journal; a copy of the article is available in 35 | http://dogma.net/markn/articles/zlibtool/zlibtool.htm 36 | 37 | The changes made in version 1.2.1.1 are documented in the file ChangeLog. 38 | 39 | Unsupported third party contributions are provided in directory "contrib". 40 | 41 | A Java implementation of zlib is available in the Java Development Kit 42 | http://java.sun.com/j2se/1.4.2/docs/api/java/util/zip/package-summary.html 43 | See the zlib home page http://www.zlib.org for details. 44 | 45 | A Perl interface to zlib written by Paul Marquess is in the 46 | CPAN (Comprehensive Perl Archive Network) sites 47 | http://www.cpan.org/modules/by-module/Compress/ 48 | 49 | A Python interface to zlib written by A.M. Kuchling is 50 | available in Python 1.5 and later versions, see 51 | http://www.python.org/doc/lib/module-zlib.html 52 | 53 | A zlib binding for TCL written by Andreas Kupries is 54 | availlable at http://www.oche.de/~akupries/soft/trf/trf_zip.html 55 | 56 | An experimental package to read and write files in .zip format, written on top 57 | of zlib by Gilles Vollant , is available in the 58 | contrib/minizip directory of zlib. 59 | 60 | 61 | Notes for some targets: 62 | 63 | - For Windows DLL versions, please see win32/DLL_FAQ.txt 64 | 65 | - For 64-bit Irix, deflate.c must be compiled without any optimization. With 66 | -O, one libpng test fails. The test works in 32 bit mode (with the -n32 67 | compiler flag). The compiler bug has been reported to SGI. 68 | 69 | - zlib doesn't work with gcc 2.6.3 on a DEC 3000/300LX under OSF/1 2.1 it works 70 | when compiled with cc. 71 | 72 | - On Digital Unix 4.0D (formely OSF/1) on AlphaServer, the cc option -std1 is 73 | necessary to get gzprintf working correctly. This is done by configure. 74 | 75 | - zlib doesn't work on HP-UX 9.05 with some versions of /bin/cc. It works with 76 | other compilers. Use "make test" to check your compiler. 77 | 78 | - gzdopen is not supported on RISCOS, BEOS and by some Mac compilers. 79 | 80 | - For PalmOs, see http://palmzlib.sourceforge.net/ 81 | 82 | - When building a shared, i.e. dynamic library on Mac OS X, the library must be 83 | installed before testing (do "make install" before "make test"), since the 84 | library location is specified in the library. 85 | 86 | 87 | Acknowledgments: 88 | 89 | The deflate format used by zlib was defined by Phil Katz. The deflate 90 | and zlib specifications were written by L. Peter Deutsch. Thanks to all the 91 | people who reported problems and suggested various improvements in zlib; 92 | they are too numerous to cite here. 93 | 94 | Copyright notice: 95 | 96 | (C) 1995-2003 Jean-loup Gailly and Mark Adler 97 | 98 | This software is provided 'as-is', without any express or implied 99 | warranty. In no event will the authors be held liable for any damages 100 | arising from the use of this software. 101 | 102 | Permission is granted to anyone to use this software for any purpose, 103 | including commercial applications, and to alter it and redistribute it 104 | freely, subject to the following restrictions: 105 | 106 | 1. The origin of this software must not be misrepresented; you must not 107 | claim that you wrote the original software. If you use this software 108 | in a product, an acknowledgment in the product documentation would be 109 | appreciated but is not required. 110 | 2. Altered source versions must be plainly marked as such, and must not be 111 | misrepresented as being the original software. 112 | 3. This notice may not be removed or altered from any source distribution. 113 | 114 | Jean-loup Gailly Mark Adler 115 | jloup@gzip.org madler@alumni.caltech.edu 116 | 117 | If you use the zlib library in a product, we would appreciate *not* 118 | receiving lengthy legal documents to sign. The sources are provided 119 | for free but without warranty of any kind. The library has been 120 | entirely written by Jean-loup Gailly and Mark Adler; it does not 121 | include third-party code. 122 | 123 | If you redistribute modified sources, we would appreciate that you include 124 | in the file ChangeLog history information documenting your changes. Please 125 | read the FAQ for more information on the distribution of modified source 126 | versions. 127 | -------------------------------------------------------------------------------- /c/zlib/inflate.h: -------------------------------------------------------------------------------- 1 | /* inflate.h -- internal inflate state definition 2 | * Copyright (C) 1995-2003 Mark Adler 3 | * For conditions of distribution and use, see copyright notice in zlib.h 4 | */ 5 | 6 | /* WARNING: this file should *not* be used by applications. It is 7 | part of the implementation of the compression library and is 8 | subject to change. Applications should only use zlib.h. 9 | */ 10 | 11 | /* define NO_GZIP when compiling if you want to disable gzip header and 12 | trailer decoding by inflate(). NO_GZIP would be used to avoid linking in 13 | the crc code when it is not needed. For shared libraries, gzip decoding 14 | should be left enabled. */ 15 | #ifndef NO_GZIP 16 | # define GUNZIP 17 | #endif 18 | 19 | /* Possible inflate modes between inflate() calls */ 20 | typedef enum { 21 | HEAD, /* i: waiting for magic header */ 22 | #ifdef GUNZIP 23 | FLAGS, /* i: waiting for method and flags (gzip) */ 24 | TIME, /* i: waiting for modification time (gzip) */ 25 | OS, /* i: waiting for extra flags and operating system (gzip) */ 26 | EXLEN, /* i: waiting for extra length (gzip) */ 27 | EXTRA, /* i: waiting for extra bytes (gzip) */ 28 | NAME, /* i: waiting for end of file name (gzip) */ 29 | COMMENT, /* i: waiting for end of comment (gzip) */ 30 | HCRC, /* i: waiting for header crc (gzip) */ 31 | #endif 32 | DICTID, /* i: waiting for dictionary check value */ 33 | DICT, /* waiting for inflateSetDictionary() call */ 34 | TYPE, /* i: waiting for type bits, including last-flag bit */ 35 | TYPEDO, /* i: same, but skip check to exit inflate on new block */ 36 | STORED, /* i: waiting for stored size (length and complement) */ 37 | COPY, /* i/o: waiting for input or output to copy stored block */ 38 | TABLE, /* i: waiting for dynamic block table lengths */ 39 | LENLENS, /* i: waiting for code length code lengths */ 40 | CODELENS, /* i: waiting for length/lit and distance code lengths */ 41 | LEN, /* i: waiting for length/lit code */ 42 | LENDO, /* i: same, but skip exit check */ 43 | LENEXT, /* i: waiting for length extra bits */ 44 | DIST, /* i: waiting for distance code */ 45 | DISTEXT, /* i: waiting for distance extra bits */ 46 | MATCH, /* o: waiting for output space to copy string */ 47 | LIT, /* o: waiting for output space to write literal */ 48 | CHECK, /* i: waiting for 32-bit check value */ 49 | #ifdef GUNZIP 50 | LENGTH, /* i: waiting for 32-bit length (gzip) */ 51 | #endif 52 | DONE, /* finished check, done -- remain here until reset */ 53 | BAD, /* got a data error -- remain here until reset */ 54 | MEM, /* got an inflate() memory error -- remain here until reset */ 55 | SYNC /* looking for synchronization bytes to restart inflate() */ 56 | } inflate_mode; 57 | 58 | /* 59 | State transitions between above modes - 60 | 61 | (most modes can go to the BAD or MEM mode -- not shown for clarity) 62 | 63 | Process header: 64 | HEAD -> (gzip) or (zlib) 65 | (gzip) -> FLAGS -> TIME -> OS -> EXLEN -> EXTRA -> NAME 66 | NAME -> COMMENT -> HCRC -> TYPE 67 | (zlib) -> DICTID or TYPE 68 | DICTID -> DICT -> TYPE 69 | Read deflate blocks: 70 | TYPE -> STORED or TABLE or LEN or CHECK 71 | STORED -> COPY -> TYPE 72 | TABLE -> LENLENS -> CODELENS -> LEN 73 | Read deflate codes: 74 | LEN -> LENEXT or LIT or TYPE 75 | LENEXT -> DIST -> DISTEXT -> MATCH -> LEN 76 | LIT -> LEN 77 | Process trailer: 78 | CHECK -> LENGTH -> DONE 79 | */ 80 | 81 | /* state maintained between inflate() calls. Approximately 7K bytes. */ 82 | struct inflate_state { 83 | inflate_mode mode; /* current inflate mode */ 84 | int last; /* true if processing last block */ 85 | int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ 86 | int havedict; /* true if dictionary provided */ 87 | int flags; /* gzip header method and flags (0 if zlib) */ 88 | unsigned long check; /* protected copy of check value */ 89 | unsigned long total; /* protected copy of output count */ 90 | /* sliding window */ 91 | unsigned wbits; /* log base 2 of requested window size */ 92 | unsigned wsize; /* window size or zero if not using window */ 93 | unsigned whave; /* valid bytes in the window */ 94 | unsigned write; /* window write index */ 95 | unsigned char FAR *window; /* allocated sliding window, if needed */ 96 | /* bit accumulator */ 97 | unsigned long hold; /* input bit accumulator */ 98 | unsigned bits; /* number of bits in "in" */ 99 | /* for string and stored block copying */ 100 | unsigned length; /* literal or length of data to copy */ 101 | unsigned offset; /* distance back to copy string from */ 102 | /* for table and code decoding */ 103 | unsigned extra; /* extra bits needed */ 104 | /* fixed and dynamic code tables */ 105 | code const FAR *lencode; /* starting table for length/literal codes */ 106 | code const FAR *distcode; /* starting table for distance codes */ 107 | unsigned lenbits; /* index bits for lencode */ 108 | unsigned distbits; /* index bits for distcode */ 109 | /* dynamic table building */ 110 | unsigned ncode; /* number of code length code lengths */ 111 | unsigned nlen; /* number of length code lengths */ 112 | unsigned ndist; /* number of distance code lengths */ 113 | unsigned have; /* number of code lengths in lens[] */ 114 | code FAR *next; /* next available space in codes[] */ 115 | unsigned short lens[320]; /* temporary storage for code lengths */ 116 | unsigned short work[288]; /* work area for code table building */ 117 | code codes[ENOUGH]; /* space for code tables */ 118 | }; 119 | -------------------------------------------------------------------------------- /c/autotools/sockinttypes.m4: -------------------------------------------------------------------------------- 1 | dnl * 2 | dnl * Copyright (c) 2001, 2003 Motoyuki Kasahara 3 | dnl * 4 | dnl * Redistribution and use in source and binary forms, with or without 5 | dnl * modification, are permitted provided that the following conditions 6 | dnl * are met: 7 | dnl * 1. Redistributions of source code must retain the above copyright 8 | dnl * notice, this list of conditions and the following disclaimer. 9 | dnl * 2. Redistributions in binary form must reproduce the above copyright 10 | dnl * notice, this list of conditions and the following disclaimer in the 11 | dnl * documentation and/or other materials provided with the distribution. 12 | dnl * 3. Neither the name of the project nor the names of its contributors 13 | dnl * may be used to endorse or promote products derived from this software 14 | dnl * without specific prior written permission. 15 | dnl * 16 | dnl * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 17 | dnl * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | dnl * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 | dnl * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORSBE 20 | dnl * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | dnl * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | dnl * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | dnl * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | dnl * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | dnl * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 | dnl * THE POSSIBILITY OF SUCH DAMAGE. 27 | dnl * 28 | 29 | dnl * 30 | dnl * Check for socklen_t. 31 | dnl * 32 | AC_DEFUN([X_TYPE_SOCKLEN_T], 33 | [AC_CACHE_CHECK([for socklen_t], ac_cv_type_socklen_t, 34 | [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 35 | #include 36 | #include ]], [[ 37 | socklen_t socklen; 38 | ]])],[ac_cv_type_socklen_t=yes],[ac_cv_type_socklen_t=no])]) 39 | if test "$ac_cv_type_socklen_t" != yes; then 40 | AC_DEFINE(socklen_t, int, 41 | [Define to `int' if or does not define.]) 42 | fi]) 43 | 44 | dnl * 45 | dnl * Check for in_port_t. 46 | dnl * 47 | AC_DEFUN([X_TYPE_IN_PORT_T], 48 | [AC_CACHE_CHECK([for in_port_t], ac_cv_type_in_port_t, 49 | [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 50 | #include 51 | #include 52 | #include ]], [[ 53 | in_port_t in_port; 54 | ]])],[ac_cv_type_in_port_t=yes],[ac_cv_type_in_port_t=no])]) 55 | if test "$ac_cv_type_in_port_t" != yes; then 56 | ac_cv_sin_port_size=unknown 57 | AC_RUN_IFELSE([AC_LANG_SOURCE([[ 58 | #include 59 | #include 60 | #include 61 | int main() { 62 | struct sockaddr_in addr; 63 | return (sizeof(addr.sin_port) == sizeof(long)) ? 0 : 1; 64 | } 65 | ]])],[ac_cv_sin_port_size=long],[],[]) 66 | AC_RUN_IFELSE([AC_LANG_SOURCE([[ 67 | #include 68 | #include 69 | #include 70 | int main() { 71 | struct sockaddr_in addr; 72 | return (sizeof(addr.sin_port) == sizeof(int)) ? 0 : 1; 73 | } 74 | ]])],[ac_cv_sin_port_size=int],[],[]) 75 | AC_RUN_IFELSE([AC_LANG_SOURCE([[ 76 | #include 77 | #include 78 | #include 79 | int main() { 80 | struct sockaddr_in addr; 81 | return (sizeof(addr.sin_port) == sizeof(short)) ? 0 : 1; 82 | } 83 | ]])],[ac_cv_sin_port_size=short],[],[]) 84 | AC_RUN_IFELSE([AC_LANG_SOURCE([[ 85 | #include 86 | #include 87 | #include 88 | int main() { 89 | struct sockaddr_in addr; 90 | return (sizeof(addr.sin_port) == sizeof(char)) ? 0 : 1; 91 | } 92 | ]])],[ac_cv_sin_port_size=char],[],[]) 93 | if test "$ac_cv_sin_port_size" = unknown; then 94 | AC_MSG_ERROR([Failed to get size of sin_port in struct sockaddr_in.]) 95 | fi 96 | AC_DEFINE_UNQUOTED(in_port_t, unsigned $ac_cv_sin_port_size, 97 | [Define to `unsigned char', `unsigned short', `unsigned int' or 98 | `unsigned long' according with size of `sin_port' in `struct sockaddr_in', 99 | if , or does not define 100 | `in_port_t'.]) 101 | fi]) 102 | 103 | dnl * 104 | dnl * Check for sa_family_t. 105 | dnl * 106 | AC_DEFUN([X_TYPE_SA_FAMILY_T], 107 | [AC_CACHE_CHECK([for sa_family_t], ac_cv_type_sa_family_t, 108 | [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 109 | #include 110 | #include ]], [[ 111 | sa_family_t sa_family; 112 | ]])],[ac_cv_type_sa_family_t=yes],[ac_cv_type_sa_family_t=no])]) 113 | if test "$ac_cv_type_sa_family_t" != yes; then 114 | ac_cv_sa_family_size=unknown 115 | AC_RUN_IFELSE([AC_LANG_SOURCE([[ 116 | #include 117 | #include 118 | int main() { 119 | struct sockaddr addr; 120 | return (sizeof(addr.sa_family) == sizeof(long)) ? 0 : 1; 121 | } 122 | ]])],[ac_cv_sa_family_size=long],[],[]) 123 | AC_RUN_IFELSE([AC_LANG_SOURCE([[ 124 | #include 125 | #include 126 | int main() { 127 | struct sockaddr addr; 128 | return (sizeof(addr.sa_family) == sizeof(int)) ? 0 : 1; 129 | } 130 | ]])],[ac_cv_sa_family_size=int],[],[]) 131 | AC_RUN_IFELSE([AC_LANG_SOURCE([[ 132 | #include 133 | #include 134 | int main() { 135 | struct sockaddr addr; 136 | return (sizeof(addr.sa_family) == sizeof(short)) ? 0 : 1; 137 | } 138 | ]])],[ac_cv_sa_family_size=short],[],[]) 139 | AC_RUN_IFELSE([AC_LANG_SOURCE([[ 140 | #include 141 | #include 142 | int main() { 143 | struct sockaddr addr; 144 | return (sizeof(addr.sa_family) == sizeof(char)) ? 0 : 1; 145 | } 146 | ]])],[ac_cv_sa_family_size=char],[],[]) 147 | if test "$ac_cv_sa_family_size" = unknown; then 148 | AC_MSG_ERROR([Failed to get size of sa_family in struct sockaddr.]) 149 | fi 150 | AC_DEFINE_UNQUOTED(sa_family_t, unsigned $ac_cv_sa_family_size, 151 | [Define to `unsigned char', `unsigned short', `unsigned int' or 152 | `unsigned long' according with size of `sa_family' in `struct sockaddr', 153 | if or does not define `sa_family_t'.]) 154 | fi]) 155 | -------------------------------------------------------------------------------- /c/libzsync/sha1.c: -------------------------------------------------------------------------------- 1 | /* $OpenBSD: sha1.c,v 1.19 2004/05/28 15:10:27 millert Exp $ */ 2 | 3 | /* 4 | * SHA-1 in C 5 | * By Steve Reid 6 | * 100% Public Domain 7 | * 8 | * Test Vectors (from FIPS PUB 180-1) 9 | * "abc" 10 | * A9993E36 4706816A BA3E2571 7850C26C 9CD0D89D 11 | * "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" 12 | * 84983E44 1C3BD26E BAAE4AA1 F95129E5 E54670F1 13 | * A million repetitions of "a" 14 | * 34AA973C D4C4DAA4 F61EEB2B DBAD2731 6534016F 15 | */ 16 | 17 | #include "zsglobal.h" 18 | 19 | #if defined(LIBC_SCCS) && !defined(lint) 20 | static const char rcsid[] = "$OpenBSD: sha1.c,v 1.19 2004/05/28 15:10:27 millert Exp $"; 21 | #endif /* LIBC_SCCS and not lint */ 22 | 23 | #include 24 | #include 25 | #include 26 | #include "sha1.h" 27 | 28 | #define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) 29 | 30 | /* Map Solaris endian stuff to something useful */ 31 | #if defined(_BIG_ENDIAN) && !defined(_BYTE_ORDER) 32 | #define LITTLE_ENDIAN 0 33 | #define BIG_ENDIAN 1 34 | #define BYTE_ORDER 1 35 | #endif 36 | 37 | /* 38 | * blk0() and blk() perform the initial expand. 39 | * I got the idea of expanding during the round function from SSLeay 40 | */ 41 | #if BYTE_ORDER == LITTLE_ENDIAN 42 | # define blk0(i) (block->l[i] = (rol(block->l[i],24)&0xFF00FF00) \ 43 | |(rol(block->l[i],8)&0x00FF00FF)) 44 | #else 45 | # define blk0(i) block->l[i] 46 | #endif 47 | #define blk(i) (block->l[i&15] = rol(block->l[(i+13)&15]^block->l[(i+8)&15] \ 48 | ^block->l[(i+2)&15]^block->l[i&15],1)) 49 | 50 | /* 51 | * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 52 | */ 53 | #define R0(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk0(i)+0x5A827999+rol(v,5);w=rol(w,30); 54 | #define R1(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=rol(w,30); 55 | #define R2(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=rol(w,30); 56 | #define R3(v,w,x,y,z,i) z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=rol(w,30); 57 | #define R4(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=rol(w,30); 58 | 59 | /* 60 | * Hash a single 512-bit block. This is the core of the algorithm. 61 | */ 62 | void 63 | SHA1Transform(uint32_t state[5], const uint8_t buffer[SHA1_BLOCK_LENGTH]) 64 | { 65 | uint32_t a, b, c, d, e; 66 | uint8_t workspace[SHA1_BLOCK_LENGTH]; 67 | typedef union { 68 | uint8_t c[64]; 69 | uint32_t l[16]; 70 | } CHAR64LONG16; 71 | CHAR64LONG16 *block = (CHAR64LONG16 *)workspace; 72 | 73 | (void)memcpy(block, buffer, SHA1_BLOCK_LENGTH); 74 | 75 | /* Copy context->state[] to working vars */ 76 | a = state[0]; 77 | b = state[1]; 78 | c = state[2]; 79 | d = state[3]; 80 | e = state[4]; 81 | 82 | /* 4 rounds of 20 operations each. Loop unrolled. */ 83 | R0(a,b,c,d,e, 0); R0(e,a,b,c,d, 1); R0(d,e,a,b,c, 2); R0(c,d,e,a,b, 3); 84 | R0(b,c,d,e,a, 4); R0(a,b,c,d,e, 5); R0(e,a,b,c,d, 6); R0(d,e,a,b,c, 7); 85 | R0(c,d,e,a,b, 8); R0(b,c,d,e,a, 9); R0(a,b,c,d,e,10); R0(e,a,b,c,d,11); 86 | R0(d,e,a,b,c,12); R0(c,d,e,a,b,13); R0(b,c,d,e,a,14); R0(a,b,c,d,e,15); 87 | R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); 88 | R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); 89 | R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); 90 | R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); 91 | R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); 92 | R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); 93 | R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); 94 | R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); 95 | R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); 96 | R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); 97 | R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); 98 | R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); 99 | R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); 100 | R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); 101 | R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); 102 | R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); 103 | 104 | /* Add the working vars back into context.state[] */ 105 | state[0] += a; 106 | state[1] += b; 107 | state[2] += c; 108 | state[3] += d; 109 | state[4] += e; 110 | 111 | /* Wipe variables */ 112 | a = b = c = d = e = 0; 113 | } 114 | 115 | 116 | /* 117 | * SHA1Init - Initialize new context 118 | */ 119 | void 120 | SHA1Init(SHA1_CTX *context) 121 | { 122 | 123 | /* SHA1 initialization constants */ 124 | context->count = 0; 125 | context->state[0] = 0x67452301; 126 | context->state[1] = 0xEFCDAB89; 127 | context->state[2] = 0x98BADCFE; 128 | context->state[3] = 0x10325476; 129 | context->state[4] = 0xC3D2E1F0; 130 | } 131 | 132 | 133 | /* 134 | * Run your data through this. 135 | */ 136 | void 137 | SHA1Update(SHA1_CTX *context, const uint8_t *data, size_t len) 138 | { 139 | size_t i, j; 140 | 141 | j = (size_t)((context->count >> 3) & 63); 142 | context->count += (len << 3); 143 | if ((j + len) > 63) { 144 | (void)memcpy(&context->buffer[j], data, (i = 64-j)); 145 | SHA1Transform(context->state, context->buffer); 146 | for ( ; i + 63 < len; i += 64) 147 | SHA1Transform(context->state, (uint8_t *)&data[i]); 148 | j = 0; 149 | } else { 150 | i = 0; 151 | } 152 | (void)memcpy(&context->buffer[j], &data[i], len - i); 153 | } 154 | 155 | 156 | /* 157 | * Add padding and return the message digest. 158 | */ 159 | void 160 | SHA1Pad(SHA1_CTX *context) 161 | { 162 | uint8_t finalcount[8]; 163 | uint8_t i; 164 | 165 | for (i = 0; i < 8; i++) { 166 | finalcount[i] = (uint8_t)((context->count >> 167 | ((7 - (i & 7)) * 8)) & 255); /* Endian independent */ 168 | } 169 | SHA1Update(context, (uint8_t *)"\200", 1); 170 | while ((context->count & 504) != 448) 171 | SHA1Update(context, (uint8_t *)"\0", 1); 172 | SHA1Update(context, finalcount, 8); /* Should cause a SHA1Transform() */ 173 | } 174 | 175 | void 176 | SHA1Final(uint8_t digest[SHA1_DIGEST_LENGTH], SHA1_CTX *context) 177 | { 178 | uint8_t i; 179 | 180 | SHA1Pad(context); 181 | if (digest) { 182 | for (i = 0; i < SHA1_DIGEST_LENGTH; i++) { 183 | digest[i] = (uint8_t) 184 | ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); 185 | } 186 | memset(context, 0, sizeof(*context)); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /c/zlib/inffixed.h: -------------------------------------------------------------------------------- 1 | /* inffixed.h -- table for decoding fixed codes 2 | * Generated automatically by makefixed(). 3 | */ 4 | 5 | /* WARNING: this file should *not* be used by applications. It 6 | is part of the implementation of the compression library and 7 | is subject to change. Applications should only use zlib.h. 8 | */ 9 | 10 | static const code lenfix[512] = { 11 | {96,7,0},{0,8,80},{0,8,16},{20,8,115},{18,7,31},{0,8,112},{0,8,48}, 12 | {0,9,192},{16,7,10},{0,8,96},{0,8,32},{0,9,160},{0,8,0},{0,8,128}, 13 | {0,8,64},{0,9,224},{16,7,6},{0,8,88},{0,8,24},{0,9,144},{19,7,59}, 14 | {0,8,120},{0,8,56},{0,9,208},{17,7,17},{0,8,104},{0,8,40},{0,9,176}, 15 | {0,8,8},{0,8,136},{0,8,72},{0,9,240},{16,7,4},{0,8,84},{0,8,20}, 16 | {21,8,227},{19,7,43},{0,8,116},{0,8,52},{0,9,200},{17,7,13},{0,8,100}, 17 | {0,8,36},{0,9,168},{0,8,4},{0,8,132},{0,8,68},{0,9,232},{16,7,8}, 18 | {0,8,92},{0,8,28},{0,9,152},{20,7,83},{0,8,124},{0,8,60},{0,9,216}, 19 | {18,7,23},{0,8,108},{0,8,44},{0,9,184},{0,8,12},{0,8,140},{0,8,76}, 20 | {0,9,248},{16,7,3},{0,8,82},{0,8,18},{21,8,163},{19,7,35},{0,8,114}, 21 | {0,8,50},{0,9,196},{17,7,11},{0,8,98},{0,8,34},{0,9,164},{0,8,2}, 22 | {0,8,130},{0,8,66},{0,9,228},{16,7,7},{0,8,90},{0,8,26},{0,9,148}, 23 | {20,7,67},{0,8,122},{0,8,58},{0,9,212},{18,7,19},{0,8,106},{0,8,42}, 24 | {0,9,180},{0,8,10},{0,8,138},{0,8,74},{0,9,244},{16,7,5},{0,8,86}, 25 | {0,8,22},{64,8,0},{19,7,51},{0,8,118},{0,8,54},{0,9,204},{17,7,15}, 26 | {0,8,102},{0,8,38},{0,9,172},{0,8,6},{0,8,134},{0,8,70},{0,9,236}, 27 | {16,7,9},{0,8,94},{0,8,30},{0,9,156},{20,7,99},{0,8,126},{0,8,62}, 28 | {0,9,220},{18,7,27},{0,8,110},{0,8,46},{0,9,188},{0,8,14},{0,8,142}, 29 | {0,8,78},{0,9,252},{96,7,0},{0,8,81},{0,8,17},{21,8,131},{18,7,31}, 30 | {0,8,113},{0,8,49},{0,9,194},{16,7,10},{0,8,97},{0,8,33},{0,9,162}, 31 | {0,8,1},{0,8,129},{0,8,65},{0,9,226},{16,7,6},{0,8,89},{0,8,25}, 32 | {0,9,146},{19,7,59},{0,8,121},{0,8,57},{0,9,210},{17,7,17},{0,8,105}, 33 | {0,8,41},{0,9,178},{0,8,9},{0,8,137},{0,8,73},{0,9,242},{16,7,4}, 34 | {0,8,85},{0,8,21},{16,8,258},{19,7,43},{0,8,117},{0,8,53},{0,9,202}, 35 | {17,7,13},{0,8,101},{0,8,37},{0,9,170},{0,8,5},{0,8,133},{0,8,69}, 36 | {0,9,234},{16,7,8},{0,8,93},{0,8,29},{0,9,154},{20,7,83},{0,8,125}, 37 | {0,8,61},{0,9,218},{18,7,23},{0,8,109},{0,8,45},{0,9,186},{0,8,13}, 38 | {0,8,141},{0,8,77},{0,9,250},{16,7,3},{0,8,83},{0,8,19},{21,8,195}, 39 | {19,7,35},{0,8,115},{0,8,51},{0,9,198},{17,7,11},{0,8,99},{0,8,35}, 40 | {0,9,166},{0,8,3},{0,8,131},{0,8,67},{0,9,230},{16,7,7},{0,8,91}, 41 | {0,8,27},{0,9,150},{20,7,67},{0,8,123},{0,8,59},{0,9,214},{18,7,19}, 42 | {0,8,107},{0,8,43},{0,9,182},{0,8,11},{0,8,139},{0,8,75},{0,9,246}, 43 | {16,7,5},{0,8,87},{0,8,23},{64,8,0},{19,7,51},{0,8,119},{0,8,55}, 44 | {0,9,206},{17,7,15},{0,8,103},{0,8,39},{0,9,174},{0,8,7},{0,8,135}, 45 | {0,8,71},{0,9,238},{16,7,9},{0,8,95},{0,8,31},{0,9,158},{20,7,99}, 46 | {0,8,127},{0,8,63},{0,9,222},{18,7,27},{0,8,111},{0,8,47},{0,9,190}, 47 | {0,8,15},{0,8,143},{0,8,79},{0,9,254},{96,7,0},{0,8,80},{0,8,16}, 48 | {20,8,115},{18,7,31},{0,8,112},{0,8,48},{0,9,193},{16,7,10},{0,8,96}, 49 | {0,8,32},{0,9,161},{0,8,0},{0,8,128},{0,8,64},{0,9,225},{16,7,6}, 50 | {0,8,88},{0,8,24},{0,9,145},{19,7,59},{0,8,120},{0,8,56},{0,9,209}, 51 | {17,7,17},{0,8,104},{0,8,40},{0,9,177},{0,8,8},{0,8,136},{0,8,72}, 52 | {0,9,241},{16,7,4},{0,8,84},{0,8,20},{21,8,227},{19,7,43},{0,8,116}, 53 | {0,8,52},{0,9,201},{17,7,13},{0,8,100},{0,8,36},{0,9,169},{0,8,4}, 54 | {0,8,132},{0,8,68},{0,9,233},{16,7,8},{0,8,92},{0,8,28},{0,9,153}, 55 | {20,7,83},{0,8,124},{0,8,60},{0,9,217},{18,7,23},{0,8,108},{0,8,44}, 56 | {0,9,185},{0,8,12},{0,8,140},{0,8,76},{0,9,249},{16,7,3},{0,8,82}, 57 | {0,8,18},{21,8,163},{19,7,35},{0,8,114},{0,8,50},{0,9,197},{17,7,11}, 58 | {0,8,98},{0,8,34},{0,9,165},{0,8,2},{0,8,130},{0,8,66},{0,9,229}, 59 | {16,7,7},{0,8,90},{0,8,26},{0,9,149},{20,7,67},{0,8,122},{0,8,58}, 60 | {0,9,213},{18,7,19},{0,8,106},{0,8,42},{0,9,181},{0,8,10},{0,8,138}, 61 | {0,8,74},{0,9,245},{16,7,5},{0,8,86},{0,8,22},{64,8,0},{19,7,51}, 62 | {0,8,118},{0,8,54},{0,9,205},{17,7,15},{0,8,102},{0,8,38},{0,9,173}, 63 | {0,8,6},{0,8,134},{0,8,70},{0,9,237},{16,7,9},{0,8,94},{0,8,30}, 64 | {0,9,157},{20,7,99},{0,8,126},{0,8,62},{0,9,221},{18,7,27},{0,8,110}, 65 | {0,8,46},{0,9,189},{0,8,14},{0,8,142},{0,8,78},{0,9,253},{96,7,0}, 66 | {0,8,81},{0,8,17},{21,8,131},{18,7,31},{0,8,113},{0,8,49},{0,9,195}, 67 | {16,7,10},{0,8,97},{0,8,33},{0,9,163},{0,8,1},{0,8,129},{0,8,65}, 68 | {0,9,227},{16,7,6},{0,8,89},{0,8,25},{0,9,147},{19,7,59},{0,8,121}, 69 | {0,8,57},{0,9,211},{17,7,17},{0,8,105},{0,8,41},{0,9,179},{0,8,9}, 70 | {0,8,137},{0,8,73},{0,9,243},{16,7,4},{0,8,85},{0,8,21},{16,8,258}, 71 | {19,7,43},{0,8,117},{0,8,53},{0,9,203},{17,7,13},{0,8,101},{0,8,37}, 72 | {0,9,171},{0,8,5},{0,8,133},{0,8,69},{0,9,235},{16,7,8},{0,8,93}, 73 | {0,8,29},{0,9,155},{20,7,83},{0,8,125},{0,8,61},{0,9,219},{18,7,23}, 74 | {0,8,109},{0,8,45},{0,9,187},{0,8,13},{0,8,141},{0,8,77},{0,9,251}, 75 | {16,7,3},{0,8,83},{0,8,19},{21,8,195},{19,7,35},{0,8,115},{0,8,51}, 76 | {0,9,199},{17,7,11},{0,8,99},{0,8,35},{0,9,167},{0,8,3},{0,8,131}, 77 | {0,8,67},{0,9,231},{16,7,7},{0,8,91},{0,8,27},{0,9,151},{20,7,67}, 78 | {0,8,123},{0,8,59},{0,9,215},{18,7,19},{0,8,107},{0,8,43},{0,9,183}, 79 | {0,8,11},{0,8,139},{0,8,75},{0,9,247},{16,7,5},{0,8,87},{0,8,23}, 80 | {64,8,0},{19,7,51},{0,8,119},{0,8,55},{0,9,207},{17,7,15},{0,8,103}, 81 | {0,8,39},{0,9,175},{0,8,7},{0,8,135},{0,8,71},{0,9,239},{16,7,9}, 82 | {0,8,95},{0,8,31},{0,9,159},{20,7,99},{0,8,127},{0,8,63},{0,9,223}, 83 | {18,7,27},{0,8,111},{0,8,47},{0,9,191},{0,8,15},{0,8,143},{0,8,79}, 84 | {0,9,255} 85 | }; 86 | 87 | static const code distfix[32] = { 88 | {16,5,1},{23,5,257},{19,5,17},{27,5,4097},{17,5,5},{25,5,1025}, 89 | {21,5,65},{29,5,16385},{16,5,3},{24,5,513},{20,5,33},{28,5,8193}, 90 | {18,5,9},{26,5,2049},{22,5,129},{64,5,0},{16,5,2},{23,5,385}, 91 | {19,5,25},{27,5,6145},{17,5,7},{25,5,1537},{21,5,97},{29,5,24577}, 92 | {16,5,4},{24,5,769},{20,5,49},{28,5,12289},{18,5,13},{26,5,3073}, 93 | {22,5,193},{64,5,0} 94 | }; 95 | -------------------------------------------------------------------------------- /c/README: -------------------------------------------------------------------------------- 1 | zsync 0.6.3 2 | =========== 3 | 4 | zsync is a file transfer program. It allows you to download a file from a 5 | remote web server, where you have a copy of an older version of the file on 6 | your computer already. zsync downloads only the new parts of the file. It uses 7 | the same algorithm as rsync. 8 | 9 | zsync does not require any special server software or a shell account on the 10 | remote system (rsync, in comparison, requires that you have an rsh or ssh 11 | account, or that the remote system runs rsyncd). Instead, it uses a control 12 | file - a .zsync file - that describes the file to be downloaded and enables 13 | zsync to work out which blocks it needs. This file can be created by the admin 14 | of the web server hosting the download, and placed alongside the file to 15 | download - it is generated once, then any downloaders with zsync can use it. 16 | Alternatively, anyone can download the file, make a .zsync and provide it to 17 | other users (this is what I am doing for the moment). 18 | 19 | The zsync web site is at http://zsync.moria.org.uk/ . There are likely to be 20 | frequent releases, so check back often. 21 | 22 | Installation 23 | ------------ 24 | 25 | See the file INSTALL for instructions on compiling and (optionally) installing 26 | zsync. 27 | 28 | As zsync is still at a very early stage, you may prefer not to install it, and 29 | to run the program where you compile it. It will work fine that way - it has no 30 | data files or libraries. The man(1) program on some systems will let you read 31 | the man pages without installing them, e.g. man -l doc/zsync.1 . 32 | 33 | zsync is free software. There is no implied support, no implied fitness for 34 | purpose, no warranty. You use it at your own risk. 35 | 36 | Use 37 | --- 38 | 39 | In its simplest form, as an end-user: 40 | 41 | zsync http://some.example.com/downloads/my-big-download.tar.zsync 42 | 43 | Someone has to make a .zsync file for the download before you can use zsync. As 44 | zsync has not been around long, there won't be many such downloads around :-). 45 | And you have to have an older copy of the file around, otherwise there is 46 | little point in using zsync - zsync normally looks in the current directory for 47 | a file of the same name as the one being downloaded (note: if the download is a 48 | .gz file, the local file should be uncompressed - use gzip -d to decompress it 49 | first). If the local file is not in the current directory or has a different 50 | name, you can specify it with -i, e.g.: 51 | 52 | zsync -i /var/lib/apt/lists/ftp.uk.debian.org_debian_dists_sarge_main_binary-i386_Packages http://zsync.moria.org.uk/s/sarge/Packages.zsync 53 | 54 | Offering zsync downloads 55 | ------------------------ 56 | 57 | Simple example: 58 | Suppose you have http://example.com/dl/some-image-0.2.iso ; which is in 59 | /var/www/downloads/ on your server. 60 | 61 | cd /var/www/downloads/ 62 | zsyncmake -u 'http://example.com/dl/some-image-0.2.iso' some-image-0.2.iso 63 | 64 | This creates some-image-0.2.iso.zsync in the same directory, and specifies the 65 | URL from which users can retrieve the full file. (Note that zsync requires both 66 | the public URL for the full download, and a local copy of the same file. Users 67 | will still need access to the full downloads - zsync merely allows then to save 68 | time by only downloading parts of the file.) 69 | 70 | A user with v0.1 of the same file can now use zsync to download only the new 71 | bits. 72 | 73 | zsync -i some-image-0.1.iso http://example.com/dl/some-image-0.2.iso.zsync 74 | 75 | Compressed example: 76 | Support you have http://example.com/downloads/big-0.2.tar.gz ; which is 77 | in /var/www/downloads/ on your server. 78 | 79 | cd /var/www/downloads/ 80 | zsyncmake -u 'http://example.com/downloads/big-0.2.tar.gz' big-0.2.tar.gz 81 | 82 | This creates big-0.2.tar.zsync in the same directory, and specifies the URL 83 | from which users can see the file. 84 | 85 | A user with v0.1 of the same file can now use zsync to download only the new 86 | bits: 87 | 88 | zsync -i <(zcat big-0.1.tar.gz) http://example.com/downloads/big-0.2.tar.zsync 89 | 90 | Feedback, Support 91 | ----------------- 92 | 93 | Mail zsync-users@lists.sourceforge.net if you have questions about zsync. And 94 | join the mailing list if you are using it a lot or are interested in the 95 | ongoing development. 96 | 97 | Copyright, Author, Acknowledgements 98 | ----------------------------------- 99 | 100 | zsync is based on the rsync algorithm, by Andrew Tridgell. It also incorporates 101 | a number of optimisations, based on ideas in academic papers by Utku Irmak, 102 | Svilen Mihaylov and Torsten Suel (primarily "Improved Single-Round Protocols 103 | for Remote File Synchronization", Sept 2004). 104 | 105 | zsync uses a large part of zlib - this code, in the zlib subdirectory, is 106 | copyright 1995-2003 Jean-loup Gailly and Mark Adler, see zlib/README for 107 | details. This code contains local changes by me that are not compatible with 108 | and not available in standard zlib; see zlib/README.zsync for an explanation. 109 | 110 | zsync includes an implementation of getaddrinfo by Motoyuki Kasahara. See 111 | getaddrinfo.c for the license, it's a BSD-style license; but normally zsync 112 | compiles against the getaddrinfo() in the system libc, so this code is not 113 | normally used. 114 | 115 | zsync also includes checksum code taken from OpenBSD. The MD4 code is public 116 | domain, by Colin Plumb and Todd C. Miller. The SHA1 code is also public 117 | domain, by Steve Reid. This code is in the libhash/ subdirectory, see the 118 | individual files for their non-copyright notices. 119 | 120 | The rest of the code (that is, everything not in the zlib and libhash 121 | subdirectories) is 122 | Copyright (C) 2004,2005,2007,2009 Colin Phipps . 123 | zsync is made available under the (clarified) Artistic License - see the file 124 | COPYING for details. 125 | 126 | Thanks also to: 127 | 128 | * Dennis Schridde, for contributing patches to improve portability. 129 | * Timothy Lee, for finding bugs and supplying patches. 130 | * Richard Kiss, for supplying a patch for MacOS X compilation. 131 | 132 | I must thank the developers above, whose code I have used. Also, I would like 133 | to thank the Free Software Foundation and its contributors, for gcc, gdb and 134 | emacs, the essential development tools. Also I would thank Sourceforge for 135 | providing the hosting facilities for the downloads and mailing list, and the 136 | compile farm for testing portability. 137 | 138 | Also, thanks to Érsek László, James Montgomerie, James Antill, 139 | saul@alien-science.org, Kent Mein, Marc Lehmann, Robert Lemmen, Mark Adler, 140 | Ricardo Correia, Karl Kalleberg, Michael Stone, Richard Lucassen, Duncan 141 | Mac-Vicar, Jari Aalto, Marcin Mirosław, Jan Varho, Loïc Minier, Gian Merlino 142 | and S Page for useful feedback and bug reports for previous versions. 143 | 144 | -------------------------------------------------------------------------------- /c/doc/zsyncmake.1: -------------------------------------------------------------------------------- 1 | .TH "zsyncmake" "1" "0.6.3" "Colin Phipps" "File Transfer" 2 | .SH "NAME" 3 | .LP 4 | zsyncmake \- Build control file for zsync(1) 5 | .SH "SYNTAX" 6 | .LP 7 | zsyncmake [ { \-z | \-Z } ] [ \-e ] [ \-C ] [ \-u \fIurl\fR ] [ \-U \fIurl\fR ] [ \-b \fIblocksize\fR ] [ \-o \fIoutfile\fR ] [ \-f \fItargetfilename\fR ] [ \-v ] \fIfilename\fP 8 | .LP 9 | zsync \-V 10 | .SH "DESCRIPTION" 11 | .LP 12 | Constructs a metafile for the zsync client program to use to perform partial file downloads. \fIfilename\fR is the file that users wish to downloads; zsyncmake constructs the appropriate metafile and writes \fIfilename\fR.zsync in the current directory. 13 | .LP 14 | zsync will need at least one URL from which to download the file content. If the .zsync will be in the same directory as the file to download, you can accept the default \- zsync includes a relative URL in the control file. If not, use the \fB\-u\fR option to specify the URL. You should also specify a URL for the uncompressed content with \-U if available, as zsync can make use of this for more efficient downloads sometimes. (You can edit the .zsync file and add these afterwards \- it has a simple key: value format in the header \- but I suggest you only do this once you are familiar with the tool.) 15 | .LP 16 | Note that zsyncmake itself does not (currently) verify the URLs or download any data, you must provide the file data locally and check the URLs yourself. 17 | .SH "OPTIONS" 18 | .LP 19 | .TP 20 | \fB\-b\fR \fIblocksize\fR 21 | Specify the blocksize to the underlying rsync algorithm. A smaller blocksize may be more efficient for files where there are likely to be lots of small, scattered changes between downloads; a larger blocksize is more efficient for files with fewer or less scattered changes. This blocksize must be a power of two. If not specified, zsyncmake chooses one which it thinks is best for this file (currently either 2048 or 4096 depending on file size) - so normally you should not need to override the default. 22 | .TP 23 | \fB\-C\fR 24 | Tells zsyncmake not to generate any instructions in the .zsync telling the client to compress the data it receives. This is implied by -z, but this option is here in case you compress a file yourself only for the transfer, but want the client to end up with the uncompressed file (e.g. you are transferring an ISO, which is held compressed on the server, but which the client cannot use unless it is uncompressed). Without -C, zsyncmake will produce directions for the client to compress the file it receives where appropriate; -C is here so you can stop it telling the client to do that. 25 | .TP 26 | \fB\-e\fR 27 | Tells zsyncmake that the client must be able to receive the exact file that was supplied. Without this option, zsyncmake only gives a weaker guarantee - that the client will receive the data it contains (e.g. it might transfer the uncompressed version of a .gz to the client). Note that this still doesn't guarantee that the client will get it - the client could ignore the directives in the zsync file, or might be incapable of exactly reproducing the compression used. But with -e you know that zsyncmake has made it possible to get the exact data - it will exit with an error if it cannot. 28 | .TP 29 | \fB\-f\fR \fIfilename\fR 30 | Set the filename to include in the output file (this is what the file will be called when a user finished downloading it). 31 | .TP 32 | \fB\-o\fR \fIoutputfile\fR 33 | Override the default output file name. 34 | .TP 35 | \fB\-u\fR \fIurl\fR 36 | Specifies the URL from which users can download the content of the supplied file. Users need the control file in order to find out what parts of the file they already have, and they need the URLs to retrieve the parts of the file that they don't already have. You can specify multiple URLs by specifying \-u multiple times. If not specified, zsync assumes that the file and the .zsync will reside in the same public directory, and includes a single relative URL. 37 | .TP 38 | \fB\-U\fR \fIurl\fR 39 | Specifies a URL corresponding to the decompressed content of the file (only applicable if it is a gzip file). zsync can sometimes download more efficiently from the uncompressed data than from the compressed data \- it will take advantage of this if available. If no URLs are specifies, zsync looks for a file without the .gz extension and assumes that this will be in the same public dir as the .zsync, and includes a relative URL to it. 40 | .TP 41 | \fB\-v\fR 42 | Enable verbose messages. 43 | .TP 44 | \fB\-V\fR 45 | Prints the version of zsync. 46 | .TP 47 | \fB\-z\fR 48 | Compress the file to transfer. Note that this overwrites any file called \fIfilename\fP.gz without warning (if you don't give a filename, e.g. because you are reading from stdin, then zsync will use the name supplied with -f, or as a last fallback, zsync-target.gz). 49 | .TP 50 | zsync can work with compressed data, and, in most cases where the data is not already compressed, it is more efficient to compress it first. While you can just compress the file to transfer with gzip, if you use this option then zsyncmake will compress the file for you, producing a .gz file which is optimised for zsync. This can be 30% more efficient at download time than compressing with gzip --best - but the compressed file will not be as small at that produced by gzip. 51 | .TP 52 | \fB\-Z\fR 53 | zsyncmake automatically looks inside gzip compressed files and exports the underlying, uncompressed data to the zsyncmake file. In testing this has proved to provide greater download efficiency. \-Z overrides the default behaviour and treats gzip files as just binary data. Use this if it is essential that the user receives the compressed data (for instance because a cryptographic signature is available only for the compressed data). zsync is typically no use if you specify \-Z, unless the gzip file was compressed with the special \-\-rsync option to make it friendly to differential transfers. 54 | .SH "EXAMPLES" 55 | .LP 56 | zsyncmake \-C \-u http://ftp.uk.debian.org/debian/dists/sarge/main/binary\-i386/Packages.gz Packages.gz 57 | .LP 58 | Note use of -C to save the client compressing the file on receipt; the Debian package system uses the file uncompressed. 59 | .LP 60 | zsyncmake \-z my-subversion-dump 61 | .LP 62 | In this case there is a large, compressible file to transfer. This creates a gzipped version of the file (optimised for zsync), and a .zsync file. A URL is automatically added assuming that the two files will be served from the same directory on the web server. 63 | .LP 64 | zsyncmake \-e \-u http://www.mirrorservice.org/sites/ftp.freebsd.org/pub/FreeBSD/ports/distfiles/zsync-0.2.2.tar.gz zsync-0.2.2.tar.gz 65 | .LP 66 | This creates a zsync referring to the named source tarball, which the client 67 | should download from the given URL. This example is for downloading a source 68 | tarball for a FreeBSD port, hence \-e is specified so the client will be able 69 | to match its md5sum. 70 | 71 | .SH "AUTHORS" 72 | .LP 73 | Colin Phipps 74 | .SH "SEE ALSO" 75 | .LP 76 | zsync(1) 77 | -------------------------------------------------------------------------------- /c/NEWS: -------------------------------------------------------------------------------- 1 | Changes in 0.6.3 2 | - Performance improvements for large (>250MB) files - on multi-gigabyte files 3 | this can be a very large improvement. 4 | - Tuning to reduce likelyhood of hitting bad performance due to bad checksum 5 | distribution. 6 | - Fix case where client would think download succeeded and try verifying it 7 | when it had really failed. 8 | - Fix a problem where the client could get into a loop retrieving data if the 9 | remote returned data inconsistent with the .zsync file. 10 | - Improvements to some of the error and progress messages. 11 | - Stop zsync requesting bytes past the end of the target file. 12 | - Fix bug where mtime would be set to something offset from the intended value 13 | due to timezones. 14 | - Fix a case where auth headers could be send to the wrong server across a 15 | redirect. Send auth headers unconditionally when username & password are 16 | provided. 17 | - Fix compilation with gcc-14. 18 | - Other optimisations. 19 | 20 | Changes in 0.6.2 21 | - fix for using zsync client on files >2GB on 32bit systems. 22 | - fix redirect handling. 23 | - improve some edge cases dealing with unusual seed data patterns. 24 | - optimise by stopping reading seed files if target file is complete. 25 | - fix infinite loop in zsyncmake when given a truncated (invalid) .gz 26 | - fix --disable-profile to configure. 27 | 28 | Changes in 0.6.1 29 | - recompression support for gzip files made with zlib:gzio.c or gzip -n 30 | - fix compilation on MacOS X 31 | - allow HTTP redirects on the target file; not sure whether this is a good idea 32 | or not... 33 | - fix unecessary transfer of whole file where file is smaller than the context 34 | size (1x or 2x blocksize) 35 | - use sequential_matches=1 when there is only one block; otherwise we're forced 36 | to transfer the whole file for files below 2kiB 37 | - fix librcksum handling of zsync streams with sequential_matches == 1; it was 38 | giving false negatives when applying the rsync algorithm, resulting in poor 39 | use of local source data when sequential_matches == 1 (which didn't actually 40 | occur in any recent version of zsync) 41 | 42 | Changes in 0.6 43 | - fix out-of-bounds memory access when processing last block of non-compressed 44 | download (patch from Timothy Lee). Also fix an error handling fault for the 45 | same. 46 | - fix "try a smaller blocksize" failures when zsyncmakeing for huge compressed 47 | files on 32bit systems 48 | - preserve mtime on downloaded files 49 | - fix potential crash when re/deallocating checksum hash in librcksum (patch 50 | from Timothy Lee) 51 | - explain status code errors better 52 | - better URL handling 53 | - add -q as a substitute for -s, as -q is more conventional (re wget). -q also 54 | suppresses the 'no relevant local data' warning now. 55 | - fix some warnings 56 | - code tidy-up and better commenting of what it is doing 57 | - tidy up autoconf use 58 | 59 | Changes in 0.5 60 | - get large file support where possible 61 | - replacement getaddrinfo on systems that lack it 62 | - fix compilation on MacOS X 63 | 64 | Changes in 0.4.3 65 | - make -s suppress summary info too 66 | - add HTTP authentication support 67 | 68 | Changes in 0.4.2 69 | - fix CAN-2005-1849 for embedded zlib 70 | - obey -s flag 71 | 72 | Changes in 0.4.1 73 | - security fix from zlib-1.2.2 74 | - generate RFC 1123 date strings in HTTP requests 75 | - include port numbers in Host headers 76 | - be slightly more accepting in proxy URL syntax 77 | - don't generate Accept-Ranges headers 78 | 79 | Changes in 0.4.0 80 | - now considered beta instead of alpha quality 81 | - fixed incorrect 100%/done display of progress bar when cleanly disconnected before end by server 82 | - updated autoconf/automake scripts 83 | 84 | Changes in 0.3.3 85 | - build in gzip in zsyncmake which produces .gz files optimised for zsync 86 | - quickly stop downloading from a URL producing bad data 87 | - -k option to keep the .zsync file, and redownload only when it changes 88 | - fixes some possible crashes that could occur with unexpected remote responses 89 | 90 | Changes in 0.3.2 91 | - automatically decompress local seed data when this is appropriate 92 | - progress bars with rate and ETA for all downloads 93 | - automatically recompress downloaded file if possible 94 | - deal better with local .zsyncs containing relative URLs 95 | 96 | Changes in 0.3.1 97 | - fix SEGV when downloading with no local data found 98 | - fix failure when incomplete final block of uncompressed stream is unknown 99 | - fix some memory leaks 100 | 101 | Changes in 0.3.0 102 | - fix endian check in libhash (works on FreeBSD again) 103 | - major code reorganisation 104 | 105 | Changes in 0.2.3 106 | - fix crash in zsyncmake with short files 107 | - "negative" hash table used to speed up processing 108 | 109 | Changes in 0.2.2 110 | - complete support for stored blocks in gzip files 111 | - update block size recommendations in man page 112 | 113 | Changes in 0.2.1 114 | - fixed bug where zsync would loop if fed more than one local source file 115 | - enabled zsync to handle gzip files containing stored (uncompressed) blocks (so we should do slightly better on gzipped binaries now - but this is still incomplete and will not work for all files) 116 | 117 | Changes in 0.2.0 118 | - major reduction in the size of the control file, due to some new optimisations 119 | - further fixes for servers refusing to do multipart/byteranges responses 120 | 121 | Changes in 0.1.6 122 | - fixes some problems with compressed file downloads and unusual server responses 123 | - improved http_proxy parsing 124 | 125 | Changes in 0.1.5 126 | - fixes some minor HTTP problems 127 | 128 | Changes in 0.1.4 129 | - fixed compilation with gcc-3.4 130 | 131 | Changes in 0.1.3 132 | - HTTP proxy support 133 | - better HTTP support - we deal better with unusual server responses 134 | 135 | Changes in 0.1.2 136 | - fixes for Solaris/sparc 137 | 138 | Changes in 0.1.1 139 | - more efficient compressed stream support 140 | - code cleanups 141 | 142 | Changes in 0.1.0 143 | - finished the cleanup of the compressed file transfer code 144 | - major improvement in efficiency for compressed file transfers 145 | - OpenSSL is no longer required 146 | - now under the v2 Artistic License 147 | 148 | Changes in 0.0.6 149 | - just code cleanups and documentation updates 150 | 151 | Changes in 0.0.5 152 | - switch to OpenSSL's MD4 code 153 | - fix checksumming at stream end 154 | - fix various portability problems 155 | 156 | Changes in 0.0.4 157 | - relative URLs in .zsync files supported 158 | - HTTP redirects are followed for the metafile 159 | - now uses both compressed and uncompressed sources to further reduce download sizes 160 | 161 | Changes in 0.0.3 162 | - fix HTTP connection termination handling 163 | - speed up local file reading 164 | 165 | Changes in 0.0.2 166 | - HTTP/1.1, with pipelining 167 | - cleaning up more of the libc streams and mmap clutter 168 | - progress displays 169 | - lots of bugfixes 170 | 171 | First release 0.0.1. 172 | 173 | -------------------------------------------------------------------------------- /c/zlib/zutil.h: -------------------------------------------------------------------------------- 1 | /* zutil.h -- internal interface and configuration of the compression library 2 | * Copyright (C) 1995-2003 Jean-loup Gailly. 3 | * For conditions of distribution and use, see copyright notice in zlib.h 4 | */ 5 | 6 | /* WARNING: this file should *not* be used by applications. It is 7 | part of the implementation of the compression library and is 8 | subject to change. Applications should only use zlib.h. 9 | */ 10 | 11 | /* @(#) $Id$ */ 12 | 13 | #ifndef ZUTIL_H 14 | #define ZUTIL_H 15 | 16 | #define ZLIB_INTERNAL 17 | #include "zlib.h" 18 | 19 | #ifdef STDC 20 | # include 21 | # include 22 | # include 23 | #endif 24 | #ifdef NO_ERRNO_H 25 | extern int errno; 26 | #else 27 | # include 28 | #endif 29 | 30 | #ifndef local 31 | # define local static 32 | #endif 33 | /* compile with -Dlocal if your debugger can't find static symbols */ 34 | 35 | typedef unsigned char uch; 36 | typedef uch FAR uchf; 37 | typedef unsigned short ush; 38 | typedef ush FAR ushf; 39 | typedef unsigned long ulg; 40 | 41 | extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ 42 | /* (size given to avoid silly warnings with Visual C++) */ 43 | 44 | #define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] 45 | 46 | #define ERR_RETURN(strm,err) \ 47 | return (strm->msg = (char*)ERR_MSG(err), (err)) 48 | /* To be used only when the state is known to be valid */ 49 | 50 | /* common constants */ 51 | 52 | #ifndef DEF_WBITS 53 | # define DEF_WBITS MAX_WBITS 54 | #endif 55 | /* default windowBits for decompression. MAX_WBITS is for compression only */ 56 | 57 | #if MAX_MEM_LEVEL >= 8 58 | # define DEF_MEM_LEVEL 8 59 | #else 60 | # define DEF_MEM_LEVEL MAX_MEM_LEVEL 61 | #endif 62 | /* default memLevel */ 63 | 64 | #define STORED_BLOCK 0 65 | #define STATIC_TREES 1 66 | #define DYN_TREES 2 67 | /* The three kinds of block type */ 68 | 69 | #define MIN_MATCH 3 70 | #define MAX_MATCH 258 71 | /* The minimum and maximum match lengths */ 72 | 73 | #define PRESET_DICT 0x20 /* preset dictionary flag in zlib header */ 74 | 75 | /* target dependencies */ 76 | 77 | #if defined(MSDOS) || (defined(WINDOWS) && !defined(WIN32)) 78 | # define OS_CODE 0x00 79 | # if defined(__TURBOC__) || defined(__BORLANDC__) 80 | # if(__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) 81 | /* Allow compilation with ANSI keywords only enabled */ 82 | void _Cdecl farfree( void *block ); 83 | void *_Cdecl farmalloc( unsigned long nbytes ); 84 | # else 85 | # include 86 | # endif 87 | # else /* MSC or DJGPP */ 88 | # include 89 | # endif 90 | #endif 91 | 92 | #ifdef AMIGA 93 | # define OS_CODE 0x01 94 | #endif 95 | 96 | #if defined(VAXC) || defined(VMS) 97 | # define OS_CODE 0x02 98 | # define F_OPEN(name, mode) \ 99 | fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") 100 | #endif 101 | 102 | #if defined(ATARI) || defined(atarist) 103 | # define OS_CODE 0x05 104 | #endif 105 | 106 | #ifdef OS2 107 | # define OS_CODE 0x06 108 | #endif 109 | 110 | #if defined(MACOS) || defined(TARGET_OS_MAC) 111 | # define OS_CODE 0x07 112 | # if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os 113 | # include /* for fdopen */ 114 | # else 115 | # ifndef fdopen 116 | # define fdopen(fd,mode) NULL /* No fdopen() */ 117 | # endif 118 | # endif 119 | #endif 120 | 121 | #ifdef TOPS20 122 | # define OS_CODE 0x0a 123 | #endif 124 | 125 | #ifdef WIN32 126 | # ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ 127 | # define OS_CODE 0x0b 128 | # endif 129 | #endif 130 | 131 | #ifdef __50SERIES /* Prime/PRIMOS */ 132 | # define OS_CODE 0x0f 133 | #endif 134 | 135 | #if defined(_BEOS_) || defined(RISCOS) 136 | # define fdopen(fd,mode) NULL /* No fdopen() */ 137 | #endif 138 | 139 | #if (defined(_MSC_VER) && (_MSC_VER > 600)) 140 | # if defined(_WIN32_WCE) 141 | # define fdopen(fd,mode) NULL /* No fdopen() */ 142 | # ifndef _PTRDIFF_T_DEFINED 143 | typedef int ptrdiff_t; 144 | # define _PTRDIFF_T_DEFINED 145 | # endif 146 | # else 147 | # define fdopen(fd,type) _fdopen(fd,type) 148 | # endif 149 | #endif 150 | 151 | /* common defaults */ 152 | 153 | #ifndef OS_CODE 154 | # define OS_CODE 0x03 /* assume Unix */ 155 | #endif 156 | 157 | #ifndef F_OPEN 158 | # define F_OPEN(name, mode) fopen((name), (mode)) 159 | #endif 160 | 161 | /* functions */ 162 | 163 | #if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) 164 | # ifndef HAVE_VSNPRINTF 165 | # define HAVE_VSNPRINTF 166 | # endif 167 | #endif 168 | #if defined(__CYGWIN__) 169 | # ifndef HAVE_VSNPRINTF 170 | # define HAVE_VSNPRINTF 171 | # endif 172 | #endif 173 | #ifndef HAVE_VSNPRINTF 174 | # ifdef MSDOS 175 | /* vsnprintf may exist on some MS-DOS compilers (DJGPP?), 176 | but for now we just assume it doesn't. */ 177 | # define NO_vsnprintf 178 | # endif 179 | # ifdef __TURBOC__ 180 | # define NO_vsnprintf 181 | # endif 182 | # ifdef WIN32 183 | /* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ 184 | # if !defined(vsnprintf) && !defined(NO_vsnprintf) 185 | # define vsnprintf _vsnprintf 186 | # endif 187 | # endif 188 | # ifdef __SASC 189 | # define NO_vsnprintf 190 | # endif 191 | #endif 192 | 193 | #ifdef HAVE_STRERROR 194 | extern char *strerror OF((int)); 195 | # define zstrerror(errnum) strerror(errnum) 196 | #else 197 | # define zstrerror(errnum) "" 198 | #endif 199 | 200 | #if defined(pyr) 201 | # define NO_MEMCPY 202 | #endif 203 | #if defined(SMALL_MEDIUM) && !defined(_MSC_VER) && !defined(__SC__) 204 | /* Use our own functions for small and medium model with MSC <= 5.0. 205 | * You may have to use the same strategy for Borland C (untested). 206 | * The __SC__ check is for Symantec. 207 | */ 208 | # define NO_MEMCPY 209 | #endif 210 | #if defined(STDC) && !defined(HAVE_MEMCPY) && !defined(NO_MEMCPY) 211 | # define HAVE_MEMCPY 212 | #endif 213 | #ifdef HAVE_MEMCPY 214 | # ifdef SMALL_MEDIUM /* MSDOS small or medium model */ 215 | # define zmemcpy _fmemcpy 216 | # define zmemcmp _fmemcmp 217 | # define zmemzero(dest, len) _fmemset(dest, 0, len) 218 | # else 219 | # define zmemcpy memcpy 220 | # define zmemcmp memcmp 221 | # define zmemzero(dest, len) memset(dest, 0, len) 222 | # endif 223 | #else 224 | extern void zmemcpy OF((Bytef* dest, const Bytef* source, uInt len)); 225 | extern int zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len)); 226 | extern void zmemzero OF((Bytef* dest, uInt len)); 227 | #endif 228 | 229 | /* Diagnostic functions */ 230 | #ifdef DEBUG 231 | # include 232 | extern int z_verbose; 233 | extern void z_error OF((char *m)); 234 | # define Assert(cond,msg) {if(!(cond)) z_error(msg);} 235 | # define Trace(x) {if (z_verbose>=0) fprintf x ;} 236 | # define Tracev(x) {if (z_verbose>0) fprintf x ;} 237 | # define Tracevv(x) {if (z_verbose>1) fprintf x ;} 238 | # define Tracec(c,x) {if (z_verbose>0 && (c)) fprintf x ;} 239 | # define Tracecv(c,x) {if (z_verbose>1 && (c)) fprintf x ;} 240 | #else 241 | # define Assert(cond,msg) 242 | # define Trace(x) 243 | # define Tracev(x) 244 | # define Tracevv(x) 245 | # define Tracec(c,x) 246 | # define Tracecv(c,x) 247 | #endif 248 | 249 | 250 | voidpf zcalloc OF((voidpf opaque, unsigned items, unsigned size)); 251 | void zcfree OF((voidpf opaque, voidpf ptr)); 252 | 253 | #define ZALLOC(strm, items, size) \ 254 | (*((strm)->zalloc))((strm)->opaque, (items), (size)) 255 | #define ZFREE(strm, addr) (*((strm)->zfree))((strm)->opaque, (voidpf)(addr)) 256 | #define TRY_FREE(s, p) {if (p) ZFREE(s, p);} 257 | 258 | #endif /* ZUTIL_H */ 259 | -------------------------------------------------------------------------------- /c/librcksum/range.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * rcksum/lib - library for using the rsync algorithm to determine 4 | * which parts of a file you have and which you need. 5 | * Copyright (C) 2004,2005,2007,2009 Colin Phipps 6 | * 7 | * This program is free software; you can redistribute it and/or modify 8 | * it under the terms of the Artistic License v2 (see the accompanying 9 | * file COPYING for the full license terms), or, at your option, any later 10 | * version of the same license. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * COPYING file for details. 16 | */ 17 | 18 | /* Manage storage of the set of ranges in the target file that we have so far 19 | * got data for. */ 20 | 21 | #include "zsglobal.h" 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #ifdef WITH_DMALLOC 28 | # include 29 | #endif 30 | 31 | #include "rcksum.h" 32 | #include "internal.h" 33 | 34 | /* r = range_before_block(self, x) 35 | * This determines which of the existing known ranges x falls in. 36 | * It returns -1 if it is inside an existing range (it doesn't tell you which 37 | * one; if you already have it, that usually is enough to know). 38 | * Or it returns 0 if x is before the 1st range; 39 | * 1 if it is between ranges 1 and 2 (array indexes 0 and 1) 40 | * ... 41 | * numranges if it is after the last range 42 | */ 43 | static int range_before_block(const struct rcksum_state* rs, zs_blockid x) { 44 | /* Lowest number and highest number block that it could be inside (0 based) */ 45 | register int min = 0, max = rs->numranges-1; 46 | 47 | /* By bisection */ 48 | for (; min<=max;) { 49 | /* Range number to compare against */ 50 | register int r = (max+min)/2; 51 | 52 | if (x > rs->ranges[2*r+1]) min = r+1; /* After range r */ 53 | else if (x < rs->ranges[2*r]) max = r-1;/* Before range r */ 54 | else return -1; /* In range r */ 55 | } 56 | 57 | /* If we reach here, we know min = max + 1 and we were below range max+1 58 | * and above range min-1. 59 | * So we're between range max and max + 1 60 | * So we return max + 1 (return value is 1 based) ( = min ) 61 | */ 62 | return min; 63 | } 64 | 65 | /* add_to_ranges(rs, blockid) 66 | * Mark the given blockid as known, updating the stored known ranges 67 | * appropriately */ 68 | void add_to_ranges(struct rcksum_state *rs, zs_blockid x) { 69 | int r = range_before_block(rs, x); 70 | 71 | if (r == -1) { 72 | /* Already have this block */ 73 | } 74 | else { 75 | rs->gotblocks++; 76 | 77 | /* If between two ranges and exactly filling the hole between them, 78 | * merge them */ 79 | if (r > 0 && r < rs->numranges 80 | && rs->ranges[2 * (r - 1) + 1] == x - 1 81 | && rs->ranges[2 * r] == x + 1) { 82 | 83 | // This block fills the gap between two areas that we have got completely. Merge the adjacent ranges 84 | rs->ranges[2 * (r - 1) + 1] = rs->ranges[2 * r + 1]; 85 | memmove(&rs->ranges[2 * r], &rs->ranges[2 * r + 2], 86 | (rs->numranges - r - 1) * sizeof(rs->ranges[0]) * 2); 87 | rs->numranges--; 88 | } 89 | 90 | /* If adjoining a range below, add to it */ 91 | else if (r > 0 && rs->numranges && rs->ranges[2 * (r - 1) + 1] == x - 1) { 92 | rs->ranges[2 * (r - 1) + 1] = x; 93 | } 94 | 95 | /* If adjoining a range above, add to it */ 96 | else if (r < rs->numranges && rs->ranges[2 * r] == x + 1) { 97 | rs->ranges[2 * r] = x; 98 | } 99 | 100 | else { /* New range for this block alone */ 101 | rs->ranges = 102 | realloc(rs->ranges, 103 | (rs->numranges + 1) * 2 * sizeof(rs->ranges[0])); 104 | memmove(&rs->ranges[2 * r + 2], &rs->ranges[2 * r], 105 | (rs->numranges - r) * 2 * sizeof(rs->ranges[0])); 106 | rs->ranges[2 * r] = rs->ranges[2 * r + 1] = x; 107 | rs->numranges++; 108 | } 109 | } 110 | #if 0 111 | { 112 | int i; 113 | for (i = 0; i < rs->numranges; i++) 114 | fprintf(stderr, "%d-%d,", rs->ranges[i * 2], rs->ranges[i * 2 + 1]); 115 | fprintf(stderr, " are the current ranges got (added %d, %d)\n\n", x, r); 116 | } 117 | #endif 118 | } 119 | 120 | /* already_got_block 121 | * Return true iff blockid x of the target file is already known */ 122 | int already_got_block(struct rcksum_state *rs, zs_blockid x) { 123 | return (range_before_block(rs, x) == -1); 124 | } 125 | 126 | /* next_blockid = next_known_block(rs, blockid) 127 | * Returns the blockid of the next block which we already have data for. 128 | * If we know the requested block, it returns the blockid given; otherwise it 129 | * will return a later blockid. 130 | * If no later blocks are known, it returns rs->numblocks (i.e. the block after 131 | * the end of the file). 132 | */ 133 | zs_blockid next_known_block(struct rcksum_state *rs, zs_blockid x) { 134 | int r = range_before_block(rs, x); 135 | if (r == -1) 136 | return x; 137 | if (r == rs->numranges) { 138 | return rs->blocks; 139 | } 140 | /* Else return first block of next known range. */ 141 | return rs->ranges[2*r]; 142 | } 143 | 144 | /* rcksum_needed_block_ranges 145 | * Return the block ranges needed to complete the target file */ 146 | zs_blockid *rcksum_needed_block_ranges(const struct rcksum_state * rs, int *num, 147 | zs_blockid from, zs_blockid to) { 148 | int i, n; 149 | int alloc_n = 100; 150 | zs_blockid *r = malloc(2 * alloc_n * sizeof(zs_blockid)); 151 | 152 | if (!r) 153 | return NULL; 154 | 155 | if (to >= rs->blocks) 156 | to = rs->blocks; 157 | r[0] = from; 158 | r[1] = to; 159 | n = 1; 160 | /* Note r[2*n-1] is the last range in our prospective list */ 161 | 162 | for (i = 0; i < rs->numranges; i++) { 163 | if (rs->ranges[2 * i] > r[2 * n - 1]) 164 | continue; 165 | if (rs->ranges[2 * i + 1] < from) 166 | continue; 167 | 168 | /* Okay, they intersect */ 169 | if (n == 1 && rs->ranges[2 * i] <= from) { /* Overlaps the start of our window */ 170 | r[0] = rs->ranges[2 * i + 1] + 1; 171 | } 172 | else { 173 | /* If the last block that we still (which is the last window end -1, due 174 | * to half-openness) then this range just cuts the end of our window */ 175 | if (rs->ranges[2 * i + 1] >= r[2 * n - 1] - 1) { 176 | r[2 * n - 1] = rs->ranges[2 * i]; 177 | } 178 | else { 179 | /* In the middle of our range, split it */ 180 | r[2 * n] = rs->ranges[2 * i + 1] + 1; 181 | r[2 * n + 1] = r[2 * n - 1]; 182 | r[2 * n - 1] = rs->ranges[2 * i]; 183 | n++; 184 | if (n == alloc_n) { 185 | zs_blockid *r2; 186 | alloc_n += 100; 187 | r2 = realloc(r, 2 * alloc_n * sizeof *r); 188 | if (!r2) { 189 | free(r); 190 | return NULL; 191 | } 192 | r = r2; 193 | } 194 | } 195 | } 196 | } 197 | r = realloc(r, 2 * n * sizeof *r); 198 | if (n == 1 && r[0] >= r[1]) 199 | n = 0; 200 | 201 | *num = n; 202 | return r; 203 | } 204 | 205 | /* rcksum_blocks_todo 206 | * Return the number of blocks still needed to complete the target file */ 207 | int rcksum_blocks_todo(const struct rcksum_state *rs) { 208 | int i, n = rs->blocks; 209 | for (i = 0; i < rs->numranges; i++) { 210 | n -= 1 + rs->ranges[2 * i + 1] - rs->ranges[2 * i]; 211 | } 212 | return n; 213 | } 214 | -------------------------------------------------------------------------------- /tools/run-tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import csv 4 | import errno 5 | import getopt 6 | import re 7 | import os 8 | import os.path 9 | import platform 10 | import shutil 11 | import subprocess 12 | import sys 13 | import time 14 | 15 | tests = [1,2,3,4,5,6,7,8,9,11,12,13,14,20,21,24,27] 16 | zsyncdir = "/home/cph/src/zsync/main/c" 17 | zsync = os.path.join(zsyncdir, "zsync") 18 | desc = None 19 | ver_suffix = None 20 | overwrite = False 21 | fails = 0 22 | skip_present = False # Skip if test results filename already present. 23 | 24 | opts, args = getopt.getopt(sys.argv[1:], '', ['zsync=', 'tests=', 'desc=', 'overwrite', 'ver-suffix=', 'skip-present']) 25 | 26 | class TestingError: 27 | def __init__(self, errstr): 28 | self._errstr = errstr 29 | def __str__(self): 30 | return self._errstr 31 | 32 | for option, value in opts: 33 | if option == '--zsync': 34 | zsync = value 35 | elif option == '--tests': 36 | tests = [int(t) for t in value.split(',')] 37 | elif option == '--ver-suffix': 38 | ver_suffix = value 39 | elif option == '--desc': 40 | desc = value 41 | elif option == '--overwrite': 42 | overwrite = True 43 | elif option == '--skip-present': 44 | skip_present = True 45 | else: 46 | raise TestingError("Unhandled argument %s=%s" % (option, value)) 47 | 48 | if args: 49 | raise TestingError("Surplus arguments: %s" % " ".join(args)) 50 | 51 | zsync_V = subprocess.check_output([zsync, "-V"]) 52 | zsync_ver = zsync_V.split(" ")[1] 53 | tmpdir = "tmp" 54 | tests_url = "http://localhost:2000/ztest/" 55 | 56 | def get_cpu_mhz(): 57 | with open("/proc/cpuinfo") as f: 58 | for l in f.readlines(): 59 | if 'MHz' in l: 60 | l = l.rstrip() 61 | return l.split(': ')[1] 62 | 63 | # Stats common to all the tests. 64 | base_stats = { 'zsync version': zsync_ver, 'extra version': ver_suffix, 65 | 'other': desc, 'platform': platform.machine(), 'cpu mhz': get_cpu_mhz(), 66 | 'fetched localhost': ('localhost' in tests_url)} 67 | 68 | used_fetched_re = re.compile(r'^used ([0-9]+) local, fetched ([0-9]+)') 69 | # Allow negative hashhit, old versions of zsync would wrap around. 70 | hashhit_re = re.compile(r'^hashhit ([-0-9]+), weakhit ([0-9]+), checksummed ([0-9]+), stronghit ([0-9]+)') 71 | blocksize_re = re.compile(r'^Blocksize: ([0-9]+)') 72 | 73 | for test in tests: 74 | # Get zsyncmake version for this .zsync file. 75 | with open(os.path.join(str(test), "get.zsync")) as f: 76 | zsyncmake_ver_str = f.readline() 77 | zsyncmake_ver = zsyncmake_ver_str.split(" ")[1].strip() 78 | zsync_size = os.fstat(f.fileno()).st_size 79 | for l in f: 80 | m = blocksize_re.match(l) 81 | if m: 82 | blocksize = int(m.group(1)) 83 | break 84 | 85 | # Clean up scratch tree. 86 | shutil.rmtree(tmpdir, ignore_errors=True) 87 | os.mkdir(tmpdir) 88 | 89 | print "--- Test %d ---" % test 90 | failed = False 91 | should_fail = os.access(os.path.join(str(test), "SHOULD-FAIL"), os.F_OK) 92 | 93 | # Build suffix for test results filenames. 94 | test_run_suffix_parts = [zsync_ver] 95 | if ver_suffix: 96 | test_run_suffix_parts.append("ver_suffix=%s" % ver_suffix) 97 | test_run_suffix_parts.append("zsyncmake=%s" % zsyncmake_ver) 98 | if desc: 99 | test_run_suffix_parts.append(" desc=%s" % desc) 100 | 101 | cmd = [zsync, '-i', os.path.join("..", str(test), "old"), tests_url + "%d/get.zsync" % test] 102 | 103 | # File for transcript of test run. 104 | transcript_filename = os.path.join(str(test), "script %s" % (" ".join(test_run_suffix_parts))) 105 | if os.access(transcript_filename, os.F_OK): 106 | if skip_present: 107 | continue 108 | if not overwrite: 109 | raise TestingError("File %s exists" % transcript_filename) 110 | 111 | with open(transcript_filename, "w") as transcript: 112 | starttime = time.time() 113 | parsed_stats = {} 114 | transcript.writelines([" ".join(cmd)]) 115 | p = subprocess.Popen(args=cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=tmpdir) 116 | 117 | # Read output and write to transcript. 118 | for l in p.stdout: 119 | print l, 120 | sys.stdout.flush() 121 | transcript.write(l) 122 | m = hashhit_re.match(l) 123 | if m: 124 | parsed_stats.update(zip( 125 | ['hashhit', 'weakhit', 'checksummed', 'stronghit'], 126 | [int(m.group(i)) for i in range(1, 5)])) 127 | m = used_fetched_re.match(l) 128 | if m: 129 | parsed_stats.update(zip( 130 | ['used local', 'fetched'], 131 | [int(m.group(i)) for i in range(1, 3)])) 132 | 133 | # Get exit status, rusage info, running time for the zsync run. 134 | pid_unused, status, rusage = os.wait4(p.pid, 0) 135 | elapsed_time = time.time() - starttime 136 | 137 | # If should_fail, expect exit status 3 from the process (could not get 138 | # all data). Otherwise expect success. 139 | if (status != 0 and not should_fail) or ((not(os.WIFEXITED(status)) or os.WEXITSTATUS(status) != 3) and should_fail): 140 | failed = True 141 | print "*** Test %d FAILED (status=%d) ***" % (test, status) 142 | transcript.write("FAIL\n") 143 | else: 144 | transcript.write("PASS\n") 145 | 146 | # Append human-readable stats to the transcript 147 | transcript.write("user=%fs\nsystem=%fs\nelapsed=%ds\n" % 148 | (rusage.ru_utime, rusage.ru_stime, elapsed_time)) 149 | transcript.write("faults: minor=%d, major=%d\n" % (rusage.ru_minflt, rusage.ru_majflt)) 150 | transcript.write("swap_out=%d\n" % (rusage.ru_nswap)) 151 | transcript.write("block io: in=%d, out=%d\n" % (rusage.ru_inblock, rusage.ru_oublock)) 152 | transcript.write("context switches: voluntary=%d, involutary=%d\n" % (rusage.ru_nvcsw, rusage.ru_nivcsw)) 153 | 154 | if failed: fails += 1 155 | # Remove downloaded file, possibly expecting failure. 156 | try: 157 | got_file = os.path.join(tmpdir, "get") 158 | if os.access("%s.gz" % got_file, os.F_OK): 159 | got_file = "%s.gz" % got_file 160 | os.unlink(got_file) 161 | except OSError, e: 162 | if e.errno != errno.ENOENT or (not should_fail and not failed): 163 | raise 164 | 165 | # Write stats csv. 166 | stats_filename = os.path.join(str(test), "stats %s.csv" % (" ".join(test_run_suffix_parts))) 167 | if os.access(stats_filename, os.F_OK) and not overwrite: 168 | raise TestingError("File %s exists" % stats_filename) 169 | with open(stats_filename, "w") as stats: 170 | sw = csv.DictWriter(stats, ['zsync version', 'extra version', 'zsyncmake version', '.zsync size', 'blocksize', 'other', 'pass', 'user time', 'system time', 'elapsed time', 'platform', 'cpu mhz', 'fetched localhost', 'used local', 'fetched', 'hashhit', 'weakhit', 'checksummed', 'stronghit', 'page faults (minor)', 'page faults (major)', 'swaps out', 'block io in', 'block io out', 'context switches (voluntary)', 'context switches (involuntary)']) 171 | test_stats = { 172 | 'zsyncmake version': zsyncmake_ver, 173 | '.zsync size': zsync_size, 174 | 'blocksize': blocksize, 175 | 'pass': not failed, 'elapsed time': elapsed_time, 176 | 'user time': rusage.ru_utime, 'system time': rusage.ru_stime, 177 | 'page faults (minor)': rusage.ru_minflt, 178 | 'page faults (major)': rusage.ru_majflt, 179 | 'swaps out': rusage.ru_nswap, 180 | 'block io in': rusage.ru_inblock, 'block io out': rusage.ru_oublock, 181 | 'context switches (voluntary)': rusage.ru_nvcsw, 182 | 'context switches (involuntary)': rusage.ru_nivcsw 183 | } 184 | test_stats.update(base_stats) 185 | test_stats.update(parsed_stats) 186 | sw.writeheader() 187 | sw.writerow(test_stats) 188 | 189 | # Clean up, and exit reflecting test status. 190 | shutil.rmtree(tmpdir, ignore_errors=True) 191 | if fails: 192 | print "%d tests FAILed" % fails 193 | sys.exit(1) 194 | -------------------------------------------------------------------------------- /c/librcksum/md4.c: -------------------------------------------------------------------------------- 1 | /* $OpenBSD: md4.c,v 1.6 2004/05/28 15:10:27 millert Exp $ */ 2 | 3 | /* 4 | * This code implements the MD4 message-digest algorithm. 5 | * The algorithm is due to Ron Rivest. This code was 6 | * written by Colin Plumb in 1993, no copyright is claimed. 7 | * This code is in the public domain; do with it what you wish. 8 | * Todd C. Miller modified the MD5 code to do MD4 based on RFC 1186. 9 | * 10 | * Equivalent code is available from RSA Data Security, Inc. 11 | * This code has been tested against that, and is equivalent, 12 | * except that you don't need to include two pages of legalese 13 | * with every copy. 14 | * 15 | * To compute the message digest of a chunk of bytes, declare an 16 | * MD4Context structure, pass it to MD4Init, call MD4Update as 17 | * needed on buffers full of bytes, and then call MD4Final, which 18 | * will fill a supplied 16-byte array with the digest. 19 | */ 20 | 21 | #if defined(LIBC_SCCS) && !defined(lint) 22 | static const char rcsid[] = "$OpenBSD: md4.c,v 1.6 2004/05/28 15:10:27 millert Exp $"; 23 | #endif /* LIBC_SCCS and not lint */ 24 | 25 | #include 26 | #include 27 | #include "md4.h" 28 | 29 | /* Map Solaris endian stuff to something useful */ 30 | #if defined(_BIG_ENDIAN) && !defined(_BYTE_ORDER) 31 | #define LITTLE_ENDIAN 0 32 | #define BIG_ENDIAN 1 33 | #define BYTE_ORDER 1 34 | #endif 35 | 36 | #pragma clang diagnostic ignored "-Wconversion" 37 | 38 | #define PUT_64BIT_LE(cp, value) do { \ 39 | (cp)[7] = (value) >> 56; \ 40 | (cp)[6] = (value) >> 48; \ 41 | (cp)[5] = (value) >> 40; \ 42 | (cp)[4] = (value) >> 32; \ 43 | (cp)[3] = (value) >> 24; \ 44 | (cp)[2] = (value) >> 16; \ 45 | (cp)[1] = (value) >> 8; \ 46 | (cp)[0] = (value); } while (0) 47 | 48 | #define PUT_32BIT_LE(cp, value) do { \ 49 | (cp)[3] = (value) >> 24; \ 50 | (cp)[2] = (value) >> 16; \ 51 | (cp)[1] = (value) >> 8; \ 52 | (cp)[0] = (value); } while (0) 53 | 54 | static uint8_t PADDING[MD4_BLOCK_LENGTH] = { 55 | 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 58 | }; 59 | 60 | /* 61 | * Start MD4 accumulation. 62 | * Set bit count to 0 and buffer to mysterious initialization constants. 63 | */ 64 | void 65 | MD4Init(MD4_CTX *ctx) 66 | { 67 | ctx->count = 0; 68 | ctx->state[0] = 0x67452301; 69 | ctx->state[1] = 0xefcdab89; 70 | ctx->state[2] = 0x98badcfe; 71 | ctx->state[3] = 0x10325476; 72 | } 73 | 74 | /* 75 | * Update context to reflect the concatenation of another buffer full 76 | * of bytes. 77 | */ 78 | void 79 | MD4Update(MD4_CTX *ctx, const unsigned char *input, size_t len) 80 | { 81 | size_t have, need; 82 | 83 | /* Check how many bytes we already have and how many more we need. */ 84 | have = (size_t)((ctx->count >> 3) & (MD4_BLOCK_LENGTH - 1)); 85 | need = MD4_BLOCK_LENGTH - have; 86 | 87 | /* Update bitcount */ 88 | ctx->count += (uint64_t)len << 3; 89 | 90 | if (len >= need) { 91 | if (have != 0) { 92 | memcpy(ctx->buffer + have, input, need); 93 | MD4Transform(ctx->state, ctx->buffer); 94 | input += need; 95 | len -= need; 96 | have = 0; 97 | } 98 | 99 | /* Process data in MD4_BLOCK_LENGTH-byte chunks. */ 100 | while (len >= MD4_BLOCK_LENGTH) { 101 | MD4Transform(ctx->state, input); 102 | input += MD4_BLOCK_LENGTH; 103 | len -= MD4_BLOCK_LENGTH; 104 | } 105 | } 106 | 107 | /* Handle any remaining bytes of data. */ 108 | if (len != 0) 109 | memcpy(ctx->buffer + have, input, len); 110 | } 111 | 112 | /* 113 | * Pad pad to 64-byte boundary with the bit pattern 114 | * 1 0* (64-bit count of bits processed, MSB-first) 115 | */ 116 | void 117 | MD4Pad(MD4_CTX *ctx) 118 | { 119 | uint8_t count[8]; 120 | size_t padlen; 121 | 122 | /* Convert count to 8 bytes in little endian order. */ 123 | PUT_64BIT_LE(count, ctx->count); 124 | 125 | /* Pad out to 56 mod 64. */ 126 | padlen = MD4_BLOCK_LENGTH - 127 | ((ctx->count >> 3) & (MD4_BLOCK_LENGTH - 1)); 128 | if (padlen < 1 + 8) 129 | padlen += MD4_BLOCK_LENGTH; 130 | MD4Update(ctx, PADDING, padlen - 8); /* padlen - 8 <= 64 */ 131 | MD4Update(ctx, count, 8); 132 | } 133 | 134 | /* 135 | * Final wrapup--call MD4Pad, fill in digest and zero out ctx. 136 | */ 137 | void 138 | MD4Final(unsigned char digest[MD4_DIGEST_LENGTH], MD4_CTX *ctx) 139 | { 140 | int i; 141 | 142 | MD4Pad(ctx); 143 | if (digest != NULL) { 144 | for (i = 0; i < 4; i++) 145 | PUT_32BIT_LE(digest + i * 4, ctx->state[i]); 146 | memset(ctx, 0, sizeof(*ctx)); 147 | } 148 | } 149 | 150 | 151 | /* The three core functions - F1 is optimized somewhat */ 152 | 153 | /* #define F1(x, y, z) (x & y | ~x & z) */ 154 | #define F1(x, y, z) (z ^ (x & (y ^ z))) 155 | #define F2(x, y, z) ((x & y) | (x & z) | (y & z)) 156 | #define F3(x, y, z) (x ^ y ^ z) 157 | 158 | /* This is the central step in the MD4 algorithm. */ 159 | #define MD4STEP(f, w, x, y, z, data, s) \ 160 | ( w += f(x, y, z) + data, w = w<>(32-s) ) 161 | 162 | /* 163 | * The core of the MD4 algorithm, this alters an existing MD4 hash to 164 | * reflect the addition of 16 longwords of new data. MD4Update blocks 165 | * the data and converts bytes into longwords for this routine. 166 | */ 167 | void 168 | MD4Transform(uint32_t state[4], const uint8_t block[MD4_BLOCK_LENGTH]) 169 | { 170 | uint32_t a, b, c, d, in[MD4_BLOCK_LENGTH / 4]; 171 | 172 | #if BYTE_ORDER == LITTLE_ENDIAN 173 | memcpy(in, block, sizeof(in)); 174 | #else 175 | for (a = 0; a < MD4_BLOCK_LENGTH / 4; a++) { 176 | in[a] = (uint32_t)( 177 | (uint32_t)(block[a * 4 + 0]) | 178 | (uint32_t)(block[a * 4 + 1]) << 8 | 179 | (uint32_t)(block[a * 4 + 2]) << 16 | 180 | (uint32_t)(block[a * 4 + 3]) << 24); 181 | } 182 | #endif 183 | 184 | a = state[0]; 185 | b = state[1]; 186 | c = state[2]; 187 | d = state[3]; 188 | 189 | MD4STEP(F1, a, b, c, d, in[ 0], 3); 190 | MD4STEP(F1, d, a, b, c, in[ 1], 7); 191 | MD4STEP(F1, c, d, a, b, in[ 2], 11); 192 | MD4STEP(F1, b, c, d, a, in[ 3], 19); 193 | MD4STEP(F1, a, b, c, d, in[ 4], 3); 194 | MD4STEP(F1, d, a, b, c, in[ 5], 7); 195 | MD4STEP(F1, c, d, a, b, in[ 6], 11); 196 | MD4STEP(F1, b, c, d, a, in[ 7], 19); 197 | MD4STEP(F1, a, b, c, d, in[ 8], 3); 198 | MD4STEP(F1, d, a, b, c, in[ 9], 7); 199 | MD4STEP(F1, c, d, a, b, in[10], 11); 200 | MD4STEP(F1, b, c, d, a, in[11], 19); 201 | MD4STEP(F1, a, b, c, d, in[12], 3); 202 | MD4STEP(F1, d, a, b, c, in[13], 7); 203 | MD4STEP(F1, c, d, a, b, in[14], 11); 204 | MD4STEP(F1, b, c, d, a, in[15], 19); 205 | 206 | MD4STEP(F2, a, b, c, d, in[ 0] + 0x5a827999, 3); 207 | MD4STEP(F2, d, a, b, c, in[ 4] + 0x5a827999, 5); 208 | MD4STEP(F2, c, d, a, b, in[ 8] + 0x5a827999, 9); 209 | MD4STEP(F2, b, c, d, a, in[12] + 0x5a827999, 13); 210 | MD4STEP(F2, a, b, c, d, in[ 1] + 0x5a827999, 3); 211 | MD4STEP(F2, d, a, b, c, in[ 5] + 0x5a827999, 5); 212 | MD4STEP(F2, c, d, a, b, in[ 9] + 0x5a827999, 9); 213 | MD4STEP(F2, b, c, d, a, in[13] + 0x5a827999, 13); 214 | MD4STEP(F2, a, b, c, d, in[ 2] + 0x5a827999, 3); 215 | MD4STEP(F2, d, a, b, c, in[ 6] + 0x5a827999, 5); 216 | MD4STEP(F2, c, d, a, b, in[10] + 0x5a827999, 9); 217 | MD4STEP(F2, b, c, d, a, in[14] + 0x5a827999, 13); 218 | MD4STEP(F2, a, b, c, d, in[ 3] + 0x5a827999, 3); 219 | MD4STEP(F2, d, a, b, c, in[ 7] + 0x5a827999, 5); 220 | MD4STEP(F2, c, d, a, b, in[11] + 0x5a827999, 9); 221 | MD4STEP(F2, b, c, d, a, in[15] + 0x5a827999, 13); 222 | 223 | MD4STEP(F3, a, b, c, d, in[ 0] + 0x6ed9eba1, 3); 224 | MD4STEP(F3, d, a, b, c, in[ 8] + 0x6ed9eba1, 9); 225 | MD4STEP(F3, c, d, a, b, in[ 4] + 0x6ed9eba1, 11); 226 | MD4STEP(F3, b, c, d, a, in[12] + 0x6ed9eba1, 15); 227 | MD4STEP(F3, a, b, c, d, in[ 2] + 0x6ed9eba1, 3); 228 | MD4STEP(F3, d, a, b, c, in[10] + 0x6ed9eba1, 9); 229 | MD4STEP(F3, c, d, a, b, in[ 6] + 0x6ed9eba1, 11); 230 | MD4STEP(F3, b, c, d, a, in[14] + 0x6ed9eba1, 15); 231 | MD4STEP(F3, a, b, c, d, in[ 1] + 0x6ed9eba1, 3); 232 | MD4STEP(F3, d, a, b, c, in[ 9] + 0x6ed9eba1, 9); 233 | MD4STEP(F3, c, d, a, b, in[ 5] + 0x6ed9eba1, 11); 234 | MD4STEP(F3, b, c, d, a, in[13] + 0x6ed9eba1, 15); 235 | MD4STEP(F3, a, b, c, d, in[ 3] + 0x6ed9eba1, 3); 236 | MD4STEP(F3, d, a, b, c, in[11] + 0x6ed9eba1, 9); 237 | MD4STEP(F3, c, d, a, b, in[ 7] + 0x6ed9eba1, 11); 238 | MD4STEP(F3, b, c, d, a, in[15] + 0x6ed9eba1, 15); 239 | 240 | state[0] += a; 241 | state[1] += b; 242 | state[2] += c; 243 | state[3] += d; 244 | } 245 | -------------------------------------------------------------------------------- /c/zlib/zutil.c: -------------------------------------------------------------------------------- 1 | /* zutil.c -- target dependent utility functions for the compression library 2 | * Copyright (C) 1995-2003 Jean-loup Gailly. 3 | * For conditions of distribution and use, see copyright notice in zlib.h 4 | */ 5 | 6 | /* @(#) $Id$ */ 7 | 8 | #include "zutil.h" 9 | 10 | #ifndef NO_DUMMY_DECL 11 | struct internal_state {int dummy;}; /* for buggy compilers */ 12 | #endif 13 | 14 | #ifndef STDC 15 | extern void exit OF((int)); 16 | #endif 17 | 18 | const char * const z_errmsg[10] = { 19 | "need dictionary", /* Z_NEED_DICT 2 */ 20 | "stream end", /* Z_STREAM_END 1 */ 21 | "", /* Z_OK 0 */ 22 | "file error", /* Z_ERRNO (-1) */ 23 | "stream error", /* Z_STREAM_ERROR (-2) */ 24 | "data error", /* Z_DATA_ERROR (-3) */ 25 | "insufficient memory", /* Z_MEM_ERROR (-4) */ 26 | "buffer error", /* Z_BUF_ERROR (-5) */ 27 | "incompatible version",/* Z_VERSION_ERROR (-6) */ 28 | ""}; 29 | 30 | 31 | const char * ZEXPORT zlibVersion() 32 | { 33 | return ZLIB_VERSION; 34 | } 35 | 36 | uLong ZEXPORT zlibCompileFlags() 37 | { 38 | uLong flags; 39 | 40 | flags = 0; 41 | switch (sizeof(uInt)) { 42 | case 2: break; 43 | case 4: flags += 1; break; 44 | case 8: flags += 2; break; 45 | default: flags += 3; 46 | } 47 | switch (sizeof(uLong)) { 48 | case 2: break; 49 | case 4: flags += 1 << 2; break; 50 | case 8: flags += 2 << 2; break; 51 | default: flags += 3 << 2; 52 | } 53 | switch (sizeof(voidpf)) { 54 | case 2: break; 55 | case 4: flags += 1 << 4; break; 56 | case 8: flags += 2 << 4; break; 57 | default: flags += 3 << 4; 58 | } 59 | switch (sizeof(z_off_t)) { 60 | case 2: break; 61 | case 4: flags += 1 << 6; break; 62 | case 8: flags += 2 << 6; break; 63 | default: flags += 3 << 6; 64 | } 65 | #ifdef DEBUG 66 | flags += 1 << 8; 67 | #endif 68 | #if defined(ASMV) || defined(ASMINF) 69 | flags += 1 << 9; 70 | #endif 71 | #ifdef ZLIB_WINAPI 72 | flags += 1 << 10; 73 | #endif 74 | #ifdef BUILDFIXED 75 | flags += 1 << 12; 76 | #endif 77 | #ifdef DYNAMIC_CRC_TABLE 78 | flags += 1 << 13; 79 | #endif 80 | #ifdef NO_GZCOMPRESS 81 | flags += 1 << 16; 82 | #endif 83 | #ifdef NO_GZIP 84 | flags += 1 << 17; 85 | #endif 86 | #ifdef PKZIP_BUG_WORKAROUND 87 | flags += 1 << 20; 88 | #endif 89 | #ifdef FASTEST 90 | flags += 1 << 21; 91 | #endif 92 | #ifdef STDC 93 | # ifdef NO_vsnprintf 94 | flags += 1 << 25; 95 | # ifdef HAS_vsprintf_void 96 | flags += 1 << 26; 97 | # endif 98 | # else 99 | # ifdef HAS_vsnprintf_void 100 | flags += 1 << 26; 101 | # endif 102 | # endif 103 | #else 104 | flags += 1 << 24; 105 | # ifdef NO_snprintf 106 | flags += 1 << 25; 107 | # ifdef HAS_sprintf_void 108 | flags += 1 << 26; 109 | # endif 110 | # else 111 | # ifdef HAS_snprintf_void 112 | flags += 1 << 26; 113 | # endif 114 | # endif 115 | #endif 116 | return flags; 117 | } 118 | 119 | #ifdef DEBUG 120 | 121 | # ifndef verbose 122 | # define verbose 0 123 | # endif 124 | int z_verbose = verbose; 125 | 126 | void z_error (m) 127 | char *m; 128 | { 129 | fprintf(stderr, "%s\n", m); 130 | exit(1); 131 | } 132 | #endif 133 | 134 | /* exported to allow conversion of error code to string for compress() and 135 | * uncompress() 136 | */ 137 | const char * ZEXPORT zError(err) 138 | int err; 139 | { 140 | return ERR_MSG(err); 141 | } 142 | 143 | #if defined(_WIN32_WCE) 144 | /* does not exist on WCE */ 145 | int errno = 0; 146 | #endif 147 | 148 | #ifndef HAVE_MEMCPY 149 | 150 | void zmemcpy(dest, source, len) 151 | Bytef* dest; 152 | const Bytef* source; 153 | uInt len; 154 | { 155 | if (len == 0) return; 156 | do { 157 | *dest++ = *source++; /* ??? to be unrolled */ 158 | } while (--len != 0); 159 | } 160 | 161 | int zmemcmp(s1, s2, len) 162 | const Bytef* s1; 163 | const Bytef* s2; 164 | uInt len; 165 | { 166 | uInt j; 167 | 168 | for (j = 0; j < len; j++) { 169 | if (s1[j] != s2[j]) return 2*(s1[j] > s2[j])-1; 170 | } 171 | return 0; 172 | } 173 | 174 | void zmemzero(dest, len) 175 | Bytef* dest; 176 | uInt len; 177 | { 178 | if (len == 0) return; 179 | do { 180 | *dest++ = 0; /* ??? to be unrolled */ 181 | } while (--len != 0); 182 | } 183 | #endif 184 | 185 | 186 | #ifdef SYS16BIT 187 | 188 | #ifdef __TURBOC__ 189 | /* Turbo C in 16-bit mode */ 190 | 191 | # define MY_ZCALLOC 192 | 193 | /* Turbo C malloc() does not allow dynamic allocation of 64K bytes 194 | * and farmalloc(64K) returns a pointer with an offset of 8, so we 195 | * must fix the pointer. Warning: the pointer must be put back to its 196 | * original form in order to free it, use zcfree(). 197 | */ 198 | 199 | #define MAX_PTR 10 200 | /* 10*64K = 640K */ 201 | 202 | local int next_ptr = 0; 203 | 204 | typedef struct ptr_table_s { 205 | voidpf org_ptr; 206 | voidpf new_ptr; 207 | } ptr_table; 208 | 209 | local ptr_table table[MAX_PTR]; 210 | /* This table is used to remember the original form of pointers 211 | * to large buffers (64K). Such pointers are normalized with a zero offset. 212 | * Since MSDOS is not a preemptive multitasking OS, this table is not 213 | * protected from concurrent access. This hack doesn't work anyway on 214 | * a protected system like OS/2. Use Microsoft C instead. 215 | */ 216 | 217 | voidpf zcalloc (voidpf opaque, unsigned items, unsigned size) 218 | { 219 | voidpf buf = opaque; /* just to make some compilers happy */ 220 | ulg bsize = (ulg)items*size; 221 | 222 | /* If we allocate less than 65520 bytes, we assume that farmalloc 223 | * will return a usable pointer which doesn't have to be normalized. 224 | */ 225 | if (bsize < 65520L) { 226 | buf = farmalloc(bsize); 227 | if (*(ush*)&buf != 0) return buf; 228 | } else { 229 | buf = farmalloc(bsize + 16L); 230 | } 231 | if (buf == NULL || next_ptr >= MAX_PTR) return NULL; 232 | table[next_ptr].org_ptr = buf; 233 | 234 | /* Normalize the pointer to seg:0 */ 235 | *((ush*)&buf+1) += ((ush)((uch*)buf-0) + 15) >> 4; 236 | *(ush*)&buf = 0; 237 | table[next_ptr++].new_ptr = buf; 238 | return buf; 239 | } 240 | 241 | void zcfree (voidpf opaque, voidpf ptr) 242 | { 243 | int n; 244 | if (*(ush*)&ptr != 0) { /* object < 64K */ 245 | farfree(ptr); 246 | return; 247 | } 248 | /* Find the original pointer */ 249 | for (n = 0; n < next_ptr; n++) { 250 | if (ptr != table[n].new_ptr) continue; 251 | 252 | farfree(table[n].org_ptr); 253 | while (++n < next_ptr) { 254 | table[n-1] = table[n]; 255 | } 256 | next_ptr--; 257 | return; 258 | } 259 | ptr = opaque; /* just to make some compilers happy */ 260 | Assert(0, "zcfree: ptr not found"); 261 | } 262 | 263 | #endif /* __TURBOC__ */ 264 | 265 | 266 | #ifdef M_I86 267 | /* Microsoft C in 16-bit mode */ 268 | 269 | # define MY_ZCALLOC 270 | 271 | #if (!defined(_MSC_VER) || (_MSC_VER <= 600)) 272 | # define _halloc halloc 273 | # define _hfree hfree 274 | #endif 275 | 276 | voidpf zcalloc (voidpf opaque, unsigned items, unsigned size) 277 | { 278 | if (opaque) opaque = 0; /* to make compiler happy */ 279 | return _halloc((long)items, size); 280 | } 281 | 282 | void zcfree (voidpf opaque, voidpf ptr) 283 | { 284 | if (opaque) opaque = 0; /* to make compiler happy */ 285 | _hfree(ptr); 286 | } 287 | 288 | #endif /* M_I86 */ 289 | 290 | #endif /* SYS16BIT */ 291 | 292 | 293 | #ifndef MY_ZCALLOC /* Any system without a special alloc function */ 294 | 295 | #ifndef STDC 296 | extern voidp malloc OF((uInt size)); 297 | extern voidp calloc OF((uInt items, uInt size)); 298 | extern void free OF((voidpf ptr)); 299 | #endif 300 | 301 | voidpf zcalloc (opaque, items, size) 302 | voidpf opaque; 303 | unsigned items; 304 | unsigned size; 305 | { 306 | if (opaque) items += size - size; /* make compiler happy */ 307 | return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : 308 | (voidpf)calloc(items, size); 309 | } 310 | 311 | void zcfree (opaque, ptr) 312 | voidpf opaque; 313 | voidpf ptr; 314 | { 315 | free(ptr); 316 | if (opaque) return; /* make compiler happy */ 317 | } 318 | 319 | #endif /* MY_ZCALLOC */ 320 | -------------------------------------------------------------------------------- /c/librcksum/hash.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * rcksum/lib - library for using the rsync algorithm to determine 4 | * which parts of a file you have and which you need. 5 | * Copyright (C) 2004,2005,2007,2009 Colin Phipps 6 | * 7 | * This program is free software; you can redistribute it and/or modify 8 | * it under the terms of the Artistic License v2 (see the accompanying 9 | * file COPYING for the full license terms), or, at your option, any later 10 | * version of the same license. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * COPYING file for details. 16 | */ 17 | 18 | /* Functions to manage the rsum and checksum values per block and set up the 19 | * hash tables of the rsum values. */ 20 | 21 | #include "zsglobal.h" 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #ifdef WITH_DMALLOC 28 | # include 29 | #endif 30 | 31 | #include "rcksum.h" 32 | #include "internal.h" 33 | 34 | /* rcksum_add_target_block(self, blockid, rsum, checksum) 35 | * Sets the stored hash values for the given blockid to the given values. 36 | */ 37 | void rcksum_add_target_block(struct rcksum_state *z, zs_blockid b, 38 | struct rsum r, void *checksum) { 39 | if (b < z->blocks) { 40 | /* Get hash entry with checksums for this block */ 41 | struct hash_entry *e = &(z->blockhashes[b]); 42 | 43 | /* Enter checksums */ 44 | memcpy(e->checksum, checksum, z->checksum_bytes); 45 | e->r.a = r.a & z->rsum_a_mask; 46 | e->r.b = r.b; 47 | 48 | /* New checksums invalidate any existing checksum hash tables */ 49 | if (z->rsum_hash) { 50 | free(z->rsum_hash); 51 | z->rsum_hash = NULL; 52 | free(z->bithash); 53 | z->bithash = NULL; 54 | } 55 | } 56 | } 57 | 58 | static void print_hashstats(const struct rcksum_state* z) { 59 | #ifdef DEBUG 60 | int i; 61 | { 62 | int num_bits_set = 0; 63 | for (i = 0; i < z->bithashmask + 1; i++) { 64 | unsigned char c; 65 | for (c = z->bithash[i]; c; c &= c - 1) 66 | num_bits_set++; 67 | } 68 | 69 | fprintf(stderr, "bithash %dKB, density %.1f%%\n", 70 | (z->bithashmask+1)/(1000 * 8), 71 | 100.0 * num_bits_set / (z->bithashmask + 1)); 72 | } 73 | { 74 | int hash_entries_used = 0; 75 | int max_depth = 0; 76 | for (i = 0; i < z->hashmask + 1; i++) { 77 | struct hash_entry* p = z->rsum_hash[i]; 78 | if (!p) continue; 79 | hash_entries_used++; 80 | int depth; 81 | for (depth = 0; p; p = p->next) 82 | depth++; 83 | if (depth > max_depth) max_depth = depth; 84 | } 85 | fprintf(stderr, 86 | "rsum hash density: %d/%d %.1f%% (depth avg: %.1f, max: %d)\n", 87 | hash_entries_used, z->hashmask+1, 88 | 100.0 * hash_entries_used / (z->hashmask + 1), 89 | z->blocks / (float)hash_entries_used, max_depth); 90 | } 91 | #if 0 92 | { /* Print blocks on "0" hash-chain */ 93 | unsigned hash = 0; 94 | const struct hash_entry *p = z->rsum_hash[hash & z->hashmask]; 95 | const struct hash_entry *first = p; 96 | int depth = 0; 97 | int in_range = 0; 98 | while (p != NULL) { 99 | zs_blockid id = get_HE_blockid(z, p); 100 | if (memcmp(p->checksum, first->checksum, sizeof(p->checksum))) { 101 | int next_is_too = 0; 102 | if (p->next) { 103 | zs_blockid next_id = get_HE_blockid(z, p->next); 104 | if (next_id == id + 1) 105 | if (memcmp(p->next->checksum, first->checksum, sizeof(p->checksum))) 106 | next_is_too = 1;; 107 | } 108 | if (!in_range) { 109 | printf("%d", id); 110 | if (next_is_too) { 111 | fputs("-", stdout); 112 | in_range = 1; 113 | } else { 114 | fputs(",", stdout); 115 | } 116 | } else { 117 | if (next_is_too) { 118 | printf("%d,", id); 119 | in_range = 0; 120 | } 121 | } 122 | } 123 | p = p->next; 124 | depth++; 125 | } 126 | printf(" (depth %d)\n", depth); 127 | } 128 | #endif 129 | #endif 130 | } 131 | 132 | /* build_hash(self) 133 | * Build hash tables to quickly lookup a block based on its rsum value. 134 | * Returns non-zero if successful. 135 | */ 136 | int build_hash(struct rcksum_state *z) { 137 | zs_blockid id; 138 | int avail_bits = z->seq_matches > 1 ? min(z->rsum_bits, 16)*2 : z->rsum_bits; 139 | int hash_bits = avail_bits; 140 | 141 | /* Pick a hash size that is a power of two and gives a load factor of <1 */ 142 | while ((1U << (hash_bits-1)) > z->blocks && hash_bits > 5) 143 | hash_bits--; 144 | 145 | /* Allocate hash based on rsum */ 146 | z->hashmask = (1U << hash_bits) - 1; 147 | z->rsum_hash = calloc(z->hashmask + 1, sizeof *(z->rsum_hash)); 148 | if (!z->rsum_hash) 149 | return 0; 150 | 151 | /* Allocate bit-table based on rsum. Aim is for 1/(1<bithashmask = (1U << hash_bits) - 1; 156 | z->bithash = calloc(z->bithashmask + 1, 1); 157 | if (!z->bithash) { 158 | free(z->rsum_hash); 159 | z->rsum_hash = NULL; 160 | return 0; 161 | } 162 | 163 | /* We want the hash function to return hash_bits bits. We will xor one 164 | * number with a second number that may have fewer than 16 bits of 165 | * available data; set up an appropriate bit shift for the second number. 166 | * This is closely tied to calc_rhash(). 167 | */ 168 | if (z->seq_matches > 1 && avail_bits < 24) { 169 | /* second number has (avail_bits/2) bits available. */ 170 | z->hash_func_shift = max(0, hash_bits - (avail_bits / 2)); 171 | } else { 172 | /* second number has avail_bits - 16 bits available. */ 173 | z->hash_func_shift = max(0, hash_bits - (avail_bits - 16)); 174 | } 175 | 176 | /* Now fill in the hash tables. 177 | * Minor point: We do this in reverse order, because we're adding entries 178 | * to the hash chains by prepending, so if we iterate over the data in 179 | * reverse then the resulting hash chains have the blocks in normal order. 180 | * That's improves our pattern of I/O when writing out identical blocks 181 | * once we are processing data; we will write them in order. */ 182 | for (id = z->blocks; id > 0;) { 183 | /* Decrement the loop variable here, and get the hash entry. */ 184 | struct hash_entry *e = z->blockhashes + (--id); 185 | 186 | /* Prepend to linked list for this hash entry */ 187 | unsigned h = calc_rhash(z, e); 188 | e->next = z->rsum_hash[h & z->hashmask]; 189 | z->rsum_hash[h & z->hashmask] = e; 190 | 191 | /* And set relevant bit in the bithash to 1 */ 192 | z->bithash[(h & z->bithashmask) >> 3] |= 1 << (h & 7); 193 | } 194 | 195 | print_hashstats(z); 196 | return 1; 197 | } 198 | 199 | /* remove_block_from_hash(self, block_id) 200 | * Remove the given data block from the rsum hash table, so it won't be 201 | * returned in a hash lookup again (e.g. because we now have the data) 202 | */ 203 | 204 | void remove_block_from_hash(struct rcksum_state *z, zs_blockid id) { 205 | struct hash_entry *t = &(z->blockhashes[id]); 206 | 207 | struct hash_entry **p = &(z->rsum_hash[calc_rhash(z, t) & z->hashmask]); 208 | 209 | while (*p != NULL) { 210 | if (*p == t) { 211 | if (t == z->rover) { 212 | z->rover = t->next; 213 | } 214 | *p = (*p)->next; 215 | return; 216 | } 217 | else { 218 | p = &((*p)->next); 219 | } 220 | } 221 | } 222 | 223 | --------------------------------------------------------------------------------