├── .gitignore ├── LICENSE ├── README.md ├── android ├── AndroidManifest.xml ├── proguard-project.txt ├── project.properties ├── res │ ├── drawable │ │ └── ic_launcher.png │ ├── layout │ │ └── activity_crsync.xml │ └── values │ │ ├── dimens.xml │ │ ├── strings.xml │ │ └── styles.xml └── src │ └── com │ └── shaddock │ └── crsync │ ├── Crsync.java │ ├── CrsyncActivity.java │ ├── CrsyncConstants.java │ ├── CrsyncInfo.java │ ├── CrsyncProvider.java │ ├── CrsyncService.java │ └── OnepieceObserver.java ├── doc ├── README.md └── crsync-基于rsync rolling算法的文件增量更新.md ├── extra ├── dictionary.c ├── dictionary.h ├── iniparser.c ├── iniparser.h ├── md5.c ├── md5.h ├── tpl.c ├── tpl.h ├── uthash.h ├── utlist.h ├── utstring.h └── win │ ├── mman.h │ └── mmap.c ├── libcurl ├── include │ └── curl │ │ ├── curl.h │ │ ├── curlbuild.h │ │ ├── curlrules.h │ │ ├── curlver.h │ │ ├── easy.h │ │ ├── mprintf.h │ │ ├── multi.h │ │ ├── stdcheaders.h │ │ └── typecheck-gcc.h └── lib │ ├── armeabi-v7a │ └── libcurl.a │ ├── m32 │ ├── libcurl.a │ ├── libcurl.dll │ └── libcurldll.a │ └── x86 │ └── libcurl.a └── src ├── crsync-build.cmd ├── crsync-console.c ├── crsync-jni.c ├── crsync.c ├── crsync.h ├── crsync.mk ├── crsync.pro ├── crsync.rc ├── crsyncver.h ├── deployment.pri ├── diff.c ├── diff.h ├── digest.c ├── digest.h ├── global.h ├── helper.c ├── helper.h ├── http.c ├── http.h ├── log.c ├── log.h ├── magnet.c ├── magnet.h ├── onepiece.c ├── onepiece.h ├── onepiecetool.c ├── onepiecetool.h ├── patch.c ├── patch.h ├── util.c └── util.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | *.obj 5 | *.elf 6 | *.d 7 | src/Makefile 8 | src/Makefile.Debug 9 | src/Makefile.Release 10 | src/object_script.crsync.Debug 11 | src/object_script.crsync.Release 12 | src/m32/ 13 | src/obj/ 14 | src/libs/ 15 | 16 | # Precompiled Headers 17 | *.gch 18 | *.pch 19 | 20 | # Executables 21 | *.exe 22 | *.out 23 | *.app 24 | *.i*86 25 | *.x86_64 26 | *.hex 27 | 28 | # Debug files 29 | *.dSYM/ 30 | 31 | # User files 32 | *.user 33 | 34 | # Android Project files 35 | android/.project 36 | android/.classpath 37 | android/.settings/ 38 | android/bin/ 39 | android/gen/ 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 chenqi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Go-joy/crsync/0af4f53c2dd2d2fe180a756f7f6b98fe6d60bf1f/README.md -------------------------------------------------------------------------------- /android/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 10 | 11 | 12 | 13 | 14 | 21 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /android/proguard-project.txt: -------------------------------------------------------------------------------- 1 | # To enable ProGuard in your project, edit project.properties 2 | # to define the proguard.config property as described in that file. 3 | # 4 | # Add project specific ProGuard rules here. 5 | # By default, the flags in this file are appended to flags specified 6 | # in ${sdk.dir}/tools/proguard/proguard-android.txt 7 | # You can edit the include path and order by changing the ProGuard 8 | # include property in project.properties. 9 | # 10 | # For more details, see 11 | # http://developer.android.com/guide/developing/tools/proguard.html 12 | 13 | # Add any project specific keep options here: 14 | 15 | # If your project uses WebView with JS, uncomment the following 16 | # and specify the fully qualified class name to the JavaScript interface 17 | # class: 18 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 19 | # public *; 20 | #} 21 | -------------------------------------------------------------------------------- /android/project.properties: -------------------------------------------------------------------------------- 1 | # This file is automatically generated by Android Tools. 2 | # Do not modify this file -- YOUR CHANGES WILL BE ERASED! 3 | # 4 | # This file must be checked in Version Control Systems. 5 | # 6 | # To customize properties used by the Ant build system edit 7 | # "ant.properties", and override values to adapt the script to your 8 | # project structure. 9 | # 10 | # To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home): 11 | #proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt 12 | 13 | # Project target. 14 | target=android-15 15 | -------------------------------------------------------------------------------- /android/res/drawable/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Go-joy/crsync/0af4f53c2dd2d2fe180a756f7f6b98fe6d60bf1f/android/res/drawable/ic_launcher.png -------------------------------------------------------------------------------- /android/res/layout/activity_crsync.xml: -------------------------------------------------------------------------------- 1 | 10 | 11 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /android/res/values/dimens.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16dp 5 | 16dp 6 | 7 | 8 | -------------------------------------------------------------------------------- /android/res/values/strings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | CrsyncAndroid 5 | 6 | 7 | -------------------------------------------------------------------------------- /android/res/values/styles.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 14 | 15 | 16 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /android/src/com/shaddock/crsync/Crsync.java: -------------------------------------------------------------------------------- 1 | package com.shaddock.crsync; 2 | 3 | import java.util.Vector; 4 | 5 | public class Crsync { 6 | 7 | public static final int Action_Idle = 0; 8 | public static final int Action_Query = 1; 9 | public static final int Action_UserConfirm = 2; 10 | public static final int Action_UpdateApp = 3; 11 | public static final int Action_UserInstall = 4; 12 | public static final int Action_UpdateRes = 5; 13 | public static final int Action_Done = 6; 14 | 15 | /* The same as crsync.h ENUM CRSYNCcode */ 16 | public static final int Code_OK = 0; 17 | public static final int Code_Failed_Init = 1; 18 | public static final int Code_Invalid_Opt = 2; 19 | public static final int Code_File_Error = 3; 20 | public static final int Code_CURL_Error = 4; 21 | public static final int Code_BUG = 100; 22 | 23 | public static final int OPT_MagnetID = 0; 24 | public static final int OPT_BaseUrl = 1; 25 | public static final int OPT_LocalApp = 2; 26 | public static final int OPT_LocalRes = 3; 27 | public static final int OPT_Action = 4; 28 | 29 | private static OnepieceObserver mObserver = null; 30 | 31 | public static void setObserver(OnepieceObserver ob) { 32 | mObserver = ob; 33 | } 34 | 35 | public static void delObserver() { 36 | mObserver = null; 37 | } 38 | 39 | static { 40 | System.loadLibrary("crsync"); 41 | } 42 | 43 | //native crsync functions 44 | public static native int JNI_onepiece_init(); 45 | public static native int JNI_onepiece_setopt(int opt, String value); 46 | public static native String JNI_onepiece_getinfo_magnet(); 47 | public static native int JNI_onepiece_perform_query(); 48 | public static native int JNI_onepiece_perform_updateapp(); 49 | public static native int JNI_onepiece_perform_updateres(); 50 | public static native void JNI_onepiece_cleanup(); 51 | 52 | public static void java_onepiece_xfer(String hash, int percent) { 53 | if(null != mObserver) { 54 | mObserver.onepiece_xfer(hash, percent); 55 | } 56 | } 57 | 58 | public static class Res { 59 | public String name = ""; 60 | public String hash = ""; 61 | public int size = 0; 62 | } 63 | 64 | public static class Magnet { 65 | public String curr_id = ""; 66 | public String next_id = ""; 67 | public String app_hash = ""; 68 | public Vector res_list = new Vector(); 69 | 70 | public static Magnet getValue(String value) { 71 | String [] s = value.split(";"); 72 | if(s.length < 3) { 73 | return null; 74 | } 75 | int i = 0; 76 | Magnet m = new Magnet(); 77 | m.curr_id = s[i++]; 78 | m.next_id = s[i++]; 79 | m.app_hash = s[i++]; 80 | while(i ri = CrsyncInfo.queryRes(getContentResolver()); 168 | sb.append("Res size : " + ri.size() + "\n"); 169 | for(CrsyncInfo.ResInfo x : ri) { 170 | sb.append(x.mName + " : " + x.mPercent + "\n"); 171 | } 172 | break; 173 | case Crsync.Action_Done: 174 | break; 175 | default: 176 | break; 177 | } 178 | mTV.setText(sb.toString()); 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /android/src/com/shaddock/crsync/CrsyncConstants.java: -------------------------------------------------------------------------------- 1 | package com.shaddock.crsync; 2 | 3 | import java.io.IOException; 4 | import java.util.logging.FileHandler; 5 | import java.util.logging.Level; 6 | import java.util.logging.Logger; 7 | import java.util.logging.SimpleFormatter; 8 | 9 | import android.net.Uri; 10 | import android.os.Environment; 11 | 12 | public class CrsyncConstants { 13 | 14 | public static final Logger logger = Logger.getLogger("crsync"); 15 | private static final boolean mDebug = true; 16 | 17 | static { 18 | logger.setLevel( mDebug ? Level.ALL : Level.OFF ); 19 | if(mDebug) { 20 | try { 21 | String file = Environment.getExternalStorageDirectory().getAbsolutePath() + "/crsync.log"; 22 | FileHandler fh = new FileHandler(file, true); 23 | fh.setFormatter(new SimpleFormatter()); 24 | logger.addHandler(fh); 25 | } catch (IOException e) { 26 | e.printStackTrace(); 27 | } 28 | } 29 | } 30 | 31 | /** The same as AndroidManifest.xml provider authority */ 32 | public static final String URIAUTHORITY = "com.shaddock.crsync.CrsyncProvider"; 33 | 34 | public static final String URIPATH_CONTENT = "content"; 35 | public static final String URIPATH_STATE = "state"; 36 | public static final String URIPATH_APP = "app"; 37 | public static final String URIPATH_RESOURCE = "res"; 38 | 39 | public static final Uri URI_BASE = Uri.parse("content://" + URIAUTHORITY); 40 | public static final Uri URI_CONTENT = Uri.parse("content://" + URIAUTHORITY + "/" + URIPATH_CONTENT); 41 | public static final Uri URI_STATE = Uri.parse("content://" + URIAUTHORITY + "/" + URIPATH_STATE); 42 | public static final Uri URI_APP = Uri.parse("content://" + URIAUTHORITY + "/" + URIPATH_APP); 43 | public static final Uri URI_RES = Uri.parse("content://" + URIAUTHORITY + "/" + URIPATH_RESOURCE); 44 | 45 | public static final String COLUMN_CONTENT_MAGNET = "content_magnet"; 46 | public static final String COLUMN_CONTENT_BASEURL = "content_baseurl"; 47 | public static final String COLUMN_CONTENT_LOCALAPP = "content_localapp"; 48 | public static final String COLUMN_CONTENT_LOCALRES = "content_localres"; 49 | 50 | public static final String COLUMN_STATE_ACTION = "state_action"; 51 | public static final String COLUMN_STATE_CODE = "state_code"; 52 | 53 | public static final String COLUMN_APP_HASH = "app_hash"; 54 | public static final String COLUMN_APP_PERCENT = "app_percent"; 55 | 56 | public static final String COLUMN_RES_NAME = "res_name"; 57 | public static final String COLUMN_RES_HASH = "res_hash"; 58 | public static final String COLUMN_RES_SIZE = "res_size"; 59 | public static final String COLUMN_RES_PERCENT = "res_percent"; 60 | 61 | } 62 | -------------------------------------------------------------------------------- /android/src/com/shaddock/crsync/CrsyncInfo.java: -------------------------------------------------------------------------------- 1 | package com.shaddock.crsync; 2 | 3 | import static com.shaddock.crsync.CrsyncConstants.logger; 4 | 5 | import java.util.Vector; 6 | 7 | import android.content.ContentResolver; 8 | import android.content.ContentValues; 9 | import android.database.Cursor; 10 | 11 | public class CrsyncInfo { 12 | public static class StateInfo { 13 | public int mAction = Crsync.Action_Idle; 14 | public int mCode = Crsync.Code_OK; 15 | 16 | public void dump() { 17 | logger.info("####StateInfo Action : " + mAction); 18 | logger.info("####StateInfo Code : " + mCode); 19 | } 20 | } 21 | 22 | public static StateInfo queryState(ContentResolver cr) { 23 | StateInfo info = new StateInfo(); 24 | 25 | Cursor c = cr.query(CrsyncConstants.URI_STATE, null, null, null, null); 26 | 27 | if(c != null && c.moveToFirst()) { 28 | int actionColumn = c.getColumnIndex(CrsyncConstants.COLUMN_STATE_ACTION); 29 | int codeColumn = c.getColumnIndex(CrsyncConstants.COLUMN_STATE_CODE); 30 | 31 | info.mAction = c.getInt( actionColumn ); 32 | info.mCode = c.getInt( codeColumn ); 33 | } else { 34 | logger.severe("####StateInfo query URI fail"); 35 | } 36 | if(c != null) { 37 | c.close(); 38 | } 39 | return info; 40 | } 41 | 42 | public static void updateState(ContentResolver cr, StateInfo info) { 43 | ContentValues values = new ContentValues(); 44 | values.put(CrsyncConstants.COLUMN_STATE_ACTION, info.mAction); 45 | values.put(CrsyncConstants.COLUMN_STATE_CODE, info.mCode); 46 | cr.update(CrsyncConstants.URI_STATE, values, null, null); 47 | } 48 | 49 | public static class ContentInfo { 50 | public String mMagnet = ""; 51 | public String mBaseUrl = ""; 52 | public String mLocalApp = ""; 53 | public String mLocalRes = ""; 54 | 55 | public void dump() { 56 | logger.info("####ContentInfo mMagnet : " + mMagnet); 57 | logger.info("####ContentInfo mBaseUrl : " + mBaseUrl); 58 | logger.info("####ContentInfo mLocalApp : " + mLocalApp); 59 | logger.info("####ContentInfo mLocalRes : " + mLocalRes); 60 | } 61 | } 62 | 63 | public static ContentInfo queryContent(ContentResolver cr) { 64 | ContentInfo info = new ContentInfo(); 65 | Cursor c = cr.query(CrsyncConstants.URI_CONTENT, null, null, null, null); 66 | 67 | if(c != null && c.moveToFirst()) { 68 | int magnetColumn = c.getColumnIndex(CrsyncConstants.COLUMN_CONTENT_MAGNET); 69 | int baseUrlColumn = c.getColumnIndex(CrsyncConstants.COLUMN_CONTENT_BASEURL); 70 | int localAppColumn = c.getColumnIndex(CrsyncConstants.COLUMN_CONTENT_LOCALAPP); 71 | int localResColumn = c.getColumnIndex(CrsyncConstants.COLUMN_CONTENT_LOCALRES); 72 | 73 | info.mMagnet = c.getString(magnetColumn); 74 | info.mBaseUrl = c.getString(baseUrlColumn); 75 | info.mLocalApp = c.getString(localAppColumn); 76 | info.mLocalRes = c.getString(localResColumn); 77 | } else { 78 | logger.severe("####ContentInfo query URI fail"); 79 | } 80 | if(c != null) { 81 | c.close(); 82 | } 83 | return info; 84 | } 85 | 86 | public static void updateContent(ContentResolver cr, ContentInfo info) { 87 | ContentValues values = new ContentValues(); 88 | values.put(CrsyncConstants.COLUMN_CONTENT_MAGNET, info.mMagnet); 89 | values.put(CrsyncConstants.COLUMN_CONTENT_BASEURL, info.mBaseUrl); 90 | values.put(CrsyncConstants.COLUMN_CONTENT_LOCALAPP, info.mLocalApp); 91 | values.put(CrsyncConstants.COLUMN_CONTENT_LOCALRES, info.mLocalRes); 92 | cr.update(CrsyncConstants.URI_CONTENT, values, null, null); 93 | } 94 | 95 | public static class AppInfo { 96 | public String mHash = ""; 97 | public int mPercent = 0; 98 | 99 | public void dump() { 100 | logger.info("####AppInfo Hash : " + mHash); 101 | logger.info("####AppInfo Percent : " + mPercent); 102 | } 103 | } 104 | 105 | public static AppInfo queryApp(ContentResolver cr) { 106 | AppInfo info = new AppInfo(); 107 | 108 | Cursor c = cr.query(CrsyncConstants.URI_APP, null, null, null, null); 109 | 110 | if(c != null && c.moveToFirst()) { 111 | int hashColumn = c.getColumnIndex(CrsyncConstants.COLUMN_APP_HASH); 112 | int percentColumn = c.getColumnIndex(CrsyncConstants.COLUMN_APP_PERCENT); 113 | 114 | info.mHash = c.getString(hashColumn); 115 | info.mPercent = c.getInt( percentColumn ); 116 | } else { 117 | logger.severe("####AppInfo query URI fail"); 118 | } 119 | if(c != null) { 120 | c.close(); 121 | } 122 | return info; 123 | } 124 | 125 | public static void updateApp(ContentResolver cr, AppInfo info) { 126 | ContentValues values = new ContentValues(); 127 | values.put(CrsyncConstants.COLUMN_APP_HASH, info.mHash); 128 | values.put(CrsyncConstants.COLUMN_APP_PERCENT, info.mPercent); 129 | cr.update(CrsyncConstants.URI_APP, values, null, null); 130 | } 131 | 132 | public static class ResInfo { 133 | public String mName = ""; 134 | public String mHash = ""; 135 | public int mSize = 0; 136 | public int mPercent = 0; 137 | 138 | public void dump() { 139 | logger.info("####ResInfo name : " + mName); 140 | logger.info("####ResInfo hash : " + mHash); 141 | logger.info("####ResInfo size : " + mSize); 142 | logger.info("####ResInfo percent : " + mPercent); 143 | } 144 | } 145 | 146 | public static Vector queryRes(ContentResolver cr) { 147 | Vector infos = new Vector(); 148 | Cursor c = cr.query(CrsyncConstants.URI_RES, null, null, null, null); 149 | if(c != null && c.moveToFirst()) { 150 | 151 | int nameColumn = c.getColumnIndex(CrsyncConstants.COLUMN_RES_NAME); 152 | int hashColumn = c.getColumnIndex(CrsyncConstants.COLUMN_RES_HASH); 153 | int sizeColumn = c.getColumnIndex(CrsyncConstants.COLUMN_RES_SIZE); 154 | int percentColumn = c.getColumnIndex(CrsyncConstants.COLUMN_RES_PERCENT); 155 | 156 | for( ; !c.isAfterLast(); c.moveToNext()) { 157 | ResInfo info = new ResInfo(); 158 | info.mName = c.getString(nameColumn); 159 | info.mHash = c.getString(hashColumn); 160 | info.mSize = c.getInt(sizeColumn); 161 | info.mPercent = c.getInt(percentColumn); 162 | infos.add(info); 163 | } 164 | } else { 165 | logger.severe("ResInfo query URI fail"); 166 | } 167 | 168 | if(c != null) { 169 | c.close(); 170 | } 171 | return infos; 172 | } 173 | 174 | public static void bulkInsertRes(ContentResolver cr, ResInfo[] infos) { 175 | ContentValues values[] = new ContentValues[infos.length]; 176 | for(int i=0; i mRes = new Vector(); 80 | 81 | @Override 82 | public boolean onCreate() { 83 | logger.info("CrsyncProvider onCreate"); 84 | //Context ctx = getContext(); 85 | //mSP = ctx.getApplicationContext().getSharedPreferences(SP_NAME, Context.MODE_PRIVATE); 86 | startCrsyncService(); 87 | return true; 88 | } 89 | /* 90 | * private static final long UPDATE_CHECK_PERIOD_MILLISECONDS = 24*60*60*1000; //24hours 91 | private static boolean isUpdateCheckRequired() { 92 | long last = IBApplication.mSP.getLong(IBApplication.KSP_update, -1); 93 | if(last == -1) { 94 | //first time to check 95 | return true; 96 | } 97 | Calendar now = Calendar.getInstance(); 98 | Calendar old = Calendar.getInstance(); 99 | old.setTimeInMillis(last); 100 | if(now.before(old)) { 101 | //device time have been changed 102 | return true; 103 | } 104 | old.setTimeInMillis(last + UPDATE_CHECK_PERIOD_MILLISECONDS); 105 | if(now.after(old)) { 106 | //pass check period 107 | return true; 108 | } 109 | return false; 110 | } 111 | * 112 | * */ 113 | @Override 114 | public Cursor query(final Uri uri, final String[] projection, final String selection, final String[] selectionArgs, final String sortOrder) { 115 | Cursor c = null; 116 | int match = sURIMatcher.match(uri); 117 | switch (match) { 118 | case CODE_STATE: 119 | { 120 | MatrixCursor mc = new MatrixCursor(CURSOR_COLUMN_STATE); 121 | mc.addRow( new Object[] {mAction, mCode}); 122 | c = mc; 123 | break; 124 | } 125 | case CODE_CONTENT: 126 | { 127 | MatrixCursor mc = new MatrixCursor(CURSOR_COLUMN_CONTENT); 128 | mc.addRow( new Object[] {mMagnet, mBaseUrl, mLocalApp, mLocalRes}); 129 | c = mc; 130 | break; 131 | } 132 | case CODE_APP: 133 | { 134 | MatrixCursor mc = new MatrixCursor(CURSOR_COLUMN_APP); 135 | mc.addRow( new Object[] {mAppHash, mAppPercent}); 136 | c = mc; 137 | break; 138 | } 139 | case CODE_RESOURCE: 140 | { 141 | MatrixCursor mc = new MatrixCursor(CURSOR_COLUMN_RESOURCE); 142 | for(CrsyncInfo.ResInfo x : mRes) { 143 | mc.addRow( new Object[] { x.mName, x.mHash, x.mSize, x.mPercent }); 144 | } 145 | c = mc; 146 | break; 147 | } 148 | default: 149 | logger.severe("Unknown URI: " + uri); 150 | throw new IllegalArgumentException("query Unknown URI: " + uri); 151 | } 152 | return c; 153 | } 154 | 155 | @Override 156 | public String getType(final Uri uri) { 157 | int match = sURIMatcher.match(uri); 158 | switch (match) { 159 | case CODE_CONTENT: 160 | case CODE_STATE: 161 | case CODE_APP: 162 | return TYPE_ITEM; 163 | case CODE_RESOURCE: 164 | return TYPE_DIR; 165 | default: 166 | logger.severe("Unknown URI: " + uri); 167 | throw new IllegalArgumentException("getType Unknown URI: " + uri); 168 | } 169 | } 170 | 171 | @Override 172 | public Uri insert(Uri uri, ContentValues values) { 173 | throw new IllegalArgumentException("insert Unimplement URI: " + uri); 174 | } 175 | 176 | @Override 177 | public int bulkInsert(Uri uri, ContentValues[] values) { 178 | int match = sURIMatcher.match(uri); 179 | switch (match) { 180 | case CODE_RESOURCE: 181 | mRes.clear(); 182 | for(ContentValues v : values) { 183 | CrsyncInfo.ResInfo r = new CrsyncInfo.ResInfo(); 184 | r.mName = v.getAsString(CrsyncConstants.COLUMN_RES_NAME); 185 | r.mHash = v.getAsString(CrsyncConstants.COLUMN_RES_HASH); 186 | r.mSize = v.getAsInteger(CrsyncConstants.COLUMN_RES_SIZE); 187 | r.mPercent = v.getAsInteger(CrsyncConstants.COLUMN_RES_PERCENT); 188 | mRes.add(r); 189 | } 190 | notifyChange(uri); 191 | return values.length; 192 | default: 193 | logger.severe("Unknown URI: " + uri); 194 | throw new IllegalArgumentException("bulkInsert Unknown URI: " + uri); 195 | } 196 | } 197 | 198 | @Override 199 | public int delete(Uri uri, String selection, String[] selectionArgs) { 200 | throw new IllegalArgumentException("delete Unimplement URI: " + uri); 201 | } 202 | 203 | @Override 204 | public int update(final Uri uri, final ContentValues values, final String selection, final String[] selectionArgs) { 205 | int match = sURIMatcher.match(uri); 206 | switch (match) { 207 | case CODE_STATE: 208 | mAction = values.getAsInteger(CrsyncConstants.COLUMN_STATE_ACTION); 209 | mCode = values.getAsInteger(CrsyncConstants.COLUMN_STATE_CODE); 210 | notifyChange(uri); 211 | return 1; 212 | case CODE_CONTENT: 213 | mMagnet = values.getAsString(CrsyncConstants.COLUMN_CONTENT_MAGNET); 214 | mBaseUrl = values.getAsString(CrsyncConstants.COLUMN_CONTENT_BASEURL); 215 | mLocalApp = values.getAsString(CrsyncConstants.COLUMN_CONTENT_LOCALAPP); 216 | mLocalRes = values.getAsString(CrsyncConstants.COLUMN_CONTENT_LOCALRES); 217 | notifyChange(uri); 218 | return 1; 219 | case CODE_APP: 220 | mAppHash = values.getAsString(CrsyncConstants.COLUMN_APP_HASH); 221 | mAppPercent = values.getAsInteger(CrsyncConstants.COLUMN_APP_PERCENT); 222 | notifyChange(uri); 223 | return 1; 224 | case CODE_RESOURCE: 225 | CrsyncInfo.ResInfo r = new CrsyncInfo.ResInfo(); 226 | r.mName = values.getAsString(CrsyncConstants.COLUMN_RES_NAME); 227 | r.mHash = values.getAsString(CrsyncConstants.COLUMN_RES_HASH); 228 | r.mSize = values.getAsInteger(CrsyncConstants.COLUMN_RES_SIZE); 229 | r.mPercent = values.getAsInteger(CrsyncConstants.COLUMN_RES_PERCENT); 230 | for (CrsyncInfo.ResInfo x : mRes) { 231 | if(x.mName.equalsIgnoreCase(r.mName) || x.mHash.equalsIgnoreCase(r.mHash)) { 232 | x.mPercent = r.mPercent; 233 | notifyChange(uri); 234 | return 1; 235 | } 236 | } 237 | logger.severe("update resource not found : " + r.toString()); 238 | return 0; 239 | default: 240 | logger.severe("Unknown URI: " + uri); 241 | throw new IllegalArgumentException("update Unknown URI: " + uri); 242 | } 243 | } 244 | 245 | private void startCrsyncService() { 246 | if(CrsyncService.isRunning) { 247 | return; 248 | } 249 | Context ctx = getContext(); 250 | ComponentName result = ctx.startService(new Intent(ctx, CrsyncService.class)); 251 | logger.info("CrsyncProvider startService : " + result); 252 | } 253 | 254 | private void notifyChange(Uri uri) { 255 | getContext().getContentResolver().notifyChange(uri, null); 256 | startCrsyncService(); 257 | } 258 | } 259 | -------------------------------------------------------------------------------- /android/src/com/shaddock/crsync/CrsyncService.java: -------------------------------------------------------------------------------- 1 | package com.shaddock.crsync; 2 | 3 | import static com.shaddock.crsync.CrsyncConstants.logger; 4 | 5 | import java.io.File; 6 | 7 | import android.app.Service; 8 | import android.content.Intent; 9 | import android.database.ContentObserver; 10 | import android.net.NetworkInfo; 11 | import android.os.Handler; 12 | import android.os.IBinder; 13 | 14 | public class CrsyncService extends Service implements OnepieceObserver { 15 | 16 | public static volatile boolean isRunning = false; 17 | 18 | private CrsyncContentObserver mObserver = null; 19 | 20 | private UpdateThread mUpdateThread = null; 21 | 22 | private boolean mPendingUpdate = false; 23 | 24 | private String mAppHash = ""; 25 | 26 | private class CrsyncContentObserver extends ContentObserver { 27 | 28 | public CrsyncContentObserver() { 29 | super(new Handler()); 30 | } 31 | 32 | @Override 33 | public boolean deliverSelfNotifications() { 34 | return false; 35 | } 36 | 37 | @Override 38 | public void onChange(final boolean selfChange) { 39 | updateFromProvider(); 40 | } 41 | 42 | } 43 | 44 | @Override 45 | public IBinder onBind(Intent intent) { 46 | throw new UnsupportedOperationException("Cannot bind to CrsyncService"); 47 | } 48 | 49 | @Override 50 | public void onCreate() { 51 | super.onCreate(); 52 | logger.info("CrsyncService onCreate"); 53 | 54 | mObserver = new CrsyncContentObserver(); 55 | getContentResolver().registerContentObserver(CrsyncConstants.URI_BASE, true, mObserver); 56 | isRunning = true; 57 | } 58 | 59 | @Override 60 | public int onStartCommand(Intent intent, int flags, int startId) { 61 | logger.info("CrsyncService onStartCommand"); 62 | int returnValue = super.onStartCommand(intent, flags, startId); 63 | updateFromProvider(); 64 | return returnValue; 65 | } 66 | 67 | @Override 68 | public void onDestroy() { 69 | logger.info("CrsyncService onDestroy"); 70 | isRunning = false; 71 | getContentResolver().unregisterContentObserver(mObserver); 72 | mObserver = null; 73 | super.onDestroy(); 74 | } 75 | 76 | private void updateFromProvider() { 77 | synchronized (this) { 78 | mPendingUpdate = true; 79 | if (mUpdateThread == null) { 80 | mUpdateThread = new UpdateThread(); 81 | mUpdateThread.start(); 82 | } 83 | } 84 | } 85 | 86 | private class UpdateThread extends Thread { 87 | public UpdateThread() { 88 | super("CrsyncServiceUpdateThread"); 89 | } 90 | 91 | @Override 92 | public void run() { 93 | while (true) { 94 | synchronized (CrsyncService.this) { 95 | if (mUpdateThread != this) { 96 | logger.severe("CrsyncService occur multiple UpdateThreads"); 97 | break; 98 | } 99 | if (!mPendingUpdate) { 100 | mUpdateThread = null; 101 | break; 102 | } 103 | mPendingUpdate = false; 104 | } 105 | Crsync.setObserver(CrsyncService.this); 106 | CrsyncInfo.StateInfo stateInfo = CrsyncInfo.queryState(getContentResolver()); 107 | handleFromProvider(stateInfo); 108 | Crsync.delObserver(); 109 | } 110 | } 111 | }// End of class UpdateThread 112 | 113 | private void handleFromProvider(CrsyncInfo.StateInfo stateInfo) { 114 | logger.info("CrsyncService handleFromProvider"); 115 | stateInfo.dump(); 116 | switch (stateInfo.mAction) { 117 | case Crsync.Action_Idle: 118 | handleIdle(stateInfo); 119 | break; 120 | case Crsync.Action_Query: 121 | handleQuery(stateInfo); 122 | break; 123 | case Crsync.Action_UserConfirm: 124 | //do nothing 125 | break; 126 | case Crsync.Action_UpdateApp: 127 | handleUpdateApp(stateInfo); 128 | break; 129 | case Crsync.Action_UserInstall: 130 | //do nothing 131 | break; 132 | case Crsync.Action_UpdateRes: 133 | handleUpdateRes(stateInfo); 134 | break; 135 | case Crsync.Action_Done: 136 | handleDone(stateInfo); 137 | break; 138 | default: 139 | logger.severe("wrong action"); 140 | break; 141 | } 142 | } 143 | 144 | private void handleIdle(CrsyncInfo.StateInfo stateInfo) { 145 | logger.info("CrsyncService handleIdle"); 146 | CrsyncInfo.ContentInfo contentInfo = CrsyncInfo.queryContent(getContentResolver()); 147 | if(contentInfo.mMagnet.isEmpty() || 148 | contentInfo.mBaseUrl.isEmpty() || 149 | contentInfo.mLocalApp.isEmpty() || 150 | contentInfo.mLocalRes.isEmpty()) { 151 | return; 152 | } 153 | 154 | new File( contentInfo.mLocalRes ).mkdirs(); 155 | 156 | int code; 157 | do { 158 | code = Crsync.JNI_onepiece_init(); 159 | if(Crsync.Code_OK != code) break; 160 | code = Crsync.JNI_onepiece_setopt(Crsync.OPT_MagnetID, contentInfo.mMagnet); 161 | if(Crsync.Code_OK != code) break; 162 | code = Crsync.JNI_onepiece_setopt(Crsync.OPT_BaseUrl, contentInfo.mBaseUrl); 163 | if(Crsync.Code_OK != code) break; 164 | code = Crsync.JNI_onepiece_setopt(Crsync.OPT_LocalApp, contentInfo.mLocalApp); 165 | if(Crsync.Code_OK != code) break; 166 | code = Crsync.JNI_onepiece_setopt(Crsync.OPT_LocalRes, contentInfo.mLocalRes); 167 | if(Crsync.Code_OK != code) break; 168 | } while(false); 169 | 170 | if(Crsync.Code_OK == code) { 171 | stateInfo.mAction = Crsync.Action_Query; 172 | stateInfo.mCode = Crsync.Code_OK; 173 | } else { 174 | stateInfo.mAction = Crsync.Action_Done; 175 | stateInfo.mCode = code; 176 | } 177 | CrsyncInfo.updateState(getContentResolver(), stateInfo); 178 | handleFromProvider(stateInfo); 179 | } 180 | 181 | private void handleQuery(CrsyncInfo.StateInfo stateInfo) { 182 | logger.info("CrsyncService handleQuery"); 183 | int code = Crsync.JNI_onepiece_perform_query(); 184 | if(Crsync.Code_OK == code) { 185 | CrsyncInfo.ContentInfo ci = CrsyncInfo.queryContent(getContentResolver()); 186 | String magnetString = Crsync.JNI_onepiece_getinfo_magnet(); 187 | logger.info("INFO_Magnet = " + magnetString); 188 | Crsync.Magnet magnet = Crsync.Magnet.getValue(magnetString); 189 | int size = magnet.res_list.size(); 190 | CrsyncInfo.ResInfo ri[] = new CrsyncInfo.ResInfo[size]; 191 | for(int i=0; i < size; i++) { 192 | CrsyncInfo.ResInfo r = new CrsyncInfo.ResInfo(); 193 | Crsync.Res res = magnet.res_list.get(i); 194 | r.mName = res.name; 195 | r.mHash = res.hash; 196 | r.mSize = res.size; 197 | r.mPercent = 0; 198 | ri[i] = r; 199 | } 200 | CrsyncInfo.bulkInsertRes(getContentResolver(), ri); 201 | if(magnet.curr_id.equalsIgnoreCase(ci.mMagnet)) { 202 | stateInfo.mAction = Crsync.Action_UpdateRes; 203 | stateInfo.mCode = Crsync.Code_OK; 204 | } else { 205 | mAppHash = magnet.app_hash; 206 | CrsyncInfo.AppInfo ai = new CrsyncInfo.AppInfo(); 207 | ai.mHash = magnet.app_hash; 208 | CrsyncInfo.updateApp(getContentResolver(), ai); 209 | stateInfo.mAction = Crsync.Action_UserConfirm; 210 | stateInfo.mCode = Crsync.Code_OK; 211 | } 212 | } else { 213 | stateInfo.mAction = Crsync.Action_Done; 214 | stateInfo.mCode = code; 215 | } 216 | CrsyncInfo.updateState(getContentResolver(), stateInfo); 217 | handleFromProvider(stateInfo); 218 | } 219 | 220 | private void handleUpdateApp(CrsyncInfo.StateInfo stateInfo) { 221 | int code = Crsync.JNI_onepiece_perform_updateapp(); 222 | if(Crsync.Code_OK == code) { 223 | stateInfo.mAction = Crsync.Action_UserInstall; 224 | } else { 225 | stateInfo.mAction = Crsync.Action_Done; 226 | } 227 | stateInfo.mCode = code; 228 | CrsyncInfo.updateState(getContentResolver(), stateInfo); 229 | handleFromProvider(stateInfo); 230 | } 231 | 232 | private void handleUpdateRes(CrsyncInfo.StateInfo stateInfo) { 233 | int code = Crsync.JNI_onepiece_perform_updateres(); 234 | stateInfo.mAction = Crsync.Action_Done; 235 | stateInfo.mCode = code; 236 | CrsyncInfo.updateState(getContentResolver(), stateInfo); 237 | handleFromProvider(stateInfo); 238 | } 239 | 240 | private void handleDone(CrsyncInfo.StateInfo stateInfo) { 241 | Crsync.JNI_onepiece_cleanup(); 242 | } 243 | 244 | @Override 245 | public void onepiece_xfer(String hash, int percent) { 246 | logger.info("onepiece_xfer : " + hash + " " + percent + "%"); 247 | if(hash.equalsIgnoreCase(mAppHash)) { 248 | CrsyncInfo.AppInfo ai = new CrsyncInfo.AppInfo(); 249 | ai.mHash = mAppHash; 250 | ai.mPercent = percent; 251 | CrsyncInfo.updateApp(getContentResolver(), ai); 252 | } else { 253 | CrsyncInfo.ResInfo ri = new CrsyncInfo.ResInfo(); 254 | ri.mHash = hash; 255 | ri.mPercent = percent; 256 | CrsyncInfo.updateRes(getContentResolver(), ri); 257 | } 258 | } 259 | 260 | } 261 | -------------------------------------------------------------------------------- /android/src/com/shaddock/crsync/OnepieceObserver.java: -------------------------------------------------------------------------------- 1 | package com.shaddock.crsync; 2 | 3 | public interface OnepieceObserver { 4 | 5 | public void onepiece_xfer(final String hash, int percent); 6 | } 7 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | # test 1 2 | 3 | File: base14009.obb to base14012.obb 4 | Device: DELL PC, Intel i3 550 (4core 3.2GHz) 4GB RAM 5 | Build: MinGW 32bit release 6 | 7 | | block size | hash bloom | hatch speed | match speed | hatch blocks | match blocks | delta size | 8 | | ---------- | :--------- | :---------: | :---------: | :----------: | :----------: | ---------: | 9 | | 2048 | 0 | 2189 ms | 14620 ms | 46328 | 40418 | 11820 KB | 10 | | 4096 | 0 | 2158 ms | 9675 ms | 23164 | 18960 | 16816 KB | 11 | | 8192 | 0 | 2137 ms | 8580 ms | 11582 | 9249 | 18664 KB | 12 | | 2048 | 20 | 2182 ms | 3753 ms | 46328 | 40418 | 11820 KB | 13 | | 4096 | 20 | 2156 ms | 2914 ms | 23164 | 18960 | 16816 KB | 14 | | 8192 | 20 | 2154 ms | 2719 ms | 11582 | 9249 | 18664 KB | 15 | -------------------------------------------------------------------------------- /doc/crsync-基于rsync rolling算法的文件增量更新.md: -------------------------------------------------------------------------------- 1 | **最终实现效果:** 2 | 1. 无版本概念,任何本地文件均可增量升级到最新.服务器不用管理多版本 3 | 2. 内存小,100M文件升级时只占用500KB内存. 4 | 5 | **使用流程:** 6 | 1. 制作新版本,上传HTTP File Server. 7 | 2. Client自动计算差异,下载差异,合并差异. 8 | 3. done! 9 | 10 | #0.缘起 11 | 目前主流的文件增量更新方案是**bs diff/patch**，以及google chrome Courgette改进方案。 12 | 无尽之剑Android版最初使用该方案，在封测期间发现存在问题: 13 | 14 | 1. 多版本运营繁琐。 15 | 假设目前先后开发了5个版本，1.0～5.0，运营中需要制作 16 | 1-2.patch 17 | 2-3.patch 18 | 3-4.patch 19 | 4-5.patch 20 | **n个版本,n-1个patch包** 21 | 或者 22 | 1-2.patch 23 | 1-3.patch 2-3.patch 24 | 1-4.patch 2-4.patch 3-4.patch 25 | 1-5.patch 2-5.patch 3-5.patch 4-5.patch 26 | **n个版本,(n-1)阶乘个patch包** 27 | 28 | 2. patch依赖本地版本文件完整性 29 | 如果本地文件损坏或被篡改，就无法增量升级，只能重新下载完整包. 30 | 31 | 3. bs diff/patch算法性能和**内存**开销太高 32 | > from [bsdiff官网](http://www.daemonology.net/bsdiff/) 33 | > bsdiff is quite memory-hungry. It requires max(17*n,9*n+m)+O(1) bytes of memory, where n is the size of the old file and m is the size of the new file. bspatch requires **n+m+O(1)** bytes. 34 | > bsdiff runs in O((n+m) log n) time; on a 200MHz Pentium Pro, building a binary patch for a 4MB file takes about 90 seconds. 35 | > bspatch runs in O(n+m) time; on the same machine, applying that patch takes about two seconds. 36 | 37 | bsdiff执行慢,可以,200M文件制作patch包需要运行64位版本，可以. 38 | bspatch执行慢,可以。需要占用**n+m**内存,这点无法接受. 39 | 无尽之剑Android版的资源包单个最大200M，在游戏运行同时进行200M+内存开销的patch操作，非常容易OOM. 40 | 41 | **那么问题来了：有没有一种版本无关，内容无关，内存开销小的增量升级方案呢?** 42 | 这就是我的**crsync**: 43 | 纯C语言实现,基于rsync rolling算法, Client Side计算diff和执行patch，通过curl进行HTTP range增量下载 44 | (同类方案有zsync,用于ubuntu package增量更新) 45 | 46 | 1. 客户端计算本地版本和服务器最新版本之间的diff，然后通过http range下载差异部分. 47 | 48 | 2. 内存开销 = 文件size/分块size * 40字节 + HashTable开销. 49 | 假设文件size 100M，分块size 16K,那么patch操作最多只需500KB内存。 50 | 51 | 3. 版本无关, 本地文件是否改动无关，只关注本地和远端之间的内容差异. 52 | 53 | #1.rsync rolling 54 | 检测2个文件的差异和重复数据，有2种策略:定长分块和变长分块. 55 | rsync算法属于前者,RDC算法属于后者,多轮rsync可以看作是对定长分块的变长优化. 56 | 57 | 定长分块是将A文件按固定块大小切分,计算校验值。在B文件中遍历,计算所有相同块大小的校验值,相同即一致，不同即差异. 58 | 以下假设文件size=100M, 分块size=1KB,测试数据基于公司开发机i7-3770 CPU. 59 | 60 | A文件切分,计算[0]~[1023], [1024]~[2047],[2048]~[3071]...的校验值，输出一张A表. 61 | B文件遍历,计算[0]~[1023], [1]~[1024],[2]~[1025],...的校验值,在A表中遍历比对.相同即重复,不同即差异. 62 | 63 | 不足1KB的数据,rsync补齐0再计算校验值，与B文件遍历计算比对. 64 | crsync不计算,直接将之存入A表末尾,当作差异. 65 | 66 | 校验算法很多,常用MD4,MD5,SHA-1,BLAKE2均可. 67 | 这样明显很慢,如何优化? 68 | 69 | rsync使用2个校验值,**弱校验**和**强校验**. 70 | 弱校验是计算一个32位的CRC值,优点: 快; 缺点: 冲突率高;校验值相同时数据可能不同. 71 | 强校验是上面说到的各种算法,优点: 冲突率低; 缺点: 慢. 72 | 首先做弱校验,值相同再做强校验.以此保证快速与准确. 73 | 遍历文件[0]~[1023], [1]~[1024],[2]~[1025],...需计算104856577次弱校验 74 | 这样依然很慢，如何优化? 75 | 76 | CRC校验算法很多,rsync使用**adler32**算法,因此有了**rolling**的特性. 77 | alder32算法由Mark Adler于1995发明,用于zlib. 78 | 一个32位的adler32值由2个16位的A和B值构成。 79 | A值=buffer中所有byte相加,对2^16取模(65536). 80 | b值=buffer中所有A相加,对2^16取模(65536). 81 | 公式如下: 82 | > A = 1 + D1 + D2 + ... + Dn (mod 65521) 83 | > B = (1 + D1) + (1 + D1 + D2) + ... + (1 + D1 + D2 + ... + Dn) (mod 65521) 84 | > = n×D1 + (n−1)×D2 + (n−2)×D3 + ... + Dn + n (mod 65521) 85 | > Adler-32(D) = B × 65536 + A 86 | 87 | 65521是65536范围内的最大素数,因此mod=65521即可. 88 | librsync(rdiff)在A和B值计算时再偏移31,rsync偏移0,crsync偏移0. 89 | 90 | 有趣的是,这个公式等价于: 91 | > A += data[i]; 92 | > B += A; 93 | 94 | rolling是什么? 95 | 计算出[0]~[1023]的校验值,再计算[1]~[1024]的校验值时,只需减去data[0],加上data[1024]即可!不需重新计算! 96 | > A -= data[0]; 97 | > A += data[1024]; 98 | > B -= 1024 * data[0]; 99 | > B += A; 100 | 101 | 如此滑动遍历,即可得到整个文件的所有分块的校验值.(大赞!) 102 | 实测文件size 100M,分块size 2K,遍历计算仅耗时275ms. 103 | 104 | #2.rsync HashTable 105 | **rolling计算足够快了,用B文件每个块的校验值check A文件的校验表,如何优化?** 106 | 107 | A文件的校验表size = 102400,冒泡?快速?折半? 108 | rsync使用二段HashTable来存储校验值,即H(A, H(B, MD5)) 109 | 用adler32的A做key, B+MD5做value, 其中再用alder32的B做key, MD5做value. 110 | 表大小2^16,满足A的值范围.冲突值用LinkList存储. 111 | 112 | 首先命中A,再命中B,最后顺序遍历LinkList命中MD5.完成! 113 | 直接使用adler32值做key,MD5做value也可以,选择合适的散列算法即可. 114 | 115 | **HashTable解决了命中率的问题,但是实际中存在大量的missing查找.如何优化?** 116 | 117 | zsync在此基础上,使用一个bitbuffer来构建"缺失表",每个分块用3个bit表示,首先做位运算,若存在再访问HashTable,否则直接跳过. 118 | 119 | crsync使用2^20的Bloom filter做faster missing. 为什么是2^20? 兼顾速度提升和内存开销(2^20 = 128KB) 120 | 121 | #3.client-side rsync 122 | rsync是C/S架构,依赖rsyncd服务端存在,执行流程是: 123 | 124 | 1. client生成本地校验表.发送给server 125 | 2. server执行rsync rolling,发送相同块的index和差异块的数据. 126 | 3. client接受指令,执行merge. 127 | 128 | 是否可以去掉rsyncd server,只依赖HttpServer(CDN)存储文件和校验表,客户端执行rsync rolling? 129 | crsync和zsync为此而生. 130 | 131 | 1. 制作新版本文件,生成校验表,一并上传CDN. 132 | 2. client下载新版本的校验表. 133 | 3. client根据本地文件和新校验表,执行rsync rolling,得到重复数据index和差异数据的块index. 134 | 4. client重用本地数据,下载差异数据,合并写入新文件. 135 | 5. done! 136 | 137 | #4.crsync VS bs diff/patch 138 | 139 | **测试对比1:** 140 | 测试文件: 无尽之剑Android版资源包, base14009.obb ~ base14012.obb 92MB～93MB 141 | 编译版本: MinGW 32bit release 142 | Bloom filter: 2^20 143 | 纯本地差异查找与合并,无网络开销 144 | 145 | 测试机: DELL PC, Intel i3 550 (4core 3.2GHz) 4GB RAM 146 | 147 | | 分块大小 | 校验表生成时间 | 校验块数量 | 差异查找时间 | 重复块数量 | 差异块大小 | 148 | | :------- | :----------: | :-------: | :---------: | :------: | --------: | 149 | | 2K | 2182 ms | 46328 | 3753 ms | 40418 | 11820 KB | 150 | | 4K | 2156 ms | 23164 | 2914 ms | 18960 | 16816 KB | 151 | | 8K | 2154 ms | 11582 | 2719 ms | 9249 | 18664 KB | 152 | 153 | 测试机 HP PC, Intel i7-3770 (8core 3.4GHz) 8GB RAM: 154 | 155 | | 分块大小 | 校验表生成时间 | 校验块数量 | 差异查找时间 | 重复块数量 | 差异块大小 | 156 | | :------- | :----------: | :-------: | :---------: | :------: | --------: | 157 | | 2K | 1745 ms | 46328 | 2974 ms | 40418 | 11820 KB | 158 | 159 | **crsync VS bspatch** 160 | 161 | | 算法 | 差异量 | 内存开销 | 162 | | :--- | :----------: | --------: | 163 | | bsdiff | 16.7MB (diff文件) | 108.7MB (92MB+16.7MB) | 164 | | crsync | **12.36MB**(校验表835KB + 差异数据11820KB) | **1.76MB** | 165 | 166 | **测试对比2:** 167 | 测试文件: [我叫MT online Android标准版](http://update2.locojoy.com/android/IamMT/150413/IamMT_200100100_locojoy_standard_ac_4.5.4.0_4540.apk) 90.7MB ~ [我叫MT online Android高清版](http://update.locojoy.com/Client/Android/MT-A/0324/IamMT_200100100_locojoy_ac_hd_4.5.4.0_4540.apk) 102MB 168 | (测试版本基于2015.4.4官网下载,目前已变更) 169 | 170 | 171 | 测试机 HP PC, Intel i7-3770 (8core 3.4GHz) 8GB RAM: 172 | 173 | | 分块大小 | 校验表生成时间 | 差异查找时间 | merge时间 | 174 | | :------- | :----------: | :---------: | --------: | 175 | | 2K | 1379 ms | 2950 ms | 90ms | 176 | 177 | **crsync VS bs diff/patch** 178 | 179 | | 算法 | 差异量 | 内存开销 | 180 | | :------- | :----------: | --------: | 181 | | bsdiff | **29.5MB** (diff文件) | 120.2MB (90.7MB+29.5MB) | 182 | | crsync | 40.8MB | **1.7MB** | 183 | 184 | **这里发现, 为什么无尽之剑资源包使用crsync得到的差异量更小?** 185 | **那么问题来了, 如何设计出适合差异更新的文件格式** 186 | 几种格式对比: 187 | 188 | 1. 整个文件不压缩,完整下载量大,更新量小,bsdiff更优,因为diff文件使用bzip2压缩格式.crsync使用原始数据http range下载. 189 | 2. 整个文件zip压缩,完整下载量大,更新量更大,这时bsdiff和crsync都无效.因为少量改动就会造成整个文件的大量差异. 190 | 3. 文件内容用zip压缩,再序列化到一起,完整下载量小,更新量小.这时crsync更优,因为bsdiff对zip压缩过的数据处理会生成额外数据量. 191 | 192 | UnrealEngine3 Android使用的obb格式为:**文件头**(贴图原始大小, 压缩大小,文件体偏移位置)+**文件体**(贴图内容zip压缩). 193 | 资源更改只影响zip压缩部分和文件头索引内容.对其他未更改资源无影响. 194 | 195 | Android APK是zip压缩格式,但是特定后缀名的文件不压缩,常见图片音视频文件为了支持流式read保持原始格式. 196 | 因此crsync对比bsdiff的差异量大些. 197 | 198 | #5.源码 199 | 纯C语言实现, 遵循C99, 核心部分仅2个文件 crsync.h crsync.c 880行代码.static-link libcurl. 200 | release版Android so 166KB, Windows mingw32 exe 299KB. 201 | API实现参考curl设计 202 | 203 | ```c 204 | CRSYNCcode crsync_global_init(); 205 | crsync_handle_t* crsync_easy_init(); 206 | 207 | CRSYNCcode crsync_easy_setopt(crsync_handle_t *handle, CRSYNCoption opt, ...); 208 | 209 | CRSYNCcode crsync_easy_perform_match(crsync_handle_t *handle); 210 | CRSYNCcode crsync_easy_perform_patch(crsync_handle_t *handle); 211 | 212 | void crsync_easy_cleanup(crsync_handle_t *handle); 213 | void crsync_global_cleanup(); 214 | ``` 215 | 216 | 源码托管在[https://github.com/9468305/crsync](https://github.com/9468305/crsync) 217 | 218 | #6.附录 219 | - [rsync官网](http://rsync.samba.org/) 220 | - [rsync算法论文](http://samba.org/~tridge/phd_thesis.pdf) 221 | - [librsync](http://librsync.sourceforge.net/) 222 | - [zsync](http://zsync.moria.org.uk/) 223 | - [BS diff/patch](http://www.daemonology.net/bsdiff/) 224 | - [Jarsync - a Java rsync implementation](http://sourceforge.net/projects/jarsync/) 225 | - [rsync教程](http://tutorials.jenkov.com/rsync/index.html) 226 | - [rsync java教程](http://javaforu.blogspot.com/2014/01/rsync-in-java-quick-and-partial-hack.html) 227 | - [REBL算法](http://www.research.ibm.com/people/f/fdouglis/papers/rebl.pdf) 228 | - [TAPER算法](http://www.usenix.org/events/fast05/tech/full_papers/jain/jain.pdf) 229 | -------------------------------------------------------------------------------- /extra/dictionary.h: -------------------------------------------------------------------------------- 1 | 2 | /*-------------------------------------------------------------------------*/ 3 | /** 4 | @file dictionary.h 5 | @author N. Devillard 6 | @brief Implements a dictionary for string variables. 7 | 8 | This module implements a simple dictionary object, i.e. a list 9 | of string/string associations. This object is useful to store e.g. 10 | informations retrieved from a configuration file (ini files). 11 | */ 12 | /*--------------------------------------------------------------------------*/ 13 | 14 | #ifndef _DICTIONARY_H_ 15 | #define _DICTIONARY_H_ 16 | 17 | /*--------------------------------------------------------------------------- 18 | Includes 19 | ---------------------------------------------------------------------------*/ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #ifdef __cplusplus 27 | extern "C" { 28 | #endif 29 | 30 | /*--------------------------------------------------------------------------- 31 | New types 32 | ---------------------------------------------------------------------------*/ 33 | 34 | 35 | /*-------------------------------------------------------------------------*/ 36 | /** 37 | @brief Dictionary object 38 | 39 | This object contains a list of string/string associations. Each 40 | association is identified by a unique string key. Looking up values 41 | in the dictionary is speeded up by the use of a (hopefully collision-free) 42 | hash function. 43 | */ 44 | /*-------------------------------------------------------------------------*/ 45 | typedef struct _dictionary_ { 46 | int n ; /** Number of entries in dictionary */ 47 | ssize_t size ; /** Storage size */ 48 | char ** val ; /** List of string values */ 49 | char ** key ; /** List of string keys */ 50 | unsigned * hash ; /** List of hash values for keys */ 51 | } dictionary ; 52 | 53 | 54 | /*--------------------------------------------------------------------------- 55 | Function prototypes 56 | ---------------------------------------------------------------------------*/ 57 | 58 | /*-------------------------------------------------------------------------*/ 59 | /** 60 | @brief Compute the hash key for a string. 61 | @param key Character string to use for key. 62 | @return 1 unsigned int on at least 32 bits. 63 | 64 | This hash function has been taken from an Article in Dr Dobbs Journal. 65 | This is normally a collision-free function, distributing keys evenly. 66 | The key is stored anyway in the struct so that collision can be avoided 67 | by comparing the key itself in last resort. 68 | */ 69 | /*--------------------------------------------------------------------------*/ 70 | unsigned dictionary_hash(const char * key); 71 | 72 | /*-------------------------------------------------------------------------*/ 73 | /** 74 | @brief Create a new dictionary object. 75 | @param size Optional initial size of the dictionary. 76 | @return 1 newly allocated dictionary objet. 77 | 78 | This function allocates a new dictionary object of given size and returns 79 | it. If you do not know in advance (roughly) the number of entries in the 80 | dictionary, give size=0. 81 | */ 82 | /*--------------------------------------------------------------------------*/ 83 | dictionary * dictionary_new(size_t size); 84 | 85 | /*-------------------------------------------------------------------------*/ 86 | /** 87 | @brief Delete a dictionary object 88 | @param d dictionary object to deallocate. 89 | @return void 90 | 91 | Deallocate a dictionary object and all memory associated to it. 92 | */ 93 | /*--------------------------------------------------------------------------*/ 94 | void dictionary_del(dictionary * vd); 95 | 96 | /*-------------------------------------------------------------------------*/ 97 | /** 98 | @brief Get a value from a dictionary. 99 | @param d dictionary object to search. 100 | @param key Key to look for in the dictionary. 101 | @param def Default value to return if key not found. 102 | @return 1 pointer to internally allocated character string. 103 | 104 | This function locates a key in a dictionary and returns a pointer to its 105 | value, or the passed 'def' pointer if no such key can be found in 106 | dictionary. The returned character pointer points to data internal to the 107 | dictionary object, you should not try to free it or modify it. 108 | */ 109 | /*--------------------------------------------------------------------------*/ 110 | const char * dictionary_get(const dictionary * d, const char * key, const char * def); 111 | 112 | 113 | /*-------------------------------------------------------------------------*/ 114 | /** 115 | @brief Set a value in a dictionary. 116 | @param d dictionary object to modify. 117 | @param key Key to modify or add. 118 | @param val Value to add. 119 | @return int 0 if Ok, anything else otherwise 120 | 121 | If the given key is found in the dictionary, the associated value is 122 | replaced by the provided one. If the key cannot be found in the 123 | dictionary, it is added to it. 124 | 125 | It is Ok to provide a NULL value for val, but NULL values for the dictionary 126 | or the key are considered as errors: the function will return immediately 127 | in such a case. 128 | 129 | Notice that if you dictionary_set a variable to NULL, a call to 130 | dictionary_get will return a NULL value: the variable will be found, and 131 | its value (NULL) is returned. In other words, setting the variable 132 | content to NULL is equivalent to deleting the variable from the 133 | dictionary. It is not possible (in this implementation) to have a key in 134 | the dictionary without value. 135 | 136 | This function returns non-zero in case of failure. 137 | */ 138 | /*--------------------------------------------------------------------------*/ 139 | int dictionary_set(dictionary * vd, const char * key, const char * val); 140 | 141 | /*-------------------------------------------------------------------------*/ 142 | /** 143 | @brief Delete a key in a dictionary 144 | @param d dictionary object to modify. 145 | @param key Key to remove. 146 | @return void 147 | 148 | This function deletes a key in a dictionary. Nothing is done if the 149 | key cannot be found. 150 | */ 151 | /*--------------------------------------------------------------------------*/ 152 | void dictionary_unset(dictionary * d, const char * key); 153 | 154 | 155 | /*-------------------------------------------------------------------------*/ 156 | /** 157 | @brief Dump a dictionary to an opened file pointer. 158 | @param d Dictionary to dump 159 | @param f Opened file pointer. 160 | @return void 161 | 162 | Dumps a dictionary onto an opened file pointer. Key pairs are printed out 163 | as @c [Key]=[Value], one per line. It is Ok to provide stdout or stderr as 164 | output file pointers. 165 | */ 166 | /*--------------------------------------------------------------------------*/ 167 | void dictionary_dump(const dictionary * d, FILE * out); 168 | 169 | #ifdef __cplusplus 170 | } 171 | #endif 172 | 173 | #endif 174 | -------------------------------------------------------------------------------- /extra/md5.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. 3 | * MD5 Message-Digest Algorithm (RFC 1321). 4 | * 5 | * Homepage: 6 | * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 7 | * 8 | * Author: 9 | * Alexander Peslyak, better known as Solar Designer 10 | * 11 | * This software was written by Alexander Peslyak in 2001. No copyright is 12 | * claimed, and the software is hereby placed in the public domain. 13 | * In case this attempt to disclaim copyright and place the software in the 14 | * public domain is deemed null and void, then the software is 15 | * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the 16 | * general public under the following terms: 17 | * 18 | * Redistribution and use in source and binary forms, with or without 19 | * modification, are permitted. 20 | * 21 | * There's ABSOLUTELY NO WARRANTY, express or implied. 22 | * 23 | * See md5.c for more information. 24 | */ 25 | 26 | #ifndef _MD5_H 27 | #define _MD5_H 28 | 29 | #define MD5_OUTBYTES 16 30 | 31 | /* Any 32-bit or wider unsigned integer data type will do */ 32 | typedef unsigned int MD5_u32plus; 33 | 34 | typedef struct { 35 | MD5_u32plus lo, hi; 36 | MD5_u32plus a, b, c, d; 37 | unsigned char buffer[64]; 38 | MD5_u32plus block[16]; 39 | } MD5_CTX; 40 | 41 | void MD5_Init(MD5_CTX *ctx); 42 | void MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size); 43 | void MD5_Final(MD5_CTX *ctx, unsigned char *result); 44 | 45 | /* Author: chenqi, <9468305@gmail.com> */ 46 | void MD5_Data(const void *data, unsigned long size, unsigned char *result); 47 | int MD5_File(const char *filename, unsigned char *result); 48 | int MD5_File_Parallel(const char *filename, unsigned char *result); 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /extra/tpl.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2005-2013, Troy D. Hanson http://troydhanson.github.com/tpl/ 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 12 | IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 13 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 14 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 15 | OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 16 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 17 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 18 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 19 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 20 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 21 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 22 | */ 23 | 24 | #ifndef TPL_H 25 | #define TPL_H 26 | 27 | #include /* size_t */ 28 | 29 | #include /* va_list */ 30 | 31 | #ifdef __INTEL_COMPILER 32 | # include 33 | #endif /* Intel Compiler efficient memcpy etc */ 34 | 35 | #ifdef _MSC_VER 36 | typedef unsigned int uint32_t; 37 | #else 38 | # include /* uint32_t */ 39 | #endif 40 | 41 | #if defined __cplusplus 42 | extern "C" { 43 | #endif 44 | 45 | #define TPL_NOLIB 46 | 47 | #ifdef _WIN32 48 | # ifdef TPL_EXPORTS 49 | # define TPL_API __declspec(dllexport) 50 | # else /* */ 51 | # ifdef TPL_NOLIB 52 | # define TPL_API 53 | # else 54 | # define TPL_API __declspec(dllimport) 55 | # endif /* TPL_NOLIB */ 56 | # endif /* TPL_EXPORTS*/ 57 | #else 58 | # define TPL_API 59 | #endif 60 | 61 | /* bit flags (external) */ 62 | #define TPL_FILE (1 << 0) 63 | #define TPL_MEM (1 << 1) 64 | #define TPL_PREALLOCD (1 << 2) 65 | #define TPL_EXCESS_OK (1 << 3) 66 | #define TPL_FD (1 << 4) 67 | #define TPL_UFREE (1 << 5) 68 | #define TPL_DATAPEEK (1 << 6) 69 | #define TPL_FXLENS (1 << 7) 70 | #define TPL_GETSIZE (1 << 8) 71 | /* do not add flags here without renumbering the internal flags! */ 72 | 73 | /* flags for tpl_gather mode */ 74 | #define TPL_GATHER_BLOCKING 1 75 | #define TPL_GATHER_NONBLOCKING 2 76 | #define TPL_GATHER_MEM 3 77 | 78 | /* Hooks for error logging, memory allocation functions and fatal */ 79 | typedef int (tpl_print_fcn)(const char *fmt, ...); 80 | typedef void *(tpl_malloc_fcn)(size_t sz); 81 | typedef void *(tpl_realloc_fcn)(void *ptr, size_t sz); 82 | typedef void (tpl_free_fcn)(void *ptr); 83 | typedef void (tpl_fatal_fcn)(const char *fmt, ...); 84 | 85 | typedef struct tpl_hook_t { 86 | tpl_print_fcn *oops; 87 | tpl_malloc_fcn *malloc; 88 | tpl_realloc_fcn *realloc; 89 | tpl_free_fcn *free; 90 | tpl_fatal_fcn *fatal; 91 | size_t gather_max; 92 | } tpl_hook_t; 93 | 94 | typedef struct tpl_node { 95 | int type; 96 | void *addr; 97 | void *data; /* r:tpl_root_data*. A:tpl_atyp*. ow:szof type */ 98 | int num; /* length of type if its a C array */ 99 | size_t ser_osz; /* serialization output size for subtree */ 100 | struct tpl_node *children; /* my children; linked-list */ 101 | struct tpl_node *next,*prev; /* my siblings (next child of my parent) */ 102 | struct tpl_node *parent; /* my parent */ 103 | } tpl_node; 104 | 105 | /* used when un/packing 'B' type (binary buffers) */ 106 | typedef struct tpl_bin { 107 | void *addr; 108 | uint32_t sz; 109 | } tpl_bin; 110 | 111 | /* for async/piecemeal reading of tpl images */ 112 | typedef struct tpl_gather_t { 113 | char *img; 114 | int len; 115 | } tpl_gather_t; 116 | 117 | /* Callback used when tpl_gather has read a full tpl image */ 118 | typedef int (tpl_gather_cb)(void *img, size_t sz, void *data); 119 | 120 | /* Prototypes */ 121 | TPL_API tpl_node *tpl_map(const char *fmt,...); /* define tpl using format */ 122 | TPL_API void tpl_free(tpl_node *r); /* free a tpl map */ 123 | TPL_API int tpl_pack(tpl_node *r, int i); /* pack the n'th packable */ 124 | TPL_API int tpl_unpack(tpl_node *r, int i); /* unpack the n'th packable */ 125 | TPL_API int tpl_dump(tpl_node *r, int mode, ...); /* serialize to mem/file */ 126 | TPL_API int tpl_load(tpl_node *r, int mode, ...); /* set mem/file to unpack */ 127 | TPL_API int tpl_Alen(tpl_node *r, int i); /* array len of packable i */ 128 | TPL_API char* tpl_peek(int mode, ...); /* sneak peek at format string */ 129 | TPL_API int tpl_gather( int mode, ...); /* non-blocking image gather */ 130 | TPL_API int tpl_jot(int mode, ...); /* quick write a simple tpl */ 131 | 132 | TPL_API tpl_node *tpl_map_va(const char *fmt, va_list ap); 133 | 134 | /* mmap record */ 135 | typedef struct tpl_mmap_rec { 136 | int fd; 137 | void *text; 138 | size_t text_sz; 139 | } tpl_mmap_rec; 140 | 141 | TPL_API int tpl_mmap_output_file(const char *filename, size_t sz, void **text_out); 142 | TPL_API int tpl_mmap_file(const char *filename, tpl_mmap_rec *map_rec); 143 | TPL_API int tpl_unmap_file(tpl_mmap_rec *mr); 144 | TPL_API void tpl_bin_malloc(tpl_bin *bin, size_t len); 145 | TPL_API void tpl_bin_free(tpl_bin *bin); 146 | 147 | #if defined __cplusplus 148 | } 149 | #endif 150 | 151 | #endif /* TPL_H */ 152 | 153 | -------------------------------------------------------------------------------- /extra/win/mman.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMAN_H_ 2 | #define _MMAN_H_ 3 | 4 | /* Protections */ 5 | #define PROT_NONE 0x00 /* no permissions */ 6 | #define PROT_READ 0x01 /* pages can be read */ 7 | #define PROT_WRITE 0x02 /* pages can be written */ 8 | #define PROT_EXEC 0x04 /* pages can be executed */ 9 | 10 | /* Sharing type and options */ 11 | #define MAP_SHARED 0x0001 /* share changes */ 12 | #define MAP_PRIVATE 0x0002 /* changes are private */ 13 | #define MAP_COPY MAP_PRIVATE /* Obsolete */ 14 | #define MAP_FIXED 0x0010 /* map addr must be exactly as requested */ 15 | #define MAP_RENAME 0x0020 /* Sun: rename private pages to file */ 16 | #define MAP_NORESERVE 0x0040 /* Sun: don't reserve needed swap area */ 17 | #define MAP_INHERIT 0x0080 /* region is retained after exec */ 18 | #define MAP_NOEXTEND 0x0100 /* for MAP_FILE, don't change file size */ 19 | #define MAP_HASSEMAPHORE 0x0200 /* region may contain semaphores */ 20 | #define MAP_STACK 0x0400 /* region grows down, like a stack */ 21 | 22 | /* Error returned from mmap() */ 23 | #define MAP_FAILED ((void *)-1) 24 | 25 | /* Flags to msync */ 26 | #define MS_ASYNC 0x01 /* perform asynchronous writes */ 27 | #define MS_SYNC 0x02 /* perform synchronous writes */ 28 | #define MS_INVALIDATE 0x04 /* invalidate cached data */ 29 | 30 | /* File modes for 'open' not defined in MinGW32 (not used by mmap) */ 31 | #ifndef S_IWGRP 32 | #define S_IWGRP 0 33 | #define S_IRGRP 0 34 | #define S_IROTH 0 35 | #endif 36 | 37 | /** 38 | * Map a file to a memory region 39 | */ 40 | void *mmap(void *addr, unsigned int len, int prot, int flags, int fd, unsigned int offset); 41 | 42 | /** 43 | * Unmap a memory region 44 | */ 45 | int munmap(void *addr, int len); 46 | 47 | /** 48 | * Synchronize a mapped region 49 | */ 50 | int msync(char *addr, int len, int flags); 51 | 52 | #endif /* _MMAN_H_ */ 53 | -------------------------------------------------------------------------------- /extra/win/mmap.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #ifdef _WIN32 4 | #include 5 | #endif 6 | #include 7 | #include "mman.h" 8 | 9 | static const char id[]="$Id: tpl.c 107 2007-04-20 17:11:29Z thanson $"; 10 | 11 | /** 12 | * @brief Map a file to a memory region 13 | * 14 | * This function emulates the POSIX mmap() using CreateFileMapping() and 15 | * MapViewOfFile() 16 | * 17 | * @param addr the suggested start address (if != 0) 18 | * @param len length of the region 19 | * @param prot region accesibility, bitwise OR of PROT_READ, PROT_WRITE, PROT_EXEC 20 | * @param flags mapping type and options (ignored) 21 | * @param fd object to be mapped into memory 22 | * @param offset offset into mapped object 23 | * @return pointer to the memory region, or NULL in case of error 24 | */ 25 | void *mmap(void *addr, unsigned int len, int prot, int flags, int fd, unsigned int offset) 26 | { 27 | (void)addr; 28 | (void)flags; 29 | (void)offset; 30 | DWORD wprot; 31 | DWORD waccess; 32 | HANDLE h; 33 | void *region; 34 | 35 | /* Translate read/write/exec flags into WIN32 constants */ 36 | switch (prot) { 37 | case PROT_READ: 38 | wprot = PAGE_READONLY; 39 | break; 40 | case PROT_EXEC: 41 | wprot = PAGE_EXECUTE_READ; 42 | break; 43 | case PROT_READ | PROT_EXEC: 44 | wprot = PAGE_EXECUTE_READ; 45 | break; 46 | case PROT_WRITE: 47 | wprot = PAGE_READWRITE; 48 | break; 49 | case PROT_READ | PROT_WRITE: 50 | wprot = PAGE_READWRITE; 51 | break; 52 | case PROT_READ | PROT_WRITE | PROT_EXEC: 53 | wprot = PAGE_EXECUTE_READWRITE; 54 | break; 55 | case PROT_WRITE | PROT_EXEC: 56 | wprot = PAGE_EXECUTE_READWRITE; 57 | break; 58 | default: 59 | return MAP_FAILED; 60 | } 61 | 62 | /* Obtaing handle to map region */ 63 | h = CreateFileMapping((HANDLE) _get_osfhandle(fd), 0, wprot, 0, len, 0); 64 | if (h == NULL) { 65 | DWORD error = GetLastError(); 66 | 67 | /* Try and translate some error codes */ 68 | switch (error) { 69 | case ERROR_ACCESS_DENIED: 70 | case ERROR_INVALID_ACCESS: 71 | errno = EACCES; 72 | break; 73 | case ERROR_OUTOFMEMORY: 74 | case ERROR_NOT_ENOUGH_MEMORY: 75 | errno = ENOMEM; 76 | break; 77 | default: 78 | errno = EINVAL; 79 | break; 80 | } 81 | return MAP_FAILED; 82 | } 83 | 84 | 85 | /* Translate sharing options into WIN32 constants */ 86 | switch (wprot) { 87 | case PAGE_READONLY: 88 | waccess = FILE_MAP_READ; 89 | break; 90 | case PAGE_READWRITE: 91 | waccess = FILE_MAP_WRITE; 92 | break; 93 | default: 94 | CloseHandle(h); 95 | return MAP_FAILED; 96 | } 97 | 98 | /* Map file and return pointer */ 99 | region = MapViewOfFile(h, waccess, 0, 0, 0); 100 | if (region == NULL) { 101 | DWORD error = GetLastError(); 102 | 103 | /* Try and translate some error codes */ 104 | switch (error) { 105 | case ERROR_ACCESS_DENIED: 106 | case ERROR_INVALID_ACCESS: 107 | errno = EACCES; 108 | break; 109 | case ERROR_INVALID_HANDLE: 110 | errno = EBADF; 111 | break; 112 | default: 113 | errno = EINVAL; 114 | break; 115 | } 116 | CloseHandle(h); 117 | return MAP_FAILED; 118 | } 119 | CloseHandle(h); /* ok to call UnmapViewOfFile after this */ 120 | 121 | /* All fine */ 122 | return region; 123 | } 124 | 125 | 126 | /** 127 | * @brief Unmap a memory region 128 | * 129 | * This is a wrapper around UnmapViewOfFile in the win32 API 130 | * 131 | * @param addr start address 132 | * @param len length of the region 133 | * @return 0 for success, -1 for error 134 | */ 135 | int munmap(void *addr, int len) 136 | { 137 | (void)len; 138 | if (UnmapViewOfFile(addr)) { 139 | return 0; 140 | } 141 | else { 142 | errno = EINVAL; 143 | return -1; 144 | } 145 | } 146 | 147 | 148 | /** 149 | * Synchronize a mapped region 150 | * 151 | * This is a wrapper around FlushViewOfFile 152 | * 153 | * @param addr start address 154 | * @param len number of bytes to flush 155 | * @param flags sync options -- currently ignored 156 | * @return 0 for success, -1 for error 157 | */ 158 | int msync(char *addr, int len, int flags) 159 | { 160 | (void)flags; 161 | if (FlushViewOfFile(addr, len) == 0) { 162 | DWORD error = GetLastError(); 163 | 164 | /* Try and translate some error codes */ 165 | switch (error) { 166 | case ERROR_INVALID_PARAMETER: 167 | errno = EINVAL; 168 | break; 169 | case ERROR_WRITE_FAULT: 170 | errno = EIO; 171 | break; 172 | default: 173 | errno = EINVAL; 174 | break; 175 | } 176 | return -1; 177 | } 178 | 179 | /* Success */ 180 | return 0; 181 | } 182 | -------------------------------------------------------------------------------- /libcurl/include/curl/curlrules.h: -------------------------------------------------------------------------------- 1 | #ifndef __CURL_CURLRULES_H 2 | #define __CURL_CURLRULES_H 3 | /*************************************************************************** 4 | * _ _ ____ _ 5 | * Project ___| | | | _ \| | 6 | * / __| | | | |_) | | 7 | * | (__| |_| | _ <| |___ 8 | * \___|\___/|_| \_\_____| 9 | * 10 | * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. 11 | * 12 | * This software is licensed as described in the file COPYING, which 13 | * you should have received as part of this distribution. The terms 14 | * are also available at http://curl.haxx.se/docs/copyright.html. 15 | * 16 | * You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 | * copies of the Software, and permit persons to whom the Software is 18 | * furnished to do so, under the terms of the COPYING file. 19 | * 20 | * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 | * KIND, either express or implied. 22 | * 23 | ***************************************************************************/ 24 | 25 | /* ================================================================ */ 26 | /* COMPILE TIME SANITY CHECKS */ 27 | /* ================================================================ */ 28 | 29 | /* 30 | * NOTE 1: 31 | * ------- 32 | * 33 | * All checks done in this file are intentionally placed in a public 34 | * header file which is pulled by curl/curl.h when an application is 35 | * being built using an already built libcurl library. Additionally 36 | * this file is also included and used when building the library. 37 | * 38 | * If compilation fails on this file it is certainly sure that the 39 | * problem is elsewhere. It could be a problem in the curlbuild.h 40 | * header file, or simply that you are using different compilation 41 | * settings than those used to build the library. 42 | * 43 | * Nothing in this file is intended to be modified or adjusted by the 44 | * curl library user nor by the curl library builder. 45 | * 46 | * Do not deactivate any check, these are done to make sure that the 47 | * library is properly built and used. 48 | * 49 | * You can find further help on the libcurl development mailing list: 50 | * http://cool.haxx.se/mailman/listinfo/curl-library/ 51 | * 52 | * NOTE 2 53 | * ------ 54 | * 55 | * Some of the following compile time checks are based on the fact 56 | * that the dimension of a constant array can not be a negative one. 57 | * In this way if the compile time verification fails, the compilation 58 | * will fail issuing an error. The error description wording is compiler 59 | * dependent but it will be quite similar to one of the following: 60 | * 61 | * "negative subscript or subscript is too large" 62 | * "array must have at least one element" 63 | * "-1 is an illegal array size" 64 | * "size of array is negative" 65 | * 66 | * If you are building an application which tries to use an already 67 | * built libcurl library and you are getting this kind of errors on 68 | * this file, it is a clear indication that there is a mismatch between 69 | * how the library was built and how you are trying to use it for your 70 | * application. Your already compiled or binary library provider is the 71 | * only one who can give you the details you need to properly use it. 72 | */ 73 | 74 | /* 75 | * Verify that some macros are actually defined. 76 | */ 77 | 78 | #ifndef CURL_SIZEOF_LONG 79 | # error "CURL_SIZEOF_LONG definition is missing!" 80 | Error Compilation_aborted_CURL_SIZEOF_LONG_is_missing 81 | #endif 82 | 83 | #ifndef CURL_TYPEOF_CURL_SOCKLEN_T 84 | # error "CURL_TYPEOF_CURL_SOCKLEN_T definition is missing!" 85 | Error Compilation_aborted_CURL_TYPEOF_CURL_SOCKLEN_T_is_missing 86 | #endif 87 | 88 | #ifndef CURL_SIZEOF_CURL_SOCKLEN_T 89 | # error "CURL_SIZEOF_CURL_SOCKLEN_T definition is missing!" 90 | Error Compilation_aborted_CURL_SIZEOF_CURL_SOCKLEN_T_is_missing 91 | #endif 92 | 93 | #ifndef CURL_TYPEOF_CURL_OFF_T 94 | # error "CURL_TYPEOF_CURL_OFF_T definition is missing!" 95 | Error Compilation_aborted_CURL_TYPEOF_CURL_OFF_T_is_missing 96 | #endif 97 | 98 | #ifndef CURL_FORMAT_CURL_OFF_T 99 | # error "CURL_FORMAT_CURL_OFF_T definition is missing!" 100 | Error Compilation_aborted_CURL_FORMAT_CURL_OFF_T_is_missing 101 | #endif 102 | 103 | #ifndef CURL_FORMAT_CURL_OFF_TU 104 | # error "CURL_FORMAT_CURL_OFF_TU definition is missing!" 105 | Error Compilation_aborted_CURL_FORMAT_CURL_OFF_TU_is_missing 106 | #endif 107 | 108 | #ifndef CURL_FORMAT_OFF_T 109 | # error "CURL_FORMAT_OFF_T definition is missing!" 110 | Error Compilation_aborted_CURL_FORMAT_OFF_T_is_missing 111 | #endif 112 | 113 | #ifndef CURL_SIZEOF_CURL_OFF_T 114 | # error "CURL_SIZEOF_CURL_OFF_T definition is missing!" 115 | Error Compilation_aborted_CURL_SIZEOF_CURL_OFF_T_is_missing 116 | #endif 117 | 118 | #ifndef CURL_SUFFIX_CURL_OFF_T 119 | # error "CURL_SUFFIX_CURL_OFF_T definition is missing!" 120 | Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_T_is_missing 121 | #endif 122 | 123 | #ifndef CURL_SUFFIX_CURL_OFF_TU 124 | # error "CURL_SUFFIX_CURL_OFF_TU definition is missing!" 125 | Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_TU_is_missing 126 | #endif 127 | 128 | /* 129 | * Macros private to this header file. 130 | */ 131 | 132 | #define CurlchkszEQ(t, s) sizeof(t) == s ? 1 : -1 133 | 134 | #define CurlchkszGE(t1, t2) sizeof(t1) >= sizeof(t2) ? 1 : -1 135 | 136 | /* 137 | * Verify that the size previously defined and expected for long 138 | * is the same as the one reported by sizeof() at compile time. 139 | */ 140 | 141 | typedef char 142 | __curl_rule_01__ 143 | [CurlchkszEQ(long, CURL_SIZEOF_LONG)]; 144 | 145 | /* 146 | * Verify that the size previously defined and expected for 147 | * curl_off_t is actually the the same as the one reported 148 | * by sizeof() at compile time. 149 | */ 150 | 151 | typedef char 152 | __curl_rule_02__ 153 | [CurlchkszEQ(curl_off_t, CURL_SIZEOF_CURL_OFF_T)]; 154 | 155 | /* 156 | * Verify at compile time that the size of curl_off_t as reported 157 | * by sizeof() is greater or equal than the one reported for long 158 | * for the current compilation. 159 | */ 160 | 161 | typedef char 162 | __curl_rule_03__ 163 | [CurlchkszGE(curl_off_t, long)]; 164 | 165 | /* 166 | * Verify that the size previously defined and expected for 167 | * curl_socklen_t is actually the the same as the one reported 168 | * by sizeof() at compile time. 169 | */ 170 | 171 | typedef char 172 | __curl_rule_04__ 173 | [CurlchkszEQ(curl_socklen_t, CURL_SIZEOF_CURL_SOCKLEN_T)]; 174 | 175 | /* 176 | * Verify at compile time that the size of curl_socklen_t as reported 177 | * by sizeof() is greater or equal than the one reported for int for 178 | * the current compilation. 179 | */ 180 | 181 | typedef char 182 | __curl_rule_05__ 183 | [CurlchkszGE(curl_socklen_t, int)]; 184 | 185 | /* ================================================================ */ 186 | /* EXTERNALLY AND INTERNALLY VISIBLE DEFINITIONS */ 187 | /* ================================================================ */ 188 | 189 | /* 190 | * CURL_ISOCPP and CURL_OFF_T_C definitions are done here in order to allow 191 | * these to be visible and exported by the external libcurl interface API, 192 | * while also making them visible to the library internals, simply including 193 | * curl_setup.h, without actually needing to include curl.h internally. 194 | * If some day this section would grow big enough, all this should be moved 195 | * to its own header file. 196 | */ 197 | 198 | /* 199 | * Figure out if we can use the ## preprocessor operator, which is supported 200 | * by ISO/ANSI C and C++. Some compilers support it without setting __STDC__ 201 | * or __cplusplus so we need to carefully check for them too. 202 | */ 203 | 204 | #if defined(__STDC__) || defined(_MSC_VER) || defined(__cplusplus) || \ 205 | defined(__HP_aCC) || defined(__BORLANDC__) || defined(__LCC__) || \ 206 | defined(__POCC__) || defined(__SALFORDC__) || defined(__HIGHC__) || \ 207 | defined(__ILEC400__) 208 | /* This compiler is believed to have an ISO compatible preprocessor */ 209 | #define CURL_ISOCPP 210 | #else 211 | /* This compiler is believed NOT to have an ISO compatible preprocessor */ 212 | #undef CURL_ISOCPP 213 | #endif 214 | 215 | /* 216 | * Macros for minimum-width signed and unsigned curl_off_t integer constants. 217 | */ 218 | 219 | #if defined(__BORLANDC__) && (__BORLANDC__ == 0x0551) 220 | # define __CURL_OFF_T_C_HLPR2(x) x 221 | # define __CURL_OFF_T_C_HLPR1(x) __CURL_OFF_T_C_HLPR2(x) 222 | # define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ 223 | __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_T) 224 | # define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ 225 | __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_TU) 226 | #else 227 | # ifdef CURL_ISOCPP 228 | # define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val ## Suffix 229 | # else 230 | # define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val/**/Suffix 231 | # endif 232 | # define __CURL_OFF_T_C_HLPR1(Val,Suffix) __CURL_OFF_T_C_HLPR2(Val,Suffix) 233 | # define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_T) 234 | # define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_TU) 235 | #endif 236 | 237 | /* 238 | * Get rid of macros private to this header file. 239 | */ 240 | 241 | #undef CurlchkszEQ 242 | #undef CurlchkszGE 243 | 244 | /* 245 | * Get rid of macros not intended to exist beyond this point. 246 | */ 247 | 248 | #undef CURL_PULL_WS2TCPIP_H 249 | #undef CURL_PULL_SYS_TYPES_H 250 | #undef CURL_PULL_SYS_SOCKET_H 251 | #undef CURL_PULL_SYS_POLL_H 252 | #undef CURL_PULL_STDINT_H 253 | #undef CURL_PULL_INTTYPES_H 254 | 255 | #undef CURL_TYPEOF_CURL_SOCKLEN_T 256 | #undef CURL_TYPEOF_CURL_OFF_T 257 | 258 | #ifdef CURL_NO_OLDIES 259 | #undef CURL_FORMAT_OFF_T /* not required since 7.19.0 - obsoleted in 7.20.0 */ 260 | #endif 261 | 262 | #endif /* __CURL_CURLRULES_H */ 263 | -------------------------------------------------------------------------------- /libcurl/include/curl/curlver.h: -------------------------------------------------------------------------------- 1 | #ifndef __CURL_CURLVER_H 2 | #define __CURL_CURLVER_H 3 | /*************************************************************************** 4 | * _ _ ____ _ 5 | * Project ___| | | | _ \| | 6 | * / __| | | | |_) | | 7 | * | (__| |_| | _ <| |___ 8 | * \___|\___/|_| \_\_____| 9 | * 10 | * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. 11 | * 12 | * This software is licensed as described in the file COPYING, which 13 | * you should have received as part of this distribution. The terms 14 | * are also available at http://curl.haxx.se/docs/copyright.html. 15 | * 16 | * You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 | * copies of the Software, and permit persons to whom the Software is 18 | * furnished to do so, under the terms of the COPYING file. 19 | * 20 | * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 | * KIND, either express or implied. 22 | * 23 | ***************************************************************************/ 24 | 25 | /* This header file contains nothing but libcurl version info, generated by 26 | a script at release-time. This was made its own header file in 7.11.2 */ 27 | 28 | /* This is the global package copyright */ 29 | #define LIBCURL_COPYRIGHT "1996 - 2015 Daniel Stenberg, ." 30 | 31 | /* This is the version number of the libcurl package from which this header 32 | file origins: */ 33 | #define LIBCURL_VERSION "7.41.0" 34 | 35 | /* The numeric version number is also available "in parts" by using these 36 | defines: */ 37 | #define LIBCURL_VERSION_MAJOR 7 38 | #define LIBCURL_VERSION_MINOR 41 39 | #define LIBCURL_VERSION_PATCH 0 40 | 41 | /* This is the numeric version of the libcurl version number, meant for easier 42 | parsing and comparions by programs. The LIBCURL_VERSION_NUM define will 43 | always follow this syntax: 44 | 45 | 0xXXYYZZ 46 | 47 | Where XX, YY and ZZ are the main version, release and patch numbers in 48 | hexadecimal (using 8 bits each). All three numbers are always represented 49 | using two digits. 1.2 would appear as "0x010200" while version 9.11.7 50 | appears as "0x090b07". 51 | 52 | This 6-digit (24 bits) hexadecimal number does not show pre-release number, 53 | and it is always a greater number in a more recent release. It makes 54 | comparisons with greater than and less than work. 55 | */ 56 | #define LIBCURL_VERSION_NUM 0x072900 57 | 58 | /* 59 | * This is the date and time when the full source package was created. The 60 | * timestamp is not stored in git, as the timestamp is properly set in the 61 | * tarballs by the maketgz script. 62 | * 63 | * The format of the date should follow this template: 64 | * 65 | * "Mon Feb 12 11:35:33 UTC 2007" 66 | */ 67 | #define LIBCURL_TIMESTAMP "Wed Feb 25 07:36:13 UTC 2015" 68 | 69 | #endif /* __CURL_CURLVER_H */ 70 | -------------------------------------------------------------------------------- /libcurl/include/curl/easy.h: -------------------------------------------------------------------------------- 1 | #ifndef __CURL_EASY_H 2 | #define __CURL_EASY_H 3 | /*************************************************************************** 4 | * _ _ ____ _ 5 | * Project ___| | | | _ \| | 6 | * / __| | | | |_) | | 7 | * | (__| |_| | _ <| |___ 8 | * \___|\___/|_| \_\_____| 9 | * 10 | * Copyright (C) 1998 - 2008, Daniel Stenberg, , et al. 11 | * 12 | * This software is licensed as described in the file COPYING, which 13 | * you should have received as part of this distribution. The terms 14 | * are also available at http://curl.haxx.se/docs/copyright.html. 15 | * 16 | * You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 | * copies of the Software, and permit persons to whom the Software is 18 | * furnished to do so, under the terms of the COPYING file. 19 | * 20 | * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 | * KIND, either express or implied. 22 | * 23 | ***************************************************************************/ 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | CURL_EXTERN CURL *curl_easy_init(void); 29 | CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); 30 | CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); 31 | CURL_EXTERN void curl_easy_cleanup(CURL *curl); 32 | 33 | /* 34 | * NAME curl_easy_getinfo() 35 | * 36 | * DESCRIPTION 37 | * 38 | * Request internal information from the curl session with this function. The 39 | * third argument MUST be a pointer to a long, a pointer to a char * or a 40 | * pointer to a double (as the documentation describes elsewhere). The data 41 | * pointed to will be filled in accordingly and can be relied upon only if the 42 | * function returns CURLE_OK. This function is intended to get used *AFTER* a 43 | * performed transfer, all results from this function are undefined until the 44 | * transfer is completed. 45 | */ 46 | CURL_EXTERN CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ...); 47 | 48 | 49 | /* 50 | * NAME curl_easy_duphandle() 51 | * 52 | * DESCRIPTION 53 | * 54 | * Creates a new curl session handle with the same options set for the handle 55 | * passed in. Duplicating a handle could only be a matter of cloning data and 56 | * options, internal state info and things like persistent connections cannot 57 | * be transferred. It is useful in multithreaded applications when you can run 58 | * curl_easy_duphandle() for each new thread to avoid a series of identical 59 | * curl_easy_setopt() invokes in every thread. 60 | */ 61 | CURL_EXTERN CURL* curl_easy_duphandle(CURL *curl); 62 | 63 | /* 64 | * NAME curl_easy_reset() 65 | * 66 | * DESCRIPTION 67 | * 68 | * Re-initializes a CURL handle to the default values. This puts back the 69 | * handle to the same state as it was in when it was just created. 70 | * 71 | * It does keep: live connections, the Session ID cache, the DNS cache and the 72 | * cookies. 73 | */ 74 | CURL_EXTERN void curl_easy_reset(CURL *curl); 75 | 76 | /* 77 | * NAME curl_easy_recv() 78 | * 79 | * DESCRIPTION 80 | * 81 | * Receives data from the connected socket. Use after successful 82 | * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. 83 | */ 84 | CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen, 85 | size_t *n); 86 | 87 | /* 88 | * NAME curl_easy_send() 89 | * 90 | * DESCRIPTION 91 | * 92 | * Sends data over the connected socket. Use after successful 93 | * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. 94 | */ 95 | CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer, 96 | size_t buflen, size_t *n); 97 | 98 | #ifdef __cplusplus 99 | } 100 | #endif 101 | 102 | #endif 103 | -------------------------------------------------------------------------------- /libcurl/include/curl/mprintf.h: -------------------------------------------------------------------------------- 1 | #ifndef __CURL_MPRINTF_H 2 | #define __CURL_MPRINTF_H 3 | /*************************************************************************** 4 | * _ _ ____ _ 5 | * Project ___| | | | _ \| | 6 | * / __| | | | |_) | | 7 | * | (__| |_| | _ <| |___ 8 | * \___|\___/|_| \_\_____| 9 | * 10 | * Copyright (C) 1998 - 2013, Daniel Stenberg, , et al. 11 | * 12 | * This software is licensed as described in the file COPYING, which 13 | * you should have received as part of this distribution. The terms 14 | * are also available at http://curl.haxx.se/docs/copyright.html. 15 | * 16 | * You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 | * copies of the Software, and permit persons to whom the Software is 18 | * furnished to do so, under the terms of the COPYING file. 19 | * 20 | * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 | * KIND, either express or implied. 22 | * 23 | ***************************************************************************/ 24 | 25 | #include 26 | #include /* needed for FILE */ 27 | 28 | #include "curl.h" 29 | 30 | #ifdef __cplusplus 31 | extern "C" { 32 | #endif 33 | 34 | CURL_EXTERN int curl_mprintf(const char *format, ...); 35 | CURL_EXTERN int curl_mfprintf(FILE *fd, const char *format, ...); 36 | CURL_EXTERN int curl_msprintf(char *buffer, const char *format, ...); 37 | CURL_EXTERN int curl_msnprintf(char *buffer, size_t maxlength, 38 | const char *format, ...); 39 | CURL_EXTERN int curl_mvprintf(const char *format, va_list args); 40 | CURL_EXTERN int curl_mvfprintf(FILE *fd, const char *format, va_list args); 41 | CURL_EXTERN int curl_mvsprintf(char *buffer, const char *format, va_list args); 42 | CURL_EXTERN int curl_mvsnprintf(char *buffer, size_t maxlength, 43 | const char *format, va_list args); 44 | CURL_EXTERN char *curl_maprintf(const char *format, ...); 45 | CURL_EXTERN char *curl_mvaprintf(const char *format, va_list args); 46 | 47 | #ifdef _MPRINTF_REPLACE 48 | # undef printf 49 | # undef fprintf 50 | # undef sprintf 51 | # undef vsprintf 52 | # undef snprintf 53 | # undef vprintf 54 | # undef vfprintf 55 | # undef vsnprintf 56 | # undef aprintf 57 | # undef vaprintf 58 | # define printf curl_mprintf 59 | # define fprintf curl_mfprintf 60 | #ifdef CURLDEBUG 61 | /* When built with CURLDEBUG we define away the sprintf functions since we 62 | don't want internal code to be using them */ 63 | # define sprintf sprintf_was_used 64 | # define vsprintf vsprintf_was_used 65 | #else 66 | # define sprintf curl_msprintf 67 | # define vsprintf curl_mvsprintf 68 | #endif 69 | # define snprintf curl_msnprintf 70 | # define vprintf curl_mvprintf 71 | # define vfprintf curl_mvfprintf 72 | # define vsnprintf curl_mvsnprintf 73 | # define aprintf curl_maprintf 74 | # define vaprintf curl_mvaprintf 75 | #endif 76 | 77 | #ifdef __cplusplus 78 | } 79 | #endif 80 | 81 | #endif /* __CURL_MPRINTF_H */ 82 | -------------------------------------------------------------------------------- /libcurl/include/curl/stdcheaders.h: -------------------------------------------------------------------------------- 1 | #ifndef __STDC_HEADERS_H 2 | #define __STDC_HEADERS_H 3 | /*************************************************************************** 4 | * _ _ ____ _ 5 | * Project ___| | | | _ \| | 6 | * / __| | | | |_) | | 7 | * | (__| |_| | _ <| |___ 8 | * \___|\___/|_| \_\_____| 9 | * 10 | * Copyright (C) 1998 - 2010, Daniel Stenberg, , et al. 11 | * 12 | * This software is licensed as described in the file COPYING, which 13 | * you should have received as part of this distribution. The terms 14 | * are also available at http://curl.haxx.se/docs/copyright.html. 15 | * 16 | * You may opt to use, copy, modify, merge, publish, distribute and/or sell 17 | * copies of the Software, and permit persons to whom the Software is 18 | * furnished to do so, under the terms of the COPYING file. 19 | * 20 | * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21 | * KIND, either express or implied. 22 | * 23 | ***************************************************************************/ 24 | 25 | #include 26 | 27 | size_t fread (void *, size_t, size_t, FILE *); 28 | size_t fwrite (const void *, size_t, size_t, FILE *); 29 | 30 | int strcasecmp(const char *, const char *); 31 | int strncasecmp(const char *, const char *, size_t); 32 | 33 | #endif /* __STDC_HEADERS_H */ 34 | -------------------------------------------------------------------------------- /libcurl/lib/armeabi-v7a/libcurl.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Go-joy/crsync/0af4f53c2dd2d2fe180a756f7f6b98fe6d60bf1f/libcurl/lib/armeabi-v7a/libcurl.a -------------------------------------------------------------------------------- /libcurl/lib/m32/libcurl.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Go-joy/crsync/0af4f53c2dd2d2fe180a756f7f6b98fe6d60bf1f/libcurl/lib/m32/libcurl.a -------------------------------------------------------------------------------- /libcurl/lib/m32/libcurl.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Go-joy/crsync/0af4f53c2dd2d2fe180a756f7f6b98fe6d60bf1f/libcurl/lib/m32/libcurl.dll -------------------------------------------------------------------------------- /libcurl/lib/m32/libcurldll.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Go-joy/crsync/0af4f53c2dd2d2fe180a756f7f6b98fe6d60bf1f/libcurl/lib/m32/libcurldll.a -------------------------------------------------------------------------------- /libcurl/lib/x86/libcurl.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Go-joy/crsync/0af4f53c2dd2d2fe180a756f7f6b98fe6d60bf1f/libcurl/lib/x86/libcurl.a -------------------------------------------------------------------------------- /src/crsync-build.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem ndk-build parameters: 3 | rem NDK_PROJECT_PATH = where is project path 4 | rem NDK_APPLICATION_MK = where is Application.mk 5 | rem APP_BUILD_SCRIPT = where is Android.mk 6 | rem APP_PLATFORM = android-3 default 7 | rem APP_ABI = all or armeabi,armeabi-v7a,x86,mips 8 | rem NDK_DEBUG = 0 or 1 9 | rem clean 10 | rem -B force rebuild 11 | rem V=1 show NDK log 12 | 13 | call %NDKROOT%/ndk-build -B NDK_PROJECT_PATH=. APP_BUILD_SCRIPT=./crsync.mk APP_ABI=armeabi-v7a,x86 APP_PLATFORM=android-15 14 | pause 15 | -------------------------------------------------------------------------------- /src/crsync-jni.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifdef ANDROID 25 | #include 26 | #include 27 | 28 | #include "global.h" 29 | #include "log.h" 30 | #include "helper.h" 31 | #include "utstring.h" 32 | #include "utlist.h" 33 | #include "util.h" 34 | #include "http.h" 35 | 36 | #define ARRAY_COUNT(x) ((int) (sizeof(x) / sizeof((x)[0]))) 37 | 38 | typedef struct JNIJavaMethods { 39 | jmethodID *JavaMethod; 40 | const char *FunctionName; 41 | const char *FunctionParams; 42 | } JNIJavaMethods; 43 | 44 | static JavaVM *gJavaVM = NULL; 45 | static jclass gJavaClass = NULL; 46 | static jmethodID gMethod_onProgress = NULL; 47 | static jmethodID gMethod_onDiff = NULL; 48 | 49 | static time_t gTime = 0; 50 | 51 | static bulkHelper_t *gBulkHelper = NULL; 52 | 53 | int crs_callback_patch(const char *basename, const unsigned int bytes, const int isComplete, const int immediate) { 54 | time_t nowTime; 55 | time(&nowTime); 56 | double diff = difftime(nowTime, gTime); 57 | if( diff < 1 && immediate == 0 ) { 58 | return 0; 59 | } 60 | gTime = nowTime; 61 | 62 | JNIEnv *env = NULL; 63 | int isCancel = 0; 64 | if ((*gJavaVM)->GetEnv(gJavaVM, (void**)&env, JNI_VERSION_1_6) == JNI_OK) { 65 | jstring jname = (*env)->NewStringUTF( env, basename ); 66 | isCancel = (*env)->CallStaticIntMethod(env, gJavaClass, gMethod_onProgress, jname, (jlong)bytes, (jint)isComplete); 67 | (*env)->DeleteLocalRef(env, jname); 68 | } 69 | return isCancel; 70 | } 71 | 72 | void crs_callback_diff(const char *basename, const unsigned int bytes, const int isComplete) { 73 | JNIEnv *env = NULL; 74 | if ((*gJavaVM)->GetEnv(gJavaVM, (void**)&env, JNI_VERSION_1_6) == JNI_OK) { 75 | jstring jname = (*env)->NewStringUTF( env, basename ); 76 | (*env)->CallStaticVoidMethod(env, gJavaClass, gMethod_onDiff, jname, (jlong)bytes, (jint)isComplete); 77 | (*env)->DeleteLocalRef(env, jname); 78 | } 79 | } 80 | 81 | jint JNI_crsync_init(JNIEnv *env, jclass clazz, jstring j_fileDir, jstring j_baseUrl) { 82 | const char *fileDir = (*env)->GetStringUTFChars(env, j_fileDir, NULL); 83 | const char *baseUrl = (*env)->GetStringUTFChars(env, j_baseUrl, NULL); 84 | 85 | log_open(); 86 | CRScode code = CRS_OK; 87 | do { 88 | code = HTTP_global_init(); 89 | if(code != CRS_OK) break; 90 | free(gBulkHelper); 91 | gBulkHelper = bulkHelper_malloc(); 92 | gBulkHelper->fileDir = strdup(fileDir); 93 | gBulkHelper->baseUrl = strdup(baseUrl); 94 | } while(0); 95 | 96 | (*env)->ReleaseStringUTFChars(env, j_fileDir, fileDir); 97 | (*env)->ReleaseStringUTFChars(env, j_baseUrl, baseUrl); 98 | return (jint)code; 99 | } 100 | 101 | jint JNI_crsync_perform_magnet(JNIEnv *env, jclass clazz) { 102 | jint r = 0;//(jint)bulkHelper_perform_magnet(gBulkHelper); 103 | return r; 104 | } 105 | 106 | jstring JNI_crsync_get_magnet(JNIEnv *env, jclass clazz) { 107 | UT_string *result = NULL; 108 | utstring_new(result); 109 | 110 | magnet_t *m = gBulkHelper->currentMagnet; 111 | if(m) { 112 | sum_t *elt=NULL; 113 | LL_FOREACH(m->file, elt) { 114 | utstring_printf(result, "%s;", elt->name); 115 | char * hashStr = Util_hex_string(elt->digest, CRS_STRONG_DIGEST_SIZE); 116 | utstring_printf(result, "%s;", hashStr); 117 | free(hashStr); 118 | utstring_printf(result, "%u;", elt->size); 119 | } 120 | } 121 | jstring jinfo = (*env)->NewStringUTF( env, utstring_body(result) ); 122 | utstring_free(result); 123 | return jinfo; 124 | } 125 | 126 | void JNI_crsync_set_magnet(JNIEnv *env, jclass clazz, jstring jMagnetString) { 127 | const char *magnetString = (*env)->GetStringUTFChars(env, jMagnetString, NULL); 128 | bulkHelper_set_magnet(gBulkHelper, magnetString); 129 | (*env)->ReleaseStringUTFChars(env, jMagnetString, magnetString); 130 | } 131 | 132 | jint JNI_crsync_perform_diff(JNIEnv *env, jclass clazz) { 133 | jint r = (jint)bulkHelper_perform_diff(gBulkHelper); 134 | return r; 135 | } 136 | 137 | jint JNI_crsync_perform_patch(JNIEnv *env, jclass clazz) { 138 | jint r = (jint)bulkHelper_perform_patch(gBulkHelper); 139 | return r; 140 | } 141 | 142 | void JNI_crsync_cleanup(JNIEnv *env, jclass clazz) { 143 | bulkHelper_free(gBulkHelper); 144 | gBulkHelper = NULL; 145 | HTTP_global_cleanup(); 146 | log_close(); 147 | } 148 | 149 | jint JNI_OnLoad(JavaVM* vm, void* reserved) { 150 | log_open(); 151 | LOGI("crsync JNI_OnLoad\n"); 152 | JNIEnv *env; 153 | if ((*vm)->GetEnv(vm, (void**)&env, JNI_VERSION_1_6) != JNI_OK) { 154 | return JNI_ERR; 155 | } 156 | 157 | // hook up native functions to string names and params 158 | JNINativeMethod NativeMethods[] = { 159 | { "JNI_crsync_init", "(Ljava/lang/String;Ljava/lang/String;)I", (void*)JNI_crsync_init }, 160 | { "JNI_crsync_get_magnet", "()Ljava/lang/String;", (void*)JNI_crsync_get_magnet }, 161 | { "JNI_crsync_set_magnet", "(Ljava/lang/String;)V", (void*)JNI_crsync_set_magnet }, 162 | { "JNI_crsync_perform_magnet", "()I", (void*)JNI_crsync_perform_magnet }, 163 | { "JNI_crsync_perform_diff", "()I", (void*)JNI_crsync_perform_diff }, 164 | { "JNI_crsync_perform_patch", "()I", (void*)JNI_crsync_perform_patch }, 165 | { "JNI_crsync_cleanup", "()V", (void*)JNI_crsync_cleanup }, 166 | }; 167 | 168 | // get the java class from JNI 169 | jclass crsyncClass = (*env)->FindClass(env, "com/shaddock/crsync/Crsync"); 170 | gJavaClass = (*env)->NewGlobalRef(env, crsyncClass); 171 | (*env)->RegisterNatives(env, gJavaClass, NativeMethods, ARRAY_COUNT(NativeMethods)); 172 | 173 | // hook up java functions to string names and param 174 | JNIJavaMethods JavaMethods[] = { 175 | { &gMethod_onProgress, "java_onProgress", "(Ljava/lang/String;JI)I" }, //public static int java_onProgress(String, long, int); 176 | { &gMethod_onDiff, "java_onDiff", "(Ljava/lang/String;JI)V" },//public static void java_onDiff(String, long, int); 177 | }; 178 | 179 | int result = 1; 180 | for( int MethodIter = 0; MethodIter < ARRAY_COUNT(JavaMethods); MethodIter++ ) { 181 | *JavaMethods[MethodIter].JavaMethod = (*env)->GetStaticMethodID(env, gJavaClass, JavaMethods[MethodIter].FunctionName, JavaMethods[MethodIter].FunctionParams); 182 | if( 0 == *JavaMethods[MethodIter].JavaMethod ) { 183 | LOGE("JavaMethod not found! %s(%s)", JavaMethods[MethodIter].FunctionName, JavaMethods[MethodIter].FunctionParams); 184 | result = 0; 185 | break; 186 | } 187 | } 188 | gJavaVM = vm; 189 | return (result == 1) ? JNI_VERSION_1_6 : JNI_ERR; 190 | } 191 | 192 | /* At Android Platform, JNI_OnUnload() will never been called */ 193 | void JNI_OnUnload(JavaVM* vm, void* InReserved) { 194 | log_close(); 195 | gJavaVM = NULL; 196 | JNIEnv *env =NULL; 197 | if( (*vm)->GetEnv(vm, (void**)&env, JNI_VERSION_1_6) == JNI_OK ) { 198 | (*env)->DeleteGlobalRef(env, gJavaClass); 199 | gJavaClass = NULL; 200 | } 201 | } 202 | 203 | #endif 204 | -------------------------------------------------------------------------------- /src/crsync.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include 25 | #if ( defined __CYGWIN__ || defined __MINGW32__ || defined _WIN32 ) 26 | # include "win/mman.h" /* mmap */ 27 | #else 28 | # include /* mmap */ 29 | #endif 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | 36 | #include "crsync.h" 37 | #include "http.h" 38 | #include "log.h" 39 | #include "util.h" 40 | 41 | CRScode crs_perform_digest(const char *srcFilename, const char *dstFilename, const uint32_t blockSize) { 42 | LOGI("begin\n"); 43 | if(srcFilename == NULL || dstFilename == NULL) { 44 | LOGE("end %d\n", CRS_PARAM_ERROR); 45 | return CRS_PARAM_ERROR; 46 | } 47 | 48 | fileDigest_t *fd = fileDigest_malloc(); 49 | CRScode code = CRS_OK; 50 | do { 51 | code = Digest_Perform(srcFilename, blockSize, fd); 52 | if(code != CRS_OK) break; 53 | code = Digest_Save(dstFilename, fd); 54 | } while(0); 55 | 56 | fileDigest_free(fd); 57 | 58 | LOGI("end %d\n", code); 59 | return code; 60 | } 61 | 62 | CRScode crs_perform_diff(const char *srcFilename, const char *dstFilename, const char *digestUrl, 63 | fileDigest_t *fd, diffResult_t *dr) { 64 | LOGI("begin\n"); 65 | 66 | if(!srcFilename || !dstFilename || !digestUrl || !fd || !dr) { 67 | LOGE("end %d\n", CRS_PARAM_ERROR); 68 | return CRS_PARAM_ERROR; 69 | } 70 | 71 | CRScode code = CRS_OK; 72 | char *digestFilename = Util_strcat(dstFilename, DIGEST_EXT); 73 | LOGI("digestFilename = %s\n", digestFilename); 74 | 75 | do { 76 | if(0 != Digest_checkfile(digestFilename)) { 77 | code = HTTP_File(digestUrl, digestFilename, 1, NULL); 78 | if(code != CRS_OK) break; 79 | } 80 | 81 | code = Digest_Load(digestFilename, fd); 82 | if(code != CRS_OK) break; 83 | code = Diff_perform(srcFilename, dstFilename, fd, dr); 84 | } while(0); 85 | 86 | free(digestFilename); 87 | fileDigest_dump(fd); 88 | diffResult_dump(dr); 89 | LOGI("end %d\n", code); 90 | return code; 91 | } 92 | 93 | CRScode crs_perform_patch(const char *srcFilename, const char *dstFilename, const char *url, 94 | const fileDigest_t *fd, const diffResult_t *dr) { 95 | LOGI("begin\n"); 96 | CRScode code = CRS_OK; 97 | code = Patch_perform(srcFilename, dstFilename, url, fd, dr); 98 | LOGI("end %d\n", code); 99 | return code; 100 | } 101 | 102 | CRScode crs_perform_update(const char *srcFilename, const char *dstFilename, const char *digestUrl, const char *url) { 103 | LOGI("begin\n"); 104 | 105 | if(!srcFilename || !dstFilename || !url) { 106 | LOGE("end %d\n", CRS_PARAM_ERROR); 107 | return CRS_PARAM_ERROR; 108 | } 109 | 110 | CRScode code = CRS_OK; 111 | 112 | fileDigest_t *fd = fileDigest_malloc(); 113 | diffResult_t *dr = diffResult_malloc(); 114 | do { 115 | code = crs_perform_diff(srcFilename, dstFilename, digestUrl, fd, dr); 116 | if(code != CRS_OK) break; 117 | code = crs_perform_patch(srcFilename, dstFilename, url, fd, dr); 118 | } while(0); 119 | 120 | fileDigest_free(fd); 121 | diffResult_free(dr); 122 | 123 | LOGI("end %d\n", code); 124 | return code; 125 | } 126 | -------------------------------------------------------------------------------- /src/crsync.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRSYNC_H 25 | #define CRSYNC_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #include "global.h" 32 | #include "crsyncver.h" /* crsync version defines */ 33 | #include "digest.h" 34 | #include "diff.h" 35 | #include "patch.h" 36 | #include "uthash.h" 37 | #include "utlist.h" 38 | #include "utstring.h" 39 | #include "tpl.h" 40 | #include "curl/curl.h" 41 | 42 | CRScode crs_perform_digest (const char *srcFilename, const char *dstFilename, const uint32_t blocksize); 43 | 44 | CRScode crs_perform_diff (const char *srcFilename, const char *dstFilename, const char *digestUrl, 45 | fileDigest_t *fd, diffResult_t *dr); 46 | 47 | CRScode crs_perform_patch (const char *srcFilename, const char *dstFilename, const char *url, 48 | const fileDigest_t *fd, const diffResult_t *dr); 49 | 50 | CRScode crs_perform_update (const char *srcFilename, const char *dstFilename, const char *digestUrl, const char *url); 51 | 52 | #if defined __cplusplus 53 | } 54 | #endif 55 | 56 | #endif // CRSYNC_H 57 | -------------------------------------------------------------------------------- /src/crsync.mk: -------------------------------------------------------------------------------- 1 | LOCAL_PATH:= $(call my-dir) 2 | 3 | include $(CLEAR_VARS) 4 | LOCAL_MODULE := curl 5 | LOCAL_SRC_FILES := $(LOCAL_PATH)/../libcurl/lib/$(TARGET_ARCH_ABI)/libcurl.a 6 | LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/../libcurl/include 7 | include $(PREBUILT_STATIC_LIBRARY) 8 | 9 | include $(CLEAR_VARS) 10 | LOCAL_MODULE := crsync 11 | LOCAL_SRC_FILES := digest.c diff.c patch.c http.c helper.c magnet.c util.c log.c crsync.c crsync-jni.c ../extra/md5.c ../extra/tpl.c 12 | LOCAL_C_INCLUDES += ../extra 13 | LOCAL_STATIC_LIBRARIES := curl 14 | LOCAL_CFLAGS += -DHASH_BLOOM=21 -DCURL_STATICLIB -std=c99 -fopenmp 15 | LOCAL_LDLIBS += -lc -lz -llog 16 | LOCAL_LDFLAGS += -fopenmp 17 | 18 | include $(BUILD_SHARED_LIBRARY) 19 | -------------------------------------------------------------------------------- /src/crsync.pro: -------------------------------------------------------------------------------- 1 | TEMPLATE = app 2 | CONFIG += console 3 | CONFIG -= app_bundle 4 | CONFIG -= qt 5 | 6 | TARGET = crsync 7 | DESTDIR = m32 8 | 9 | SOURCES += \ 10 | digest.c \ 11 | diff.c \ 12 | patch.c \ 13 | http.c \ 14 | magnet.c \ 15 | helper.c \ 16 | util.c \ 17 | log.c \ 18 | crsync.c \ 19 | crsync-console.c \ 20 | ../extra/md5.c \ 21 | ../extra/tpl.c \ 22 | ../extra/win/mmap.c \ 23 | ../extra/dictionary.c \ 24 | ../extra/iniparser.c 25 | 26 | HEADERS += \ 27 | global.h \ 28 | digest.h \ 29 | diff.h \ 30 | patch.h \ 31 | http.h \ 32 | magnet.h \ 33 | helper.h \ 34 | util.h \ 35 | log.h \ 36 | crsync.h \ 37 | crsyncver.h \ 38 | ../extra/md5.h \ 39 | ../extra/tpl.h \ 40 | ../extra/win/mman.h \ 41 | ../extra/uthash.h \ 42 | ../extra/utstring.h \ 43 | ../extra/utlist.h \ 44 | ../extra/dictionary.h \ 45 | ../extra/iniparser.h 46 | 47 | DEFINES += HASH_BLOOM=21 CURL_STATICLIB 48 | INCLUDEPATH += $${_PRO_FILE_PWD_}/../libcurl/include 49 | LIBS += -L$${_PRO_FILE_PWD_}/../libcurl/lib/m32 -lcurl -lws2_32 50 | 51 | INCLUDEPATH += $${_PRO_FILE_PWD_}/../extra/ 52 | 53 | QMAKE_CFLAGS += -O3 -std=c99 -fopenmp 54 | QMAKE_LFLAGS += -static -static-libgcc -fopenmp 55 | 56 | win32:RC_FILE = crsync.rc 57 | 58 | include(deployment.pri) 59 | qtcAddDeployment() 60 | -------------------------------------------------------------------------------- /src/crsync.rc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Go-joy/crsync/0af4f53c2dd2d2fe180a756f7f6b98fe6d60bf1f/src/crsync.rc -------------------------------------------------------------------------------- /src/crsyncver.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef __CRSYNCVER_H 25 | #define __CRSYNCVER_H 26 | 27 | /* This is the global package copyright */ 28 | #define CRSYNC_COPYRIGHT "2015 chenqi, <9468305@qq.com>." 29 | 30 | /* This is the version number of the crsync package from which this header file origins: */ 31 | #define CRSYNC_VERSION "0.1.0" 32 | 33 | /* The numeric version number is also available "in parts" by using these defines: */ 34 | #define CRSYNC_VERSION_MAJOR 0 35 | #define CRSYNC_VERSION_MINOR 1 36 | #define CRSYNC_VERSION_PATCH 0 37 | 38 | /* This is the numeric version of the crsync version number, meant for easier 39 | parsing and comparions by programs. The CRSYNC_VERSION_NUM define will 40 | always follow this syntax: 41 | 42 | 0xXXYYZZ 43 | 44 | Where XX, YY and ZZ are the main version, release and patch numbers in 45 | hexadecimal (using 8 bits each). All three numbers are always represented 46 | using two digits. 1.2.3 would appear as "0x010203". 47 | 48 | This 6-digit (24 bits) hexadecimal number does not show pre-release number, 49 | and it is always a greater number in a more recent release. It makes 50 | comparisons with greater than and less than work. 51 | */ 52 | #define CRSYNC_VERSION_NUM 0x000100 53 | 54 | #endif //__CRSYNCVER_H 55 | -------------------------------------------------------------------------------- /src/deployment.pri: -------------------------------------------------------------------------------- 1 | # This file was generated by an application wizard of Qt Creator. 2 | # The code below handles deployment to Android and Maemo, aswell as copying 3 | # of the application data to shadow build directories on desktop. 4 | # It is recommended not to modify this file, since newer versions of Qt Creator 5 | # may offer an updated version of it. 6 | 7 | defineTest(qtcAddDeployment) { 8 | for(deploymentfolder, DEPLOYMENTFOLDERS) { 9 | item = item$${deploymentfolder} 10 | greaterThan(QT_MAJOR_VERSION, 4) { 11 | itemsources = $${item}.files 12 | } else { 13 | itemsources = $${item}.sources 14 | } 15 | $$itemsources = $$eval($${deploymentfolder}.source) 16 | itempath = $${item}.path 17 | $$itempath= $$eval($${deploymentfolder}.target) 18 | export($$itemsources) 19 | export($$itempath) 20 | DEPLOYMENT += $$item 21 | } 22 | 23 | MAINPROFILEPWD = $$PWD 24 | 25 | android-no-sdk { 26 | for(deploymentfolder, DEPLOYMENTFOLDERS) { 27 | item = item$${deploymentfolder} 28 | itemfiles = $${item}.files 29 | $$itemfiles = $$eval($${deploymentfolder}.source) 30 | itempath = $${item}.path 31 | $$itempath = /data/user/qt/$$eval($${deploymentfolder}.target) 32 | export($$itemfiles) 33 | export($$itempath) 34 | INSTALLS += $$item 35 | } 36 | 37 | target.path = /data/user/qt 38 | 39 | export(target.path) 40 | INSTALLS += target 41 | } else:android { 42 | for(deploymentfolder, DEPLOYMENTFOLDERS) { 43 | item = item$${deploymentfolder} 44 | itemfiles = $${item}.files 45 | $$itemfiles = $$eval($${deploymentfolder}.source) 46 | itempath = $${item}.path 47 | $$itempath = /assets/$$eval($${deploymentfolder}.target) 48 | export($$itemfiles) 49 | export($$itempath) 50 | INSTALLS += $$item 51 | } 52 | 53 | x86 { 54 | target.path = /libs/x86 55 | } else: armeabi-v7a { 56 | target.path = /libs/armeabi-v7a 57 | } else { 58 | target.path = /libs/armeabi 59 | } 60 | 61 | export(target.path) 62 | INSTALLS += target 63 | } else:win32 { 64 | copyCommand = 65 | for(deploymentfolder, DEPLOYMENTFOLDERS) { 66 | source = $$MAINPROFILEPWD/$$eval($${deploymentfolder}.source) 67 | source = $$replace(source, /, \\) 68 | sourcePathSegments = $$split(source, \\) 69 | target = $$OUT_PWD/$$eval($${deploymentfolder}.target)/$$last(sourcePathSegments) 70 | target = $$replace(target, /, \\) 71 | target ~= s,\\\\\\.?\\\\,\\, 72 | !isEqual(source,$$target) { 73 | !isEmpty(copyCommand):copyCommand += && 74 | isEqual(QMAKE_DIR_SEP, \\) { 75 | copyCommand += $(COPY_DIR) \"$$source\" \"$$target\" 76 | } else { 77 | source = $$replace(source, \\\\, /) 78 | target = $$OUT_PWD/$$eval($${deploymentfolder}.target) 79 | target = $$replace(target, \\\\, /) 80 | copyCommand += test -d \"$$target\" || mkdir -p \"$$target\" && cp -r \"$$source\" \"$$target\" 81 | } 82 | } 83 | } 84 | !isEmpty(copyCommand) { 85 | copyCommand = @echo Copying application data... && $$copyCommand 86 | copydeploymentfolders.commands = $$copyCommand 87 | first.depends = $(first) copydeploymentfolders 88 | export(first.depends) 89 | export(copydeploymentfolders.commands) 90 | QMAKE_EXTRA_TARGETS += first copydeploymentfolders 91 | } 92 | } else:ios { 93 | copyCommand = 94 | for(deploymentfolder, DEPLOYMENTFOLDERS) { 95 | source = $$MAINPROFILEPWD/$$eval($${deploymentfolder}.source) 96 | source = $$replace(source, \\\\, /) 97 | target = $CODESIGNING_FOLDER_PATH/$$eval($${deploymentfolder}.target) 98 | target = $$replace(target, \\\\, /) 99 | sourcePathSegments = $$split(source, /) 100 | targetFullPath = $$target/$$last(sourcePathSegments) 101 | targetFullPath ~= s,/\\.?/,/, 102 | !isEqual(source,$$targetFullPath) { 103 | !isEmpty(copyCommand):copyCommand += && 104 | copyCommand += mkdir -p \"$$target\" 105 | copyCommand += && cp -r \"$$source\" \"$$target\" 106 | } 107 | } 108 | !isEmpty(copyCommand) { 109 | copyCommand = echo Copying application data... && $$copyCommand 110 | !isEmpty(QMAKE_POST_LINK): QMAKE_POST_LINK += ";" 111 | QMAKE_POST_LINK += "$$copyCommand" 112 | export(QMAKE_POST_LINK) 113 | } 114 | } else:unix { 115 | maemo5 { 116 | desktopfile.files = $${TARGET}.desktop 117 | desktopfile.path = /usr/share/applications/hildon 118 | icon.files = $${TARGET}64.png 119 | icon.path = /usr/share/icons/hicolor/64x64/apps 120 | } else:!isEmpty(MEEGO_VERSION_MAJOR) { 121 | desktopfile.files = $${TARGET}_harmattan.desktop 122 | desktopfile.path = /usr/share/applications 123 | icon.files = $${TARGET}80.png 124 | icon.path = /usr/share/icons/hicolor/80x80/apps 125 | } else { # Assumed to be a Desktop Unix 126 | copyCommand = 127 | for(deploymentfolder, DEPLOYMENTFOLDERS) { 128 | source = $$MAINPROFILEPWD/$$eval($${deploymentfolder}.source) 129 | source = $$replace(source, \\\\, /) 130 | macx { 131 | target = $$OUT_PWD/$${TARGET}.app/Contents/Resources/$$eval($${deploymentfolder}.target) 132 | } else { 133 | target = $$OUT_PWD/$$eval($${deploymentfolder}.target) 134 | } 135 | target = $$replace(target, \\\\, /) 136 | sourcePathSegments = $$split(source, /) 137 | targetFullPath = $$target/$$last(sourcePathSegments) 138 | targetFullPath ~= s,/\\.?/,/, 139 | !isEqual(source,$$targetFullPath) { 140 | !isEmpty(copyCommand):copyCommand += && 141 | copyCommand += $(MKDIR) \"$$target\" 142 | copyCommand += && $(COPY_DIR) \"$$source\" \"$$target\" 143 | } 144 | } 145 | !isEmpty(copyCommand) { 146 | copyCommand = @echo Copying application data... && $$copyCommand 147 | copydeploymentfolders.commands = $$copyCommand 148 | first.depends = $(first) copydeploymentfolders 149 | export(first.depends) 150 | export(copydeploymentfolders.commands) 151 | QMAKE_EXTRA_TARGETS += first copydeploymentfolders 152 | } 153 | } 154 | !isEmpty(target.path) { 155 | installPrefix = $${target.path} 156 | } else { 157 | installPrefix = /opt/$${TARGET} 158 | } 159 | for(deploymentfolder, DEPLOYMENTFOLDERS) { 160 | item = item$${deploymentfolder} 161 | itemfiles = $${item}.files 162 | $$itemfiles = $$eval($${deploymentfolder}.source) 163 | itempath = $${item}.path 164 | $$itempath = $${installPrefix}/$$eval($${deploymentfolder}.target) 165 | export($$itemfiles) 166 | export($$itempath) 167 | INSTALLS += $$item 168 | } 169 | 170 | !isEmpty(desktopfile.path) { 171 | export(icon.files) 172 | export(icon.path) 173 | export(desktopfile.files) 174 | export(desktopfile.path) 175 | INSTALLS += icon desktopfile 176 | } 177 | 178 | isEmpty(target.path) { 179 | target.path = $${installPrefix}/bin 180 | export(target.path) 181 | } 182 | INSTALLS += target 183 | } 184 | 185 | export (ICON) 186 | export (INSTALLS) 187 | export (DEPLOYMENT) 188 | export (LIBS) 189 | export (QMAKE_EXTRA_TARGETS) 190 | } 191 | 192 | -------------------------------------------------------------------------------- /src/diff.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include "diff.h" 30 | #include "uthash.h" 31 | #include "log.h" 32 | 33 | typedef struct diffHash_t { 34 | uint32_t weak; //first key 35 | int32_t seq; //second key 36 | uint8_t *strong; //reference pointer 37 | struct diffHash_t *sub; //sub HashTable 38 | UT_hash_handle hh; 39 | } diffHash_t; 40 | 41 | static diffHash_t* diffHash_malloc() { 42 | diffHash_t *dh = calloc(1, sizeof(diffHash_t)); 43 | return dh; 44 | } 45 | 46 | static void diffHash_free(diffHash_t **dh) { 47 | diffHash_t *sumItem=NULL, *sumTemp=NULL, *sumIter=NULL, *sumTemp2=NULL; 48 | HASH_ITER(hh, *dh, sumItem, sumTemp) { 49 | HASH_ITER(hh, sumItem->sub, sumIter, sumTemp2 ) { 50 | HASH_DEL(sumItem->sub, sumIter); 51 | free(sumIter); 52 | } 53 | HASH_DEL(*dh, sumItem); 54 | free(sumItem); 55 | } 56 | } 57 | 58 | diffResult_t* diffResult_malloc() { 59 | diffResult_t *dr = calloc(1, sizeof(diffResult_t)); 60 | return dr; 61 | } 62 | 63 | void diffResult_free(diffResult_t *dr) { 64 | if(dr) { 65 | free(dr->offsets); 66 | free(dr); 67 | } 68 | } 69 | 70 | void diffResult_dump(const diffResult_t *dr) { 71 | if(dr){ 72 | LOGI("totalNum = %d\n", dr->totalNum); 73 | LOGI("matchNum = %d\n", dr->matchNum); 74 | LOGI("cacheNum = %d\n", dr->cacheNum); 75 | LOGI("missNum = %d\n", dr->totalNum - dr->matchNum - dr->cacheNum); 76 | } else { 77 | LOGI("none\n"); 78 | } 79 | } 80 | 81 | static diffHash_t* Diff_hash(const fileDigest_t *fd) { 82 | diffHash_t *dh = NULL; 83 | diffHash_t *item = NULL, *temp = NULL; 84 | uint32_t blockNum = fd->fileSize / fd->blockSize; 85 | 86 | for(size_t i=0; iweak = fd->blockDigest[i].weak; 90 | item->seq = i; 91 | item->strong = fd->blockDigest[i].strong; 92 | 93 | HASH_FIND_INT(dh, &item->weak, temp); 94 | if (!temp) { 95 | HASH_ADD_INT( dh, weak, item ); 96 | } else { 97 | HASH_ADD_INT( temp->sub, seq, item ); 98 | } 99 | } 100 | return dh; 101 | } 102 | 103 | 104 | #define DIFF_PARALLELISM_DEGREE 4 105 | 106 | static void Diff_match(const char *filename, const fileDigest_t *fd, const diffHash_t **dh, diffResult_t *dr) { 107 | dr->totalNum = fd->fileSize / fd->blockSize; 108 | dr->matchNum = 0; 109 | dr->cacheNum = 0; 110 | dr->offsets = malloc(dr->totalNum * sizeof(int32_t)); 111 | memset(dr->offsets, -1, dr->totalNum * sizeof(int32_t)); 112 | 113 | struct stat st; 114 | if(stat(filename, &st)!=0 || (size_t)st.st_size <= fd->blockSize*DIFF_PARALLELISM_DEGREE) { 115 | // file not exist || small file 116 | return; 117 | } 118 | 119 | const size_t parallel_size = (st.st_size + DIFF_PARALLELISM_DEGREE - 1) / DIFF_PARALLELISM_DEGREE; 120 | 121 | #pragma omp parallel shared(fd, dh, dr), num_threads(DIFF_PARALLELISM_DEGREE) 122 | { 123 | size_t id__ = omp_get_thread_num(); 124 | FILE *file = fopen(filename, "rb"); 125 | if(file) 126 | { 127 | size_t read_begin = 0; 128 | size_t read_end = 0; 129 | if(id__ == 0) { 130 | read_begin = 0; 131 | read_end = parallel_size; 132 | } else if(id__ == DIFF_PARALLELISM_DEGREE-1) { 133 | read_begin = id__ * parallel_size - fd->blockSize + 1; 134 | read_end = st.st_size; 135 | } else { 136 | read_begin = id__ * parallel_size - fd->blockSize + 1; 137 | read_end = (id__+1) * parallel_size; 138 | } 139 | size_t read_len = read_end - read_begin; 140 | size_t offset = read_begin; 141 | 142 | unsigned char *buf1 = malloc(fd->blockSize); 143 | unsigned char *buf2 = malloc(fd->blockSize); 144 | 145 | size_t r; 146 | 147 | r = fseek(file, read_begin, SEEK_SET); 148 | if(r != 0) { 149 | LOGE("error fseek\n"); 150 | } 151 | r = fread(buf1, 1, fd->blockSize, file); 152 | read_len -= fd->blockSize; 153 | if(r != fd->blockSize) { 154 | LOGE("error fread\n"); 155 | } 156 | uint32_t weak; 157 | uint8_t strong[CRS_STRONG_DIGEST_SIZE]; 158 | diffHash_t *sumItem = NULL, *sumIter = NULL, *sumTemp = NULL; 159 | 160 | //Digest_match_first 161 | Digest_CalcWeak_Data(buf1, fd->blockSize, &weak); 162 | HASH_FIND_INT( *dh, &weak, sumItem ); 163 | if(sumItem) { 164 | Digest_CalcStrong_Data(buf1, fd->blockSize, strong); 165 | if (0 == memcmp(strong, sumItem->strong, CRS_STRONG_DIGEST_SIZE)) { 166 | dr->offsets[sumItem->seq] = offset; 167 | } 168 | HASH_ITER(hh, sumItem->sub, sumIter, sumTemp) { 169 | if (0 == memcmp(strong, sumIter->strong, CRS_STRONG_DIGEST_SIZE)) { 170 | dr->offsets[sumIter->seq] = offset; 171 | } 172 | } 173 | } 174 | 175 | //Digest_match_loop 176 | while(read_len >= fd->blockSize) { 177 | r = fread(buf2, 1, fd->blockSize, file); 178 | if(r != fd->blockSize) { 179 | LOGE("error fread\n"); 180 | } 181 | read_len -= fd->blockSize; 182 | 183 | for(size_t i=0; iblockSize;) { 184 | Digest_CalcWeak_Roll(buf1[i], buf2[i], fd->blockSize, &weak); 185 | ++i; 186 | ++offset; 187 | HASH_FIND_INT( *dh, &weak, sumItem ); 188 | if(sumItem) { 189 | Digest_CalcStrong_Data2(buf1, buf2, fd->blockSize, i, strong); 190 | if (0 == memcmp(strong, sumItem->strong, CRS_STRONG_DIGEST_SIZE)) { 191 | dr->offsets[sumItem->seq] = offset; 192 | } 193 | HASH_ITER(hh, sumItem->sub, sumIter, sumTemp) { 194 | if (0 == memcmp(strong, sumIter->strong, CRS_STRONG_DIGEST_SIZE)) { 195 | dr->offsets[sumIter->seq] = offset; 196 | } 197 | } 198 | } 199 | } 200 | //switch buffer 201 | uint8_t *tmpbuf = buf1; 202 | buf1 = buf2; 203 | buf2 = tmpbuf; 204 | } 205 | 206 | //Digest_match_end 207 | if(read_len > 0) { 208 | r = fread(buf2, 1, read_len, file); 209 | if(r != read_len) { 210 | LOGE("error fread\n"); 211 | } 212 | for(size_t i=0; iblockSize, &weak); 214 | ++i; 215 | ++offset; 216 | HASH_FIND_INT( *dh, &weak, sumItem ); 217 | if(sumItem) { 218 | Digest_CalcStrong_Data2(buf1, buf2, fd->blockSize, i, strong); 219 | if (0 == memcmp(strong, sumItem->strong, CRS_STRONG_DIGEST_SIZE)) { 220 | dr->offsets[sumItem->seq] = offset; 221 | } 222 | HASH_ITER(hh, sumItem->sub, sumIter, sumTemp) { 223 | if (0 == memcmp(strong, sumIter->strong, CRS_STRONG_DIGEST_SIZE)) { 224 | dr->offsets[sumIter->seq] = offset; 225 | } 226 | } 227 | } 228 | } 229 | } 230 | 231 | free(buf1); 232 | free(buf2); 233 | fclose(file); 234 | }//end of if(file) 235 | }//end of omp parallel (DIFF_PARALLELISM_DEGREE) 236 | 237 | for(int32_t i=0; i< dr->totalNum; ++i) { 238 | if(dr->offsets[i] >= 0) { 239 | dr->matchNum++; 240 | } 241 | } 242 | } 243 | 244 | static CRScode Diff_cache(const char *dstFilename, const fileDigest_t *fd, diffResult_t *dr) { 245 | if(!dstFilename || !fd || !dr) { 246 | LOGE("end %d\n", CRS_PARAM_ERROR); 247 | return CRS_PARAM_ERROR; 248 | } 249 | 250 | if(0 != access(dstFilename, F_OK)) { 251 | LOGI("end file not exist\n"); 252 | return CRS_OK; 253 | } 254 | 255 | struct stat st; 256 | if(stat(dstFilename, &st) != 0) { 257 | LOGE("dstFile stat fail %s\n", strerror(errno)); 258 | LOGE("%s\n", dstFilename); 259 | //should return CRS_FILE_ERROR, but I do not want break workflow; 260 | return CRS_OK; 261 | } 262 | 263 | if((size_t)st.st_size != fd->fileSize) { 264 | if(0 != truncate(dstFilename, fd->fileSize)) { 265 | LOGE("dest file truncate %dBytes error %s\n", fd->fileSize, strerror(errno)); 266 | //should return CRS_FILE_ERROR, but I do not want break workflow; 267 | return CRS_OK; 268 | } 269 | } 270 | 271 | FILE *f = fopen(dstFilename, "rb"); 272 | if(!f){ 273 | LOGE("dest file fopen rb error %s\n", strerror(errno)); 274 | //should return CRS_FILE_ERROR, but I do not want break workflow; 275 | return CRS_OK; 276 | } 277 | 278 | uint8_t *buf = malloc(fd->blockSize); 279 | uint8_t hash[CRS_STRONG_DIGEST_SIZE]; 280 | 281 | for(int i=0; itotalNum; ++i) { 282 | if(dr->offsets[i] == -1) { 283 | fseek(f, i*fd->blockSize, SEEK_SET); 284 | fread(buf, 1, fd->blockSize, f); 285 | 286 | Digest_CalcStrong_Data(buf, fd->blockSize, hash); 287 | if(0 == memcmp(hash, fd->blockDigest[i].strong, CRS_STRONG_DIGEST_SIZE)) { 288 | dr->offsets[i] = -2; 289 | dr->cacheNum++; 290 | } 291 | } 292 | } 293 | free(buf); 294 | fclose(f); 295 | return CRS_OK; 296 | } 297 | 298 | CRScode Diff_perform(const char *srcFilename, const char *dstFilename, const fileDigest_t *fd, diffResult_t *dr) { 299 | LOGI("begin\n"); 300 | if(!srcFilename || !dstFilename || !fd || !dr) { 301 | LOGE("end %d\n", CRS_PARAM_ERROR); 302 | return CRS_PARAM_ERROR; 303 | } 304 | 305 | CRScode code = CRS_OK; 306 | diffHash_t *dh = Diff_hash(fd); 307 | 308 | Diff_match(srcFilename, fd, (const diffHash_t **)&dh, dr); 309 | 310 | Diff_cache(dstFilename, fd, dr); 311 | 312 | diffHash_free(&dh); 313 | 314 | LOGI("end %d\n", code); 315 | return code; 316 | } 317 | -------------------------------------------------------------------------------- /src/diff.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRS_DIFF_H 25 | #define CRS_DIFF_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #include 32 | 33 | #include "global.h" 34 | #include "digest.h" 35 | 36 | typedef struct diffResult_t { 37 | int32_t totalNum; //should be fileDigest_t.fileSize / fileDigest_t.blockSize; 38 | int32_t matchNum; //calc from fileDigest_t.offsets, compare to source file 39 | int32_t cacheNum; //dst file already got 40 | int32_t *offsets; //performed result, -1(default) miss, -2 cache, >=0 offset at source file; 41 | } diffResult_t; 42 | 43 | diffResult_t* diffResult_malloc(); 44 | void diffResult_free(diffResult_t *dr); 45 | 46 | void diffResult_dump(const diffResult_t *dr); 47 | 48 | CRScode Diff_perform(const char *srcFilename, const char *dstFilename, const fileDigest_t *fd, diffResult_t *dr); 49 | 50 | #if defined __cplusplus 51 | } 52 | #endif 53 | 54 | #endif // CRS_DIFF_H 55 | -------------------------------------------------------------------------------- /src/digest.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include 25 | #include 26 | 27 | #include "digest.h" 28 | #include "md5.h" 29 | //#include "blake2.h" 30 | #include "md5.h" 31 | #include "log.h" 32 | #include "util.h" 33 | #include "tpl.h" 34 | 35 | const char *DIGEST_EXT = ".sum"; 36 | 37 | void Digest_CalcWeak_Data(const uint8_t *data, const uint32_t size, uint32_t *out) { 38 | uint32_t i = 0, a = 0, b = 0; 39 | const uint8_t *p = data; 40 | for ( ; i < size - 4; i += 4) { 41 | b += 4 * (a + p[i]) + 3 * p[i + 1] + 2 * p[i + 2] + p[i + 3]; 42 | a += p[i + 0] + p[i + 1] + p[i + 2] + p[i + 3]; 43 | } 44 | for ( ; i < size; i++) { 45 | a += p[i]; 46 | b += a; 47 | } 48 | *out = (a & 0xffff) | (b << 16); 49 | } 50 | 51 | void Digest_CalcWeak_Roll(const uint8_t out, const uint8_t in, const uint32_t blockSize, uint32_t *weak) { 52 | uint32_t a = *weak & 0xffff; 53 | uint32_t b = *weak >> 16; 54 | 55 | a += in - out; 56 | b += a - blockSize * out; 57 | 58 | *weak = (a & 0xffff) | (b << 16); 59 | } 60 | 61 | void Digest_CalcStrong_Data(const uint8_t *data, const uint32_t size, uint8_t *out) { 62 | MD5_Data(data, size, out); 63 | /* 64 | blake2b_state ctx; 65 | blake2b_init(&ctx, CRS_STRONG_DIGEST_SIZE); 66 | blake2b_update(&ctx, data, size); 67 | blake2b_final(&ctx, (uint8_t *)out, CRS_STRONG_DIGEST_SIZE);*/ 68 | } 69 | 70 | void Digest_CalcStrong_Data2(const uint8_t *buf1, const uint8_t *buf2, const uint32_t size, const uint32_t offset, uint8_t *out) { 71 | MD5_CTX ctx; 72 | MD5_Init(&ctx); 73 | MD5_Update(&ctx, buf1+offset, size-offset); 74 | MD5_Update(&ctx, buf2, offset); 75 | MD5_Final(&ctx, out); 76 | /* 77 | blake2b_state ctx; 78 | blake2b_init(&ctx, CRS_STRONG_DIGEST_SIZE); 79 | blake2b_update(&ctx, buf1+offset, size-offset); 80 | blake2b_update(&ctx, buf2, offset); 81 | blake2b_final(&ctx, (uint8_t *)out, CRS_STRONG_DIGEST_SIZE);*/ 82 | } 83 | 84 | int Digest_CalcStrong_File(const char *filename, uint8_t *out) { 85 | return MD5_File(filename, out); 86 | //return blake2b_File(filename, out, CRS_STRONG_DIGEST_SIZE); 87 | } 88 | 89 | fileDigest_t* fileDigest_malloc() { 90 | return calloc(1, sizeof(fileDigest_t)); 91 | } 92 | 93 | void fileDigest_free(fileDigest_t* fd) { 94 | if(fd) { 95 | free(fd->blockDigest); 96 | free(fd->restData); 97 | free(fd); 98 | } 99 | } 100 | 101 | void fileDigest_dump(const fileDigest_t* fd) { 102 | if(fd) { 103 | LOGI("fileSize = %d\n", fd->fileSize); 104 | LOGI("blockSize = %d KiB\n", fd->blockSize/1024); 105 | char *hashString = Util_hex_string(fd->fileDigest, CRS_STRONG_DIGEST_SIZE); 106 | LOGI("fileDigest = %s\n", hashString); 107 | free(hashString); 108 | size_t restSize = (fd->blockSize == 0) ? 0 : (fd->fileSize % fd->blockSize); 109 | LOGI("restSize = %d Bytes\n", restSize); 110 | } else { 111 | LOGI("none\n"); 112 | } 113 | } 114 | 115 | CRScode Digest_Perform(const char *filename, const uint32_t blockSize, fileDigest_t *fd) { 116 | LOGI("begin\n"); 117 | 118 | if(!filename || blockSize == 0 || !fd) { 119 | LOGE("end %d\n", CRS_PARAM_ERROR); 120 | return CRS_PARAM_ERROR; 121 | } 122 | 123 | struct stat st; 124 | if(stat(filename, &st)!=0 || st.st_size==0){ 125 | // file not exist || file size is zero 126 | LOGE("end %s stat\n", filename); 127 | return CRS_FILE_ERROR; 128 | } 129 | 130 | FILE *f = fopen(filename, "rb"); 131 | if(!f) { 132 | LOGE("end %s fopen\n", filename); 133 | return CRS_FILE_ERROR; 134 | } 135 | 136 | CRScode code = CRS_OK; 137 | uint8_t *buf = malloc(blockSize); 138 | uint32_t blockNum = st.st_size / blockSize; 139 | uint32_t restSize = st.st_size % blockSize; 140 | uint8_t *restData = (restSize > 0) ? malloc(restSize) : NULL; 141 | digest_t *digests = (blockNum > 0) ? malloc(sizeof(digest_t) * blockNum) : NULL; 142 | 143 | for(uint32_t i=0; i< blockNum; ++i) { 144 | if(fread(buf, 1, blockSize, f) == blockSize) { 145 | Digest_CalcWeak_Data(buf, blockSize, &digests[i].weak); 146 | Digest_CalcStrong_Data(buf, blockSize, digests[i].strong); 147 | } else { 148 | code = CRS_FILE_ERROR; 149 | LOGE("error %s fread\n", filename); 150 | break; 151 | } 152 | } 153 | 154 | if(code == CRS_OK && restSize > 0) { 155 | if(fread(buf, 1, restSize, f) == restSize) { 156 | memcpy(restData, buf, restSize); 157 | } else { 158 | code = CRS_FILE_ERROR; 159 | LOGE("error %s fread\n", filename); 160 | } 161 | } 162 | free(buf); 163 | fclose(f); 164 | 165 | if(code != CRS_OK) { 166 | LOGE("end %d\n", code); 167 | return code; 168 | } 169 | 170 | uint8_t fileDigest[CRS_STRONG_DIGEST_SIZE]; 171 | if(0 != Digest_CalcStrong_File(filename, fileDigest)) { 172 | LOGE("end %s Digest_CalcStrong_File\n", filename); 173 | return CRS_FILE_ERROR; 174 | } 175 | 176 | fd->fileSize = st.st_size; 177 | fd->blockSize = blockSize; 178 | fd->blockDigest = digests; 179 | fd->restData = restData; 180 | memcpy(fd->fileDigest, fileDigest, CRS_STRONG_DIGEST_SIZE); 181 | 182 | LOGI("end %d\n", code); 183 | return code; 184 | } 185 | 186 | static const char *DIGEST_TPLMAP_FORMAT = "uuc#BA(uc#)"; 187 | 188 | CRScode Digest_Load(const char *filename, fileDigest_t *fd) { 189 | LOGI("begin\n"); 190 | 191 | if(!filename || !fd) { 192 | LOGE("end %d\n", CRS_PARAM_ERROR); 193 | return CRS_PARAM_ERROR; 194 | } 195 | 196 | if(0 != Digest_checkfile(filename)) { 197 | LOGI("end %s miss\n", filename); 198 | return CRS_FILE_ERROR; 199 | } 200 | 201 | CRScode code = CRS_OK; 202 | tpl_bin tb = {NULL, 0}; 203 | digest_t digest; 204 | 205 | tpl_node *tn = tpl_map( DIGEST_TPLMAP_FORMAT, 206 | &fd->fileSize, 207 | &fd->blockSize, 208 | fd->fileDigest, 209 | CRS_STRONG_DIGEST_SIZE, 210 | &tb, 211 | &digest.weak, 212 | &digest.strong, 213 | CRS_STRONG_DIGEST_SIZE); 214 | if(0 == tpl_load(tn, TPL_FILE, filename)) { 215 | tpl_unpack(tn, 0); 216 | 217 | uint32_t blockNum = fd->fileSize / fd->blockSize; 218 | fd->blockDigest = (blockNum > 0) ? malloc(sizeof(digest_t) * blockNum) : NULL; 219 | fd->restData = tb.addr; 220 | 221 | for (uint32_t i = 0; i < blockNum; i++) { 222 | tpl_unpack(tn, 1); 223 | fd->blockDigest[i].weak = digest.weak; 224 | memcpy(fd->blockDigest[i].strong, digest.strong, CRS_STRONG_DIGEST_SIZE); 225 | } 226 | } else { 227 | LOGE("error tpl_load %s\n", filename); 228 | code = CRS_FILE_ERROR; 229 | } 230 | tpl_free(tn); 231 | 232 | LOGI("end %d\n", code); 233 | return code; 234 | } 235 | 236 | CRScode Digest_Save(const char *filename, fileDigest_t *fd) { 237 | LOGI("begin\n"); 238 | 239 | if(!filename || !fd) { 240 | LOGE("end %d\n", CRS_PARAM_ERROR); 241 | return CRS_PARAM_ERROR; 242 | } 243 | 244 | CRScode code = CRS_OK; 245 | tpl_bin tb = {NULL, 0}; 246 | tb.addr = fd->restData; 247 | tb.sz = fd->fileSize % fd->blockSize; 248 | 249 | digest_t digest; 250 | 251 | tpl_node *tn = tpl_map( DIGEST_TPLMAP_FORMAT, 252 | &fd->fileSize, 253 | &fd->blockSize, 254 | fd->fileDigest, 255 | CRS_STRONG_DIGEST_SIZE, 256 | &tb, 257 | &digest.weak, 258 | &digest.strong, 259 | CRS_STRONG_DIGEST_SIZE); 260 | tpl_pack(tn, 0); 261 | 262 | uint32_t blockNum = fd->fileSize / fd->blockSize; 263 | for (uint32_t i = 0; i < blockNum; ++i) { 264 | digest.weak = fd->blockDigest[i].weak; 265 | memcpy(digest.strong, fd->blockDigest[i].strong, CRS_STRONG_DIGEST_SIZE); 266 | tpl_pack(tn, 1); 267 | } 268 | 269 | if(0 != tpl_dump(tn, TPL_FILE, filename)) { 270 | LOGE("error tpl_dump %s\n", filename); 271 | code = CRS_FILE_ERROR; 272 | } 273 | tpl_free(tn); 274 | 275 | LOGI("end %d\n", code); 276 | return code; 277 | } 278 | 279 | int Digest_checkfile(const char *filename) { 280 | return Util_tplcmp(filename, DIGEST_TPLMAP_FORMAT); 281 | } 282 | -------------------------------------------------------------------------------- /src/digest.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRS_DIGEST_H 25 | #define CRS_DIGEST_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #include 32 | 33 | #include "global.h" 34 | 35 | extern const char *DIGEST_EXT; 36 | 37 | void Digest_CalcWeak_Data(const uint8_t *data, const uint32_t len, uint32_t *out); 38 | void Digest_CalcWeak_Roll(const uint8_t out, const uint8_t in, const uint32_t blockSize, uint32_t *weak); 39 | 40 | void Digest_CalcStrong_Data(const uint8_t *data, const uint32_t len, uint8_t *out); 41 | void Digest_CalcStrong_Data2(const uint8_t *buf1, const uint8_t *buf2, const uint32_t size, const uint32_t offset, uint8_t *out); 42 | int Digest_CalcStrong_File(const char *filename, uint8_t *out); 43 | 44 | typedef struct digest_t { 45 | uint8_t strong[CRS_STRONG_DIGEST_SIZE]; // strong digest (md5, blake2 etc.) 46 | uint32_t weak; // Adler32, used for Rolling calc 47 | } digest_t; 48 | 49 | typedef struct fileDigest_t { 50 | uint32_t fileSize; //file size 51 | uint32_t blockSize; //block size 52 | uint8_t fileDigest[CRS_STRONG_DIGEST_SIZE]; //file strong sum 53 | digest_t *blockDigest; //every block's rsum_t data 54 | uint8_t *restData; //rest binary data, size = fileSize % blockSize 55 | } fileDigest_t; 56 | 57 | fileDigest_t* fileDigest_malloc(); 58 | void fileDigest_free(fileDigest_t* fd); 59 | void fileDigest_dump(const fileDigest_t* fd); 60 | 61 | CRScode Digest_Perform(const char *filename, const uint32_t blockSize, fileDigest_t *fd); 62 | CRScode Digest_Load(const char *filename, fileDigest_t *fd); 63 | CRScode Digest_Save(const char *filename, fileDigest_t *fd); 64 | int Digest_checkfile(const char *filename); 65 | 66 | #if defined __cplusplus 67 | } 68 | #endif 69 | 70 | #endif // CRS_DIGEST_H 71 | -------------------------------------------------------------------------------- /src/global.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRS_GLOBAL_H 25 | #define CRS_GLOBAL_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | typedef enum { 32 | CRS_OK = 0, 33 | CRS_INIT_ERROR, //mostly curl init error 34 | CRS_PARAM_ERROR, //input parameters wrong 35 | CRS_FILE_ERROR, //file io error 36 | CRS_HTTP_ERROR, //http io error 37 | CRS_USER_CANCEL, //user cancel (manual operation, Network Policy) 38 | 39 | CRS_BUG = 100, //Internal error 40 | } CRScode; 41 | 42 | #define CRS_STRONG_DIGEST_SIZE 16 43 | 44 | extern void crs_callback_diff (const char *basename, const unsigned int bytes, const int isComplete); 45 | extern int crs_callback_patch (const char *basename, const unsigned int bytes, const int isComplete, const int immediate); 46 | 47 | #if defined __cplusplus 48 | } 49 | #endif 50 | 51 | #endif // CRS_GLOBAL_H 52 | -------------------------------------------------------------------------------- /src/helper.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRS_HELPER_H 25 | #define CRS_HELPER_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #include 32 | 33 | #include "global.h" 34 | #include "magnet.h" 35 | #include "diff.h" 36 | 37 | CRScode helper_perform_version(); 38 | 39 | typedef struct helper_t { 40 | //here is constant ref pointer, never change 41 | char *fileDir; //file directory, end with '/', ref to bulkhelper_t.fileDir 42 | char *baseUrl; //base url, end with '/', ref to bulkhelper_t.baseUrl 43 | 44 | //here is constant, never change 45 | char *fileName; //file name 46 | uint32_t fileSize; //whole file size 47 | uint8_t fileDigest[CRS_STRONG_DIGEST_SIZE]; //whole file digest 48 | struct helper_t *next; //used by bulkhelper with utlist(single-link) 49 | 50 | //here is variable, which may change in diff and patch 51 | 52 | uint32_t cacheSize; //local, cache, same file size 53 | int isComplete; //0 not; 1 complete 54 | fileDigest_t *fd; 55 | diffResult_t *dr; 56 | 57 | } helper_t; 58 | 59 | helper_t* helper_malloc (); 60 | void helper_free (helper_t *h); 61 | 62 | CRScode helper_perform_diff (helper_t *h); 63 | 64 | CRScode helper_perform_patch(helper_t *h); 65 | 66 | typedef struct bulkHelper_t { 67 | //here is constant, never change 68 | char *fileDir; 69 | char *baseUrl; 70 | 71 | //here is bulk file struct 72 | magnet_t *currentMagnet; //currVersion's magnet 73 | helper_t *currentBulk; //used with utlist(signle-link) 74 | 75 | } bulkHelper_t; 76 | 77 | bulkHelper_t * bulkHelper_malloc (); 78 | void bulkHelper_free (bulkHelper_t *bh); 79 | 80 | //CRScode bulkHelper_perform_magnet (bulkHelper_t *bh); 81 | CRScode bulkHelper_set_magnet (bulkHelper_t *bh, const char *magnetString); 82 | 83 | CRScode bulkHelper_perform_diff (bulkHelper_t *bh); 84 | 85 | CRScode bulkHelper_perform_patch (bulkHelper_t *bh); 86 | 87 | #if defined __cplusplus 88 | } 89 | #endif 90 | 91 | #endif // CRS_HELPER_H 92 | -------------------------------------------------------------------------------- /src/http.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include 25 | #include 26 | #include 27 | 28 | #include "http.h" 29 | #include "util.h" 30 | #include "log.h" 31 | 32 | CRScode HTTP_global_init() { 33 | CURLcode code = curl_global_init(CURL_GLOBAL_DEFAULT); 34 | return (code == CURLE_OK) ? CRS_OK : CRS_INIT_ERROR; 35 | } 36 | 37 | void HTTP_global_cleanup() { 38 | curl_global_cleanup(); 39 | } 40 | 41 | #if 0 42 | static const char s_infotype[CURLINFO_END][3] = {"* ", "< ", "> ", "{ ", "} ", "{ ", "} " }; 43 | static int HTTP_curl_debug(CURL *curl, curl_infotype type, char *data, size_t size, void *userptr) { 44 | (void)curl; 45 | (void)size; 46 | (void)userptr; 47 | switch (type) { 48 | case CURLINFO_TEXT: 49 | case CURLINFO_HEADER_IN: 50 | case CURLINFO_HEADER_OUT: 51 | LOGD("%s: %s\n", s_infotype[type], data); 52 | break; 53 | default: 54 | break; 55 | } 56 | return 0; 57 | } 58 | #endif 59 | 60 | static void HTTP_curl_setopt(CURL *curl) { 61 | #if 0 62 | curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); 63 | curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, HTTP_curl_debug); 64 | #endif 65 | curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); 66 | curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP); /* http protocol only */ 67 | curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); /* request failure on HTTP response >= 400 */ 68 | curl_easy_setopt(curl, CURLOPT_AUTOREFERER, 1L); /* allow auto referer */ 69 | curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); /* allow follow location */ 70 | curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L); /* allow redir 5 times */ 71 | curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10L); /* connection timeout */ 72 | /*Do not setup CURLOPT_TIMEOUT, since range and file download may cost lots of time*/ 73 | /*curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L);*/ 74 | } 75 | 76 | static const char *response200 = "HTTP/1.0 200 OK"; 77 | static const char *response200_1 = "HTTP/1.1 200 OK"; 78 | static const size_t responselen = 15; //as above strlen 79 | 80 | static size_t header_callback(void *data, size_t size, size_t nmemb, void *userp) 81 | { 82 | (void)userp; 83 | size_t realSize = size * nmemb; 84 | size_t minSize = realSize < responselen ? realSize : responselen; 85 | if( 0 == strncasecmp((const char*)data, response200, minSize) || 86 | 0 == strncasecmp((const char*)data, response200_1, minSize) ) { 87 | LOGE("range response 200\n"); 88 | return 0; 89 | } 90 | return realSize; 91 | } 92 | 93 | CURLcode HTTP_Range(CURL *curl, const char *url, const char *range, void *callback, void *data) { 94 | HTTP_curl_setopt(curl); 95 | curl_easy_setopt(curl, CURLOPT_URL, url); 96 | curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L); 97 | curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (void*)header_callback); 98 | curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, callback); 99 | curl_easy_setopt(curl, CURLOPT_WRITEDATA, data); 100 | curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); 101 | curl_easy_setopt(curl, CURLOPT_RANGE, range); 102 | 103 | CURLcode curlcode = curl_easy_perform(curl); 104 | if(CURLE_OK != curlcode) { 105 | LOGI("curlcode %d\n", curlcode); 106 | } 107 | 108 | curl_easy_reset(curl); 109 | return curlcode; 110 | } 111 | 112 | typedef struct filecache_t { 113 | const char *name; 114 | long bytes; 115 | FILE *file; 116 | } filecache_t; 117 | 118 | static size_t HTTP_writefile_func(void *ptr, size_t size, size_t nmemb, void *userdata) 119 | { 120 | filecache_t * cache = (filecache_t*)userdata; 121 | size_t w = fwrite(ptr, size, nmemb, cache->file); 122 | cache->bytes += w; 123 | int isCancel = 0; 124 | if(cache->name) { 125 | isCancel = crs_callback_patch(cache->name, cache->bytes, 0, 0); 126 | } 127 | return (isCancel == 0) ? w : 0; 128 | } 129 | 130 | CRScode HTTP_File(const char *url, const char *filename, int retry, const char *cbname) { 131 | if(!url || !filename) { 132 | LOGE("%d\n", CRS_PARAM_ERROR); 133 | return CRS_PARAM_ERROR; 134 | } 135 | 136 | CURL *curl = curl_easy_init(); 137 | if(!curl) { 138 | LOGE("%d\n", CRS_INIT_ERROR); 139 | return CRS_INIT_ERROR; 140 | } 141 | 142 | CRScode code = CRS_OK; 143 | filecache_t cache; 144 | cache.name = cbname; 145 | 146 | struct stat st; 147 | while(retry-- >= 0) { 148 | if(0 == stat(filename, &st)) { 149 | cache.bytes = st.st_size; 150 | } else { 151 | cache.bytes = 0L; 152 | } 153 | 154 | FILE *f = fopen(filename, "ab+"); 155 | if(!f) { 156 | code = CRS_FILE_ERROR; 157 | break; 158 | } 159 | cache.file = f; 160 | 161 | HTTP_curl_setopt(curl); 162 | curl_easy_setopt(curl, CURLOPT_URL, url); 163 | curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L); 164 | curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, HTTP_writefile_func); 165 | curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&cache); 166 | curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); 167 | curl_easy_setopt(curl, CURLOPT_RESUME_FROM, cache.bytes); 168 | 169 | CURLcode curlcode = curl_easy_perform(curl); 170 | fclose(f); 171 | curl_easy_reset(curl); 172 | 173 | switch(curlcode) { 174 | case CURLE_OK: 175 | code = CRS_OK; 176 | break; 177 | case CURLE_HTTP_RETURNED_ERROR: // server connect OK, remote file not exist 178 | retry = -1; 179 | code = CRS_HTTP_ERROR; 180 | break; 181 | case CURLE_OPERATION_TIMEDOUT: //timeout, go on 182 | retry++; 183 | code = CRS_HTTP_ERROR; 184 | break; 185 | default: 186 | code = CRS_HTTP_ERROR; 187 | break; 188 | } 189 | 190 | if(code == CRS_OK) { 191 | break; 192 | } 193 | LOGI("curlcode %d\n", curlcode); 194 | 195 | }//end of while(retry) 196 | curl_easy_cleanup(curl); 197 | 198 | return code; 199 | } 200 | -------------------------------------------------------------------------------- /src/http.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRS_HTTP_H 25 | #define CRS_HTTP_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #include "global.h" 32 | #include "curl/curl.h" 33 | 34 | CRScode HTTP_global_init(); 35 | void HTTP_global_cleanup(); 36 | 37 | //callback should be int (Range_callback)(void *data, size_t size, size_t nmemb, void *userp); 38 | CURLcode HTTP_Range(CURL *curl, const char *url, const char *range, void *callback, void *data); 39 | 40 | CRScode HTTP_File(const char *url, const char *filename, int retry, const char *cbname); 41 | 42 | #if defined __cplusplus 43 | } 44 | #endif 45 | 46 | #endif // CRS_HTTP_H 47 | -------------------------------------------------------------------------------- /src/log.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include 25 | 26 | #include "log.h" 27 | #include "utlist.h" 28 | 29 | #if defined ANDROID 30 | # define LOG_FILENAME "/sdcard/crsync_core.log" 31 | #else 32 | # define LOG_FILENAME "crsync_core.log" 33 | #endif 34 | 35 | FILE *logfile = NULL; 36 | 37 | void log_open() { 38 | #if CRSYNC_DEBUG 39 | if(!logfile) logfile = fopen(LOG_FILENAME, "at+"); 40 | #endif 41 | } 42 | 43 | void log_close() { 44 | #if CRSYNC_DEBUG 45 | if(logfile) { 46 | fflush(logfile); 47 | fclose(logfile); 48 | logfile = NULL; 49 | } 50 | #endif 51 | } 52 | 53 | void log_timestamp(char *ts) { 54 | time_t t; 55 | time(&t); 56 | #if defined ANDROID 57 | struct tm tmv; 58 | localtime_r(&t, &tmv); 59 | strftime(ts, LOG_TIME_STRING_SIZE, "%Y%m%d %H:%M:%S", &tmv); 60 | #else 61 | struct tm *tmv = localtime(&t); 62 | strftime(ts, LOG_TIME_STRING_SIZE, "%Y%m%d %H:%M:%S", tmv); 63 | #endif 64 | } 65 | -------------------------------------------------------------------------------- /src/log.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRSYNC_LOG_H 25 | #define CRSYNC_LOG_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #include 32 | #include 33 | 34 | #define CRSYNC_DEBUG 1 35 | 36 | extern FILE *logfile; 37 | 38 | void log_open(); 39 | void log_close(); 40 | 41 | #define LOG_TIME_STRING_SIZE 20 42 | void log_timestamp(char *ts); 43 | 44 | #if defined ANDROID 45 | #include 46 | # define LOG_TAG "crsync_core" 47 | # define LEVEL_D ANDROID_LOG_DEBUG 48 | # define LEVEL_I ANDROID_LOG_INFO 49 | # define LEVEL_W ANDROID_LOG_WARN 50 | # define LEVEL_E ANDROID_LOG_ERROR 51 | # define LOG_PRINT(level, fmt, ...) __android_log_print(level, LOG_TAG, "[%s]: " fmt, __func__, ##__VA_ARGS__) 52 | #else 53 | # define LEVEL_D 1 54 | # define LEVEL_I 2 55 | # define LEVEL_W 3 56 | # define LEVEL_E 4 57 | # define LOG_PRINT(level, fmt, ...) printf("%s %d [%s]: " fmt, ts, level, __func__, ##__VA_ARGS__) 58 | #endif 59 | 60 | #define LOG_FILE(level, fmt, ...) \ 61 | if(logfile) { \ 62 | fprintf(logfile, "%s %d [%s]: " fmt, ts, level, __func__, ##__VA_ARGS__); \ 63 | fflush(logfile); \ 64 | } 65 | 66 | #define LOG_OUTPUT(level, fmt, ...) \ 67 | do{\ 68 | char ts[LOG_TIME_STRING_SIZE];\ 69 | log_timestamp(ts);\ 70 | LOG_PRINT(level, fmt, ##__VA_ARGS__);\ 71 | LOG_FILE(level, fmt, ##__VA_ARGS__);\ 72 | } while(0) 73 | 74 | #if CRSYNC_DEBUG 75 | # define LOGD(fmt, ...) LOG_OUTPUT(LEVEL_D, fmt, ##__VA_ARGS__) 76 | # define LOGI(fmt, ...) LOG_OUTPUT(LEVEL_I, fmt, ##__VA_ARGS__) 77 | # define LOGW(fmt, ...) LOG_OUTPUT(LEVEL_W, fmt, ##__VA_ARGS__) 78 | # define LOGE(fmt, ...) LOG_OUTPUT(LEVEL_E, fmt, ##__VA_ARGS__) 79 | #else 80 | # define LOGD(...) 81 | # define LOGI(...) 82 | # define LOGW(...) 83 | # define LOGE(...) 84 | #endif //CRSYNC_DEBUG 85 | 86 | #if defined __cplusplus 87 | } 88 | #endif 89 | 90 | #endif // CRSYNC_LOG_H 91 | -------------------------------------------------------------------------------- /src/magnet.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include 25 | #include 26 | 27 | #include "magnet.h" 28 | #include "utlist.h" 29 | #include "tpl.h" 30 | #include "util.h" 31 | #include "log.h" 32 | 33 | const char *MAGNET_EXT = ".fdi"; 34 | static const char *MAGNET_TPL_FORMAT = "ssA(suc#)"; 35 | 36 | sum_t* sum_malloc() { 37 | return calloc(1, sizeof(sum_t)); 38 | } 39 | 40 | void sum_free(sum_t *s) { 41 | sum_t *elt, *tmp; 42 | LL_FOREACH_SAFE(s,elt,tmp) { 43 | LL_DELETE(s,elt); 44 | free(elt->name); 45 | free(elt); 46 | } 47 | } 48 | 49 | magnet_t* magnet_malloc() { 50 | return calloc(1, sizeof(magnet_t)); 51 | } 52 | 53 | void magnet_free(magnet_t *m) { 54 | if(m) { 55 | free(m->currVersion); 56 | free(m->nextVersion); 57 | sum_free(m->file); 58 | } 59 | } 60 | 61 | void magnet_toString(magnet_t *m, UT_string **str) { 62 | if(!m || !str || !*str) 63 | return; 64 | UT_string *s = *str; 65 | utstring_clear(s); 66 | sum_t *elt; 67 | LL_FOREACH(m->file, elt) { 68 | char *hash = Util_hex_string(elt->digest, CRS_STRONG_DIGEST_SIZE); 69 | utstring_printf(s, "%s;%s;%u;", elt->name, hash, elt->size); 70 | free(hash); 71 | } 72 | } 73 | 74 | magnet_t* magnet_fromString(UT_string **str) { 75 | if(!str || !*str) { 76 | return NULL; 77 | } 78 | magnet_t *m = magnet_malloc(); 79 | UT_string *s = *str; 80 | int findPos = -1; 81 | int startPos = 0; 82 | int order = 0; 83 | 84 | sum_t *sum = NULL; 85 | char findStr[128];//should enough mostly 86 | 87 | while(-1 != (findPos = utstring_find(s, startPos, ";", 1))) { 88 | if(!sum) 89 | sum = sum_malloc(); 90 | strncpy(findStr, utstring_body(s) + startPos, findPos - startPos); 91 | findStr[findPos - startPos] = '\0'; 92 | 93 | switch(order) { 94 | case 0: { 95 | sum->name = strdup(findStr); 96 | break; 97 | } 98 | case 1: { 99 | unsigned char *digest = Util_string_hex(findStr); 100 | memcpy(sum->digest, digest, CRS_STRONG_DIGEST_SIZE); 101 | free(digest); 102 | break; 103 | } 104 | case 2: { 105 | sum->size = (unsigned int)atoi(findStr); 106 | LL_APPEND(m->file, sum); 107 | sum = NULL; 108 | break; 109 | } 110 | } 111 | order = (order >= 2) ? 0 : order+1; 112 | startPos = findPos + 1; 113 | } 114 | 115 | return m; 116 | } 117 | 118 | CRScode magnet_load(magnet_t *m, const char *file) { 119 | LOGI("begin\n"); 120 | CRScode code = CRS_OK; 121 | char *name = NULL; 122 | unsigned int size = 0; 123 | unsigned char digest[CRS_STRONG_DIGEST_SIZE]; 124 | tpl_node *tn = tpl_map(MAGNET_TPL_FORMAT, 125 | &m->currVersion, 126 | &m->nextVersion, 127 | &name, 128 | &size, 129 | &digest, 130 | CRS_STRONG_DIGEST_SIZE); 131 | if (0 == tpl_load(tn, TPL_FILE, file) ) { 132 | tpl_unpack(tn, 0); 133 | while (tpl_unpack(tn, 1) > 0) { 134 | sum_t *s = sum_malloc(); 135 | s->name = strdup(name); 136 | s->size = size; 137 | memcpy(s->digest, digest, CRS_STRONG_DIGEST_SIZE); 138 | LL_APPEND(m->file, s); 139 | } 140 | } else { 141 | code = CRS_FILE_ERROR; 142 | } 143 | tpl_free(tn); 144 | LOGI("end %d\n", code); 145 | return code; 146 | } 147 | 148 | CRScode magnet_save(magnet_t *m, const char *file) { 149 | LOGI("begin\n"); 150 | char *name = NULL; 151 | unsigned int size = 0; 152 | unsigned char digest[CRS_STRONG_DIGEST_SIZE]; 153 | tpl_node *tn = tpl_map(MAGNET_TPL_FORMAT, 154 | &m->currVersion, 155 | &m->nextVersion, 156 | &name, 157 | &size, 158 | &digest, 159 | CRS_STRONG_DIGEST_SIZE); 160 | tpl_pack(tn, 0); 161 | 162 | sum_t *elt=NULL; 163 | LL_FOREACH(m->file,elt) { 164 | name = elt->name; 165 | size = elt->size; 166 | memcpy(digest,elt->digest, CRS_STRONG_DIGEST_SIZE); 167 | tpl_pack(tn, 1); 168 | } 169 | int result = tpl_dump(tn, TPL_FILE, file); 170 | tpl_free(tn); 171 | CRScode code = (-1 == result) ? CRS_FILE_ERROR : CRS_OK; 172 | LOGI("end %d\n", code); 173 | return code; 174 | } 175 | 176 | int Magnet_checkfile(const char *filename) { 177 | return Util_tplcmp(filename, MAGNET_TPL_FORMAT); 178 | } 179 | -------------------------------------------------------------------------------- /src/magnet.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRS_MAGNET_H 25 | #define CRS_MAGNET_H 26 | 27 | #include "global.h" 28 | #include "utstring.h" 29 | 30 | extern const char *MAGNET_EXT; 31 | 32 | typedef struct sum_t { 33 | char *name; 34 | unsigned int size; 35 | unsigned char digest[CRS_STRONG_DIGEST_SIZE]; 36 | struct sum_t *next; //utlist(single-link) 37 | } sum_t; 38 | 39 | sum_t* sum_malloc(); 40 | void sum_free(sum_t *s); 41 | 42 | typedef struct magnet_t { 43 | char *currVersion; 44 | char *nextVersion; 45 | sum_t *file; //utlist(single-link) 46 | } magnet_t; 47 | 48 | magnet_t* magnet_malloc(); 49 | void magnet_free(magnet_t *m); 50 | 51 | void magnet_toString(magnet_t *m, UT_string **str); 52 | magnet_t* magnet_fromString(UT_string **str); 53 | 54 | CRScode magnet_load(magnet_t *m, const char *file); 55 | CRScode magnet_save(magnet_t *m, const char *file); 56 | 57 | int Magnet_checkfile(const char *filename); 58 | 59 | #endif // CRS_MAGNET_H 60 | 61 | -------------------------------------------------------------------------------- /src/onepiece.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRSYNC_ONEPIECE_H 25 | #define CRSYNC_ONEPIECE_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #include "crsync.h" 32 | 33 | #define MAGNET_SUFFIX ".magnet" 34 | #define MAGNET_TPLMAP_FORMAT "sssA(ss)" 35 | 36 | typedef struct res_t { 37 | char *name; 38 | char *hash; 39 | uint32_t size; 40 | uint32_t diff_size; 41 | struct res_t *next; 42 | } res_t; 43 | 44 | void res_free(res_t *res); 45 | 46 | typedef struct oldmagnet_t { 47 | char *curr_id; /* current magnet info id */ 48 | char *next_id; /* next magnet info id */ 49 | res_t *app; /* android apk hash */ 50 | res_t *res_list; /* resource list */ 51 | } oldmagnet_t; 52 | 53 | oldmagnet_t* onepiece_magnet_malloc(); 54 | void onepiece_magnet_free(oldmagnet_t *m); 55 | 56 | CRSYNCcode onepiece_magnet_load(const char *filename, oldmagnet_t *magnet); 57 | CRSYNCcode onepiece_magnet_save(const char *filename, oldmagnet_t *magnet); 58 | 59 | typedef struct onepiece_t onepiece_t; 60 | 61 | typedef enum { 62 | ONEPIECEOPT_STARTID = 0, 63 | ONEPIECEOPT_BASEURL, 64 | ONEPIECEOPT_LOCALAPP, 65 | ONEPIECEOPT_LOCALRES, 66 | ONEPIECEOPT_NEVERUPDATE, 67 | ONEPIECEOPT_XFER, 68 | } ONEPIECEoption; 69 | 70 | typedef enum { 71 | ONEPIECEINFO_MAGNET = 0, 72 | } ONEPIECEinfo; 73 | 74 | CRSYNCcode onepiece_init(); 75 | CRSYNCcode onepiece_setopt(ONEPIECEoption opt, void *value); 76 | oldmagnet_t* onepiece_getinfo_magnet(); 77 | CRSYNCcode onepiece_perform_query(); 78 | CRSYNCcode onepiece_perform_MatchApp(); 79 | CRSYNCcode onepiece_perform_PatchApp(); 80 | CRSYNCcode onepiece_perform_MatchRes(); 81 | CRSYNCcode onepiece_perform_PatchRes(); 82 | //CRSYNCcode onepiece_perform_updateapp(); 83 | //CRSYNCcode onepiece_perform_updateres(); 84 | void onepiece_cleanup(); 85 | 86 | #if defined __cplusplus 87 | } 88 | #endif 89 | 90 | #endif // CRSYNC_ONEPIECE_H 91 | -------------------------------------------------------------------------------- /src/onepiecetool.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include 25 | #include 26 | 27 | #include "onepiecetool.h" 28 | #include "log.h" 29 | 30 | static const uint32_t Default_BLOCK_SIZE = 8*1024; //default block size to 8K 31 | 32 | onepiecetool_option_t* onepiecetool_option_malloc() { 33 | onepiecetool_option_t* op = calloc(1, sizeof(onepiecetool_option_t)); 34 | op->block_size = Default_BLOCK_SIZE; 35 | return op; 36 | } 37 | 38 | void resname_free(resname_t *res) { 39 | resname_t *elt=NULL, *tmp=NULL; 40 | LL_FOREACH_SAFE(res, elt, tmp) { 41 | LL_DELETE(res, elt); 42 | free(elt->name); 43 | free(elt); 44 | } 45 | } 46 | 47 | void onepiecetool_option_free(onepiecetool_option_t *option) { 48 | if(option) { 49 | free(option->curr_id); 50 | free(option->next_id); 51 | free(option->app_name); 52 | free(option->res_dir); 53 | resname_free(option->res_list); 54 | free(option->output_dir); 55 | free(option); 56 | } 57 | } 58 | 59 | /* 1. copy input file to output dir with hashname 60 | 2. move input file's rsum file to output dir with hashname.rsum 61 | */ 62 | static CRSYNCcode util_movefile(const char *inputfile, const char *outputdir, const char *hash) { 63 | CRSYNCcode code = CRSYNCE_FILE_ERROR; 64 | UT_string *input = NULL; 65 | utstring_new(input); 66 | utstring_printf(input, "%s%s", inputfile, RSUM_SUFFIX); 67 | 68 | UT_string *output = NULL; 69 | utstring_new(output); 70 | utstring_printf(output, "%s%s", outputdir, hash); 71 | 72 | do { 73 | tpl_mmap_rec recin = {-1, NULL, 0}; 74 | tpl_mmap_rec recout = {-1, NULL, 0}; 75 | if(0 == tpl_mmap_file(inputfile, &recin)) { 76 | recout.fd = tpl_mmap_output_file(utstring_body(output), recin.text_sz, &recout.text); 77 | if(-1 != recout.fd) { 78 | memcpy(recout.text, recin.text, recin.text_sz); 79 | tpl_unmap_file(&recout); 80 | code = CRSYNCE_OK; 81 | } 82 | tpl_unmap_file(&recin); 83 | } 84 | if(CRSYNCE_OK != code) break; 85 | 86 | utstring_printf(output, "%s", RSUM_SUFFIX); 87 | int i = rename(utstring_body(input), utstring_body(output)); 88 | code = (0 == i) ? CRSYNCE_OK : CRSYNCE_FILE_ERROR; 89 | } while (0); 90 | 91 | utstring_free(input); 92 | utstring_free(output); 93 | return code; 94 | } 95 | 96 | CRSYNCcode onepiecetool_perform(onepiecetool_option_t *option) { 97 | LOGI("onepiecetool_perform\n"); 98 | CRSYNCcode code = CRSYNCE_OK; 99 | 100 | UT_string *input = NULL; 101 | utstring_new(input); 102 | 103 | UT_string *output = NULL; 104 | utstring_new(output); 105 | 106 | UT_string *hash = NULL; 107 | utstring_new(hash); 108 | 109 | LOGI("clean up %s\n", option->output_dir); 110 | DIR *dirp = opendir(option->output_dir); 111 | if(dirp) { 112 | struct dirent *direntp = NULL; 113 | while ((direntp = readdir(dirp)) != NULL) { 114 | if(0 == strcmp(direntp->d_name, ".") || 115 | 0 == strcmp(direntp->d_name, "..")) { 116 | continue; 117 | } 118 | LOGD("remove %s\n", direntp->d_name); 119 | utstring_clear(output); 120 | utstring_printf(output, "%s%s", option->output_dir, direntp->d_name); 121 | remove(utstring_body(output)); 122 | } 123 | closedir(dirp); 124 | } else { 125 | #ifndef _WIN32 126 | mkdir(option->output_dir, S_IRUSR|S_IWUSR|S_IWGRP|S_IRGRP|S_IROTH); 127 | #else 128 | mkdir(option->output_dir); 129 | #endif 130 | } 131 | 132 | oldmagnet_t *magnet = onepiece_magnet_malloc(); 133 | magnet->curr_id = strdup(option->curr_id); 134 | magnet->next_id = strdup(option->next_id); 135 | 136 | do { 137 | LOGI("generate app rsum file\n"); 138 | LOGI("perform %s\n", option->app_name); 139 | code = crsync_rsum_generate(option->app_name, option->block_size, hash); 140 | if(CRSYNCE_OK != code) break; 141 | res_t *app = calloc(1, sizeof(res_t)); 142 | app->hash = strdup(utstring_body(hash)); 143 | LL_APPEND(magnet->app, app); 144 | code = util_movefile(option->app_name, option->output_dir, utstring_body(hash)); 145 | if(CRSYNCE_OK != code) break; 146 | 147 | LOGI("generate resource rsum file\n"); 148 | resname_t *elt=NULL; 149 | LL_FOREACH(option->res_list, elt) { 150 | LOGI("perform %s\n", elt->name); 151 | utstring_clear(input); 152 | utstring_printf(input, "%s%s", option->res_dir, elt->name); 153 | code = crsync_rsum_generate(utstring_body(input), option->block_size, hash); 154 | if(CRSYNCE_OK != code) break; 155 | struct stat file_info; 156 | stat(utstring_body(input), &file_info); 157 | code = util_movefile(utstring_body(input), option->output_dir, utstring_body(hash)); 158 | if(CRSYNCE_OK != code) break; 159 | 160 | res_t *res = calloc(1, sizeof(res_t)); 161 | res->name = strdup(elt->name); 162 | res->hash = strdup(utstring_body(hash)); 163 | res->size = file_info.st_size; 164 | LL_APPEND(magnet->res_list, res); 165 | } 166 | if(CRSYNCE_OK != code) break; 167 | 168 | LOGI("generate magnet file\n"); 169 | utstring_clear(output); 170 | utstring_printf(output, "%s%s%s", option->output_dir, option->curr_id, MAGNET_SUFFIX); 171 | code = onepiece_magnet_save(utstring_body(output), magnet); 172 | } while (0); 173 | 174 | onepiece_magnet_free(magnet); 175 | utstring_free(input); 176 | utstring_free(output); 177 | utstring_free(hash); 178 | LOGI("onepiecetool_perform code = %d\n", code); 179 | return code; 180 | } 181 | -------------------------------------------------------------------------------- /src/onepiecetool.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef ONEPIECE_TOOL_H 25 | #define ONEPIECE_TOOL_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #include "onepiece.h" 32 | 33 | typedef struct resname_t { 34 | char *name; 35 | struct resname_t *next; 36 | } resname_t; 37 | 38 | void resname_free(resname_t *res); 39 | 40 | typedef struct onepiecetool_option_t { 41 | char *curr_id; //current magnet info id 42 | char *next_id; //next magnet info id 43 | char *app_name; //app full filename 44 | char *res_dir; //resource directory, end with '/' 45 | resname_t *res_list; //resource file list 46 | char *output_dir; //output directory 47 | uint32_t block_size; //block size, default 8K 48 | } onepiecetool_option_t; 49 | 50 | onepiecetool_option_t* onepiecetool_option_malloc(); 51 | void onepiecetool_option_free(onepiecetool_option_t *option); 52 | 53 | CRSYNCcode onepiecetool_perform(onepiecetool_option_t *option); 54 | 55 | #if defined __cplusplus 56 | } 57 | #endif 58 | 59 | #endif // ONEPIECE_TOOL_H 60 | -------------------------------------------------------------------------------- /src/patch.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include "patch.h" 31 | #include "util.h" 32 | #include "log.h" 33 | #include "http.h" 34 | 35 | static CRScode Patch_match(const char *srcFilename, const char *dstFilename, 36 | const fileDigest_t *fd, const diffResult_t *dr) { 37 | LOGI("begin\n"); 38 | if(!srcFilename || !dstFilename || !fd || !dr) { 39 | LOGE("end %d\n", CRS_PARAM_ERROR); 40 | return CRS_PARAM_ERROR; 41 | } 42 | 43 | FILE *f1 = fopen(srcFilename, "rb"); 44 | if(!f1){ 45 | LOGE("source file fopen error %s\n", strerror(errno)); 46 | return CRS_FILE_ERROR; 47 | } 48 | 49 | FILE *f2 = fopen(dstFilename, "rb+"); 50 | if(!f2){ 51 | LOGE("dest file fopen error %s\n", strerror(errno)); 52 | fclose(f1); 53 | return CRS_FILE_ERROR; 54 | } 55 | 56 | CRScode code = CRS_OK; 57 | uint8_t *buf = malloc(fd->blockSize); 58 | 59 | for(int i=0; itotalNum; ++i) { 60 | if(dr->offsets[i] >= 0) { 61 | fseek(f1, dr->offsets[i], SEEK_SET); 62 | fread(buf, 1, fd->blockSize, f1); 63 | 64 | fseek(f2, i*fd->blockSize, SEEK_SET); 65 | fwrite(buf, 1, fd->blockSize, f2); 66 | } 67 | } 68 | 69 | size_t restSize = fd->fileSize % fd->blockSize; 70 | if(restSize > 0){ 71 | 72 | fseek(f2, -restSize, SEEK_END); 73 | fwrite(fd->restData, 1, restSize, f2); 74 | } 75 | 76 | free(buf); 77 | fclose(f1); 78 | fclose(f2); 79 | LOGI("end %d\n", code); 80 | return code; 81 | } 82 | 83 | //continuous blocks, used for reduce http range frequency 84 | typedef struct combineblock_t { 85 | size_t pos; //block start position 86 | size_t got; //data fwrite size, used for HTTP retry 87 | size_t len; //block length 88 | } combineblock_t; 89 | 90 | static uint32_t Patch_missCombine(const diffResult_t *dr, combineblock_t *cb, uint32_t blockSize) { 91 | uint32_t combineNum = 0; 92 | int missNum = dr->totalNum - dr->matchNum - dr->cacheNum; 93 | if(missNum == 0) { 94 | LOGW("missNum is Zero, maybe cache done!\n"); 95 | return combineNum; 96 | } 97 | if(missNum < 0) { 98 | LOGE("WTF: matchNum %d bigger than totalNum %d\n", dr->matchNum, dr->totalNum); 99 | return combineNum; 100 | } 101 | int i; 102 | uint32_t j; 103 | for(i=0; itotalNum; ++i) { 104 | if(dr->offsets[i] == -1) { 105 | for(j=0; j < combineNum; ++j) { 106 | if(cb[j].pos + cb[j].len == i * blockSize) { 107 | cb[j].len += blockSize; 108 | break; 109 | } 110 | } 111 | if(j == combineNum) { 112 | cb[combineNum].pos = i*blockSize; 113 | cb[combineNum].len = blockSize; 114 | ++combineNum; 115 | } 116 | } 117 | } 118 | LOGI("combineblocks Num = %d\n", combineNum); 119 | return combineNum; 120 | } 121 | 122 | typedef struct rangedata_t { 123 | combineblock_t *cb; //ref to one Patch_miss() 124 | FILE *file; //ref to one Patch_miss() 125 | char *basename; //ref to one Patch_miss() 126 | size_t cacheBytes; 127 | } rangedata_t; 128 | 129 | static size_t Range_callback(void *data, size_t size, size_t nmemb, void *userp) { 130 | rangedata_t *rd = (rangedata_t*)userp; 131 | size_t realSize = size * nmemb; 132 | fseek(rd->file, rd->cb->pos + rd->cb->got, SEEK_SET); 133 | fwrite(data, size, nmemb, rd->file); 134 | rd->cb->got += realSize; 135 | rd->cacheBytes += realSize; 136 | int isCancel = crs_callback_patch(rd->basename, rd->cacheBytes, 0, 0); 137 | return (isCancel == 0) ? realSize : 0; 138 | } 139 | 140 | static CRScode Patch_miss(const char *srcFilename, const char *dstFilename, const char *url, 141 | const fileDigest_t *fd, const diffResult_t *dr) { 142 | LOGI("begin\n"); 143 | if(!srcFilename || !dstFilename || !fd || !dr) { 144 | LOGE("end %d\n", CRS_PARAM_ERROR); 145 | return CRS_PARAM_ERROR; 146 | } 147 | 148 | int missNum = dr->totalNum - dr->matchNum - dr->cacheNum; 149 | if(missNum == 0) { 150 | LOGI("end missblocks = 0\n"); 151 | return CRS_OK; 152 | } 153 | 154 | FILE *f = fopen(dstFilename, "rb+"); 155 | if(!f){ 156 | LOGE("%s\n", dstFilename); 157 | LOGE("end fopen error %s\n", strerror(errno)); 158 | return CRS_FILE_ERROR; 159 | } 160 | 161 | //combine continuous blocks, no more than missNum 162 | combineblock_t *cb = calloc(missNum, sizeof(combineblock_t) ); 163 | uint32_t cbNum = Patch_missCombine(dr, cb, fd->blockSize); 164 | 165 | CRScode code = CRS_OK; 166 | rangedata_t rd; 167 | rd.file = f; 168 | rd.cacheBytes = fd->fileSize - (missNum * fd->blockSize); 169 | 170 | //Some compiler maybe change basename() param 171 | //Do not free(basename) since it maybe inside of fullname 172 | char *tempname = strdup(srcFilename); 173 | rd.basename = basename(tempname); 174 | CURL *curl = curl_easy_init(); 175 | for(uint32_t i=0; i< cbNum; ++i) { 176 | rd.cb = &cb[i]; 177 | char range[32]; 178 | long rangeFrom; 179 | long rangeTo; 180 | 181 | int retry = 10; 182 | while(retry-- > 0) { 183 | rangeFrom = cb[i].pos + cb[i].got; 184 | rangeTo = cb[i].pos + cb[i].len - 1; 185 | snprintf(range, 32, "%ld-%ld", rangeFrom, rangeTo); 186 | 187 | if(rangeFrom >= rangeTo) { 188 | LOGE("range request %ld-%ld wrong\n", rangeFrom, rangeTo); 189 | LOGE("This should not happend! But continue NextBlock\n"); 190 | code = CRS_OK; 191 | break; 192 | } 193 | 194 | CURLcode curlcode = HTTP_Range(curl, url, range, (void*)Range_callback, &rd); 195 | 196 | switch(curlcode) { 197 | case CURLE_OK: 198 | code = CRS_OK; 199 | break; 200 | case CURLE_HTTP_RETURNED_ERROR: 201 | LOGE("HTTP request/response header wrong!\n"); 202 | retry = -1; 203 | code = CRS_HTTP_ERROR; 204 | break; 205 | case CURLE_OPERATION_TIMEDOUT: //timeout 206 | retry++; 207 | code = CRS_HTTP_ERROR; 208 | break; 209 | default: 210 | code = CRS_HTTP_ERROR; 211 | break; 212 | } 213 | 214 | if(CRS_OK == code) { //got it 215 | break; 216 | } else { 217 | LOGE("curl code %d\n", curlcode); 218 | } 219 | }//end of while 220 | 221 | }//end of for 222 | 223 | curl_easy_cleanup(curl); 224 | free(tempname); 225 | free(cb); 226 | fclose(f); 227 | LOGI("end %d\n", code); 228 | return code; 229 | } 230 | 231 | CRScode Patch_perform(const char *srcFilename, const char *dstFilename, const char *url, 232 | const fileDigest_t *fd, const diffResult_t *dr) { 233 | LOGI("begin\n"); 234 | 235 | if(!srcFilename || !dstFilename || !url || !fd || !dr) { 236 | LOGE("end %d\n", CRS_PARAM_ERROR); 237 | return CRS_PARAM_ERROR; 238 | } 239 | 240 | CRScode code = CRS_OK; 241 | do { 242 | if(0 != access(srcFilename, F_OK)) { 243 | LOGE("src file not exist %s\n", strerror(errno)); 244 | LOGE("%s\n", srcFilename); 245 | code = CRS_FILE_ERROR; 246 | break; 247 | } 248 | if(0 != access(dstFilename, F_OK)) { 249 | FILE *f = fopen(dstFilename, "ab+"); 250 | if(f) { 251 | fclose(f); 252 | } else { 253 | LOGE("dst file create fail %s\n", strerror(errno)); 254 | LOGE("%s\n", dstFilename); 255 | code = CRS_FILE_ERROR; 256 | break; 257 | } 258 | } 259 | struct stat st; 260 | if(stat(dstFilename, &st) != 0) { 261 | LOGE("dst file stat fail %s\n", strerror(errno)); 262 | LOGE("%s\n", dstFilename); 263 | code = CRS_FILE_ERROR; 264 | break; 265 | } 266 | if((size_t)st.st_size != fd->fileSize) { 267 | if(0 != truncate(dstFilename, fd->fileSize)) { 268 | LOGE("dest file truncate %dBytes error %s\n", fd->fileSize, strerror(errno)); 269 | code = CRS_FILE_ERROR; 270 | break; 271 | } 272 | } 273 | 274 | //Patch_match Blocks 275 | code = Patch_match(srcFilename, dstFilename, fd, dr); 276 | if(code != CRS_OK) break; 277 | 278 | //Patch_miss Blocks 279 | code = Patch_miss(srcFilename, dstFilename, url, fd, dr); 280 | if(code != CRS_OK) break; 281 | 282 | char *tempname = strdup(srcFilename); 283 | char *name = basename(tempname); 284 | crs_callback_patch(name, fd->fileSize, 0, 1); 285 | free(tempname); 286 | 287 | uint8_t hash[CRS_STRONG_DIGEST_SIZE]; 288 | Digest_CalcStrong_File(dstFilename, hash); 289 | char * hashString = Util_hex_string(hash, CRS_STRONG_DIGEST_SIZE); 290 | LOGI("fileDigest = %s\n", hashString); 291 | free(hashString); 292 | code = (0 == memcmp(hash, fd->fileDigest, CRS_STRONG_DIGEST_SIZE)) ? CRS_OK : CRS_BUG ; 293 | 294 | } while (0); 295 | 296 | LOGI("end %d\n", code); 297 | return code; 298 | } 299 | -------------------------------------------------------------------------------- /src/patch.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRS_PATCH_H 25 | #define CRS_PATCH_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | #include "diff.h" 32 | 33 | CRScode Patch_perform(const char *srcFilename, const char *dstFilename, const char *url, 34 | const fileDigest_t *fd, const diffResult_t *dr); 35 | 36 | #if defined __cplusplus 37 | } 38 | #endif 39 | 40 | #endif // CRS_PATCH_H 41 | 42 | -------------------------------------------------------------------------------- /src/util.c: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include "util.h" 32 | #include "tpl.h" 33 | #include "log.h" 34 | 35 | static const char* s_hexTable[256] = { 36 | "00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0a", "0b", "0c", "0d", "0e", "0f", 37 | "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "1a", "1b", "1c", "1d", "1e", "1f", 38 | "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "2a", "2b", "2c", "2d", "2e", "2f", 39 | "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "3a", "3b", "3c", "3d", "3e", "3f", 40 | "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "4a", "4b", "4c", "4d", "4e", "4f", 41 | "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "5a", "5b", "5c", "5d", "5e", "5f", 42 | "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "6a", "6b", "6c", "6d", "6e", "6f", 43 | "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "7a", "7b", "7c", "7d", "7e", "7f", 44 | "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "8a", "8b", "8c", "8d", "8e", "8f", 45 | "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "9a", "9b", "9c", "9d", "9e", "9f", 46 | "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9", "aa", "ab", "ac", "ad", "ae", "af", 47 | "b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9", "ba", "bb", "bc", "bd", "be", "bf", 48 | "c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "ca", "cb", "cc", "cd", "ce", "cf", 49 | "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9", "da", "db", "dc", "dd", "de", "df", 50 | "e0", "e1", "e2", "e3", "e4", "e5", "e6", "e7", "e8", "e9", "ea", "eb", "ec", "ed", "ee", "ef", 51 | "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "fa", "fb", "fc", "fd", "fe", "ff" 52 | }; 53 | 54 | char* Util_hex_string(const unsigned char *in, const unsigned int inlen) { 55 | unsigned int outlen = inlen * 2; 56 | char *out = malloc(outlen+1); 57 | for(unsigned int i=0; i= '0' && c <= '9') return c - '0'; 67 | if(c >= 'a' && c <= 'f') return c - 'a' + 10; 68 | if(c >= 'A' && c <= 'F') return c - 'A' + 10; 69 | return 255; 70 | } 71 | 72 | unsigned char* Util_string_hex(const char *in) { 73 | unsigned char *retval; 74 | unsigned char *p; 75 | int len, i; 76 | 77 | len = strlen(in) / 2; 78 | retval = malloc(len+1); 79 | for(i=0, p = (unsigned char *) in; i 0) { 124 | fwrite(buf, 1, size, d); 125 | } 126 | free(buf); 127 | 128 | fclose(s); 129 | fclose(d); 130 | 131 | struct stat stSrc, stDst; 132 | if(stat(src, &stSrc) == 0 && stat(dst, &stDst) == 0) { 133 | if(stSrc.st_size == stDst.st_size) { 134 | return 0; 135 | } 136 | } 137 | LOGE("stat error %s, %s", src, dst); 138 | return -1; 139 | } 140 | 141 | int Util_filemove(const char *src, const char *dst) { 142 | if(0 == Util_filecpy(src, dst)) { 143 | if(0 == remove(src)) { 144 | return 0; 145 | } else { 146 | LOGE("remove %s\n", strerror(errno)); 147 | return -1; 148 | } 149 | } else { 150 | LOGE("filecpy fail\n"); 151 | return -1; 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 chenqi 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #ifndef CRS_UTIL_H 25 | #define CRS_UTIL_H 26 | 27 | #if defined __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | char* Util_hex_string(const unsigned char *in, const unsigned int inlen); 32 | 33 | unsigned char* Util_string_hex(const char *in); 34 | 35 | char* Util_strcat(const char* s1, const char *s2); 36 | 37 | int Util_tplcmp(const char *filename, const char* fmt); 38 | 39 | int Util_filecpy(const char *src, const char *dst); 40 | 41 | int Util_filemove(const char *src, const char *dst); 42 | 43 | #if defined __cplusplus 44 | } 45 | #endif 46 | 47 | #endif // CRS_UTIL_H 48 | --------------------------------------------------------------------------------