├── FileSync
    ├── .classpath
    ├── .project
    ├── .settings
    │   ├── org.eclipse.core.resources.prefs
    │   └── org.eclipse.jdt.core.prefs
    ├── docs
    │   └── note.txt
    └── src
    │   └── com
    │       └── hjh
    │           └── file
    │               └── sync
    │                   ├── core
    │                       ├── FSConfig.java
    │                       ├── KeyGeneral.java
    │                       ├── SyncFolderInfo.java
    │                       └── SyncItem.java
    │                   ├── main
    │                       └── Main.java
    │                   ├── process
    │                       ├── CancelControl.java
    │                       ├── IProcessListener.java
    │                       ├── ProcessPrinter.java
    │                       └── SimpleProcessListener.java
    │                   └── util
    │                       ├── FileUtils.java
    │                       ├── LogHelper.java
    │                       └── MD5.java
├── README.md
├── fileSync.jar
└── run.png


/FileSync/.classpath:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <classpath>
3 | 	<classpathentry kind="src" path="src"/>
4 | 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
5 | 	<classpathentry kind="output" path="bin"/>
6 | </classpath>
7 | 


--------------------------------------------------------------------------------
/FileSync/.project:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <projectDescription>
 3 | 	<name>FileSync</name>
 4 | 	<comment></comment>
 5 | 	<projects>
 6 | 	</projects>
 7 | 	<buildSpec>
 8 | 		<buildCommand>
 9 | 			<name>org.eclipse.jdt.core.javabuilder</name>
10 | 			<arguments>
11 | 			</arguments>
12 | 		</buildCommand>
13 | 	</buildSpec>
14 | 	<natures>
15 | 		<nature>org.eclipse.jdt.core.javanature</nature>
16 | 	</natures>
17 | </projectDescription>
18 | 


--------------------------------------------------------------------------------
/FileSync/.settings/org.eclipse.core.resources.prefs:
--------------------------------------------------------------------------------
1 | eclipse.preferences.version=1
2 | encoding/<project>=UTF-8
3 | 


--------------------------------------------------------------------------------
/FileSync/.settings/org.eclipse.jdt.core.prefs:
--------------------------------------------------------------------------------
 1 | eclipse.preferences.version=1
 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
 3 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
 4 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
 5 | org.eclipse.jdt.core.compiler.compliance=1.7
 6 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate
 7 | org.eclipse.jdt.core.compiler.debug.localVariable=generate
 8 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate
 9 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
10 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
11 | org.eclipse.jdt.core.compiler.source=1.7
12 | 


--------------------------------------------------------------------------------
/FileSync/docs/note.txt:
--------------------------------------------------------------------------------
1 | The main class is 'com.hjh.file.sync.main.Main'
2 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/core/FSConfig.java:
--------------------------------------------------------------------------------
 1 | package com.hjh.file.sync.core;
 2 | 
 3 | import java.io.File;
 4 | import java.io.FileInputStream;
 5 | import java.io.FileOutputStream;
 6 | import java.io.IOException;
 7 | import java.io.UnsupportedEncodingException;
 8 | import java.util.UUID;
 9 | 
10 | import com.hjh.file.sync.util.LogHelper;
11 | 
12 | /**
13 |  * 
14 |  * 属性配置
15 |  * 
16 |  * @author 洪 qq:2260806429
17 |  * 
18 |  */
19 | public class FSConfig {
20 | 
21 | 	public static final String TEST_PATH = "E:\\testsync_source";
22 | 
23 | 	public static final String CACHE_ID_FILE = ".hsync";
24 | 
25 | 	public static final long BLOCK_SIZE = 1024 * 1024 * 10; // 10M的块大小
26 | 
27 | 	public final static String INFO_SEP = ":";
28 | 	public final static String CONTENT_SEP = ";";
29 | 
30 | 	public final static int FOLDER_INFO_LEN = 2;
31 | 	public final static int NAME_INDEX = 0;
32 | 	public final static int TIME_INDEX = 1;
33 | 	public final static int SIZE_INDEX = 2;
34 | 	public final static int CONTENT_INDEX = 3;
35 | 
36 | 	public static boolean isCompareBinary() { // 是否二进制比较，否则时间比较
37 | 		return false;
38 | 	}
39 | 
40 | 	public static boolean isIgnore(File file) { // 文件是否忽略
41 | 		if (file.getName().equals(CACHE_ID_FILE)) {
42 | 			return true;
43 | 		}
44 | 		return false;
45 | 	}
46 | 
47 | 	private static File cacheFile(String path, File target) {
48 | 		try {
49 | 			return new File(new File(target, CACHE_ID_FILE), UUID
50 | 					.nameUUIDFromBytes(path.getBytes("utf-8")).toString());
51 | 		} catch (UnsupportedEncodingException e) {
52 | 			LogHelper.error(e);
53 | 		}
54 | 		throw new RuntimeException("生成缓存路径失败:" + path);
55 | 	}
56 | 
57 | 	public static String getCache(String path, File target) throws IOException {
58 | 		File cacheFile = cacheFile(path, target);
59 | 		if (cacheFile.isFile()) {
60 | 			File source = new File(target.getAbsolutePath() + path);
61 | 			if (source.lastModified() == cacheFile.lastModified()) {
62 | 				FileInputStream in = new FileInputStream(cacheFile);
63 | 				byte[] cache = new byte[(int) cacheFile.length()];
64 | 				try {
65 | 					in.read(cache);
66 | 				} finally {
67 | 					in.close();
68 | 				}
69 | 				return new String(cache);
70 | 			} else {
71 | 				LogHelper.warn("缓存KEY文件过期:" + path);
72 | 			}
73 | 		}
74 | 		return null;
75 | 	}
76 | 
77 | 	public static void cache(String id, File target) throws IOException {
78 | 		String[] arr = id.split(INFO_SEP);
79 | 		long size = Long.parseLong(arr[SIZE_INDEX]);
80 | 		if (size > BLOCK_SIZE * 10) {
81 | 			String path = arr[NAME_INDEX];
82 | 			String content = arr[CONTENT_INDEX];
83 | 			File save = cacheFile(path, target);
84 | 			if (!save.getParentFile().exists()) {
85 | 				save.getParentFile().mkdir();
86 | 			}
87 | 			FileOutputStream out = new FileOutputStream(save);
88 | 			try {
89 | 				out.write(content.getBytes());
90 | 			} finally {
91 | 				out.close();
92 | 			}
93 | 			save.setLastModified(new File(target.getAbsolutePath() + path)
94 | 					.lastModified());
95 | 		}
96 | 	}
97 | 
98 | }
99 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/core/KeyGeneral.java:
--------------------------------------------------------------------------------
  1 | package com.hjh.file.sync.core;
  2 | 
  3 | import java.io.File;
  4 | import java.io.FileInputStream;
  5 | import java.io.IOException;
  6 | 
  7 | import com.hjh.file.sync.process.IProcessListener;
  8 | import com.hjh.file.sync.process.SimpleProcessListener;
  9 | import com.hjh.file.sync.util.LogHelper;
 10 | import com.hjh.file.sync.util.MD5;
 11 | 
 12 | /**
 13 |  * 文件指纹计算
 14 |  * 
 15 |  * @author 洪 qq:2260806429
 16 |  */
 17 | public class KeyGeneral {
 18 | 
 19 | 	public static void main(String argv[]) throws IOException {
 20 | 		final String path = "G:\\disk";
 21 | 		// FSConfig.TEST_PATH;
 22 | 		for (File item : new File(path).listFiles()) {
 23 | 			if (item.isFile()) {
 24 | 				genId(item.getAbsolutePath(), new File(path));
 25 | 			}
 26 | 		}
 27 | 	}
 28 | 
 29 | 	private static void genId(String path, File root) throws IOException {
 30 | 		final IProcessListener listener = new SimpleProcessListener(null,
 31 | 				new File(path).isDirectory() ? 0 : new File(path).length());
 32 | 		listener.print("gen id " + new File(path).getName());
 33 | 		String id = id(path, listener, root, true);
 34 | 		LogHelper.info("ID:" + id);
 35 | 	}
 36 | 
 37 | 	public static String id(String filePath, IProcessListener listener,
 38 | 			File root, boolean fromcache) throws IOException {
 39 | 		File file = new File(filePath);
 40 | 		StringBuffer idBuf = new StringBuffer();
 41 | 		try {
 42 | 			idBuf.append(file.getCanonicalPath());
 43 | 		} catch (IOException e) {
 44 | 			throw new RuntimeException("获取路径失败:" + filePath, e);
 45 | 		}
 46 | 		idBuf.append(FSConfig.INFO_SEP);
 47 | 		idBuf.append(file.lastModified());
 48 | 		boolean tocache = false;
 49 | 		if (file.isFile()) {
 50 | 			idBuf.append(FSConfig.INFO_SEP);
 51 | 			idBuf.append(file.length());
 52 | 			idBuf.append(FSConfig.INFO_SEP);
 53 | 			String key = null;
 54 | 			if (fromcache) {
 55 | 				key = FSConfig.getCache(
 56 | 						filePath.substring(root.getAbsolutePath().length()),
 57 | 						root);
 58 | 				if (null != key) {
 59 | 					listener.work(file.length());
 60 | 				}
 61 | 			}
 62 | 			if (null == key) {
 63 | 				key = key(filePath, listener);
 64 | 				tocache = null != key;
 65 | 			}
 66 | 			if (null == key) {
 67 | 				return null;
 68 | 			}
 69 | 			idBuf.append(key);
 70 | 		}
 71 | 		String id = idBuf.toString().substring(root.getAbsolutePath().length());
 72 | 		if (tocache) {
 73 | 			FSConfig.cache(id, root);
 74 | 		}
 75 | 		return id;
 76 | 	}
 77 | 
 78 | 	public static String key(String filePath, IProcessListener listener)
 79 | 			throws IOException {
 80 | 
 81 | 		StringBuffer keyBuf = new StringBuffer();
 82 | 		File file = new File(filePath);
 83 | 		FileInputStream in = new FileInputStream(file);
 84 | 		byte[] cache = new byte[(int) FSConfig.BLOCK_SIZE];
 85 | 		try {
 86 | 			int len = 0;
 87 | 			while (true) {
 88 | 				if (listener.isCancel()) {
 89 | 					return null;
 90 | 				}
 91 | 				len = in.read(cache);
 92 | 				if (len <= 0) {
 93 | 					break;
 94 | 				}
 95 | 				keyBuf.append(FSConfig.CONTENT_SEP);
 96 | 				keyBuf.append(MD5.md5(cache, 0, len));
 97 | 				listener.work(len);
 98 | 			}
 99 | 		} finally {
100 | 			in.close();
101 | 		}
102 | 
103 | 		byte[] allKeys = keyBuf.toString().getBytes();
104 | 		String sumkey = MD5.md5(allKeys, 0, allKeys.length);
105 | 
106 | 		return sumkey + keyBuf.toString();
107 | 	}
108 | 
109 | }
110 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/core/SyncFolderInfo.java:
--------------------------------------------------------------------------------
  1 | package com.hjh.file.sync.core;
  2 | 
  3 | import java.io.File;
  4 | import java.io.IOException;
  5 | import java.util.ArrayList;
  6 | import java.util.Collections;
  7 | import java.util.List;
  8 | 
  9 | import com.hjh.file.sync.process.IProcessListener;
 10 | import com.hjh.file.sync.process.SimpleProcessListener;
 11 | 
 12 | /**
 13 |  * 目录信息
 14 |  * 
 15 |  * @author 洪 qq:2260806429
 16 |  */
 17 | public class SyncFolderInfo {
 18 | 
 19 | 	public static void main(String argv[]) throws IOException {
 20 | 		final String path = FSConfig.TEST_PATH;
 21 | 		final IProcessListener listener = new SimpleProcessListener();
 22 | 		listener.print("test scan");
 23 | 		System.out.println(new SyncFolderInfo(new File(path)).scan(listener)
 24 | 				.printInfo());
 25 | 	}
 26 | 
 27 | 	private File folder;
 28 | 	private long folderCount;
 29 | 	private long fileCount;
 30 | 	private long dataSize;
 31 | 	private List<String> fileKeys;
 32 | 
 33 | 	public SyncFolderInfo(File folder) {
 34 | 		this.folder = folder;
 35 | 	}
 36 | 
 37 | 	private void reset() {
 38 | 		folderCount = 0;
 39 | 		fileCount = 0;
 40 | 		dataSize = 0;
 41 | 		fileKeys = new ArrayList<String>();
 42 | 	}
 43 | 
 44 | 	public String printInfo() {
 45 | 		StringBuffer buf = new StringBuffer();
 46 | 		buf.append(String.format("folder %d file %d dataSize %d \r\n",
 47 | 				folderCount, fileCount, dataSize));
 48 | 		for (String item : fileKeys) {
 49 | 			buf.append(item);
 50 | 			buf.append("\r\n");
 51 | 		}
 52 | 		return buf.toString();
 53 | 	}
 54 | 
 55 | 	public List<SyncItem> sync(SyncFolderInfo target) {
 56 | 		List<SyncItem> ret = new ArrayList<SyncItem>();
 57 | 		int maxSource = this.fileKeys.size();
 58 | 		int maxTarget = target.fileKeys.size();
 59 | 		for (int i = 0, j = 0; i < maxSource || j < maxTarget;) {
 60 | 			String keysource = i >= maxSource ? null : this.fileKeys.get(i);
 61 | 			String keytarget = j >= maxTarget ? null : target.fileKeys.get(j);
 62 | 			if (keysource == null) {
 63 | 				ret.add(new SyncItem(null, keytarget));
 64 | 				j++;
 65 | 			} else if (keytarget == null) {
 66 | 				ret.add(new SyncItem(keysource, null));
 67 | 				i++;
 68 | 			} else {
 69 | 				String pathsource = keysource.split(FSConfig.INFO_SEP)[FSConfig.NAME_INDEX];
 70 | 				String pathtarget = keytarget.split(FSConfig.INFO_SEP)[FSConfig.NAME_INDEX];
 71 | 				int cmp = pathsource.compareTo(pathtarget);
 72 | 				if (0 == cmp) {
 73 | 					if (!keysource.equals(keytarget)) {
 74 | 						ret.add(new SyncItem(keysource, keytarget));
 75 | 					}
 76 | 					i++;
 77 | 					j++;
 78 | 				} else if (cmp > 0) {
 79 | 					ret.add(new SyncItem(null, keytarget));
 80 | 					j++;
 81 | 				} else {
 82 | 					ret.add(new SyncItem(keysource, null));
 83 | 					i++;
 84 | 				}
 85 | 			}
 86 | 		}
 87 | 		return ret;
 88 | 	}
 89 | 
 90 | 	public SyncFolderInfo scan(IProcessListener listener) throws IOException {
 91 | 		reset();
 92 | 		if (!this.folder.exists()) {
 93 | 			listener.updateTotalSize(0);
 94 | 			return this;
 95 | 		}
 96 | 		count(folder, listener);
 97 | 		if (listener.isCancel()) {
 98 | 			return this;
 99 | 		}
100 | 		listener.updateTotalSize(dataSize);
101 | 		key(folder, listener);
102 | 		Collections.sort(fileKeys);
103 | 		return this;
104 | 	}
105 | 
106 | 	private void key(File file, IProcessListener listener) throws IOException {
107 | 		if (FSConfig.isIgnore(file)) {
108 | 			return;
109 | 		}
110 | 		String id = KeyGeneral.id(file.getAbsolutePath(), listener, folder,
111 | 				true);
112 | 		if (id.length() > 0) {
113 | 			fileKeys.add(id);
114 | 		}
115 | 		if (file.isDirectory()) {
116 | 			if (listener.isCancel()) {
117 | 				return;
118 | 			}
119 | 			File[] list = file.listFiles();
120 | 			if (null != list) {
121 | 				for (File item : list) {
122 | 					key(item, listener);
123 | 					if (listener.isCancel()) {
124 | 						return;
125 | 					}
126 | 				}
127 | 			}
128 | 		}
129 | 	}
130 | 
131 | 	private void count(File file, IProcessListener listener) {
132 | 		if (FSConfig.isIgnore(file)) {
133 | 			return;
134 | 		}
135 | 		if (file.isDirectory()) {
136 | 			folderCount++;
137 | 			if (listener.isCancel()) {
138 | 				return;
139 | 			}
140 | 			File[] list = file.listFiles();
141 | 			if (null != list) {
142 | 				for (File item : list) {
143 | 					count(item, listener);
144 | 					if (listener.isCancel()) {
145 | 						return;
146 | 					}
147 | 				}
148 | 			}
149 | 		} else {
150 | 			fileCount++;
151 | 			dataSize += file.length();
152 | 		}
153 | 	}
154 | 
155 | }
156 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/core/SyncItem.java:
--------------------------------------------------------------------------------
  1 | package com.hjh.file.sync.core;
  2 | 
  3 | import java.io.File;
  4 | import java.io.FileInputStream;
  5 | import java.io.FileOutputStream;
  6 | import java.io.IOException;
  7 | import java.io.RandomAccessFile;
  8 | 
  9 | import com.hjh.file.sync.process.IProcessListener;
 10 | import com.hjh.file.sync.util.LogHelper;
 11 | 
 12 | /**
 13 |  * 同步项
 14 |  * 
 15 |  * @author 洪 qq:2260806429
 16 |  */
 17 | public class SyncItem {
 18 | 
 19 | 	private File targetFile;
 20 | 	public String from;
 21 | 	public String to;
 22 | 
 23 | 	public SyncItem(String from, String to) {
 24 | 		this.from = from;
 25 | 		this.to = to;
 26 | 	}
 27 | 
 28 | 	public long getSyncSize() {
 29 | 		if (isDel()) {
 30 | 			return 0;
 31 | 		}
 32 | 		String[] fromArr = from.split(FSConfig.INFO_SEP);
 33 | 		long fromSize = fromArr.length == FSConfig.FOLDER_INFO_LEN ? 0 : Long
 34 | 				.parseLong(fromArr[FSConfig.SIZE_INDEX]);
 35 | 		if (isCopy() || 0 == fromSize) {
 36 | 			return fromSize;
 37 | 		}
 38 | 		long size = 0;
 39 | 		String[] fromContent = fromArr[FSConfig.CONTENT_INDEX]
 40 | 				.split(FSConfig.CONTENT_SEP);
 41 | 		String[] toContent = to.split(FSConfig.INFO_SEP)[FSConfig.CONTENT_INDEX]
 42 | 				.split(FSConfig.CONTENT_SEP);
 43 | 		for (int i = 1; i < fromContent.length; i++) {
 44 | 			if (i < toContent.length) {
 45 | 				if (toContent[i].equals(fromContent[i])) {
 46 | 					continue;
 47 | 				}
 48 | 			}
 49 | 			if (i == fromContent.length - 1) {
 50 | 				size += (fromSize - FSConfig.BLOCK_SIZE * (i - 1));
 51 | 			} else {
 52 | 				size += FSConfig.BLOCK_SIZE;
 53 | 			}
 54 | 		}
 55 | 		return size;
 56 | 	}
 57 | 
 58 | 	public String toString() {
 59 | 		if (isDel()) {
 60 | 			return "del:" + to;
 61 | 		}
 62 | 		if (isCopy()) {
 63 | 			return "add:" + from;
 64 | 		}
 65 | 		return "update:" + from + ":" + to;
 66 | 	}
 67 | 
 68 | 	public boolean isDel() {
 69 | 		return null == from;
 70 | 	}
 71 | 
 72 | 	public boolean isCopy() {
 73 | 		return null == to;
 74 | 	}
 75 | 
 76 | 	public boolean isUpdate() {
 77 | 		return !isDel() && !isCopy();
 78 | 	}
 79 | 
 80 | 	public void sync(File sourceFile, File targetFile,
 81 | 			IProcessListener listener_sync) throws IOException {
 82 | 		this.targetFile = targetFile;
 83 | 		if (listener_sync.isCancel()) {
 84 | 			return;
 85 | 		}
 86 | 		if (isDel()) {
 87 | 			del(toPath(targetFile), listener_sync);
 88 | 		} else if (isCopy()) {
 89 | 			copy(fromPath(sourceFile), toPath(targetFile), listener_sync);
 90 | 		} else {
 91 | 			update(fromPath(sourceFile), toPath(targetFile), listener_sync);
 92 | 		}
 93 | 	}
 94 | 
 95 | 	private String fromPath(File sourceFile) {
 96 | 		return sourceFile.getAbsolutePath()
 97 | 				+ from.split(FSConfig.INFO_SEP)[FSConfig.NAME_INDEX];
 98 | 	}
 99 | 
100 | 	private String toPath(File targetFile) {
101 | 		return targetFile.getAbsolutePath()
102 | 				+ (to == null ? from : to).split(FSConfig.INFO_SEP)[FSConfig.NAME_INDEX];
103 | 	}
104 | 
105 | 	private void del(String path, IProcessListener listener_sync) {
106 | 		File file = new File(path);
107 | 		if (!file.exists()) {
108 | 			return;
109 | 		}
110 | 		if (file.isDirectory()) {
111 | 			File list[] = file.listFiles();
112 | 			if (null != list) {
113 | 				for (File item : list) {
114 | 					if (listener_sync.isCancel()) {
115 | 						return;
116 | 					}
117 | 					del(item.getAbsolutePath(), listener_sync);
118 | 				}
119 | 			}
120 | 		}
121 | 		if (listener_sync.isCancel()) {
122 | 			return;
123 | 		}
124 | 		if (!file.delete()) {
125 | 			LogHelper.warn("删除文件失败:" + file.getAbsolutePath());
126 | 		}
127 | 	}
128 | 
129 | 	private void copy(String fromPath, String toPath,
130 | 			IProcessListener listener_sync) throws IOException {
131 | 		if (listener_sync.isCancel()) {
132 | 			return;
133 | 		}
134 | 		File from = new File(fromPath);
135 | 		File to = new File(toPath);
136 | 		if (!from.exists()) {
137 | 			LogHelper.warn("文件不存在:" + from.getAbsolutePath());
138 | 			return;
139 | 		}
140 | 		if (from.isDirectory()) {
141 | 			if (to.isFile()) {
142 | 				to.delete();
143 | 			}
144 | 			if (!to.isDirectory()) {
145 | 				to.mkdir();
146 | 			}
147 | 			to.setLastModified(from.lastModified());
148 | 		} else {
149 | 			if (to.isDirectory()) {
150 | 				del(toPath, listener_sync);
151 | 			}
152 | 			FileInputStream in = new FileInputStream(from);
153 | 			FileOutputStream out = new FileOutputStream(to);
154 | 			byte[] cache = new byte[(int) FSConfig.BLOCK_SIZE];
155 | 			try {
156 | 				int len = 0;
157 | 				while (true) {
158 | 					if (listener_sync.isCancel()) {
159 | 						return;
160 | 					}
161 | 					len = in.read(cache);
162 | 					if (len <= 0) {
163 | 						break;
164 | 					}
165 | 					out.write(cache, 0, len);
166 | 					listener_sync.work(len);
167 | 				}
168 | 				out.close();
169 | 				out = null;
170 | 				to.setLastModified(from.lastModified());
171 | 				FSConfig.cache(this.from, targetFile);
172 | 			} finally {
173 | 				in.close();
174 | 				if (null != out) {
175 | 					out.close();
176 | 				}
177 | 			}
178 | 		}
179 | 	}
180 | 
181 | 	private void update(String fromPath, String toPath,
182 | 			IProcessListener listener_sync) throws IOException {
183 | 		if (listener_sync.isCancel()) {
184 | 			return;
185 | 		}
186 | 		File from = new File(fromPath);
187 | 		File to = new File(toPath);
188 | 		if (!from.exists()) {
189 | 			LogHelper.warn("文件不存在:" + from.getAbsolutePath());
190 | 			return;
191 | 		}
192 | 		if (from.isDirectory()) {
193 | 			if (to.isFile()) {
194 | 				to.delete();
195 | 			}
196 | 			if (!to.isDirectory()) {
197 | 				to.mkdir();
198 | 			}
199 | 			to.setLastModified(from.lastModified());
200 | 		} else {
201 | 			if (to.isDirectory()) {
202 | 				del(toPath, listener_sync);
203 | 			}
204 | 			RandomAccessFile in = new RandomAccessFile(from, "r");
205 | 			RandomAccessFile out = new RandomAccessFile(to, "rw");
206 | 			byte[] cache = new byte[(int) FSConfig.BLOCK_SIZE];
207 | 			try {
208 | 				int len = 0;
209 | 				String[] fromArr = this.from.split(FSConfig.INFO_SEP);
210 | 				String[] fromContent = fromArr[FSConfig.CONTENT_INDEX]
211 | 						.split(FSConfig.CONTENT_SEP);
212 | 				String[] toContent = this.to.split(FSConfig.INFO_SEP)[FSConfig.CONTENT_INDEX]
213 | 						.split(FSConfig.CONTENT_SEP);
214 | 				for (int i = 1; i < fromContent.length; i++) {
215 | 					if (i < toContent.length) {
216 | 						if (toContent[i].equals(fromContent[i])) {
217 | 							continue;
218 | 						}
219 | 					}
220 | 					long pos = FSConfig.BLOCK_SIZE * (i - 1);
221 | 					in.seek(pos);
222 | 					len = in.read(cache);
223 | 					out.seek(pos);
224 | 					out.write(cache, 0, len);
225 | 					listener_sync.work(len);
226 | 				}
227 | 				out.setLength(from.length());
228 | 				out.close();
229 | 				out = null;
230 | 				to.setLastModified(from.lastModified());
231 | 				FSConfig.cache(this.from, targetFile);
232 | 			} finally {
233 | 				in.close();
234 | 				if (null != out) {
235 | 					out.close();
236 | 				}
237 | 			}
238 | 		}
239 | 	}
240 | 
241 | }


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/main/Main.java:
--------------------------------------------------------------------------------
  1 | package com.hjh.file.sync.main;
  2 | 
  3 | import java.io.File;
  4 | import java.io.IOException;
  5 | import java.util.List;
  6 | 
  7 | import com.hjh.file.sync.core.FSConfig;
  8 | import com.hjh.file.sync.core.SyncFolderInfo;
  9 | import com.hjh.file.sync.core.SyncItem;
 10 | import com.hjh.file.sync.process.CancelControl;
 11 | import com.hjh.file.sync.process.IProcessListener;
 12 | import com.hjh.file.sync.process.SimpleProcessListener;
 13 | import com.hjh.file.sync.util.FileUtils;
 14 | import com.hjh.file.sync.util.LogHelper;
 15 | 
 16 | /**
 17 |  * 程序入口
 18 |  * 
 19 |  * @author 洪 qq:2260806429
 20 |  */
 21 | public class Main {
 22 | 
 23 | 	public static void main(String argv[]) throws IOException {
 24 | 		File sourceFile = null;
 25 | 		File targetFile = null;
 26 | 		if (argv.length == 2) {
 27 | 			sourceFile = new File(argv[0]);
 28 | 			targetFile = new File(argv[1]);
 29 | 		} else {
 30 | 			LogHelper.info("请传入参数");
 31 | 			return;
 32 | 		}
 33 | 		if (!sourceFile.exists()) {
 34 | 			LogHelper.info("源对象不存在:" + sourceFile.getAbsolutePath());
 35 | 			return;
 36 | 		}
 37 | 		long start = System.currentTimeMillis();
 38 | 		try {
 39 | 			sync(sourceFile, targetFile);
 40 | 		} finally {
 41 | 			long cost = System.currentTimeMillis() - start;
 42 | 			try {
 43 | 				Thread.sleep(500);
 44 | 			} catch (InterruptedException e) {
 45 | 			}
 46 | 			LogHelper.info("同步完成耗时:" + printCostTime(cost));
 47 | 		}
 48 | 	}
 49 | 
 50 | 	public static String printCostTime(long cost) {
 51 | 		if (cost < 1000) {
 52 | 			return cost + "毫秒";
 53 | 		}
 54 | 		cost = cost / 1000;
 55 | 		if (cost < 60) {
 56 | 			return cost + "秒";
 57 | 		}
 58 | 		if (cost / 60 < 10) {
 59 | 			return cost / 60 + "分" + cost % 60 + "秒";
 60 | 		}
 61 | 		return cost / 60 / 60 + "小时" + cost / 60 % 60 + "分" + cost % 60 % 60
 62 | 				+ "秒";
 63 | 	}
 64 | 
 65 | 	public static void sync(File sourceFile, File targetFile)
 66 | 			throws IOException {
 67 | 		final CancelControl cancelControl = new CancelControl();
 68 | 		LogHelper.info("Sync from \"" + sourceFile.getAbsolutePath()
 69 | 				+ "\" TO \"" + targetFile.getAbsolutePath() + "\"");
 70 | 		final IProcessListener listener_source = new SimpleProcessListener(
 71 | 				cancelControl);
 72 | 		listener_source.print("source scan");
 73 | 		File cacheFile = new File(sourceFile, FSConfig.CACHE_ID_FILE);
 74 | 		if (cacheFile.isDirectory()) {
 75 | 			FileUtils.del(cacheFile);
 76 | 		}
 77 | 		final SyncFolderInfo source = new SyncFolderInfo(sourceFile);
 78 | 
 79 | 		final IProcessListener listener_target = new SimpleProcessListener(
 80 | 				cancelControl);
 81 | 		listener_target.print("target scan");
 82 | 		final SyncFolderInfo target = new SyncFolderInfo(targetFile);
 83 | 
 84 | 		final int[] finishcount = new int[1];
 85 | 
 86 | 		new Thread() {
 87 | 			public void run() {
 88 | 				try {
 89 | 					source.scan(listener_source);
 90 | 				} catch (IOException e) {
 91 | 					LogHelper.error(e);
 92 | 					cancelControl.cancel = true;
 93 | 				} finally {
 94 | 					finishcount[0]++;
 95 | 				}
 96 | 			}
 97 | 		}.start();
 98 | 
 99 | 		new Thread() {
100 | 			public void run() {
101 | 				try {
102 | 					target.scan(listener_target);
103 | 				} catch (IOException e) {
104 | 					LogHelper.error(e);
105 | 					cancelControl.cancel = true;
106 | 				} finally {
107 | 					finishcount[0]++;
108 | 				}
109 | 			}
110 | 		}.start();
111 | 
112 | 		while (finishcount[0] != 2) {
113 | 			try {
114 | 				Thread.sleep(1000);
115 | 			} catch (InterruptedException e) {
116 | 				LogHelper.error(e);
117 | 			}
118 | 		}
119 | 
120 | 		try {
121 | 			Thread.sleep(500);
122 | 		} catch (InterruptedException e) {
123 | 			LogHelper.error(e);
124 | 		}
125 | 
126 | 		long size = 0;
127 | 		List<SyncItem> items = source.sync(target);
128 | 		for (SyncItem item : items) {
129 | 			size += item.getSyncSize();
130 | 		}
131 | 		LogHelper.info("total sync size:"
132 | 				+ (size * 100 / (1024 * 1024) * 1.0 / 100) + "M");
133 | 		if (!targetFile.exists()) {
134 | 			targetFile.mkdir();
135 | 		}
136 | 		if (!targetFile.exists()) {
137 | 			throw new RuntimeException("创建目标文件夹失败");
138 | 		}
139 | 
140 | 		final IProcessListener listener_sync = new SimpleProcessListener(
141 | 				cancelControl, size);
142 | 		listener_sync.print("sync");
143 | 		for (SyncItem item : items) {
144 | 			if (listener_sync.isCancel()) {
145 | 				break;
146 | 			}
147 | 			item.sync(sourceFile, targetFile, listener_sync);
148 | 		}
149 | 
150 | 	}
151 | 
152 | }
153 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/process/CancelControl.java:
--------------------------------------------------------------------------------
1 | package com.hjh.file.sync.process;
2 | 
3 | public class CancelControl {
4 | 	public boolean cancel = false;
5 | 
6 | 	public boolean isCancel() {
7 | 		return cancel;
8 | 	}
9 | }


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/process/IProcessListener.java:
--------------------------------------------------------------------------------
 1 | package com.hjh.file.sync.process;
 2 | 
 3 | /**
 4 |  * 
 5 |  * 进度监听
 6 |  * 
 7 |  * @author 洪 qq:2260806429
 8 |  * 
 9 |  */
10 | public interface IProcessListener {
11 | 
12 | 	public void updateTotalSize(long totalSize);
13 | 
14 | 	public void work(long size);
15 | 
16 | 	public boolean isFinish();
17 | 
18 | 	public boolean isCancel();
19 | 
20 | 	public double getPercent();
21 | 
22 | 	public void print(String name);
23 | 
24 | }
25 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/process/ProcessPrinter.java:
--------------------------------------------------------------------------------
 1 | package com.hjh.file.sync.process;
 2 | 
 3 | import com.hjh.file.sync.util.LogHelper;
 4 | 
 5 | /**
 6 |  * @author 洪 qq:2260806429
 7 |  */
 8 | class ProcessPrinter {
 9 | 
10 | 	public boolean done;
11 | 	public String name;
12 | 
13 | 	public ProcessPrinter(String name) {
14 | 		this.name = name;
15 | 	}
16 | 
17 | 	public void start(final IProcessListener listener) {
18 | 		new Thread() {
19 | 			public void run() {
20 | 				done = false;
21 | 				try {
22 | 					do {
23 | 						LogHelper.info(name + " work:"
24 | 								+ String.format("%5.2f", listener.getPercent())
25 | 								+ "%");
26 | 						if (listener.isFinish()) {
27 | 							break;
28 | 						}
29 | 						try {
30 | 							Thread.sleep(500);
31 | 						} catch (InterruptedException e) {
32 | 							LogHelper.error(e);
33 | 						}
34 | 					} while (true);
35 | 				} finally {
36 | 					done = true;
37 | 				}
38 | 			}
39 | 		}.start();
40 | 	}
41 | 
42 | }
43 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/process/SimpleProcessListener.java:
--------------------------------------------------------------------------------
 1 | package com.hjh.file.sync.process;
 2 | 
 3 | /**
 4 |  * @author 洪 qq:2260806429
 5 |  */
 6 | public class SimpleProcessListener implements IProcessListener {
 7 | 
 8 | 	private CancelControl cancelControl;
 9 | 	private String name = "";
10 | 	private long totalsize;
11 | 	private long worksize = 0;
12 | 
13 | 	public SimpleProcessListener() {
14 | 		this(null);
15 | 	}
16 | 
17 | 	public SimpleProcessListener(CancelControl cancelControl) {
18 | 		this(cancelControl, 0);
19 | 	}
20 | 
21 | 	public SimpleProcessListener(CancelControl cancelControl, long totalSize) {
22 | 		this.cancelControl = cancelControl;
23 | 		this.totalsize = totalSize;
24 | 	}
25 | 
26 | 	@Override
27 | 	public void work(long size) {
28 | 		worksize += size;
29 | 	}
30 | 
31 | 	@Override
32 | 	public boolean isFinish() {
33 | 		return (totalsize != 0 && totalsize == worksize) || isCancel();
34 | 	}
35 | 
36 | 	@Override
37 | 	public double getPercent() {
38 | 		return ((int) (worksize * 1.0 / totalsize * 10000)) * 1.0 / 100;
39 | 	}
40 | 
41 | 	@Override
42 | 	public boolean isCancel() {
43 | 		return cancelControl == null ? false : cancelControl.isCancel();
44 | 	}
45 | 
46 | 	@Override
47 | 	public void updateTotalSize(long totalSize) {
48 | 		this.totalsize = totalSize;
49 | 		touchPrint();
50 | 	}
51 | 
52 | 	@Override
53 | 	public void print(String name) {
54 | 		this.name = name;
55 | 		touchPrint();
56 | 	}
57 | 
58 | 	private void touchPrint() {
59 | 		if (null != name && 0 != name.length() && 0 != this.totalsize) {
60 | 			new ProcessPrinter(name).start(this);
61 | 		}
62 | 	}
63 | 
64 | }
65 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/util/FileUtils.java:
--------------------------------------------------------------------------------
 1 | package com.hjh.file.sync.util;
 2 | 
 3 | import java.io.File;
 4 | 
 5 | public class FileUtils {
 6 | 
 7 | 	public static void del(File file) {
 8 | 		if (!file.exists()) {
 9 | 			return;
10 | 		}
11 | 		if (file.isDirectory()) {
12 | 			File list[] = file.listFiles();
13 | 			if (null != list) {
14 | 				for (File item : list) {
15 | 					del(item);
16 | 				}
17 | 			}
18 | 		}
19 | 		if (!file.delete()) {
20 | 			LogHelper.warn("删除文件失败:" + file.getAbsolutePath());
21 | 		}
22 | 	}
23 | 
24 | }
25 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/util/LogHelper.java:
--------------------------------------------------------------------------------
 1 | package com.hjh.file.sync.util;
 2 | 
 3 | /**
 4 |  * 日志工具
 5 |  * @author 洪 qq:2260806429
 6 |  */
 7 | public class LogHelper {
 8 | 
 9 | 	public static void info(String msg) {
10 | 		System.out.println("[INFO]" + msg);
11 | 	}
12 | 
13 | 	public static void warn(String msg) {
14 | 		System.out.println("[WARN]" + msg);
15 | 	}
16 | 
17 | 	public static void error(Exception e) {
18 | 		e.printStackTrace();
19 | 	}
20 | 
21 | }
22 | 


--------------------------------------------------------------------------------
/FileSync/src/com/hjh/file/sync/util/MD5.java:
--------------------------------------------------------------------------------
 1 | package com.hjh.file.sync.util;
 2 | 
 3 | import java.security.MessageDigest;
 4 | import java.security.NoSuchAlgorithmException;
 5 | 
 6 | /**
 7 |  * @author 洪 qq:2260806429
 8 |  */
 9 | public class MD5 {
10 | 
11 | 	public static String md5(byte[] datas, int pos, int len) {
12 | 		try {
13 | 			MessageDigest md = MessageDigest.getInstance("MD5");
14 | 			md.update(datas, pos, len);
15 | 			return toString(md);
16 | 		} catch (NoSuchAlgorithmException e) {
17 | 			throw new RuntimeException(e);
18 | 		}
19 | 	}
20 | 
21 | 	private static String toString(MessageDigest md) {
22 | 		byte b[] = md.digest();
23 | 		int i;
24 | 		StringBuffer buf = new StringBuffer("");
25 | 		for (int offset = 0; offset < b.length; offset++) {
26 | 			i = b[offset];
27 | 			if (i < 0)
28 | 				i += 256;
29 | 			if (i < 16)
30 | 				buf.append("0");
31 | 			buf.append(Integer.toHexString(i));
32 | 		}
33 | 		return buf.toString();
34 | 	}
35 | 
36 | }
37 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # file-sync
 2 | 数据同步工具<br/>
 3 | 
 4 | 一.问题背景<br/>
 5 | 经常碰到要同步数据的情况，而系统自带的复制功能又不能实现增量同步，每次都要做全量复制，发生异常情况后只能重头再来，非常麻烦，优其是对那种大文件的处理，更是耗时。<br/>
 6 | 二.解決方案<br/>
 7 | 1.计算源目录数据指纹<br/>
 8 | 2.计算目标目录数据指纹<br/>
 9 | 3.对比指纹数据，找出差异项，得到需要添加,删除或更新的文件列表，计算出需要更新的数据大小<br/>
10 | 4.挨个同步差异项，如果碰到大文件，则缓存其指纹数据到目标文件夹中，供下次同步数据时使用<br/>
11 | 三.数据指纹说明<br/>
12 | 数据指纹顾名思义，就是对某一文件夹或文件的唯一标识，其格式为:<br/>
13 | 文件相对路径＋:(分隔符)＋修改日期＋:+数据长度＋:+內容指纹<br/>
14 | 內容指纹是由多个内容块的md5组成<br/>
15 | 内容块就是对大文件进行分割处理，每次比较数据，最小的同步对象就是內容块，避免对整个文件做处理，也是实现增量同步的关健点<br/>
16 | 
17 | <br/>
18 | <br/>
19 | 
20 | <br/>
21 | 使用说明：<br/>
22 | 1.安装Jre<br/>
23 | 2.下载fileSync.jar<br/>
24 | 3.运行命令：java -jar fileSync.jar 源目录 目标目录<br/>
25 | <br/>
26 | 特性说明：<br/>
27 | 可增量同步，程序会检测源目录和目标目录已经同步好的数据，<br/>
28 | 对比之后继续同步未同步的数据<br/>
29 | 
30 | <br/>
31 | <br/>
32 | <img src="https://raw.githubusercontent.com/xxonehjh/file-sync/master/run.png"/><br/>


--------------------------------------------------------------------------------
/fileSync.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xxonehjh/file-sync/1c8fcc4b576d8a461a038d0c4066d15e883574da/fileSync.jar


--------------------------------------------------------------------------------
/run.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/xxonehjh/file-sync/1c8fcc4b576d8a461a038d0c4066d15e883574da/run.png


--------------------------------------------------------------------------------