();
125 | for(String sid : sids) {
126 | ids.add(NumberUtils.toLong(sid));
127 | }
128 | dataService.dealDataByIds(ids, null);
129 | String advice="重发了"+ids.toString();
130 | ByteBuf buf = copiedBuffer(advice, CharsetUtil.UTF_8);
131 | // Build the response object.
132 | FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, buf);
133 |
134 | response.headers().set(CONTENT_TYPE, "text/html; charset=UTF-8");
135 | response.headers().set(CONTENT_LENGTH, buf.readableBytes());
136 |
137 | // Write the response.
138 | ctx.channel().writeAndFlush(response);
139 | }
140 | }
141 |
142 | @Override
143 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
144 | ctx.channel().close();
145 | }
146 |
147 | @Override
148 | protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception {
149 | messageReceived(ctx, msg);
150 | }
151 | }
152 | }
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/common/Startup.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.common;
2 |
3 | import java.util.concurrent.atomic.AtomicInteger;
4 |
5 | import org.springframework.context.ApplicationContext;
6 | import org.springframework.context.support.ClassPathXmlApplicationContext;
7 | /**
8 | * 包含各种启动选项：包括加载spring上下文，是客户端需要直接调用的一个类
9 | *
10 | *
11 | * .==. .==.
12 | * //'^\\ //^'\\
13 | * // ^^\(\__/)/^ ^^\\
14 | * //^ ^^ ^/6 6\ ^^^ \\
15 | * //^ ^^ ^/( .. )\^ ^^ \\
16 | * // ^^ ^/\|v""v|/\^^ ^ \\
17 | * // ^^/\/ / '~~' \ \/\^ ^\\
18 | * ----------------------------------------
19 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
20 | *
21 | * @author 静儿(987489055@qq.com)
22 | *
23 | */
24 | public class Startup {
25 | /**
26 | * spring 上下文：路径请放于classpath的spring文件夹下
27 | */
28 | public static ApplicationContext context = new ClassPathXmlApplicationContext("spring/applicationContext.xml");//加载spring task
29 | /**
30 | * 当前运行中的全量线程计数器
31 | */
32 | public static final AtomicInteger threadNumber = new AtomicInteger(0);
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/exception/EpiphanyException.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.exception;
2 | /**
3 | *
4 | *
5 | *
6 | * .==. .==.
7 | * //'^\\ //^'\\
8 | * // ^^\(\__/)/^ ^^\\
9 | * //^ ^^ ^/6 6\ ^^^ \\
10 | * //^ ^^ ^/( .. )\^ ^^ \\
11 | * // ^^ ^/\|v""v|/\^^ ^ \\
12 | * // ^^/\/ / '~~' \ \/\^ ^\\
13 | * ----------------------------------------
14 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
15 | *
16 | * @author 静儿(987489055@qq.com)
17 | * 定义一个此功能抛出的业务异常
18 | */
19 | public class EpiphanyException extends RuntimeException {
20 | static final long serialVersionUID = 1L;
21 |
22 | /** Constructs a new runtime exception with {@code null} as its
23 | * detail message. The cause is not initialized, and may subsequently be
24 | * initialized by a call to {@link #initCause}.
25 | */
26 | public EpiphanyException() {
27 | super();
28 | }
29 |
30 | /** Constructs a new runtime exception with the specified detail message.
31 | * The cause is not initialized, and may subsequently be initialized by a
32 | * call to {@link #initCause}.
33 | *
34 | * @param message the detail message. The detail message is saved for
35 | * later retrieval by the {@link #getMessage()} method.
36 | */
37 | public EpiphanyException(String message) {
38 | super(message);
39 | }
40 |
41 | /**
42 | * Constructs a new runtime exception with the specified detail message and
43 | * cause. Note that the detail message associated with
44 | * {@code cause} is not automatically incorporated in
45 | * this runtime exception's detail message.
46 | *
47 | * @param message the detail message (which is saved for later retrieval
48 | * by the {@link #getMessage()} method).
49 | * @param cause the cause (which is saved for later retrieval by the
50 | * {@link #getCause()} method). (A null value is
51 | * permitted, and indicates that the cause is nonexistent or
52 | * unknown.)
53 | * @since 1.4
54 | */
55 | public EpiphanyException(String message, Throwable cause) {
56 | super(message, cause);
57 | }
58 |
59 | /** Constructs a new runtime exception with the specified cause and a
60 | * detail message of (cause==null ? null : cause.toString())
61 | * (which typically contains the class and detail message of
62 | * cause ). This constructor is useful for runtime exceptions
63 | * that are little more than wrappers for other throwables.
64 | *
65 | * @param cause the cause (which is saved for later retrieval by the
66 | * {@link #getCause()} method). (A null value is
67 | * permitted, and indicates that the cause is nonexistent or
68 | * unknown.)
69 | * @since 1.4
70 | */
71 | public EpiphanyException(Throwable cause) {
72 | super(cause);
73 | }
74 |
75 | /**
76 | * Constructs a new runtime exception with the specified detail
77 | * message, cause, suppression enabled or disabled, and writable
78 | * stack trace enabled or disabled.
79 | *
80 | * @param message the detail message.
81 | * @param cause the cause. (A {@code null} value is permitted,
82 | * and indicates that the cause is nonexistent or unknown.)
83 | * @param enableSuppression whether or not suppression is enabled
84 | * or disabled
85 | * @param writableStackTrace whether or not the stack trace should
86 | * be writable
87 | *
88 | * @since 1.7
89 | */
90 | protected EpiphanyException(String message, Throwable cause,
91 | boolean enableSuppression,
92 | boolean writableStackTrace) {
93 | super(message, cause, enableSuppression, writableStackTrace);
94 | }
95 |
96 | /**
97 | * Fills in the execution stack trace. This method records within this
98 | * {@code Throwable} object information about the current state of
99 | * the stack frames for the current thread.
100 | *
101 | *
If the stack trace of this {@code Throwable} {@linkplain
102 | * Throwable#Throwable(String, Throwable, boolean, boolean) is not
103 | * writable}, calling this method has no effect.
104 | *
105 | * @return a reference to this {@code Throwable} instance.
106 | * @see java.lang.Throwable#printStackTrace()
107 | */
108 | @Override
109 | public Throwable fillInStackTrace() {
110 | return this;
111 | }
112 | }
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/model/KeyValue.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.model;
2 |
3 | public class KeyValue {
4 | private String key;
5 | private String value;
6 | public String getKey() {
7 | return key;
8 | }
9 | public void setKey(String key) {
10 | this.key = key;
11 | }
12 | public String getValue() {
13 | return value;
14 | }
15 | public void setValue(String value) {
16 | this.value = value;
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/pool/LimitedKeyedPoolableObjectFactory.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.pool;
2 |
3 | import org.apache.commons.pool.BaseKeyedPoolableObjectFactory;
4 | import org.apache.commons.pool.KeyedObjectPool;
5 | import org.apache.commons.pool.KeyedObjectPoolFactory;
6 | import org.apache.commons.pool.KeyedPoolableObjectFactory;
7 | import org.apache.commons.pool.impl.StackKeyedObjectPoolFactory;
8 | import org.slf4j.Logger;
9 | import org.slf4j.LoggerFactory;
10 |
11 | import com.brmayi.epiphany.exception.EpiphanyException;
12 |
13 | /**
14 | *
15 | *
16 | *
17 | * .==. .==.
18 | * //'^\\ //^'\\
19 | * // ^^\(\__/)/^ ^^\\
20 | * //^ ^^ ^/6 6\ ^^^ \\
21 | * //^ ^^ ^/( .. )\^ ^^ \\
22 | * // ^^ ^/\|v""v|/\^^ ^ \\
23 | * // ^^/\/ / '~~' \ \/\^ ^\\
24 | * ----------------------------------------
25 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
26 | *
27 | * @author 静儿(987489055@qq.com)
28 | *
29 | */
30 | public class LimitedKeyedPoolableObjectFactory extends BaseKeyedPoolableObjectFactory {
31 | private static final Logger logger = LoggerFactory.getLogger(LimitedKeyedPoolableObjectFactory.class);
32 |
33 | private static final int MAX_NUM_ACTIVE = 100;
34 |
35 | private static final KeyedPoolableObjectFactory limitedFactory = new LimitedKeyedPoolableObjectFactory();
36 |
37 | private static final KeyedObjectPoolFactory limitedPoolFactory = new StackKeyedObjectPoolFactory(limitedFactory);
38 |
39 | public static KeyedObjectPool objectPool = limitedPoolFactory.createPool();
40 |
41 | @Override
42 | public Runnable makeObject(String paramK) throws EpiphanyException {
43 | try {
44 | if(objectPool.getNumActive()>=MAX_NUM_ACTIVE) {
45 | logger.error("limitedObjectPool active exceeded!");
46 | objectPool.clear();
47 | return null;
48 | }
49 | @SuppressWarnings("unchecked")
50 | Class cls = (Class) Class.forName(paramK);
51 | Runnable obj = cls.newInstance();
52 | logger.info("create class:{}", paramK);
53 | return obj;
54 | } catch(Exception e) {
55 | logger.error("limitedObjectPool exception", e);
56 | throw new EpiphanyException(e);
57 | }
58 | }
59 |
60 | @Override
61 | public void destroyObject(String key, Runnable obj){
62 | obj = null;
63 | }
64 | }
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/pool/UnlimitedKeyedPoolableObjectFactory.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.pool;
2 |
3 | import org.apache.commons.pool.BaseKeyedPoolableObjectFactory;
4 | import org.apache.commons.pool.KeyedObjectPool;
5 | import org.apache.commons.pool.KeyedObjectPoolFactory;
6 | import org.apache.commons.pool.KeyedPoolableObjectFactory;
7 | import org.apache.commons.pool.impl.StackKeyedObjectPoolFactory;
8 | import org.slf4j.Logger;
9 | import org.slf4j.LoggerFactory;
10 |
11 | import com.brmayi.epiphany.exception.EpiphanyException;
12 |
13 | /**
14 | *
15 | *
16 | *
17 | * .==. .==.
18 | * //'^\\ //^'\\
19 | * // ^^\(\__/)/^ ^^\\
20 | * //^ ^^ ^/6 6\ ^^^ \\
21 | * //^ ^^ ^/( .. )\^ ^^ \\
22 | * // ^^ ^/\|v""v|/\^^ ^ \\
23 | * // ^^/\/ / '~~' \ \/\^ ^\\
24 | * ----------------------------------------
25 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
26 | *
27 | * @author 静儿(987489055@qq.com)
28 | *
29 | */
30 | public class UnlimitedKeyedPoolableObjectFactory extends BaseKeyedPoolableObjectFactory {
31 | private static final Logger logger = LoggerFactory.getLogger(UnlimitedKeyedPoolableObjectFactory.class);
32 |
33 | private static final KeyedPoolableObjectFactory unlimitedFactory = new UnlimitedKeyedPoolableObjectFactory();
34 |
35 | private static final KeyedObjectPoolFactory unlimitedPoolFactory = new StackKeyedObjectPoolFactory(unlimitedFactory);
36 |
37 | public static KeyedObjectPool objectPool = unlimitedPoolFactory.createPool();
38 |
39 | @Override
40 | public Runnable makeObject(String paramK) throws EpiphanyException {
41 | try {
42 | @SuppressWarnings("unchecked")
43 | Class cls = (Class) Class.forName(paramK);
44 | Runnable obj = cls.newInstance();
45 | logger.info("create class:{}", paramK);
46 | return obj;
47 | } catch(Exception e) {
48 | logger.error("unlimitedObjectPool exception", e);
49 | throw new EpiphanyException(e);
50 | }
51 | }
52 |
53 | @Override
54 | public void destroyObject(String key, Runnable obj) throws Exception {
55 | obj = null;
56 | }
57 | }
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/serializer/ObjectRedisSerializer.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.serializer;
2 |
3 | import java.io.ByteArrayInputStream;
4 | import java.io.ByteArrayOutputStream;
5 | import java.io.ObjectInputStream;
6 | import java.io.ObjectOutputStream;
7 | import java.nio.charset.Charset;
8 |
9 | import org.springframework.data.redis.serializer.RedisSerializer;
10 | import org.springframework.data.redis.serializer.SerializationException;
11 |
12 | import com.brmayi.epiphany.exception.EpiphanyException;
13 | import com.brmayi.epiphany.util.GzCompressUtil;
14 | /**
15 | * 序列化工具
16 | *
17 | *
18 | * .==. .==.
19 | * //'^\\ //^'\\
20 | * // ^^\(\__/)/^ ^^\\
21 | * //^ ^^ ^/6 6\ ^^^ \\
22 | * //^ ^^ ^/( .. )\^ ^^ \\
23 | * // ^^ ^/\|v""v|/\^^ ^ \\
24 | * // ^^/\/ / '~~' \ \/\^ ^\\
25 | * ----------------------------------------
26 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
27 | *
28 | * @author 静儿(987489055@qq.com)
29 | *
30 | */
31 | public class ObjectRedisSerializer implements RedisSerializer {
32 | /**
33 | * 对象序列化为字符串
34 | * @param obj jvm对象
35 | * @return 序列化后的字符串
36 | * @throws EpiphanyException 非检查异常
37 | */
38 | @Override
39 | public byte[] serialize(Object obj) throws SerializationException {
40 | if(obj==null) {
41 | return null;
42 | }
43 | try ( ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
44 | ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteArrayOutputStream); ){
45 | objectOutputStream.writeObject(obj);
46 | return GzCompressUtil.compress(byteArrayOutputStream.toByteArray());
47 | } catch (Exception e) {
48 | throw new EpiphanyException(e);
49 | }
50 | }
51 |
52 |
53 | /**
54 | * 字符串反序列化为对象
55 | * @param redisData 序列化的字符串
56 | * @return jvm对象
57 | * @throws EpiphanyException 非检查异常
58 | */
59 | @Override
60 | public Object deserialize(byte[] redisData) throws SerializationException {
61 | if(redisData==null) {
62 | return null;
63 | }
64 | Object newObj = null;
65 | redisData = GzCompressUtil.decompress(redisData);
66 | try ( ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(redisData);
67 | ObjectInputStream objectInputStream = new ObjectInputStream(byteArrayInputStream); ){
68 | newObj = objectInputStream.readObject();
69 | } catch (Exception e) {
70 | throw new EpiphanyException(e);
71 | }
72 | return newObj;
73 | }
74 |
75 | }
76 |
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/service/DataService.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.service;
2 |
3 | import java.util.List;
4 |
5 | import com.brmayi.epiphany.exception.EpiphanyException;
6 | /**
7 | *
8 | * 通用文件处理类：这是业务代码的核心类
9 | *
10 | * .==. .==.
11 | * //'^\\ //^'\\
12 | * // ^^\(\__/)/^ ^^\\
13 | * //^ ^^ ^/6 6\ ^^^ \\
14 | * //^ ^^ ^/( .. )\^ ^^ \\
15 | * // ^^ ^/\|v""v|/\^^ ^ \\
16 | * // ^^/\/ / '~~' \ \/\^ ^\\
17 | * ----------------------------------------
18 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
19 | *
20 | * @author 静儿(987489055@qq.com)
21 | *
22 | */
23 | public interface DataService {
24 | /**
25 | * 根据ID进行业务数据处理,因为增量和全量复用此接口，可以在实现类中做判断，如果path为null为增量，否则为全量
26 | * @param dealIds 处理ID
27 | * @param path 要保存到的磁盘路径，不需要保存磁盘，可以为null
28 | * @throws EpiphanyException 抛出通用异常
29 | */
30 | public void dealDataByIds(List dealIds, String path) throws EpiphanyException;
31 |
32 | /**
33 | * 根据时间区间获取id列表
34 | * @param beginTime 开始时间
35 | * @param endTime 结束时间
36 | * @return id列表
37 | * @throws EpiphanyException 抛出通用异常
38 | */
39 | public List getIds(String beginTime, String endTime) throws EpiphanyException;
40 |
41 | /**
42 | * 根据id区间获取id列表
43 | * @param beginId 开始ID
44 | * @param endId 结束ID
45 | * @return id列表
46 | * @throws EpiphanyException 抛出通用异常
47 | */
48 | public List getIdsByBeginEnd(long beginId, long endId) throws EpiphanyException;
49 |
50 | /**
51 | * 根据开始结束ID处理数据
52 | * @param beginId 开始ID
53 | * @param endId 结束ID
54 | * @param path 要保存到的磁盘路径，不需要保存磁盘，可以为null
55 | * @throws EpiphanyException 抛出通用异常
56 | */
57 | public void dealDataByBeginEnd(long beginId, long endId, String path) throws EpiphanyException;
58 |
59 | /**
60 | * 取得最大ID
61 | * @return 最大ID
62 | * @throws EpiphanyException 抛出通用异常
63 | */
64 | public long getMaxId() throws EpiphanyException;
65 |
66 |
67 | /**
68 | * 取得最小ID
69 | * @return 最小ID
70 | * @throws EpiphanyException 抛出通用异常
71 | */
72 | public long getMinId() throws EpiphanyException;
73 |
74 | /**
75 | * 取得优先运行的数据
76 | * @return 数据ID队列
77 | * @throws EpiphanyException 抛出通用异常
78 | */
79 | public List getNiceQueue() throws EpiphanyException;
80 | }
81 |
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/util/EpiphanyFileUtil.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.util;
2 |
3 | import java.io.File;
4 | import java.io.IOException;
5 | import java.io.RandomAccessFile;
6 | import java.text.SimpleDateFormat;
7 | import java.util.Calendar;
8 |
9 | import org.slf4j.Logger;
10 | import org.slf4j.LoggerFactory;
11 |
12 | import com.brmayi.epiphany.exception.EpiphanyException;
13 |
14 | /**
15 | * 文件处理工具类
16 | *
17 | *
18 | * .==. .==.
19 | * //'^\\ //^'\\
20 | * // ^^\(\__/)/^ ^^\\
21 | * //^ ^^ ^/6 6\ ^^^ \\
22 | * //^ ^^ ^/( .. )\^ ^^ \\
23 | * // ^^ ^/\|v""v|/\^^ ^ \\
24 | * // ^^/\/ / '~~' \ \/\^ ^\\
25 | * ----------------------------------------
26 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
27 | *
28 | * @author 静儿(987489055@qq.com)
29 | *
30 | */
31 | public class EpiphanyFileUtil {
32 | private final static Logger LOGGER = LoggerFactory.getLogger(EpiphanyFileUtil.class);
33 |
34 | /**
35 | * 或缺文件路径
36 | * @param filePath 文件根路径
37 | * @return 加上当前时间目录的路径
38 | */
39 | public static String getPath(String filePath) {
40 | SimpleDateFormat sdfForDir = new SimpleDateFormat("yyyyMMdd");
41 | Calendar date = Calendar.getInstance();
42 | String dateFormat = sdfForDir.format(date.getTime());
43 | return new StringBuilder(filePath).append("/").append(dateFormat).append("/").toString();
44 | }
45 |
46 | /**
47 | * 创建磁盘目录
48 | * @param filePath 文件根路径
49 | * @return 加上当前时间目录的路径
50 | */
51 | public static String createPath(String filePath) {
52 | SimpleDateFormat sdfForDir = new SimpleDateFormat("yyyyMMdd");
53 | Calendar date = Calendar.getInstance();
54 | String dateFormat = sdfForDir.format(date.getTime());
55 | StringBuffer path = new StringBuffer(filePath).append("/").append(dateFormat);
56 | File f = new File(path.toString());
57 | if(f.exists()) {
58 | String[] children = f.list();
59 | for(String c : children) {
60 | new File(f, c).delete();
61 | }
62 | } else {
63 | f.mkdirs();
64 | }
65 | return path.append("/").toString();
66 | }
67 |
68 |
69 | /**
70 | * 将json数据写入磁盘文件，分批写释放内存
71 | * @param path 文件全路径
72 | * @param content 写入内容
73 | */
74 | public static void writeToFile(String path, String content) throws EpiphanyException{
75 | try (RandomAccessFile randomFile = new RandomAccessFile(path, "rw");){// 打开一个随机访问文件流，按读写方式
76 | // 文件长度，字节数
77 | long fileLength = randomFile.length();
78 | //将写文件指针移到文件尾。
79 | randomFile.seek(fileLength);
80 | randomFile.write(content.getBytes("UTF8"));
81 | } catch (IOException e) {
82 | LOGGER.error("写入全量专辑文件失败", e);
83 | throw new EpiphanyException(e);
84 | }
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/util/GzCompressUtil.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.util;
2 |
3 | import java.io.BufferedInputStream;
4 | import java.io.ByteArrayOutputStream;
5 | import java.io.FileInputStream;
6 | import java.io.FileNotFoundException;
7 | import java.io.FileOutputStream;
8 | import java.io.IOException;
9 | import java.util.concurrent.ExecutorService;
10 | import java.util.concurrent.Executors;
11 | import java.util.zip.Deflater;
12 | import java.util.zip.GZIPOutputStream;
13 | import java.util.zip.Inflater;
14 |
15 | import org.slf4j.Logger;
16 | import org.slf4j.LoggerFactory;
17 |
18 | import com.brmayi.epiphany.exception.EpiphanyException;
19 | import com.brmayi.epiphany.pool.UnlimitedKeyedPoolableObjectFactory;
20 | import com.brmayi.epiphany.util.sub.GzCompress;
21 |
22 | /**
23 | *
24 | *
25 | *
26 | * .==. .==.
27 | * //'^\\ //^'\\
28 | * // ^^\(\__/)/^ ^^\\
29 | * //^ ^^ ^/6 6\ ^^^ \\
30 | * //^ ^^ ^/( .. )\^ ^^ \\
31 | * // ^^ ^/\|v""v|/\^^ ^ \\
32 | * // ^^/\/ / '~~' \ \/\^ ^\\
33 | * ----------------------------------------
34 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
35 | *
36 | * 这里单独将压缩提出来是因为压缩计算量特别大，非常占用CPU，所以需要限制线程数
37 | * 建议将大文件写入磁盘，这时候速度和压缩比，后者更重要，所以采用gzip而没有支持deflate实现
38 | * @author 静儿(987489055@qq.com)
39 | *
40 | */
41 | public class GzCompressUtil {
42 | private final static Logger LOGGER = LoggerFactory.getLogger(GzCompressUtil.class);
43 | private static final ExecutorService fixedThreadPoolForGz = Executors.newFixedThreadPool(6);
44 |
45 |
46 | /**
47 | * 压缩
48 | *
49 | * @param data
50 | * 待压缩数据
51 | * @return byte[] 压缩后的数据
52 | */
53 | public static byte[] compress(byte[] data) {
54 | byte[] output = new byte[0];
55 |
56 | Deflater compresser = new Deflater();
57 | compresser.reset();
58 | compresser.setInput(data);
59 | compresser.finish();
60 | try (ByteArrayOutputStream bos = new ByteArrayOutputStream(data.length)){
61 | byte[] buf = new byte[data.length];
62 | while (!compresser.finished()) {
63 | int i = compresser.deflate(buf);
64 | bos.write(buf, 0, i);
65 | }
66 | output = bos.toByteArray();
67 | } catch (Exception e) {
68 | output = data;
69 | LOGGER.error("队列消息压缩错误", e);
70 | }
71 | compresser.end();
72 | return output;
73 | }
74 |
75 | /**
76 | * 解压缩
77 | *
78 | * @param data 待压缩的数据
79 | * @return byte[] 解压缩后的数据
80 | */
81 | public static byte[] decompress(byte[] data) {
82 | byte[] output = new byte[0];
83 |
84 | Inflater decompresser = new Inflater();
85 | decompresser.reset();
86 | decompresser.setInput(data);
87 |
88 | try (ByteArrayOutputStream o = new ByteArrayOutputStream(data.length);){
89 | byte[] buf = new byte[1024];
90 | while (!decompresser.finished()) {
91 | int i = decompresser.inflate(buf);
92 | o.write(buf, 0, i);
93 | }
94 | output = o.toByteArray();
95 | } catch (Exception e) {
96 | output = data;
97 | e.printStackTrace();
98 | }
99 | decompresser.end();
100 | return output;
101 | }
102 | /**
103 | * 执行gz压缩
104 | * @param path 文件路径
105 | * @throws EpiphanyException 抛出压缩异常
106 | */
107 | @Deprecated
108 | public static void gzCompress(String path) throws EpiphanyException {
109 | try {
110 | GzCompress gzCompressTask = (GzCompress) UnlimitedKeyedPoolableObjectFactory.objectPool.borrowObject(GzCompress.class.getName());
111 | gzCompressTask.setPath(path);
112 | fixedThreadPoolForGz.execute(gzCompressTask);//压缩
113 | } catch(Exception e) {
114 | LOGGER.error("gzcompress exception", e);
115 | throw new EpiphanyException(e);
116 | }
117 | }
118 |
119 | /**
120 | * 同步压缩，为了好计算是否终止状态，将来真正有需求可以支持同步和异步两种模式
121 | * @param path 文件路径
122 | * @throws EpiphanyException 抛出压缩异常
123 | */
124 | public static void compressFile(String path) throws EpiphanyException {
125 | try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path));
126 | GZIPOutputStream out = new GZIPOutputStream(new FileOutputStream(path+".gz"));){
127 | byte[] array = new byte[1024];
128 | int number = -1;
129 | while((number = in.read(array, 0, array.length)) != -1) {
130 | out.write(array, 0, number);
131 | }
132 | } catch (FileNotFoundException e) {
133 | LOGGER.error("压缩错误", e);
134 | throw new EpiphanyException(e);
135 | } catch (IOException e) {
136 | LOGGER.error("压缩错误", e);
137 | throw new EpiphanyException(e);
138 | }
139 | }
140 | }
141 |
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/util/TimerUtil.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.util;
2 |
3 | import java.util.Calendar;
4 | import java.util.Date;
5 | import java.util.Timer;
6 | import java.util.TimerTask;
7 | /**
8 | *
9 | *
10 | *
11 | * .==. .==.
12 | * //'^\\ //^'\\
13 | * // ^^\(\__/)/^ ^^\\
14 | * //^ ^^ ^/6 6\ ^^^ \\
15 | * //^ ^^ ^/( .. )\^ ^^ \\
16 | * // ^^ ^/\|v""v|/\^^ ^ \\
17 | * // ^^/\/ / '~~' \ \/\^ ^\\
18 | * ----------------------------------------
19 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
20 | *
21 | * @author 静儿(987489055@qq.com)
22 | *
23 | */
24 | public class TimerUtil {
25 | private static final long PERIOD_DAY = 24 * 60 * 60 * 1000;
26 | private static final Timer timer = new Timer();
27 |
28 | /**
29 | * 每天执行一次的定时器
30 | * @param task 要执行的定时任务
31 | * @param hour 几点执行
32 | * @param minute 几分执行
33 | * @param second 几秒执行
34 | */
35 | public static void runEveryday(TimerTask task, int hour, int minute, int second) {
36 | Calendar calendar = Calendar.getInstance();
37 | calendar.set(Calendar.HOUR_OF_DAY, hour);
38 | calendar.set(Calendar.MINUTE, minute);
39 | calendar.set(Calendar.SECOND, second);
40 | Date date=calendar.getTime(); //第一次执行定时任务的时间
41 | if (date.before(new Date())) {
42 | date = addDay(date, 1);
43 | }
44 | timer.schedule(task,date,PERIOD_DAY);
45 | }
46 |
47 | // 增加或减少天数
48 | private static Date addDay(Date date, int num) {
49 | Calendar startDT = Calendar.getInstance();
50 | startDT.setTime(date);
51 | startDT.add(Calendar.DAY_OF_MONTH, num);
52 | return startDT.getTime();
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/src/main/java/com/brmayi/epiphany/util/sub/GzCompress.java:
--------------------------------------------------------------------------------
1 | package com.brmayi.epiphany.util.sub;
2 |
3 | import java.io.BufferedInputStream;
4 | import java.io.FileInputStream;
5 | import java.io.FileNotFoundException;
6 | import java.io.FileOutputStream;
7 | import java.io.IOException;
8 | import java.util.zip.GZIPOutputStream;
9 |
10 | import org.slf4j.Logger;
11 | import org.slf4j.LoggerFactory;
12 |
13 | import com.brmayi.epiphany.util.GzCompressUtil;
14 | /**
15 | *
16 | *
17 | *
18 | * .==. .==.
19 | * //'^\\ //^'\\
20 | * // ^^\(\__/)/^ ^^\\
21 | * //^ ^^ ^/6 6\ ^^^ \\
22 | * //^ ^^ ^/( .. )\^ ^^ \\
23 | * // ^^ ^/\|v""v|/\^^ ^ \\
24 | * // ^^/\/ / '~~' \ \/\^ ^\\
25 | * ----------------------------------------
26 | * HERE BE DRAGONS WHICH CAN CREATE MIRACLE
27 | *
28 | * @author 静儿(987489055@qq.com)
29 | *
30 | */
31 | public class GzCompress implements Runnable {
32 | private String path;
33 | private final static Logger LOGGER = LoggerFactory.getLogger(GzCompressUtil.class);
34 | /**
35 | * hadoop支持gz格式自解压，gz可以压缩到原文本文件的1/7
36 | *
37 | * @param path 压缩的文件路径
38 | */
39 | public void setPath(String path) {
40 | this.path = path;
41 | }
42 |
43 | /**
44 | * hadoop支持gz格式自解压，gz可以压缩到原文本文件的1/7
45 | *
46 | */
47 | @Override
48 | public void run() {
49 | try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path));
50 | GZIPOutputStream out = new GZIPOutputStream(new FileOutputStream(path+".gz"));){
51 | byte[] array = new byte[1024];
52 | int number = -1;
53 | while((number = in.read(array, 0, array.length)) != -1) {
54 | out.write(array, 0, number);
55 | }
56 | } catch (FileNotFoundException e) {
57 | LOGGER.error("压缩错误", e);
58 | } catch (IOException e) {
59 | LOGGER.error("压缩错误", e);
60 | }
61 | }
62 | }
--------------------------------------------------------------------------------