├── src ├── main │ ├── resources │ │ ├── testJson │ │ │ └── testJson.json │ │ ├── cookie │ │ │ └── index.txt │ │ ├── config.rmi │ │ └── config.sys │ └── java │ │ └── org │ │ └── decaywood │ │ ├── entity │ │ ├── trend │ │ │ ├── ITrendBlock.java │ │ │ ├── Trend.java │ │ │ ├── MarketIndexTrend.java │ │ │ ├── CubeTrend.java │ │ │ ├── ShareHoldersTrend.java │ │ │ └── Rebalancing.java │ │ ├── DeepCopy.java │ │ ├── Entry.java │ │ ├── selectorQuota │ │ │ ├── QuotaHead.java │ │ │ ├── AbstractQuotaNode.java │ │ │ ├── QuotaChainNode.java │ │ │ ├── BasicQuota.java │ │ │ ├── XueQiuQuota.java │ │ │ └── MarketQuotationsQuota.java │ │ ├── User.java │ │ ├── Industry.java │ │ ├── Comment.java │ │ ├── CompanyInfo.java │ │ ├── CapitalFlow.java │ │ ├── PostInfo.java │ │ ├── Cube.java │ │ └── LongHuBangInfo.java │ │ ├── timeWaitingStrategy │ │ ├── TimeWaitingStrategy.java │ │ └── DefaultTimeWaitingStrategy.java │ │ ├── utils │ │ ├── RequestParaBuilder.java │ │ ├── MathUtils.java │ │ ├── JsonParser.java │ │ ├── StringUtils.java │ │ ├── URLMapper.java │ │ ├── HttpRequestHelper.java │ │ ├── DateParser.java │ │ └── FileLoader.java │ │ ├── mapper │ │ ├── user │ │ │ └── UserSetMapper.java │ │ ├── dateFirst │ │ │ └── DateToLongHuBangStockMapper.java │ │ ├── AbstractMapper.java │ │ ├── industryFirst │ │ │ └── IndustryToStocksMapper.java │ │ ├── comment │ │ │ └── CommentSetMapper.java │ │ ├── stockFirst │ │ │ ├── StockToCapitalFlowEntryMapper.java │ │ │ ├── StockToStockWithAttributeMapper.java │ │ │ ├── StockToLongHuBangMapper.java │ │ │ ├── StockToStockWithShareHolderTrendMapper.java │ │ │ ├── StockToVIPFollowerCountEntryMapper.java │ │ │ ├── StockToStockWithCompanyInfoMapper.java │ │ │ └── StockToStockWithStockTrendMapper.java │ │ └── cubeFirst │ │ │ ├── CubeToCubeWithLastBalancingMapper.java │ │ │ └── CubeToCubeWithTrendMapper.java │ │ ├── AbstractRequester.java │ │ ├── filter │ │ ├── PageKeyFilter.java │ │ └── AbstractFilter.java │ │ ├── collector │ │ ├── DateRangeCollector.java │ │ ├── CommissionIndustryCollector.java │ │ ├── AbstractCollector.java │ │ ├── StockCommentCollector.java │ │ ├── StockSlectorBaseCollector.java │ │ ├── UserCommentCollector.java │ │ ├── CommentCollector.java │ │ ├── StockScopeHotRankCollector.java │ │ ├── HuShenNewsRefCollector.java │ │ ├── MostProfitableCubeCollector.java │ │ └── MarketQuotationsRankCollector.java │ │ ├── acceptor │ │ └── AbstractAcceptor.java │ │ ├── CookieProcessor.java │ │ └── GlobalSystemConfigLoader.java └── test │ └── java │ ├── GlobalSystemConfigLoaderTest.java │ ├── utilTest │ ├── FileLoaderTest.java │ ├── StringUtilsTest.java │ └── DateParserTest.java │ ├── collectTest │ ├── StockCommentCollectorTest.java │ ├── HuShenNewsRefCollectorTest.java │ ├── CommissionIndustryCollectorTest.java │ ├── DateRangeCollectorTest.java │ ├── MostProfitableCubeCollectorTest.java │ ├── MarketQuotationsRankCollectorTest.java │ └── StockScopeHotRankCollectorTest.java │ ├── mapperTest │ ├── StockToStockWithStockTrendMapperTest.java │ ├── StockToCapitalFlowEntryMapperTest.java │ ├── StockToVIPFollowerCountEntryMapperTest.java │ ├── CubeToCubeWithLastBalancingMapperTest.java │ ├── IndustryToStocksMapperTest.java │ ├── DateToLongHuBangStockMapperTest.java │ ├── StockToLongHuBangMapperTest.java │ ├── TestCaseGenerator.java │ ├── StockToStockWithCompanyInfoMapperTest.java │ ├── CubeToCubeWithTrendMapperTest.java │ ├── StockToStockWithShareHolderTrendMapperTest.java │ └── StockToStockWithAttributeMapperTest.java │ ├── filterTest │ └── PageKeyFilterTest.java │ └── entryFirst │ └── UserInfoToDBAcceptor.java ├── info ├── structure.png └── RMI.md ├── .gitignore ├── LICENSE └── pom.xml /src/main/resources/testJson/testJson.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/main/resources/cookie/index.txt: -------------------------------------------------------------------------------- 1 | 内部文件系统定位文件，勿删！！！ -------------------------------------------------------------------------------- /info/structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/decaywood/XueQiuSuperSpider/HEAD/info/structure.png -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/trend/ITrendBlock.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.trend; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * @author: decaywood 7 | * @date: 2015/12/13 17:19. 8 | */ 9 | public interface ITrendBlock extends Serializable { 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/DeepCopy.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * @author: decaywood 7 | * @date: 2015/11/24 19:52 8 | */ 9 | public interface DeepCopy extends Serializable { 10 | 11 | R copy(); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/resources/config.rmi: -------------------------------------------------------------------------------- 1 | ## JVM配置文件 2 | ## '##' 为注释符号，代表此行被忽略 3 | ## 此文件为RMI配置文件 4 | 5 | ## 客户端请求服务IP地址（IP:port，可添加多个服务IP以进行负载分流 xxx.xxx.xxx.xxx:xxx形式 不能为localhost ） 6 | master_req_ip = 192.168.1.100:7779 7 | 8 | 9 | ## 服务端接受服务IP地址（IP:port 不能为localhost ） 10 | slave_rcv_ip = 192.168.1.100:7779 11 | -------------------------------------------------------------------------------- /src/main/resources/config.sys: -------------------------------------------------------------------------------- 1 | ## JVM配置文件 2 | ## '##' 为注释符号，代表此行被忽略 3 | 4 | ## 并行流底层线程数 5 | java.util.concurrent.ForkJoinPool.common.parallelism = 4 6 | cookies = add your own cookie fetched from browser 7 | ## areaCode = 86 可选 默认国内 8 | ## userID = 186xxxxxxxx 9 | ## password = xxxx 10 | ## rememberMe = true 可选 默认开启 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # excluding gitignore in sub folders 2 | */.gitignore 3 | 4 | # eclipse setting files 5 | /.settings 6 | */.settings 7 | .project 8 | .classpath 9 | maven-eclipse.xml 10 | 11 | # idea 12 | /.idea 13 | */.idea 14 | *.iml 15 | .DS_Store 16 | 17 | # maven target folder 18 | /target 19 | */target 20 | 21 | 22 | -------------------------------------------------------------------------------- /src/test/java/GlobalSystemConfigLoaderTest.java: -------------------------------------------------------------------------------- 1 | import org.decaywood.GlobalSystemConfigLoader; 2 | import org.junit.Test; 3 | 4 | /** 5 | * @author: decaywood 6 | * @date: 2015/12/11 13:33 7 | */ 8 | public class GlobalSystemConfigLoaderTest { 9 | 10 | @Test 11 | public void test() { 12 | GlobalSystemConfigLoader.loadConfig(); 13 | // Assert.assertEquals("192.168.1.155:7779", GlobalSystemConfigLoader.getRMIConfig("server_rcv_ip")); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/timeWaitingStrategy/TimeWaitingStrategy.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.timeWaitingStrategy; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/24 16:16 6 | */ 7 | 8 | /** 9 | * 超时等待策略 10 | */ 11 | public interface TimeWaitingStrategy { 12 | 13 | /** 14 | * 等待逻辑 15 | * @param loopTime 循环到loopTime次 16 | */ 17 | void waiting(int loopTime); 18 | 19 | /** 20 | * 最多重试次数 21 | */ 22 | int retryTimes(); 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/Entry.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/30 21:35. 6 | */ 7 | public class Entry { 8 | 9 | private final K key; 10 | private final V value; 11 | 12 | public Entry(K key, V value) { 13 | this.key = key; 14 | this.value = value; 15 | } 16 | 17 | public K getKey() { 18 | return key; 19 | } 20 | 21 | public V getValue() { 22 | return value; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/utilTest/FileLoaderTest.java: -------------------------------------------------------------------------------- 1 | package utilTest; 2 | 3 | import org.decaywood.utils.FileLoader; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | import java.io.File; 8 | 9 | /** 10 | * @author: decaywood 11 | * @date: 2015/12/2 10:52 12 | */ 13 | public class FileLoaderTest { 14 | 15 | @Test 16 | public void testFileLoader() { 17 | FileLoader.updateCookie("I am a cookie", "baidu"); 18 | File file = FileLoader.loadFile("config.sys"); 19 | Assert.assertTrue(file.exists()); 20 | } 21 | 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/trend/Trend.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.trend; 2 | 3 | import org.decaywood.entity.DeepCopy; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * @author: decaywood 9 | * @date: 2015/11/26 9:57 10 | */ 11 | 12 | 13 | /** 14 | * 抽象类， 保存历史趋势 15 | */ 16 | public abstract class Trend implements DeepCopy { 17 | 18 | protected final List history; 19 | 20 | public Trend(List history) { 21 | if(history == null) throw new IllegalArgumentException(); 22 | this.history = history; 23 | } 24 | 25 | public List getHistory() { 26 | return history; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/collectTest/StockCommentCollectorTest.java: -------------------------------------------------------------------------------- 1 | package collectTest; 2 | 3 | import org.decaywood.collector.StockCommentCollector; 4 | import org.decaywood.entity.Comment; 5 | import org.decaywood.entity.PostInfo; 6 | import org.decaywood.mapper.comment.CommentSetMapper; 7 | import org.junit.Assert; 8 | import org.junit.Test; 9 | 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | 13 | /** 14 | * @author decaywood 15 | * @date 2020/10/7 22:05 16 | */ 17 | public class StockCommentCollectorTest { 18 | 19 | @Test 20 | public void test() { 21 | List infos = new StockCommentCollector("SH688180", StockCommentCollector.SortType.alpha, 1, 1).get(); 22 | Assert.assertEquals(1, infos.size()); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/StockToStockWithStockTrendMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.Stock; 4 | import org.decaywood.mapper.stockFirst.StockToStockWithStockTrendMapper; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import java.rmi.RemoteException; 9 | import java.util.List; 10 | 11 | /** 12 | * @author: decaywood 13 | * @date: 2015/11/25 20:05 14 | */ 15 | public class StockToStockWithStockTrendMapperTest { 16 | 17 | @Test 18 | public void testFunc() { 19 | StockToStockWithStockTrendMapper mapper = new StockToStockWithStockTrendMapper(); 20 | List stocks = TestCaseGenerator.generateStocks(); 21 | boolean res = stocks.stream().map(mapper.andThen(Stock::getStockTrend)).noneMatch(x -> x.getHistory().isEmpty()); 22 | Assert.assertTrue(res); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/StockToCapitalFlowEntryMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.Stock; 4 | import org.decaywood.mapper.stockFirst.StockToCapitalFlowEntryMapper; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import java.rmi.RemoteException; 9 | import java.util.List; 10 | 11 | /** 12 | * @author: decaywood 13 | * @date: 2015/12/2 12:34 14 | */ 15 | public class StockToCapitalFlowEntryMapperTest { 16 | 17 | @Test 18 | public void testFunc() { 19 | StockToCapitalFlowEntryMapper mapper = new StockToCapitalFlowEntryMapper(); 20 | List stocks = TestCaseGenerator.generateStocks(); 21 | boolean match = stocks 22 | .stream() 23 | .map(mapper) 24 | .allMatch(x -> x.getValue().getFiveDayInflows().size() == 5); 25 | Assert.assertTrue(match); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/filterTest/PageKeyFilterTest.java: -------------------------------------------------------------------------------- 1 | package filterTest; 2 | 3 | import org.decaywood.filter.PageKeyFilter; 4 | import org.junit.Test; 5 | 6 | import java.net.URL; 7 | import java.util.regex.Matcher; 8 | import java.util.regex.Pattern; 9 | 10 | /** 11 | * @author: decaywood 12 | * @date: 2015/12/3 10:26 13 | */ 14 | public class PageKeyFilterTest { 15 | 16 | 17 | @Test 18 | public void testFunc() throws Exception { 19 | URL url = new URL("http://xueqiu.com/2306062563/60965010"); 20 | PageKeyFilter filter = new PageKeyFilter("协鑫集成", true); 21 | boolean res = filter.test(url); 22 | System.out.println(res); 23 | } 24 | 25 | @Test 26 | public void regexTest() { 27 | Pattern pattern = Pattern.compile("哥"); 28 | Matcher matcher = pattern.matcher("阿迪啊无法无法阿哥挨饿不是人"); 29 | boolean res = matcher.find(); 30 | System.out.println(res); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/collectTest/HuShenNewsRefCollectorTest.java: -------------------------------------------------------------------------------- 1 | package collectTest; 2 | 3 | import org.decaywood.collector.HuShenNewsRefCollector; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | import java.net.URL; 8 | import java.rmi.RemoteException; 9 | import java.util.List; 10 | 11 | /** 12 | * @author: decaywood 13 | * @date: 2015/12/2 22:18. 14 | */ 15 | public class HuShenNewsRefCollectorTest { 16 | 17 | @Test 18 | public void testCorrectArg() { 19 | new HuShenNewsRefCollector(HuShenNewsRefCollector.Topic.TOTAL, 3); 20 | } 21 | 22 | 23 | @Test(expected = IllegalArgumentException.class) 24 | public void testWrongArgument2() { 25 | new HuShenNewsRefCollector(HuShenNewsRefCollector.Topic.TOTAL, 0); 26 | } 27 | 28 | @Test 29 | public void testFunc() { 30 | List res = new HuShenNewsRefCollector().get(); 31 | Assert.assertFalse(res.isEmpty()); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/StockToVIPFollowerCountEntryMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.Entry; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.mapper.stockFirst.StockToVIPFollowerCountEntryMapper; 6 | import org.junit.Test; 7 | 8 | import java.rmi.RemoteException; 9 | import java.util.List; 10 | 11 | /** 12 | * @author: decaywood 13 | * @date: 2015/11/30 17:06 14 | */ 15 | public class StockToVIPFollowerCountEntryMapperTest { 16 | 17 | 18 | @Test 19 | public void testFunc() { 20 | 21 | List stocks = TestCaseGenerator.generateStocks(); 22 | StockToVIPFollowerCountEntryMapper mapper = new StockToVIPFollowerCountEntryMapper(); 23 | stocks.parallelStream().forEach(x -> { 24 | Entry count = mapper.apply(x); 25 | System.out.println(x.getStockName() + "粉丝过万的关注者人数为： " + count.getValue() + " 人"); 26 | }); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/test/java/collectTest/CommissionIndustryCollectorTest.java: -------------------------------------------------------------------------------- 1 | package collectTest; 2 | 3 | import org.decaywood.collector.CommissionIndustryCollector; 4 | import org.decaywood.entity.Industry; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import java.util.List; 9 | 10 | /** 11 | * @author: decaywood 12 | * @date: 2015/11/25 10:45 13 | */ 14 | public class CommissionIndustryCollectorTest { 15 | 16 | @Test 17 | public void test() throws Exception { 18 | CommissionIndustryCollector commissionIndustryCollector = new CommissionIndustryCollector(); 19 | List list = commissionIndustryCollector.collectLogic(); 20 | Assert.assertEquals(list.size(), 77); 21 | } 22 | 23 | @Test 24 | public void testNull() throws Exception { 25 | CommissionIndustryCollector commissionIndustryCollector = new CommissionIndustryCollector(null); 26 | List list = commissionIndustryCollector.collectLogic(); 27 | Assert.assertEquals(list.size(), 77); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/selectorQuota/QuotaHead.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.selectorQuota; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/28 12:48 6 | */ 7 | public class QuotaHead extends AbstractQuotaNode { 8 | 9 | private String category = "SH"; 10 | private String exchange = ""; //市场 11 | private String areacode = ""; //地域 12 | private String indcode = ""; //板块代码 13 | private String orderby = "symbol"; //排序方式 14 | private String order = "desc"; 15 | private String page = "1"; 16 | 17 | 18 | 19 | 20 | @Override 21 | StringBuilder builderSelf() { 22 | StringBuilder builder = new StringBuilder("?"); 23 | append(builder, "category", category); 24 | append(builder, "exchange", exchange); 25 | append(builder, "areacode", areacode); 26 | append(builder, "indcode", indcode); 27 | append(builder, "orderby", orderby); 28 | append(builder, "order", order); 29 | append(builder, "page", page); 30 | 31 | return builder; 32 | } 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/CubeToCubeWithLastBalancingMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/26 16:08 6 | */ 7 | public class CubeToCubeWithLastBalancingMapperTest { 8 | 9 | // @Test(expected = IllegalArgumentException.class) 10 | // public void testWrongArgument() { 11 | // 12 | // CubeToCubeWithLastBalancingMapper cubeCubeFirst = 13 | // new CubeToCubeWithLastBalancingMapper(-1); 14 | // Cube cube = new Cube("446272", "港股低价倍增", "ZH221124"); 15 | // Cube mapped = cubeCubeFirst.apply(cube); 16 | // } 17 | // 18 | // @Test 19 | // public void testFunction() { 20 | // 21 | // CubeToCubeWithLastBalancingMapper cubeCubeFirst = 22 | // new CubeToCubeWithLastBalancingMapper(5); 23 | // Cube cube = new Cube("446272", "港股低价倍增", "ZH221124"); 24 | // Cube mapped = cubeCubeFirst.apply(cube); 25 | // Assert.assertNotNull(mapped.getRebalancing()); 26 | // Assert.assertFalse(mapped.getRebalancing().getHistory().isEmpty()); 27 | // 28 | // } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/selectorQuota/AbstractQuotaNode.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.selectorQuota; 2 | 3 | import org.decaywood.utils.EmptyObject; 4 | 5 | /** 6 | * @author: decaywood 7 | * @date: 2015/11/28 15:36 8 | */ 9 | 10 | /** 11 | * 股票指标节点抽象类，负责对指标参数的组装 12 | */ 13 | public abstract class AbstractQuotaNode implements QuotaChainNode { 14 | 15 | private QuotaChainNode next = EmptyObject.emptyQuotaChainNode; 16 | 17 | @Override 18 | public QuotaChainNode getNext() { 19 | return next; 20 | } 21 | 22 | @Override 23 | public void setNext(QuotaChainNode next) { 24 | if(next == null) return; 25 | if(!end()) getNext().setNext(next); 26 | else this.next = next; 27 | } 28 | 29 | @Override 30 | public String generateQuotaRequest() { 31 | StringBuilder builder = builderSelf(); 32 | if(!this.end()) 33 | builder.append(this.getNext().generateQuotaRequest()); 34 | builder.deleteCharAt(builder.length() - 1); 35 | return builder.toString(); 36 | } 37 | 38 | abstract StringBuilder builderSelf(); 39 | } 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) [2016] [decaywood] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/utils/RequestParaBuilder.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.utils; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/23 16:57 6 | */ 7 | public class RequestParaBuilder { 8 | 9 | private StringBuilder config; 10 | 11 | public RequestParaBuilder(String target) { 12 | this.config = new StringBuilder(target); 13 | this.config.append('?'); 14 | } 15 | 16 | public RequestParaBuilder addParameter(String key, String val) { 17 | this.config.append(key).append("=").append(val).append("&"); 18 | return this; 19 | } 20 | 21 | public RequestParaBuilder addParameter(String key, boolean val) { 22 | this.config.append(key).append("=").append(val).append("&"); 23 | return this; 24 | } 25 | 26 | public RequestParaBuilder addParameter(String key, int val) { 27 | this.config.append(key).append("=").append(val).append("&"); 28 | return this; 29 | } 30 | 31 | public RequestParaBuilder addParameter(String key, long val) { 32 | this.config.append(key).append("=").append(val).append("&"); 33 | return this; 34 | } 35 | 36 | public String build() { 37 | return this.config.substring(0, config.length() - 1); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/IndustryToStocksMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.Industry; 4 | import org.decaywood.mapper.industryFirst.IndustryToStocksMapper; 5 | import org.decaywood.utils.EmptyObject; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import java.rmi.RemoteException; 10 | import java.util.List; 11 | 12 | /** 13 | * @author: decaywood 14 | * @date: 2015/11/25 19:04 15 | */ 16 | public class IndustryToStocksMapperTest { 17 | 18 | 19 | @Test 20 | public void testFunction() { 21 | List industries = TestCaseGenerator.generateIndustries(); 22 | IndustryToStocksMapper mapper = new IndustryToStocksMapper(); 23 | Assert.assertTrue(industries.stream() 24 | .map(mapper) 25 | .noneMatch(List::isEmpty)); 26 | } 27 | 28 | @Test 29 | public void testNull() { 30 | IndustryToStocksMapper mapper = new IndustryToStocksMapper(); 31 | Assert.assertNotNull(mapper.apply(null)); 32 | Assert.assertNotNull(mapper.apply(EmptyObject.emptyIndustry)); 33 | Assert.assertTrue(mapper.apply(null).isEmpty()); 34 | Assert.assertTrue(mapper.apply(EmptyObject.emptyIndustry).isEmpty()); 35 | } 36 | 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/DateToLongHuBangStockMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.Stock; 4 | import org.decaywood.mapper.dateFirst.DateToLongHuBangStockMapper; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import java.rmi.RemoteException; 9 | import java.util.Calendar; 10 | import java.util.List; 11 | 12 | /** 13 | * @author: decaywood 14 | * @date: 2015/11/27 11:12 15 | */ 16 | public class DateToLongHuBangStockMapperTest { 17 | 18 | @Test 19 | public void testFunction() { 20 | DateToLongHuBangStockMapper mapper = new DateToLongHuBangStockMapper(null); 21 | Calendar calendar = Calendar.getInstance(); 22 | calendar.set(2015, Calendar.NOVEMBER, 26); 23 | List stocks = mapper.apply(calendar.getTime()); 24 | Assert.assertFalse(stocks.isEmpty()); 25 | } 26 | 27 | @Test 28 | public void testNoExchangeDate() { 29 | 30 | DateToLongHuBangStockMapper mapper = new DateToLongHuBangStockMapper(null); 31 | Calendar calendar = Calendar.getInstance(); 32 | calendar.set(2015, Calendar.NOVEMBER, 21); 33 | List stocks = mapper.apply(calendar.getTime()); 34 | Assert.assertTrue(stocks.isEmpty()); 35 | 36 | 37 | } 38 | 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/utils/MathUtils.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.utils; 2 | 3 | import java.util.Arrays; 4 | 5 | /** 6 | * @author: decaywood 7 | * @date: 2015/12/9 20:38 8 | */ 9 | public abstract class MathUtils { 10 | 11 | public static double min(double... nums) { 12 | if(nums.length == 0) return Double.MIN_VALUE; 13 | return Arrays.stream(nums).min().getAsDouble(); 14 | } 15 | 16 | public static int min(int... nums) { 17 | if(nums.length == 0) return Integer.MIN_VALUE; 18 | return Arrays.stream(nums).min().getAsInt(); 19 | } 20 | 21 | 22 | public static long min(long... nums) { 23 | if(nums.length == 0) return Long.MIN_VALUE; 24 | return Arrays.stream(nums).min().getAsLong(); 25 | } 26 | 27 | 28 | public static double max(double... nums) { 29 | if(nums.length == 0) return Double.MIN_VALUE; 30 | return Arrays.stream(nums).max().getAsDouble(); 31 | } 32 | 33 | public static int max(int... nums) { 34 | if(nums.length == 0) return Integer.MIN_VALUE; 35 | return Arrays.stream(nums).max().getAsInt(); 36 | } 37 | 38 | 39 | public static long max(long... nums) { 40 | if(nums.length == 0) return Long.MIN_VALUE; 41 | return Arrays.stream(nums).max().getAsLong(); 42 | } 43 | 44 | 45 | 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/collectTest/DateRangeCollectorTest.java: -------------------------------------------------------------------------------- 1 | package collectTest; 2 | 3 | import org.decaywood.collector.DateRangeCollector; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | import java.rmi.RemoteException; 8 | import java.util.Calendar; 9 | import java.util.Date; 10 | import java.util.List; 11 | 12 | /** 13 | * @author: decaywood 14 | * @date: 2015/11/27 10:17 15 | */ 16 | public class DateRangeCollectorTest { 17 | 18 | @Test(expected = IllegalArgumentException.class) 19 | public void testWrongArgument() { 20 | Calendar calendar = Calendar.getInstance(); 21 | calendar.set(2015, Calendar.SEPTEMBER, 21); 22 | Date from = calendar.getTime(); 23 | calendar.set(2016, Calendar.OCTOBER, 24); 24 | Date to = calendar.getTime(); 25 | 26 | new DateRangeCollector(to, from); 27 | } 28 | 29 | 30 | @Test 31 | public void functionTest() { 32 | 33 | Calendar calendar = Calendar.getInstance(); 34 | 35 | calendar.set(2015, Calendar.SEPTEMBER, 21); 36 | Date from = calendar.getTime(); 37 | 38 | calendar.set(2015, Calendar.OCTOBER, 21); 39 | Date to = calendar.getTime(); 40 | List strings = new DateRangeCollector(from, to).get(); 41 | 42 | Assert.assertTrue(strings.size() == 31); 43 | 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/utils/JsonParser.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.utils; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.springframework.util.ReflectionUtils; 5 | import org.springframework.util.StringUtils; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Optional; 10 | import java.util.function.BiConsumer; 11 | import java.util.function.Supplier; 12 | 13 | /** 14 | * @author decaywood 15 | * @date 2020/10/7 17:15 16 | */ 17 | public abstract class JsonParser { 18 | 19 | public static List parseArray(Supplier supplier, BiConsumer consumer, JsonNode jsonNode) { 20 | List res = new ArrayList<>(); 21 | for (JsonNode node : jsonNode) { 22 | T parse = parse(supplier, node); 23 | consumer.accept(parse, node); 24 | res.add(parse); 25 | } 26 | return res; 27 | } 28 | 29 | public static T parse(Supplier supplier, JsonNode jsonNode) { 30 | T t = supplier.get(); 31 | ReflectionUtils.doWithFields(t.getClass(), field -> { 32 | ReflectionUtils.makeAccessible(field); 33 | String v = Optional.ofNullable(jsonNode.get(field.getName())).map(JsonNode::asText).orElse(null); 34 | field.set(t, v); 35 | }, f -> f.getType() == String.class); 36 | return t; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/User.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | /** 4 | * @author decaywood 5 | * @date 2020/10/7 17:37 6 | */ 7 | public class User { 8 | 9 | public interface UserSetter { 10 | void setUser(User user); 11 | String getUserId(); 12 | } 13 | 14 | /** 昵称 */ 15 | private String screen_name; 16 | /** 粉丝数量 */ 17 | private String followers_count; 18 | /** 自选股票数 */ 19 | private String stocks_count; 20 | /** 关注人数 */ 21 | private String friends_count; 22 | 23 | public String getScreen_name() { 24 | return screen_name; 25 | } 26 | 27 | public void setScreen_name(String screen_name) { 28 | this.screen_name = screen_name; 29 | } 30 | 31 | public String getFollowers_count() { 32 | return followers_count; 33 | } 34 | 35 | public void setFollowers_count(String followers_count) { 36 | this.followers_count = followers_count; 37 | } 38 | 39 | public String getStocks_count() { 40 | return stocks_count; 41 | } 42 | 43 | public void setStocks_count(String stocks_count) { 44 | this.stocks_count = stocks_count; 45 | } 46 | 47 | public String getFriends_count() { 48 | return friends_count; 49 | } 50 | 51 | public void setFriends_count(String friends_count) { 52 | this.friends_count = friends_count; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/user/UserSetMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.user; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.User; 5 | import org.decaywood.mapper.AbstractMapper; 6 | import org.decaywood.utils.JsonParser; 7 | import org.decaywood.utils.RequestParaBuilder; 8 | import org.decaywood.utils.URLMapper; 9 | import org.springframework.util.StringUtils; 10 | 11 | import java.net.URL; 12 | 13 | /** 14 | * 用户信息组装器 15 | * @author decaywood 16 | * @date 2020/10/7 17:41 17 | */ 18 | public class UserSetMapper extends AbstractMapper { 19 | 20 | public UserSetMapper() { 21 | super(null); 22 | } 23 | 24 | @Override 25 | protected User.UserSetter mapLogic(User.UserSetter obj) throws Exception { 26 | String userId = obj.getUserId(); 27 | if (StringUtils.isEmpty(userId)) { 28 | return obj; 29 | } 30 | String target = URLMapper.USER_INFO_JSON.toString(); 31 | RequestParaBuilder builder = new RequestParaBuilder(target) 32 | .addParameter("id", userId); 33 | URL url = new URL(builder.build()); 34 | String json = request(url); 35 | JsonNode node = mapper.readTree(json); 36 | User user = JsonParser.parse(User::new, node); 37 | obj.setUser(user); 38 | return obj; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/AbstractRequester.java: -------------------------------------------------------------------------------- 1 | package org.decaywood; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.decaywood.timeWaitingStrategy.DefaultTimeWaitingStrategy; 5 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 6 | import org.decaywood.utils.HttpRequestHelper; 7 | 8 | import java.io.IOException; 9 | import java.net.URL; 10 | import java.rmi.RemoteException; 11 | 12 | /** 13 | * 请求发送 14 | * @author: decaywood 15 | * @date: 2015/12/4 13:35 16 | */ 17 | public abstract class AbstractRequester { 18 | 19 | static { 20 | /** 21 | * 加载全局配置 22 | */ 23 | GlobalSystemConfigLoader.loadConfig(); 24 | } 25 | 26 | protected String webSite; 27 | 28 | protected TimeWaitingStrategy strategy; 29 | 30 | protected ObjectMapper mapper; 31 | 32 | public AbstractRequester(TimeWaitingStrategy strategy, String webSite) { 33 | super(); 34 | this.webSite = webSite; 35 | this.strategy = strategy == null ? new DefaultTimeWaitingStrategy() : strategy; 36 | this.mapper = new ObjectMapper(); 37 | 38 | } 39 | 40 | protected String request(URL url) throws IOException { 41 | return new HttpRequestHelper(webSite).request(url); 42 | } 43 | 44 | protected String requestWithoutGzip(URL url) throws IOException { 45 | return new HttpRequestHelper(webSite, false).request(url); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/filter/PageKeyFilter.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.filter; 2 | 3 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 4 | 5 | import java.net.URL; 6 | import java.rmi.RemoteException; 7 | import java.util.regex.Matcher; 8 | import java.util.regex.Pattern; 9 | 10 | /** 11 | * @author: decaywood 12 | * @date: 2015/12/3 9:45 13 | */ 14 | public class PageKeyFilter extends AbstractFilter { 15 | 16 | private String key; 17 | private boolean regex; 18 | 19 | 20 | 21 | public PageKeyFilter(String key, boolean regex) { 22 | this(null, key, regex); 23 | } 24 | 25 | 26 | /** 27 | * @param strategy 超时等待策略（null则设置为默认等待策略） 28 | * @param key 关键字 29 | * @param regex 是否启用正则表达式 30 | */ 31 | public PageKeyFilter(TimeWaitingStrategy strategy, String key, boolean regex) { 32 | super(strategy); 33 | this.key = key; 34 | this.regex = regex; 35 | } 36 | 37 | @Override 38 | protected boolean filterLogic(URL url) throws Exception { 39 | if (url == null) return false; 40 | 41 | String pageContent = request(url); 42 | boolean res; 43 | 44 | if(regex) { 45 | Pattern pattern = Pattern.compile(key); 46 | Matcher matcher = pattern.matcher(pageContent); 47 | res = matcher.find(); 48 | } else res = pageContent.contains(key); 49 | 50 | return res; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/timeWaitingStrategy/DefaultTimeWaitingStrategy.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.timeWaitingStrategy; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/24 16:21 6 | */ 7 | 8 | /** 9 | * 默认超时等待策略 10 | */ 11 | public class DefaultTimeWaitingStrategy implements TimeWaitingStrategy { 12 | 13 | private final long timeWaitingThreshold; 14 | private final long timeWaiting; 15 | 16 | private final int retryTime; 17 | 18 | public DefaultTimeWaitingStrategy() { 19 | this(10000, 500, 10); 20 | } 21 | 22 | 23 | /** 24 | * 25 | * @param timeWaitingThreshold 超时等待阈值（最多等待阈值指定时间然后进入下一次请求尝试） 26 | * @param timeWaiting 起始等待时间 27 | * @param retryTime 重试次数（超过次数抛出超时异常） 28 | */ 29 | public DefaultTimeWaitingStrategy(final long timeWaitingThreshold, long timeWaiting, int retryTime) { 30 | this.timeWaitingThreshold = timeWaitingThreshold; 31 | this.timeWaiting = timeWaiting; 32 | this.retryTime = retryTime; 33 | } 34 | 35 | 36 | @Override 37 | public void waiting(int loopTime) { 38 | try { 39 | 40 | long sleepTime = this.timeWaiting * (2 << loopTime); 41 | sleepTime = Math.min(sleepTime, timeWaitingThreshold); 42 | Thread.sleep(sleepTime); 43 | 44 | } catch (InterruptedException e) { 45 | e.printStackTrace(); 46 | } 47 | } 48 | 49 | @Override 50 | public int retryTimes() { 51 | return retryTime; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/selectorQuota/QuotaChainNode.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.selectorQuota; 2 | 3 | import org.decaywood.utils.DateParser; 4 | import org.decaywood.utils.EmptyObject; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @author: decaywood 10 | * @date: 2015/11/28 13:03 11 | */ 12 | 13 | /** 14 | * 股票选择指标统一接口 15 | */ 16 | public interface QuotaChainNode extends Serializable { 17 | 18 | 19 | //获取下一个指标节点 20 | QuotaChainNode getNext(); 21 | //设置指标节点 22 | void setNext(QuotaChainNode next); 23 | //生成请求字符串 24 | String generateQuotaRequest(); 25 | 26 | 27 | //------------------------------------ 以下为系统方法 --------------------------------------------- 28 | 29 | 30 | default String getTimePrefix() { 31 | return "." + DateParser.getTimePrefix(true); 32 | } 33 | 34 | default boolean end() { 35 | return getNext() == EmptyObject.emptyQuotaChainNode; 36 | } 37 | 38 | default QuotaChainNode append(StringBuilder builder, String paramName, String param) { 39 | if("ALL".equals(param)) return this; 40 | builder.append(paramName).append("=").append(param).append("&"); 41 | return this; 42 | } 43 | 44 | default QuotaChainNode appendWithTimePrefix(StringBuilder builder, String paramName, String param) { 45 | if("ALL".equals(param)) return this; 46 | builder.append(paramName.concat(getTimePrefix())).append("=").append(param).append("&"); 47 | return this; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/StockToLongHuBangMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.LongHuBangInfo; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.mapper.stockFirst.StockToLongHuBangMapper; 6 | import org.junit.Assert; 7 | import org.junit.Rule; 8 | import org.junit.Test; 9 | import org.junit.rules.ExpectedException; 10 | 11 | import java.util.Calendar; 12 | import java.util.Date; 13 | 14 | /** 15 | * @author: decaywood 16 | * @date: 2015/11/27 15:46 17 | */ 18 | public class StockToLongHuBangMapperTest { 19 | 20 | @Rule 21 | public ExpectedException thrown= ExpectedException.none(); 22 | 23 | @Test() 24 | public void testWrongArgument() throws Exception{ 25 | 26 | Stock stock = new Stock("中飞股份", "SZ300489"); 27 | StockToLongHuBangMapper mapper = new StockToLongHuBangMapper(); 28 | thrown.expect(IllegalArgumentException.class); 29 | thrown.expectMessage("lost parameter: stockQueryDate"); 30 | mapper.mapLogic(stock); 31 | } 32 | 33 | @Test 34 | public void testFunction() { 35 | Stock stock = new Stock("中飞股份", "SZ300489"); 36 | 37 | Calendar calendar = Calendar.getInstance(); 38 | calendar.set(2015, Calendar.NOVEMBER, 25); 39 | Date since = calendar.getTime(); 40 | stock.setStockQueryDate(since); 41 | 42 | LongHuBangInfo info = new StockToLongHuBangMapper().apply(stock); 43 | 44 | Assert.assertFalse(info.getTopBuyList().isEmpty()); 45 | Assert.assertFalse(info.getTopSaleList().isEmpty()); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/Industry.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/23 13:42 6 | */ 7 | 8 | /** 9 | * 行业板块 10 | */ 11 | public class Industry implements DeepCopy { 12 | 13 | private final String industryName;//板块名字 14 | 15 | private final String industryInfo;//板块代码 16 | 17 | 18 | public Industry(final String industryName, final String industrySiteURL) { 19 | this.industryName = industryName; 20 | this.industryInfo = industrySiteURL; 21 | } 22 | 23 | 24 | 25 | public String getIndustryName() { 26 | return industryName; 27 | } 28 | 29 | public String getIndustryInfo() { 30 | return industryInfo; 31 | } 32 | 33 | @Override 34 | public boolean equals(Object o) { 35 | if (this == o) return true; 36 | if (o == null || getClass() != o.getClass()) return false; 37 | 38 | Industry industry = (Industry) o; 39 | 40 | return industryName.equals(industry.industryName) && industryInfo.equals(industry.industryInfo); 41 | 42 | } 43 | 44 | @Override 45 | public int hashCode() { 46 | int result = industryName.hashCode(); 47 | result = 31 * result + industryInfo.hashCode(); 48 | return result; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return "industryName = " + industryName + " " + "industryInfo = " + industryInfo; 54 | } 55 | 56 | @Override 57 | public Industry copy() { 58 | return new Industry(industryName, industryInfo); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/utils/StringUtils.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.utils; 2 | 3 | import java.util.Arrays; 4 | 5 | /** 6 | * @author: decaywood 7 | * @date: 2015/11/26 10:19 8 | */ 9 | public abstract class StringUtils { 10 | 11 | public static boolean isNull(String string) { 12 | return !isNotNull(string); 13 | } 14 | 15 | public static boolean isNotNull(String string) { 16 | return string != null; 17 | } 18 | 19 | public static boolean isNumeric(String test) { 20 | return test != null 21 | && test.length() > 0 22 | && test.chars().allMatch(Character::isDigit); 23 | } 24 | 25 | public static boolean nullOrEmpty(String... args) { 26 | return Arrays.stream(args) 27 | .anyMatch(x -> x == null || x.length() == 0 || EmptyObject.emptyString.equals(x)); 28 | } 29 | 30 | public static String string2Unicode(String string) { 31 | 32 | StringBuilder unicode = new StringBuilder(); 33 | for (int i = 0; i < string.length(); i++) { 34 | char c = string.charAt(i); 35 | unicode.append("\\u").append(Integer.toHexString(c)); 36 | } 37 | return unicode.toString(); 38 | } 39 | 40 | public static String unicode2String(String unicode) { 41 | 42 | StringBuilder string = new StringBuilder(); 43 | String[] hex = unicode.split("\\\\u"); 44 | for (int i = 1; i < hex.length; i++) { 45 | int data = Integer.parseInt(hex[i], 16); 46 | string.append((char) data); 47 | } 48 | return string.toString(); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/utilTest/StringUtilsTest.java: -------------------------------------------------------------------------------- 1 | package utilTest; 2 | 3 | import org.decaywood.utils.EmptyObject; 4 | import org.decaywood.utils.StringUtils; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | /** 9 | * @author: decaywood 10 | * @date: 2015/11/26 10:23 11 | */ 12 | public class StringUtilsTest { 13 | 14 | @Test 15 | public void digitCheckTest() { 16 | /* 17 | '-' is not acceptable 18 | */ 19 | String case1 = "125125125135"; 20 | String case2 = "-1245124124"; // not acceptable 21 | String case3 = "afwaw12412aw"; 22 | String case4 = "12124w124124"; 23 | String case5 = ""; 24 | 25 | Assert.assertTrue(StringUtils.isNumeric(case1)); 26 | Assert.assertFalse(StringUtils.isNumeric(case2)); 27 | Assert.assertFalse(StringUtils.isNumeric(case3)); 28 | Assert.assertFalse(StringUtils.isNumeric(case4)); 29 | Assert.assertFalse(StringUtils.isNumeric(case5)); 30 | Assert.assertFalse(StringUtils.isNumeric(null)); 31 | 32 | } 33 | 34 | @Test 35 | public void notNullOrEmptyCheckTest() { 36 | Assert.assertTrue(StringUtils.nullOrEmpty("xx", null, "xxxx")); 37 | Assert.assertTrue(StringUtils.nullOrEmpty("xx", null, EmptyObject.emptyString)); 38 | Assert.assertTrue(StringUtils.nullOrEmpty("xx", "", "xx")); 39 | Assert.assertFalse(StringUtils.nullOrEmpty("xx", "xx", "xxx")); 40 | } 41 | 42 | @Test 43 | public void unicodeTest() { 44 | Assert.assertEquals("雪球超级爬虫", StringUtils.unicode2String(StringUtils.string2Unicode("雪球超级爬虫"))); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/TestCaseGenerator.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.Cube; 4 | import org.decaywood.entity.Industry; 5 | import org.decaywood.entity.Stock; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | /** 11 | * @author: decaywood 12 | * @date: 2015/11/25 19:04 13 | */ 14 | public class TestCaseGenerator { 15 | 16 | public static List generateIndustries() { 17 | List industries = new ArrayList<>(); 18 | industries.add(new Industry("有色金属冶炼和压延加工业","#exchange=CN&plate=1_3_66&firstName=1&secondName=1_3&level2code=C32")); 19 | industries.add(new Industry("渔业","#exchange=CN&plate=1_3_67&firstName=1&secondName=1_3&level2code=A04")); 20 | industries.add(new Industry("医药制造业","#exchange=CN&plate=1_3_68&firstName=1&secondName=1_3&level2code=C27")); 21 | industries.add(new Industry("资本市场服务","#exchange=CN&plate=1_3_69&firstName=1&secondName=1_3&level2code=J67")); 22 | return industries; 23 | } 24 | 25 | 26 | public static List generateStocks() { 27 | List stocks = new ArrayList<>(); 28 | stocks.add(new Stock("银之杰", "SZ300085")); 29 | stocks.add(new Stock("利德曼","SZ300289")); 30 | stocks.add(new Stock("国元证券","SZ000728")); 31 | return stocks; 32 | } 33 | 34 | public static List generateCube() { 35 | List cubes = new ArrayList<>(); 36 | cubes.add(new Cube("xxx", "xxx", "ZH128412")); //沪深组合 37 | cubes.add(new Cube("xxx", "xxx", "ZH102164")); //港股组合 38 | cubes.add(new Cube("xxx", "xxx", "ZH739627")); //美股组合 39 | return cubes; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/StockToStockWithCompanyInfoMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.CompanyInfo; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.mapper.stockFirst.StockToStockWithCompanyInfoMapper; 6 | import org.decaywood.utils.StringUtils; 7 | import org.junit.Assert; 8 | import org.junit.Test; 9 | 10 | import java.rmi.RemoteException; 11 | import java.util.List; 12 | import java.util.stream.Collectors; 13 | 14 | /** 15 | * @author: decaywood 16 | * @date: 2015/12/4 16:00 17 | */ 18 | public class StockToStockWithCompanyInfoMapperTest { 19 | 20 | 21 | @Test 22 | public void testFunc() { 23 | 24 | StockToStockWithCompanyInfoMapper mapper = new StockToStockWithCompanyInfoMapper(); 25 | List stocks = TestCaseGenerator.generateStocks(); 26 | List companyInfos = stocks.stream() 27 | .map(mapper.andThen(Stock::getCompanyInfo)) 28 | .collect(Collectors.toList()); 29 | for (CompanyInfo companyInfo : companyInfos) { 30 | Assert.assertFalse(StringUtils.nullOrEmpty(companyInfo.getBizscope())); 31 | Assert.assertFalse(StringUtils.nullOrEmpty(companyInfo.getCompsname())); 32 | Assert.assertFalse(StringUtils.nullOrEmpty(companyInfo.getFounddate())); 33 | Assert.assertFalse(StringUtils.nullOrEmpty(companyInfo.getMajorbiz())); 34 | Assert.assertFalse(StringUtils.nullOrEmpty(companyInfo.getOrgtype())); 35 | Assert.assertFalse(StringUtils.nullOrEmpty(companyInfo.getRegion())); 36 | Assert.assertFalse(companyInfo.getTqCompIndustryList().isEmpty()); 37 | } 38 | 39 | } 40 | 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/DateRangeCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 4 | 5 | import java.rmi.RemoteException; 6 | import java.util.ArrayList; 7 | import java.util.Calendar; 8 | import java.util.Date; 9 | import java.util.List; 10 | 11 | /** 12 | * @author: decaywood 13 | * @date: 2015/11/27 0:27 14 | */ 15 | 16 | /** 17 | * 时间范围收集器，获取一段时间的Date 18 | */ 19 | public class DateRangeCollector extends AbstractCollector > { 20 | 21 | private final Date from; 22 | private final Date to; 23 | 24 | public DateRangeCollector(Date from, Date to) { 25 | this(null, from, to); 26 | } 27 | 28 | 29 | /** 30 | * 31 | * @param strategy 超时等待策略（null则设置为默认等待策略） 32 | * @param from 起始时间 33 | * @param to 结束时间 34 | */ 35 | public DateRangeCollector(TimeWaitingStrategy strategy, Date from, Date to) { 36 | super(strategy); 37 | if(from == null || to == null || from.after(to)) throw new IllegalArgumentException(); 38 | this.from = from; 39 | this.to = to; 40 | } 41 | 42 | @Override 43 | public List collectLogic() throws Exception { 44 | 45 | List dates = new ArrayList<>(); 46 | Calendar calendar = Calendar.getInstance(); 47 | StringBuilder builder = new StringBuilder(); 48 | 49 | for (Date i = from; i.before(to) || i.equals(to); ) { 50 | 51 | dates.add(i); 52 | 53 | calendar.setTime(i); 54 | builder.delete(0, builder.length()); 55 | calendar.add(Calendar.DATE, 1); 56 | i = calendar.getTime(); 57 | 58 | } 59 | 60 | return dates; 61 | 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/CubeToCubeWithTrendMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.Cube; 4 | import org.decaywood.mapper.cubeFirst.CubeToCubeWithTrendMapper; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import java.rmi.RemoteException; 9 | import java.util.Calendar; 10 | import java.util.Date; 11 | import java.util.List; 12 | 13 | /** 14 | * @author: decaywood 15 | * @date: 2015/11/26 11:15 16 | */ 17 | public class CubeToCubeWithTrendMapperTest { 18 | 19 | @Test(expected = IllegalArgumentException.class) 20 | public void testWrongArgument() { 21 | 22 | Calendar calendar = Calendar.getInstance(); 23 | 24 | calendar.set(2014, Calendar.AUGUST, 5); 25 | Date since = calendar.getTime(); 26 | 27 | calendar.set(2015, Calendar.MAY, 28); 28 | Date until = calendar.getTime(); 29 | 30 | new CubeToCubeWithTrendMapper(until, since); 31 | 32 | } 33 | 34 | @Test 35 | public void testFunction() { 36 | 37 | Calendar calendar = Calendar.getInstance(); 38 | 39 | calendar.set(2014, Calendar.AUGUST, 5); 40 | Date since = calendar.getTime(); 41 | 42 | calendar.set(2015, Calendar.MAY, 28); 43 | Date until = calendar.getTime(); 44 | 45 | CubeToCubeWithTrendMapper cubeCubeFirst = new CubeToCubeWithTrendMapper(since, until); 46 | 47 | List cubes = TestCaseGenerator.generateCube(); 48 | 49 | cubes.stream() 50 | .map(cubeCubeFirst) 51 | .peek(Assert::assertNotNull) 52 | .map(Cube::getCubeTrend) 53 | .forEach(cubeTrend -> Assert.assertNotNull(cubeTrend.getHistory())); 54 | 55 | cubes.stream() 56 | .map(cubeCubeFirst) 57 | .peek(Assert::assertNotNull) 58 | .map(Cube::getMarketIndexTrend) 59 | .forEach(cubeTrend -> Assert.assertNotNull(cubeTrend.getHistory())); 60 | 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/CommissionIndustryCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import org.decaywood.entity.Industry; 4 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 5 | import org.decaywood.utils.URLMapper; 6 | import org.jsoup.Jsoup; 7 | import org.jsoup.nodes.Document; 8 | import org.jsoup.nodes.Element; 9 | import org.jsoup.select.Elements; 10 | 11 | import java.net.URL; 12 | import java.rmi.RemoteException; 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | 16 | /** 17 | * @author: decaywood 18 | * @date: 2015/11/23 10:50 19 | */ 20 | 21 | /** 22 | * 板块收集器，收集证监会所有板块类型编号以及名称 23 | */ 24 | public class CommissionIndustryCollector extends AbstractCollector> { 25 | 26 | public CommissionIndustryCollector() { 27 | this(null); 28 | } 29 | 30 | /** 31 | *@param strategy 超时等待策略（null则设置为默认等待策略） 32 | */ 33 | public CommissionIndustryCollector(TimeWaitingStrategy strategy) { 34 | super(strategy); 35 | } 36 | 37 | @Override 38 | public List collectLogic() throws Exception { 39 | 40 | List res = new ArrayList<>(); 41 | 42 | String target = URLMapper.COMPREHENSIVE_PAGE.toString(); 43 | 44 | String content = request(new URL(target)); 45 | Document doc = Jsoup.parse(content); 46 | Elements element = doc.getElementsByClass("second-nav") 47 | .get(1).children() 48 | .get(2).children() 49 | .get(3).children() 50 | .select("a"); 51 | StringBuilder builder = new StringBuilder(); 52 | for (Element ele : element) { 53 | if (!ele.hasAttr("title") || !ele.hasAttr("href")) continue; 54 | builder.append(ele.attr("href")); 55 | res.add(new Industry(ele.attr("title"), builder.toString())); 56 | builder.delete(0, builder.length()); 57 | } 58 | 59 | return res; 60 | } 61 | 62 | 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/filter/AbstractFilter.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.filter; 2 | 3 | import org.decaywood.AbstractRequester; 4 | import org.decaywood.CookieProcessor; 5 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 6 | import org.decaywood.utils.URLMapper; 7 | 8 | import java.io.IOException; 9 | import java.rmi.RemoteException; 10 | import java.util.function.Predicate; 11 | 12 | /** 13 | * @author: decaywood 14 | * @date: 2015/12/3 10:01 15 | */ 16 | 17 | /** 18 | * 过滤器，特别是要访问网页的过滤器，可以继承此抽象类 19 | */ 20 | public abstract class AbstractFilter extends AbstractRequester implements 21 | Predicate, 22 | CookieProcessor { 23 | 24 | 25 | protected abstract boolean filterLogic(T t) throws Exception; 26 | 27 | 28 | public AbstractFilter(TimeWaitingStrategy strategy) { 29 | this(strategy, URLMapper.MAIN_PAGE.toString()); 30 | } 31 | 32 | public AbstractFilter(TimeWaitingStrategy strategy, String webSite) { 33 | super(strategy, webSite); 34 | } 35 | 36 | 37 | @Override 38 | public boolean test(T t) { 39 | 40 | System.out.println(getClass().getSimpleName() + " filtering..."); 41 | 42 | 43 | boolean res = false; 44 | int retryTime = this.strategy.retryTimes(); 45 | 46 | try { 47 | int loopTime = 1; 48 | boolean needRMI = true; 49 | 50 | while (retryTime > loopTime) { 51 | try { 52 | res = filterLogic(t); 53 | break; 54 | } catch (Exception e) { 55 | if(!(e instanceof IOException)) throw e; 56 | System.out.println("Filter: Network busy Retrying -> " + loopTime + " times"); 57 | updateCookie(webSite); 58 | this.strategy.waiting(loopTime++); 59 | } 60 | } 61 | 62 | } catch (Exception e) { 63 | e.printStackTrace(); 64 | } 65 | return res; 66 | 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/utils/URLMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.utils; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/24 18:54 6 | */ 7 | public enum URLMapper { 8 | 9 | /*-------------------------------- Xue Qiu --------------------------------------*/ 10 | MAIN_PAGE("https://xueqiu.com"), 11 | STOCK_MAIN_PAGE("https://xueqiu.com/query/v1/symbol/search/status"), 12 | USER_INFO_JSON("https://xueqiu.com/user/show.json"), 13 | USER_COMMENT_JSON("https://xueqiu.com/v4/statuses/user_timeline.json"), 14 | COMMENTS_INFO_JSON("https://xueqiu.com/statuses/comments.json"), 15 | COMPREHENSIVE_PAGE("https://xueqiu.com/hq"), 16 | HU_SHEN_NEWS_REF_JSON("https://xueqiu.com/statuses/topic.json"), 17 | STOCK_SHAREHOLDERS_JSON("https://xueqiu.com/stock/f10/shareholdernum.json"), 18 | STOCK_SELECTOR_JSON("https://xueqiu.com/stock/screener/screen.json"), 19 | LONGHUBANG_JSON("https://xueqiu.com/stock/f10/bizunittrdinfo.json"), 20 | STOCK_INDUSTRY_JSON("https://xueqiu.com/stock/f10/compinfo.json"), 21 | CUBE_REBALANCING_JSON("https://xueqiu.com/cubes/rebalancing/history.json"), 22 | CUBE_TREND_JSON("https://xueqiu.com/cubes/nav_daily/all.json"), 23 | CUBES_RANK_JSON("https://xueqiu.com/cubes/discover/rank/cube/list.json"), 24 | MARKET_QUOTATIONS_RANK_JSON("https://xueqiu.com/stock/quote_order.json"), 25 | SCOPE_STOCK_RANK_JSON("https://xueqiu.com/stock/rank.json"), 26 | STOCK_TREND_JSON("https://xueqiu.com/stock/forchartk/stocklist.json"), 27 | STOCK_JSON("https://xueqiu.com/v4/stock/quote.json"), 28 | INDUSTRY_JSON("https://xueqiu.com/service/v5/stock/screener/quote/list"), 29 | 30 | /*-------------------------------- NetEase --------------------------------------*/ 31 | 32 | NETEASE_MAIN_PAGE("https://quotes.money.163.com/stock"), 33 | STOCK_CAPITAL_FLOW("https://quotes.money.163.com/service/zjlx_chart.html"); 34 | 35 | 36 | URLMapper(String URL) { 37 | this.URL = URL; 38 | } 39 | 40 | private String URL; 41 | 42 | @Override 43 | public String toString() { 44 | return URL; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/Comment.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | import java.util.List; 4 | import java.util.Objects; 5 | 6 | /** 7 | * 评论模型 8 | * @author decaywood 9 | * @date 2020/10/7 19:18 10 | */ 11 | public class Comment implements User.UserSetter { 12 | 13 | public interface CommentSetter { 14 | void addComments(List comments); 15 | String getPostId(); 16 | int getReplyCnt(); 17 | } 18 | 19 | /** 评论id */ 20 | private String id; 21 | 22 | /** 用户信息 */ 23 | private User user; 24 | 25 | /** 引用 */ 26 | private Comment reply_comment; 27 | 28 | /** 评论内容 */ 29 | private String text; 30 | 31 | /** 用户id */ 32 | private String user_id; 33 | 34 | public String getId() { 35 | return id; 36 | } 37 | 38 | public void setId(String id) { 39 | this.id = id; 40 | } 41 | 42 | public User getUser() { 43 | return user; 44 | } 45 | 46 | public String getText() { 47 | return text; 48 | } 49 | 50 | public void setText(String text) { 51 | this.text = text; 52 | } 53 | 54 | public String getUser_id() { 55 | return user_id; 56 | } 57 | 58 | public void setUser_id(String user_id) { 59 | this.user_id = user_id; 60 | } 61 | 62 | public Comment getReply_comment() { 63 | return reply_comment; 64 | } 65 | 66 | public void setReply_comment(Comment reply_comment) { 67 | this.reply_comment = reply_comment; 68 | } 69 | 70 | @Override 71 | public void setUser(User user) { 72 | this.user = user; 73 | } 74 | 75 | @Override 76 | public String getUserId() { 77 | return user_id; 78 | } 79 | 80 | @Override 81 | public boolean equals(Object o) { 82 | if (this == o) return true; 83 | if (o == null || getClass() != o.getClass()) return false; 84 | Comment comment = (Comment) o; 85 | return id.equals(comment.id); 86 | } 87 | 88 | @Override 89 | public int hashCode() { 90 | return Objects.hash(id); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/test/java/entryFirst/UserInfoToDBAcceptor.java: -------------------------------------------------------------------------------- 1 | package entryFirst; 2 | 3 | import org.decaywood.acceptor.AbstractAcceptor; 4 | import org.decaywood.entity.Entry; 5 | import org.decaywood.entity.Stock; 6 | import org.decaywood.utils.DatabaseAccessor; 7 | 8 | import java.rmi.RemoteException; 9 | import java.sql.Connection; 10 | import java.sql.PreparedStatement; 11 | 12 | 13 | /** 14 | * @author: decaywood 15 | * @date: 2015/12/1 13:27 16 | */ 17 | 18 | /** 19 | * 示例类， 接收信息放入数据库 20 | */ 21 | public class UserInfoToDBAcceptor extends AbstractAcceptor> { 22 | 23 | public UserInfoToDBAcceptor() {} 24 | 25 | /** 26 | 27 | 28 | CREATE TABLE `xueqiuspider`.`stock_vip_followers` ( 29 | `id` INT NOT NULL AUTO_INCREMENT, 30 | `stock_id` VARCHAR(45) NULL, 31 | `stock_name` VARCHAR(45) NULL, 32 | `vip_id` VARCHAR(45) NULL, 33 | `vip_count` INT NULL, 34 | PRIMARY KEY (`id`), 35 | UNIQUE INDEX `id_UNIQUE` (`id` ASC), 36 | UNIQUE INDEX `stock_id_UNIQUE` (`stock_id` ASC), 37 | UNIQUE INDEX `stock_name_UNIQUE` (`stock_name` ASC)); 38 | 39 | */ 40 | 41 | @Override 42 | protected void consumLogic(Entry entry) throws Exception{ 43 | Stock stock = entry.getKey(); 44 | int k = entry.getValue(); 45 | Connection connection = DatabaseAccessor.Holder.ACCESSOR.getConnection(); 46 | StringBuilder builder = new StringBuilder(); 47 | builder.append("insert into stock_vip_followers ") 48 | .append("(stock_id, stock_name, vip_count) ") 49 | .append("values (?, ?, ?)") 50 | .append("on duplicate key update vip_count=?"); 51 | String sql = builder.toString(); 52 | PreparedStatement statement = connection.prepareStatement(sql); 53 | statement.setString(1, stock.getStockNo()); 54 | statement.setString(2, stock.getStockName()); 55 | statement.setInt(3, k); 56 | statement.setInt(4, k); 57 | statement.execute(); 58 | DatabaseAccessor.Holder.ACCESSOR.returnConnection(connection); 59 | 60 | } 61 | 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/acceptor/AbstractAcceptor.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.acceptor; 2 | 3 | import org.decaywood.AbstractRequester; 4 | import org.decaywood.CookieProcessor; 5 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 6 | import org.decaywood.utils.URLMapper; 7 | 8 | import java.io.IOException; 9 | import java.rmi.RemoteException; 10 | import java.util.function.Consumer; 11 | import java.util.function.Function; 12 | 13 | /** 14 | * @author: decaywood 15 | * @date: 2015/11/30 22:22. 16 | */ 17 | 18 | /** 19 | * 接受处理完的数据流并进行分析，既可以当作整个生命周期的中点也可以当作中间组件使用 20 | */ 21 | public abstract class AbstractAcceptor extends AbstractRequester implements 22 | Consumer, 23 | Function, 24 | CookieProcessor { 25 | 26 | public AbstractAcceptor() { 27 | this(null); 28 | } 29 | 30 | public AbstractAcceptor(TimeWaitingStrategy strategy) { 31 | this(strategy, URLMapper.MAIN_PAGE.toString()); 32 | } 33 | 34 | public AbstractAcceptor(TimeWaitingStrategy strategy, String webSite) { 35 | super(strategy, webSite); 36 | } 37 | 38 | protected abstract void consumLogic(T t) throws Exception; 39 | 40 | 41 | @Override 42 | public void accept(T t) { 43 | 44 | System.out.println(getClass().getSimpleName() + " accepting..."); 45 | 46 | int retryTime = this.strategy.retryTimes(); 47 | 48 | try { 49 | int loopTime = 1; 50 | 51 | while (retryTime > loopTime) { 52 | try { 53 | consumLogic(t); 54 | break; 55 | } catch (Exception e) { 56 | if(!(e instanceof IOException)) throw e; 57 | System.out.println("Acceptor: Network busy Retrying -> " + loopTime + " times"); 58 | updateCookie(webSite); 59 | this.strategy.waiting(loopTime++); 60 | } 61 | } 62 | 63 | } catch (Exception e) { 64 | e.printStackTrace(); 65 | } 66 | 67 | } 68 | 69 | @Override 70 | public T apply(T t) { 71 | accept(t); 72 | return t; 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/AbstractCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import org.decaywood.AbstractRequester; 4 | import org.decaywood.CookieProcessor; 5 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 6 | import org.decaywood.utils.URLMapper; 7 | 8 | import java.io.IOException; 9 | import java.util.function.Supplier; 10 | 11 | /** 12 | * @author: decaywood 13 | * @date: 2015/11/23 13:51 14 | */ 15 | 16 | /** 17 | * 整个框架数据收集生命周期的起点，负责对数据进行原始定位与收集 18 | * 通过收集器可以定位你所关注的数据类型，继而进行进一步的数据挖掘 19 | * 如果要贡献模块，强烈建议继承此类，它有进行数据收集所需的API 20 | * 供调用。有完备的超时重传机制以及等待策略 21 | */ 22 | public abstract class AbstractCollector extends AbstractRequester implements 23 | Supplier, 24 | CookieProcessor { 25 | 26 | /** 27 | * 收集器收集逻辑,由子类实现 28 | */ 29 | protected abstract T collectLogic() throws Exception; 30 | 31 | 32 | public AbstractCollector(TimeWaitingStrategy strategy) { 33 | this(strategy, URLMapper.MAIN_PAGE.toString()); 34 | } 35 | 36 | 37 | /** 38 | * @param strategy 超时等待策略（null则设置为默认等待策略） 39 | * @param webSite 站点（默认为雪球网首页，可拓展其他财经网站--作用为获取cookie） 40 | */ 41 | public AbstractCollector(TimeWaitingStrategy strategy, String webSite) { 42 | super(strategy, webSite); 43 | } 44 | 45 | 46 | @Override 47 | public T get() { 48 | System.out.println(getClass().getSimpleName() + " collecting..."); 49 | T res = null; 50 | int retryTime = this.strategy.retryTimes(); 51 | try { 52 | int loopTime = 1; 53 | while (retryTime > loopTime) { 54 | try { 55 | res = collectLogic(); 56 | break; 57 | } catch (Exception e) { 58 | if(!(e instanceof IOException)) throw e; 59 | System.out.println("Collector: Network busy Retrying -> " + loopTime + " times"); 60 | updateCookie(webSite); 61 | this.strategy.waiting(loopTime++); 62 | } 63 | } 64 | } catch (Exception e) { 65 | e.printStackTrace(); 66 | } 67 | return res; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/dateFirst/DateToLongHuBangStockMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.dateFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.mapper.AbstractMapper; 6 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 7 | import org.decaywood.utils.DateParser; 8 | import org.decaywood.utils.RequestParaBuilder; 9 | import org.decaywood.utils.URLMapper; 10 | 11 | import java.net.URL; 12 | import java.rmi.RemoteException; 13 | import java.util.ArrayList; 14 | import java.util.Date; 15 | import java.util.List; 16 | 17 | /** 18 | * @author: decaywood 19 | * @date: 2015/11/27 10:48 20 | */ 21 | 22 | 23 | /** 24 | * 当前日期 -> 龙虎榜映射器 25 | */ 26 | public class DateToLongHuBangStockMapper extends AbstractMapper> { 27 | 28 | public DateToLongHuBangStockMapper() { 29 | this(null); 30 | } 31 | 32 | /** 33 | * @param strategy 超时等待策略（null则设置为默认等待策略） 34 | */ 35 | public DateToLongHuBangStockMapper(TimeWaitingStrategy strategy) { 36 | super(strategy); 37 | } 38 | 39 | @Override 40 | public List mapLogic(Date date) throws Exception { 41 | 42 | String dateParam = DateParser.getTimePrefix(date, false); 43 | 44 | String target = URLMapper.LONGHUBANG_JSON.toString(); 45 | RequestParaBuilder builder = new RequestParaBuilder(target) 46 | .addParameter("date", dateParam); 47 | URL url = new URL(builder.build()); 48 | String json = request(url); 49 | JsonNode node = mapper.readTree(json); 50 | return processNode(node, date); 51 | 52 | } 53 | 54 | private List processNode(JsonNode node, Date date) { 55 | List stocks = new ArrayList<>(); 56 | for (JsonNode jsonNode : node.get("list")) { 57 | 58 | String name = jsonNode.get("name").asText(); 59 | String symbol = jsonNode.get("symbol").asText(); 60 | Stock stock = new Stock(name, symbol); 61 | stock.setStockQueryDate(date); 62 | 63 | stocks.add(stock); 64 | 65 | } 66 | return stocks; 67 | } 68 | 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/StockCommentCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.PostInfo; 5 | import org.decaywood.utils.JsonParser; 6 | import org.decaywood.utils.RequestParaBuilder; 7 | import org.decaywood.utils.StringUtils; 8 | import org.decaywood.utils.URLMapper; 9 | 10 | import java.net.URL; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | /** 15 | * 某只标的下的用户评论收集器 16 | * @author decaywood 17 | * @date 2020/10/7 16:04 18 | */ 19 | public class StockCommentCollector extends AbstractCollector> { 20 | 21 | public enum SortType { 22 | // 最新时间 23 | time, 24 | // 热度排名 25 | alpha 26 | } 27 | 28 | private String symbol; 29 | 30 | private int pageSize; 31 | 32 | private int page; 33 | 34 | private String sort; 35 | 36 | public StockCommentCollector(String symbol, SortType type, int page, int pageSize) { 37 | super(null); 38 | this.symbol = symbol; 39 | this.page = page; 40 | this.pageSize = pageSize; 41 | this.sort = type.toString(); 42 | } 43 | 44 | @Override 45 | protected List collectLogic() throws Exception { 46 | if (StringUtils.isNull(symbol) || page < 1 || pageSize <= 0) { 47 | throw new IllegalArgumentException("symbol is null or page size is zero"); 48 | } 49 | String target = URLMapper.STOCK_MAIN_PAGE.toString(); 50 | RequestParaBuilder builder = new RequestParaBuilder(target); 51 | builder.addParameter("count", pageSize); 52 | builder.addParameter("symbol", symbol); 53 | builder.addParameter("sort", sort); 54 | builder.addParameter("page", page); 55 | URL url = new URL(builder.build()); 56 | String json = requestWithoutGzip(url); 57 | JsonNode node = mapper.readTree(json); 58 | List postInfos = new ArrayList<>(); 59 | for (JsonNode jsonNode : node.get("list")) { 60 | PostInfo postInfo = JsonParser.parse(PostInfo::new, jsonNode); 61 | postInfos.add(postInfo); 62 | } 63 | return postInfos; 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/StockSlectorBaseCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.entity.selectorQuota.QuotaChainNode; 6 | import org.decaywood.entity.selectorQuota.QuotaHead; 7 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 8 | import org.decaywood.utils.URLMapper; 9 | 10 | import java.net.URL; 11 | import java.rmi.RemoteException; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | /** 16 | * @author: decaywood 17 | * @date: 2015/11/28 12:40 18 | */ 19 | 20 | /** 21 | * 股票指标选择收集器，根据具体指标对股票进行选择。 22 | */ 23 | public class StockSlectorBaseCollector extends AbstractCollector> { 24 | 25 | /** 26 | * 指标链，指标分多种类型，不同类型可挂靠在指标链头节点上 27 | * 收集器会根据指标链的指标进行统计并收集符合指标链所有指标的个股 28 | */ 29 | QuotaChainNode head; 30 | 31 | public StockSlectorBaseCollector() { 32 | this(null); 33 | } 34 | 35 | /** 36 | * 37 | * @param strategy 超时等待策略（null则设置为默认等待策略） 38 | */ 39 | public StockSlectorBaseCollector(TimeWaitingStrategy strategy) { 40 | super(strategy); 41 | head = new QuotaHead(); 42 | } 43 | 44 | /** 45 | * @param node 需要添加到指标链上的股票指标 46 | */ 47 | public void addQuotaChainNode(QuotaChainNode node) { 48 | head.setNext(node); 49 | } 50 | 51 | @Override 52 | public List collectLogic() throws Exception { 53 | String target = URLMapper.STOCK_SELECTOR_JSON.toString(); 54 | URL url = new URL(target + head.generateQuotaRequest()); 55 | 56 | String json = request(url); 57 | JsonNode node = mapper.readTree(json); 58 | return processNode(node); 59 | 60 | } 61 | 62 | private List processNode(JsonNode node) { 63 | JsonNode jsonNode = node.get("list"); 64 | List stocks = new ArrayList<>(); 65 | for (JsonNode n : jsonNode) { 66 | String symbol = n.get("symbol").asText(); 67 | String name = n.get("name").asText(); 68 | Stock stock = new Stock(name, symbol); 69 | stocks.add(stock); 70 | } 71 | return stocks; 72 | } 73 | 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/AbstractMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper; 2 | 3 | import org.decaywood.AbstractRequester; 4 | import org.decaywood.CookieProcessor; 5 | import org.decaywood.entity.DeepCopy; 6 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 7 | import org.decaywood.utils.URLMapper; 8 | 9 | import java.io.IOException; 10 | import java.rmi.RemoteException; 11 | import java.util.function.Function; 12 | 13 | /** 14 | * @author: decaywood 15 | * @date: 2015/11/24 16:56 16 | */ 17 | 18 | public abstract class AbstractMapper extends AbstractRequester implements 19 | Function, 20 | CookieProcessor { 21 | 22 | 23 | protected abstract R mapLogic(T t) throws Exception; 24 | 25 | 26 | public AbstractMapper(TimeWaitingStrategy strategy) { 27 | this(strategy, URLMapper.MAIN_PAGE.toString()); 28 | } 29 | 30 | public AbstractMapper(TimeWaitingStrategy strategy, String webSite) { 31 | super(strategy, webSite); 32 | } 33 | 34 | @Override 35 | public R apply(T t) { 36 | 37 | if (t != null) 38 | System.out.println(getClass().getSimpleName() + " mapping -> " + t.getClass().getSimpleName()); 39 | 40 | R res = null; 41 | int retryTime = this.strategy.retryTimes(); 42 | 43 | try { 44 | 45 | int loopTime = 1; 46 | boolean needRMI = true; 47 | 48 | if(t != null) //noinspection unchecked 49 | t = t instanceof DeepCopy ? ((DeepCopy) t).copy() : t; 50 | 51 | while (retryTime > loopTime) { 52 | try { 53 | res = mapLogic(t); 54 | needRMI = false; 55 | break; 56 | } catch (Exception e) { 57 | if (!(e instanceof IOException)) throw e; 58 | System.out.println("Mapper: Network busy Retrying -> " + loopTime + " times" + " " + this.getClass().getSimpleName()); 59 | updateCookie(webSite); 60 | this.strategy.waiting(loopTime++); 61 | } 62 | } 63 | 64 | } catch (Exception e) { 65 | e.printStackTrace(); 66 | } 67 | return res; 68 | 69 | } 70 | 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/utilTest/DateParserTest.java: -------------------------------------------------------------------------------- 1 | package utilTest; 2 | 3 | import org.decaywood.utils.DateParser; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | import java.util.Calendar; 8 | import java.util.Date; 9 | 10 | /** 11 | * @author: decaywood 12 | * @date: 2015/11/29 22:10. 13 | */ 14 | public class DateParserTest { 15 | 16 | 17 | @Test 18 | public void testGetTimePrefixFunc() { 19 | 20 | Calendar calendar = Calendar.getInstance(); 21 | calendar.set(2015, Calendar.JANUARY, 31); 22 | 23 | Assert.assertEquals("20141231", DateParser.getTimePrefix(calendar.getTime(), true)); 24 | Assert.assertEquals("20150131", DateParser.getTimePrefix(calendar.getTime(), false)); 25 | 26 | calendar.set(2015, Calendar.MARCH, 31); 27 | Assert.assertEquals("20141231", DateParser.getTimePrefix(calendar.getTime(), true)); 28 | 29 | calendar.set(2015, Calendar.JUNE, 30); 30 | Assert.assertEquals("20150331", DateParser.getTimePrefix(calendar.getTime(), true)); 31 | 32 | calendar.set(2015, Calendar.SEPTEMBER, 30); 33 | Assert.assertEquals("20150630", DateParser.getTimePrefix(calendar.getTime(), true)); 34 | 35 | calendar.set(2015, Calendar.DECEMBER, 30); 36 | Assert.assertEquals("20150930", DateParser.getTimePrefix(calendar.getTime(), true)); 37 | } 38 | 39 | @Test 40 | public void testGetDateFunc() { 41 | 42 | Calendar calendar1 = Calendar.getInstance(); 43 | calendar1.set(2015, Calendar.JANUARY, 31); 44 | 45 | Calendar calendar2 = Calendar.getInstance(); 46 | Date date = DateParser.parseDate("20150131"); 47 | calendar2.setTime(date); 48 | 49 | Assert.assertEquals(calendar1.get(Calendar.YEAR), calendar2.get(Calendar.YEAR)); 50 | Assert.assertEquals(calendar1.get(Calendar.MONTH), calendar2.get(Calendar.MONTH)); 51 | Assert.assertEquals(calendar1.get(Calendar.DATE), calendar2.get(Calendar.DATE)); 52 | } 53 | 54 | @Test(expected = IllegalArgumentException.class) 55 | public void wrongArgumentTest1() { 56 | DateParser.parseDate("2015730"); 57 | } 58 | 59 | @Test(expected = IllegalArgumentException.class) 60 | public void wrongArgumentTest2() { 61 | DateParser.parseDate("201x730"); 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/UserCommentCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.PostInfo; 5 | import org.decaywood.utils.JsonParser; 6 | import org.decaywood.utils.RequestParaBuilder; 7 | import org.decaywood.utils.URLMapper; 8 | import org.springframework.util.StringUtils; 9 | 10 | import java.net.URL; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.stream.Collectors; 14 | 15 | /** 16 | * @author decaywood 17 | * @date 2020/10/9 12:39 18 | */ 19 | public class UserCommentCollector extends AbstractCollector> { 20 | 21 | private String userId; 22 | 23 | /** 评论页码 */ 24 | private int fromPage; 25 | /** 评论页码 */ 26 | private int toPage; 27 | 28 | private int pageSize; 29 | 30 | public UserCommentCollector(String userId, int fromPage, int toPage, int pageSize) { 31 | super(null); 32 | this.userId = userId; 33 | this.fromPage = fromPage; 34 | this.toPage = toPage; 35 | this.pageSize = pageSize; 36 | } 37 | 38 | @Override 39 | protected List collectLogic() throws Exception { 40 | if (StringUtils.isEmpty(userId) || fromPage < 1 || fromPage > toPage) { 41 | throw new IllegalArgumentException("userId is null or page size illegal"); 42 | } 43 | String target = URLMapper.USER_COMMENT_JSON.toString(); 44 | List postInfos = new ArrayList<>(); 45 | for (int i = fromPage; i <= toPage; i++) { 46 | RequestParaBuilder builder = new RequestParaBuilder(target); 47 | builder.addParameter("page", i); 48 | builder.addParameter("user_id", userId); 49 | builder.addParameter("count", pageSize); 50 | URL url = new URL(builder.build()); 51 | String json = request(url); 52 | JsonNode node = mapper.readTree(json); 53 | toPage = Math.min(toPage, Integer.parseInt(node.get("maxPage").asText())); 54 | for (JsonNode jsonNode : node.get("statuses")) { 55 | PostInfo postInfo = JsonParser.parse(PostInfo::new, jsonNode); 56 | postInfos.add(postInfo); 57 | } 58 | } 59 | return postInfos; 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/CompanyInfo.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * @author: decaywood 8 | * @date: 2015/12/4 14:38 9 | */ 10 | 11 | /** 12 | * 公司信息 13 | */ 14 | public class CompanyInfo implements DeepCopy { 15 | 16 | private final String compsname;//公司名称 17 | private final String orgtype;//组织形式 18 | private final String founddate;//成立日期 19 | private final String bizscope;//经营范围 20 | private final String majorbiz;//主营业务 21 | private final String region;//地区代码 22 | private final List tqCompIndustryList;//所属板块 23 | 24 | 25 | /** 26 | * 27 | * @param compsname 公司名称 28 | * @param orgtype 组织形式 29 | * @param founddate 成立日期 30 | * @param bizscope 经营范围 31 | * @param majorbiz 主营业务 32 | * @param region 地区代码 33 | * @param tqCompIndustryList 所属板块 34 | */ 35 | public CompanyInfo(String compsname, 36 | String orgtype, 37 | String founddate, 38 | String bizscope, 39 | String majorbiz, 40 | String region, 41 | List tqCompIndustryList) { 42 | this.compsname = compsname; 43 | this.orgtype = orgtype; 44 | this.founddate = founddate; 45 | this.bizscope = bizscope; 46 | this.majorbiz = majorbiz; 47 | this.region = region; 48 | this.tqCompIndustryList = tqCompIndustryList; 49 | } 50 | 51 | public String getCompsname() { 52 | return compsname; 53 | } 54 | 55 | public String getOrgtype() { 56 | return orgtype; 57 | } 58 | 59 | public String getFounddate() { 60 | return founddate; 61 | } 62 | 63 | public String getBizscope() { 64 | return bizscope; 65 | } 66 | 67 | public String getMajorbiz() { 68 | return majorbiz; 69 | } 70 | 71 | public String getRegion() { 72 | return region; 73 | } 74 | 75 | public List getTqCompIndustryList() { 76 | return tqCompIndustryList; 77 | } 78 | 79 | @Override 80 | public CompanyInfo copy() { 81 | return new CompanyInfo( 82 | compsname, 83 | orgtype, 84 | founddate, 85 | bizscope, 86 | majorbiz, 87 | region, 88 | new ArrayList<>(tqCompIndustryList)); 89 | } 90 | 91 | 92 | } 93 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | decaywood 8 | XueQiuSpider 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 2.5.1 13 | 1.15.3 14 | 2.11.3 15 | 4.12 16 | 8.0.28 17 | 1.1.0 18 | 5.2.9.RELEASE 19 | 20 | 21 | 22 | 23 | 24 | org.apache.maven.plugins 25 | maven-compiler-plugin 26 | ${maven.compiler.plugin.version} 27 | 28 | 1.8 29 | 1.8 30 | UTF-8 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | org.springframework 40 | spring-beans 41 | ${spring-version} 42 | 43 | 44 | 45 | 46 | mysql 47 | mysql-connector-java 48 | ${jdbc-driver-version} 49 | 50 | 51 | 52 | org.jsoup 53 | jsoup 54 | ${jsoup-version} 55 | 56 | 57 | 58 | com.fasterxml.jackson.core 59 | jackson-databind 60 | ${jackson-version} 61 | 62 | 63 | 64 | junit 65 | junit 66 | ${junit-version} 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/selectorQuota/BasicQuota.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.selectorQuota; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/28 12:48 6 | */ 7 | 8 | /** 9 | * 股票基本指标节点 10 | */ 11 | public class BasicQuota extends AbstractQuotaNode { 12 | 13 | 14 | private String mc = "ALL";//总市值(亿) 15 | private String fmc = "ALL";//流通市值(亿) 16 | private String pettm = "ALL";//动态市盈率(倍) 17 | private String pelyr = "ALL";//静态市盈率(倍) 18 | private String eps = "ALL";//每股收益 19 | private String bps = "ALL";//每股净资产 20 | private String roediluted = "ALL";//净资产收益率 21 | private String netprofit = "ALL";//净利润(万) 22 | private String dy = "ALL";//股息率(%) 23 | private String pb = "ALL";//市净率(倍) 24 | 25 | 26 | //设置总市值范围 27 | public void setMc(double from, double to) { 28 | mc = from + "_" + to; 29 | } 30 | 31 | //设置流通市值范围 32 | public void setFmc(double from, double to) { 33 | fmc = from + "_" + to; 34 | } 35 | //设置动态市盈率范围 36 | public void setPettm(double from, double to) { 37 | pettm = from + "_" + to; 38 | } 39 | //设置静态市盈率范围 40 | public void setPelyr(double from, double to) { 41 | pelyr = from + "_" + to; 42 | } 43 | //设置每股收益范围 44 | public void setEps(double from, double to) { 45 | eps = from + "_" + to; 46 | } 47 | //设置每股净资产范围 48 | public void setBps(double from, double to) { 49 | bps = from + "_" + to; 50 | } 51 | //设置净资产收益率范围 52 | public void setRoediluted(double from, double to) { 53 | roediluted = from + "_" + to; 54 | } 55 | //设置净利润范围 56 | public void setNetprofit(double from, double to) { 57 | netprofit = from + "_" + to; 58 | } 59 | //设置股息率范围 60 | public void setDy(double from, double to) { 61 | dy = from + "_" + to; 62 | } 63 | //设置市净率范围 64 | public void setPb(double from, double to) { 65 | pb = from + "_" + to; 66 | } 67 | 68 | 69 | @Override 70 | StringBuilder builderSelf() { 71 | StringBuilder builder = new StringBuilder(); 72 | append(builder, "mc", mc); 73 | append(builder, "fmc", fmc); 74 | append(builder, "pettm", pettm); 75 | append(builder, "pelyr", pelyr); 76 | appendWithTimePrefix(builder, "eps", eps); 77 | appendWithTimePrefix(builder, "bps", bps); 78 | appendWithTimePrefix(builder, "roediluted", roediluted); 79 | appendWithTimePrefix(builder, "netprofit", netprofit); 80 | append(builder, "dy", dy); 81 | append(builder, "pb", pb); 82 | return builder; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/CommentCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.node.NullNode; 5 | import org.decaywood.entity.Comment; 6 | import org.decaywood.entity.User; 7 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 8 | import org.decaywood.utils.JsonParser; 9 | import org.decaywood.utils.RequestParaBuilder; 10 | import org.decaywood.utils.URLMapper; 11 | import org.springframework.util.CollectionUtils; 12 | 13 | import java.net.URL; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | 17 | /** 18 | * @author decaywood 19 | * @date 2020/11/7 12:59 20 | */ 21 | public class CommentCollector extends AbstractCollector> { 22 | 23 | private String postId; 24 | 25 | public CommentCollector(String postId) { 26 | super(null); 27 | this.postId = postId; 28 | } 29 | 30 | @Override 31 | protected List collectLogic() throws Exception { 32 | String target = URLMapper.COMMENTS_INFO_JSON.toString(); 33 | int page = 1; 34 | List res = new ArrayList<>(); 35 | while (true) { 36 | int cnt = 20; 37 | RequestParaBuilder builder = new RequestParaBuilder(target) 38 | .addParameter("id", postId) 39 | .addParameter("count", cnt) 40 | .addParameter("page", page) 41 | .addParameter("reply", true) 42 | .addParameter("split", true); 43 | URL url = new URL(builder.build()); 44 | String json = request(url); 45 | JsonNode node = mapper.readTree(json); 46 | List comments = JsonParser.parseArray(Comment::new, (t, n) -> { 47 | User user = JsonParser.parse(User::new, n.get("user")); 48 | JsonNode replyComment = n.get("reply_comment"); 49 | if (!(replyComment instanceof NullNode)) { 50 | Comment reply_comment = JsonParser.parse(Comment::new, replyComment); 51 | User replyUser = JsonParser.parse(User::new, replyComment.get("user")); 52 | reply_comment.setUser(replyUser); 53 | t.setReply_comment(reply_comment); 54 | } 55 | t.setUser(user); 56 | }, node.get("comments")); 57 | if (CollectionUtils.isEmpty(comments)) { 58 | break; 59 | } 60 | res.addAll(comments); 61 | page++; 62 | } 63 | return res; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/CookieProcessor.java: -------------------------------------------------------------------------------- 1 | package org.decaywood; 2 | 3 | import org.decaywood.utils.FileLoader; 4 | import org.decaywood.utils.RequestParaBuilder; 5 | 6 | import java.net.HttpURLConnection; 7 | import java.net.URL; 8 | 9 | /** 10 | * @author: decaywood 11 | * @date: 2016/04/10 20:01 12 | */ 13 | public interface CookieProcessor { 14 | 15 | 16 | default void updateCookie(String website) throws Exception { 17 | 18 | GlobalSystemConfigLoader.loadConfig(); 19 | 20 | String areacode = System.getProperty("areaCode"); 21 | String userID = System.getProperty("userID"); 22 | String passwd = System.getProperty("password"); 23 | String cookies = System.getProperty("cookies"); 24 | boolean rememberMe = Boolean.parseBoolean(System.getProperty("rememberMe")); 25 | 26 | HttpURLConnection connection = null; 27 | if (userID != null && passwd != null) { 28 | connection = login(areacode, userID, passwd, rememberMe); 29 | } 30 | try { 31 | connection = connection == null ? 32 | (HttpURLConnection) new URL(website).openConnection() : connection; 33 | connection.connect(); 34 | 35 | String cookie = cookies != null ? cookies : connection.getHeaderFields().get("Set-Cookie") 36 | .stream() 37 | .map(x -> x.split(";")[0].concat(";")) 38 | .filter(x -> x.contains("token=") || x.contains("s=")) 39 | .reduce("", String::concat); 40 | FileLoader.updateCookie(cookie, website); 41 | } finally { 42 | if (connection != null) connection.disconnect(); 43 | } 44 | 45 | } 46 | 47 | default HttpURLConnection login(String areacode, 48 | String userID, 49 | String passwd, 50 | boolean rememberMe) throws Exception { 51 | 52 | areacode = areacode == null ? "86" : areacode; 53 | if (userID == null || passwd == null) { 54 | throw new IllegalArgumentException("null parameter: userID or password"); 55 | } 56 | 57 | RequestParaBuilder builder = new RequestParaBuilder("http://xueqiu.com/user/login") 58 | .addParameter("areacode", areacode) 59 | .addParameter("telephone", userID) 60 | .addParameter("password", passwd) 61 | .addParameter("remember_me", rememberMe ? "on" : "off"); 62 | 63 | URL url = new URL(builder.build()); 64 | return (HttpURLConnection) url.openConnection(); 65 | } 66 | 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/selectorQuota/XueQiuQuota.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.selectorQuota; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/28 15:17 6 | */ 7 | 8 | /** 9 | * 雪球指标节点 10 | */ 11 | public class XueQiuQuota extends AbstractQuotaNode { 12 | 13 | 14 | private String follow = "ALL"; //累计关注人数 15 | private String tweet = "ALL"; //累计讨论次数 16 | private String deal = "ALL"; //累计交易分享数 17 | private String follow7d = "ALL"; //一周新增关注 18 | private String tweet7d = "ALL"; //一周新增讨论数 19 | private String deal7d = "ALL"; //一周新增交易分享数 20 | private String follow7dpct = "ALL"; //一周关注增长率(%) 21 | private String tweet7dpct = "ALL"; //一周讨论增长率(%) 22 | private String deal7dpct = "ALL"; //一周交易分享增长率(%) 23 | 24 | private QuotaChainNode next; 25 | 26 | //设置累计关注人数范围 27 | public void setFollow(double from, double to) { 28 | this.follow = from + "_" + to; 29 | } 30 | 31 | //设置累计讨论次数范围 32 | public void setTweet(double from, double to) { 33 | this.tweet = from + "_" + to; 34 | } 35 | 36 | //设置累计交易分享数范围 37 | public void setDeal(double from, double to) { 38 | this.deal = from + "_" + to; 39 | } 40 | 41 | //设置一周新增关注范围 42 | public void setFollow7d(double from, double to) { 43 | this.follow7d = from + "_" + to; 44 | } 45 | 46 | //设置一周新增讨论数范围 47 | public void setTweet7d(double from, double to) { 48 | this.tweet7d = from + "_" + to; 49 | } 50 | 51 | //设置一周新增交易分享数范围 52 | public void setDeal7d(double from, double to) { 53 | this.deal7d = from + "_" + to; 54 | } 55 | 56 | //设置一周关注增长率范围 57 | public void setFollow7dpct(double from, double to) { 58 | this.follow7dpct = from + "_" + to; 59 | } 60 | 61 | //设置一周讨论增长率范围 62 | public void setTweet7dpct(double from, double to) { 63 | this.tweet7dpct = from + "_" + to; 64 | } 65 | 66 | //设置一周交易分享增长率范围 67 | public void setDeal7dpct(double from, double to) { 68 | this.deal7dpct = from + "_" + to; 69 | } 70 | 71 | 72 | 73 | @Override 74 | StringBuilder builderSelf() { 75 | StringBuilder builder = new StringBuilder(); 76 | 77 | append(builder, "follow", follow); 78 | append(builder, "tweet", tweet); 79 | append(builder, "deal", deal); 80 | 81 | append(builder, "follow7d", follow7d); 82 | append(builder, "tweet7d", tweet7d); 83 | append(builder, "deal7d", deal7d); 84 | 85 | append(builder, "follow7dpct", follow7dpct); 86 | append(builder, "tweet7dpct", tweet7dpct); 87 | append(builder, "deal7dpct", deal7dpct); 88 | 89 | return builder; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/trend/MarketIndexTrend.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.trend; 2 | 3 | import org.decaywood.utils.StringUtils; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * @author: decaywood 9 | * @date: 2015/11/26 10:03 10 | */ 11 | 12 | /** 13 | * 大盘历史走线 14 | */ 15 | public class MarketIndexTrend extends Trend { 16 | 17 | 18 | private final String symbol; 19 | private final String name; 20 | private final String from; 21 | private final String to; 22 | 23 | /** 24 | * 25 | * @param symbol 大盘代码 26 | * @param name 大盘名字 27 | * @param from 走线计算起始时间 28 | * @param to 走线计算结束时间 29 | * @param history 走线集合 30 | */ 31 | public MarketIndexTrend(String symbol, String name, String from, String to, List history) { 32 | super(history); 33 | 34 | if(symbol == null || name == null) 35 | throw new IllegalArgumentException(); 36 | 37 | this.name = name; 38 | this.symbol = symbol; 39 | this.from = from; 40 | this.to = to; 41 | } 42 | 43 | 44 | /** 45 | * 趋势集合单元 46 | */ 47 | public static class TrendBlock implements ITrendBlock { 48 | private final String time; 49 | private final String date; 50 | private final String value; 51 | private final String percent; 52 | 53 | 54 | /** 55 | * 56 | * @param time 节点时间 57 | * @param date 节点日期 58 | * @param value 涨幅 59 | * @param percent 涨幅（%） 60 | */ 61 | public TrendBlock(String time, String date, String value, String percent) { 62 | if(StringUtils.nullOrEmpty(time, date, value, percent)) 63 | throw new IllegalArgumentException(); 64 | this.time = time; 65 | this.date = date; 66 | this.value = value; 67 | this.percent = percent; 68 | } 69 | 70 | public String getTime() { 71 | return time; 72 | } 73 | 74 | public String getDate() { 75 | return date; 76 | } 77 | 78 | public String getValue() { 79 | return value; 80 | } 81 | 82 | public String getPercent() { 83 | return percent; 84 | } 85 | } 86 | 87 | public String getSymbol() { 88 | return symbol; 89 | } 90 | 91 | public String getName() { 92 | return name; 93 | } 94 | 95 | @Override 96 | public MarketIndexTrend copy() { 97 | return new MarketIndexTrend(symbol, from, to, name, history); 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/StockToStockWithShareHolderTrendMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.Stock; 4 | import org.decaywood.entity.trend.Trend; 5 | import org.decaywood.mapper.stockFirst.StockToStockWithShareHolderTrendMapper; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import java.rmi.RemoteException; 10 | import java.util.Calendar; 11 | import java.util.Date; 12 | import java.util.List; 13 | 14 | /** 15 | * @author: decaywood 16 | * @date: 2015/11/30 10:33 17 | */ 18 | public class StockToStockWithShareHolderTrendMapperTest { 19 | 20 | @Test 21 | public void testFunc() { 22 | 23 | List stocks = TestCaseGenerator.generateStocks(); 24 | StockToStockWithShareHolderTrendMapper mapper = new StockToStockWithShareHolderTrendMapper(); 25 | 26 | boolean b = stocks.stream() 27 | .map(mapper.andThen(Stock::getShareHoldersTrend).andThen(Trend::getHistory)) 28 | .noneMatch(List::isEmpty); 29 | 30 | Assert.assertTrue(b); 31 | 32 | } 33 | 34 | @Test(expected = IllegalArgumentException.class) 35 | public void testWrongArgument() { 36 | 37 | Calendar calendar = Calendar.getInstance(); 38 | 39 | calendar.set(2014, Calendar.AUGUST, 5); 40 | Date since = calendar.getTime(); 41 | 42 | calendar.set(2015, Calendar.MAY, 28); 43 | Date until = calendar.getTime(); 44 | 45 | StockToStockWithShareHolderTrendMapper mapper = 46 | new StockToStockWithShareHolderTrendMapper(until, since); 47 | 48 | } 49 | 50 | @Test 51 | public void testRangeFunc() { 52 | List stocks = TestCaseGenerator.generateStocks(); 53 | StockToStockWithShareHolderTrendMapper mapper = new StockToStockWithShareHolderTrendMapper(); 54 | 55 | int count1 = stocks.stream() 56 | .map(mapper.andThen(Stock::getShareHoldersTrend).andThen(Trend::getHistory)) 57 | .mapToInt(List::size).reduce(Integer::sum).getAsInt(); 58 | 59 | Calendar calendar = Calendar.getInstance(); 60 | calendar.set(2014, Calendar.AUGUST, 5); 61 | Date since = calendar.getTime(); 62 | 63 | calendar.set(2015, Calendar.MAY, 28); 64 | Date until = calendar.getTime(); 65 | 66 | StockToStockWithShareHolderTrendMapper rangeMapper = 67 | new StockToStockWithShareHolderTrendMapper(since, until); 68 | 69 | int count2 = stocks.stream() 70 | .map(rangeMapper.andThen(Stock::getShareHoldersTrend).andThen(Trend::getHistory)) 71 | .mapToInt(List::size).reduce(Integer::sum).getAsInt(); 72 | 73 | Assert.assertTrue(count2 < count1); 74 | 75 | } 76 | 77 | 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/industryFirst/IndustryToStocksMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.industryFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Industry; 5 | import org.decaywood.entity.Stock; 6 | import org.decaywood.mapper.AbstractMapper; 7 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 8 | import org.decaywood.utils.EmptyObject; 9 | import org.decaywood.utils.RequestParaBuilder; 10 | import org.decaywood.utils.URLMapper; 11 | 12 | import java.net.URL; 13 | import java.rmi.RemoteException; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | 17 | /** 18 | * @author: decaywood 19 | * @date: 2015/11/23 14:04 20 | */ 21 | 22 | /** 23 | * 行业 -> 该行业股票 映射器 24 | */ 25 | public class IndustryToStocksMapper extends AbstractMapper> { 26 | 27 | /** 28 | * @param strategy 超时等待策略（null则设置为默认等待策略） 29 | */ 30 | public IndustryToStocksMapper(TimeWaitingStrategy strategy) { 31 | super(strategy); 32 | } 33 | 34 | public IndustryToStocksMapper() { 35 | this(null); 36 | } 37 | 38 | @Override 39 | public List mapLogic(Industry industry) throws Exception { 40 | 41 | if(industry == null || industry == EmptyObject.emptyIndustry) return new ArrayList<>(); 42 | 43 | String target = URLMapper.INDUSTRY_JSON.toString(); 44 | RequestParaBuilder builder = new RequestParaBuilder(target); 45 | builder.addParameter("page", 1) 46 | .addParameter("size", 500) 47 | .addParameter("order", "desc") 48 | .addParameter("order_by", "percent") 49 | .addParameter("exchange", "CN") 50 | .addParameter("market", "CN"); 51 | String info = industry.getIndustryInfo(); 52 | if (info.startsWith("#")) info = info.substring(1); 53 | for (String s : info.split("&")) { 54 | String[] keyAndVal = s.split("="); 55 | if ("level2code".equalsIgnoreCase(keyAndVal[0])) { 56 | builder.addParameter("ind_code", keyAndVal[1]); 57 | } 58 | } 59 | builder.addParameter("_", System.currentTimeMillis()); 60 | URL url = new URL(builder.build()); 61 | 62 | String json = request(url); 63 | JsonNode jsonNode = mapper.readTree(json); 64 | 65 | return parserJson(jsonNode); 66 | 67 | } 68 | 69 | 70 | private List parserJson(JsonNode node) { 71 | 72 | List stocks = new ArrayList<>(); 73 | 74 | JsonNode data = node.get("data"); 75 | for (JsonNode jsonNode : data.get("list")) { 76 | Stock stock = new Stock(jsonNode.get("name").asText(), jsonNode.get("symbol").asText()); 77 | stocks.add(stock); 78 | } 79 | return stocks; 80 | 81 | } 82 | 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/comment/CommentSetMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.comment; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.node.NullNode; 5 | import org.decaywood.collector.StockCommentCollector; 6 | import org.decaywood.entity.Comment; 7 | import org.decaywood.entity.PostInfo; 8 | import org.decaywood.entity.User; 9 | import org.decaywood.mapper.AbstractMapper; 10 | import org.decaywood.utils.JsonParser; 11 | import org.decaywood.utils.RequestParaBuilder; 12 | import org.decaywood.utils.URLMapper; 13 | import org.springframework.util.StringUtils; 14 | 15 | import java.net.URL; 16 | import java.util.List; 17 | import java.util.stream.Collectors; 18 | 19 | /** 20 | * @author decaywood 21 | * @date 2020/10/7 19:18 22 | */ 23 | public class CommentSetMapper extends AbstractMapper { 24 | 25 | private int pageSize; 26 | 27 | public CommentSetMapper() { 28 | this(Integer.MAX_VALUE); 29 | } 30 | 31 | public CommentSetMapper(int pageSize) { 32 | super(null); 33 | this.pageSize = pageSize; 34 | } 35 | 36 | @Override 37 | protected T mapLogic(T commentSetter) throws Exception { 38 | String postId = commentSetter.getPostId(); 39 | String target = URLMapper.COMMENTS_INFO_JSON.toString(); 40 | int replyCnt = commentSetter.getReplyCnt(); 41 | int page = 1; 42 | while (replyCnt > 0) { 43 | int cnt = Math.min(replyCnt, pageSize); 44 | RequestParaBuilder builder = new RequestParaBuilder(target) 45 | .addParameter("id", postId) 46 | .addParameter("count", cnt) 47 | .addParameter("page", page) 48 | .addParameter("reply", true) 49 | .addParameter("split", true); 50 | URL url = new URL(builder.build()); 51 | String json = request(url); 52 | JsonNode node = mapper.readTree(json); 53 | List comments = JsonParser.parseArray(Comment::new, (t, n) -> { 54 | User user = JsonParser.parse(User::new, n.get("user")); 55 | JsonNode replyComment = n.get("reply_comment"); 56 | if (!(replyComment instanceof NullNode)) { 57 | Comment reply_comment = JsonParser.parse(Comment::new, replyComment); 58 | User replyUser = JsonParser.parse(User::new, replyComment.get("user")); 59 | reply_comment.setUser(replyUser); 60 | t.setReply_comment(reply_comment); 61 | } 62 | t.setUser(user); 63 | }, node.get("comments")); 64 | commentSetter.addComments(comments); 65 | page++; 66 | replyCnt -= cnt; 67 | } 68 | return commentSetter; 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/trend/CubeTrend.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.trend; 2 | 3 | import org.decaywood.utils.StringUtils; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | /** 9 | * @author: decaywood 10 | * @date: 2015/11/25 23:16. 11 | */ 12 | 13 | /** 14 | * 股票组合收益历史走线 15 | */ 16 | public class CubeTrend extends Trend { 17 | 18 | 19 | private final String symbol; 20 | private final String name; 21 | private final String from; 22 | private final String to; 23 | 24 | 25 | /** 26 | * 27 | * @param symbol 组合代码 28 | * @param name 组合名字 29 | * @param from 走线计算起始时间 30 | * @param to 走线计算结束时间 31 | * @param history 走线集合 32 | */ 33 | public CubeTrend(String symbol, String name, String from, String to, List history) { 34 | super(history); 35 | if(symbol == null || name == null || from == null || to == null) 36 | throw new IllegalArgumentException(); 37 | this.symbol = symbol; 38 | this.name = name; 39 | this.from = from; 40 | this.to = to; 41 | } 42 | 43 | public String getSymbol() { 44 | return symbol; 45 | } 46 | 47 | public String getName() { 48 | return name; 49 | } 50 | 51 | public String getFrom() { 52 | return from; 53 | } 54 | 55 | public String getTo() { 56 | return to; 57 | } 58 | 59 | /** 60 | * 趋势集合单元 61 | */ 62 | public static class TrendBlock implements ITrendBlock { 63 | 64 | 65 | private final String time; 66 | private final String date; 67 | private final String value; 68 | private final String percent; 69 | 70 | 71 | /** 72 | * 73 | * @param time 节点时间 74 | * @param date 节点日期 75 | * @param value 涨幅 76 | * @param percent 涨幅（%） 77 | */ 78 | public TrendBlock(String time, String date, String value, String percent) { 79 | if(StringUtils.nullOrEmpty(time, date, value, percent)) 80 | throw new IllegalArgumentException(); 81 | this.time = time; 82 | this.date = date; 83 | this.value = value; 84 | this.percent = percent; 85 | } 86 | 87 | public String getTime() { 88 | return time; 89 | } 90 | 91 | public String getDate() { 92 | return date; 93 | } 94 | 95 | public String getValue() { 96 | return value; 97 | } 98 | 99 | public String getPercent() { 100 | return percent; 101 | } 102 | } 103 | 104 | @Override 105 | public CubeTrend copy() { 106 | return new CubeTrend(symbol, name, from, to, new ArrayList<>(history)); 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/utils/HttpRequestHelper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.utils; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.io.InputStreamReader; 7 | import java.net.HttpURLConnection; 8 | import java.net.URL; 9 | import java.nio.charset.StandardCharsets; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | import java.util.zip.GZIPInputStream; 13 | 14 | /** 15 | * @author: decaywood 16 | * @date: 2015/11/23 14:27 17 | */ 18 | public class HttpRequestHelper { 19 | 20 | private Map config; 21 | private boolean post; 22 | private boolean gzip; 23 | 24 | public HttpRequestHelper(String webSite) { 25 | this(webSite, true); 26 | } 27 | 28 | public HttpRequestHelper(String webSite, boolean gzip) { 29 | this.config = new HashMap<>(); 30 | if (gzip) { 31 | this.gzipDecode() 32 | .addToHeader("Accept-Encoding", "gzip,deflate,sdch"); 33 | } else { 34 | this.addToHeader("Accept-Encoding", "utf-8"); 35 | } 36 | this.addToHeader("Referer", webSite) 37 | .addToHeader("Cookie", FileLoader.loadCookie(webSite)) 38 | .addToHeader("Host", "xueqiu.com"); 39 | } 40 | 41 | public HttpRequestHelper post() { 42 | this.post = true; 43 | return this; 44 | } 45 | 46 | public HttpRequestHelper gzipDecode() { 47 | this.gzip = true; 48 | return this; 49 | } 50 | 51 | public HttpRequestHelper addToHeader(String key, String val) { 52 | this.config.put(key, val); 53 | return this; 54 | } 55 | 56 | public HttpRequestHelper addToHeader(String key, int val) { 57 | this.config.put(key, String.valueOf(val)); 58 | return this; 59 | } 60 | 61 | public String request(URL url) throws IOException { 62 | return request(url, this.config); 63 | } 64 | 65 | 66 | public String request(URL url, Map config) throws IOException { 67 | HttpURLConnection httpURLConn = null; 68 | try { 69 | httpURLConn = (HttpURLConnection) url.openConnection(); 70 | if (post) httpURLConn.setRequestMethod("POST"); 71 | httpURLConn.setDoOutput(true); 72 | for (Map.Entry entry : config.entrySet()) 73 | httpURLConn.setRequestProperty(entry.getKey(), entry.getValue()); 74 | httpURLConn.connect(); 75 | InputStream in = httpURLConn.getInputStream(); 76 | if (gzip) in = new GZIPInputStream(in); 77 | BufferedReader bd = new BufferedReader(new InputStreamReader(in)); 78 | StringBuilder builder = new StringBuilder(); 79 | String text; 80 | while ((text = bd.readLine()) != null) builder.append(text); 81 | return builder.toString(); 82 | } finally { 83 | if (httpURLConn != null) httpURLConn.disconnect(); 84 | } 85 | } 86 | 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/test/java/collectTest/MostProfitableCubeCollectorTest.java: -------------------------------------------------------------------------------- 1 | package collectTest; 2 | 3 | import org.decaywood.collector.MostProfitableCubeCollector; 4 | import org.decaywood.entity.Cube; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import java.rmi.RemoteException; 9 | import java.util.List; 10 | 11 | /** 12 | * @author: decaywood 13 | * @date: 2015/11/25 22:46. 14 | */ 15 | public class MostProfitableCubeCollectorTest { 16 | 17 | 18 | @Test 19 | public void testMarket() { 20 | doTest(MostProfitableCubeCollector.Market.CN); 21 | doTest(MostProfitableCubeCollector.Market.HK); 22 | doTest(MostProfitableCubeCollector.Market.US); 23 | } 24 | 25 | @Test 26 | public void testORDER_BY() { 27 | doTest(MostProfitableCubeCollector.ORDER_BY.DAILY); 28 | doTest(MostProfitableCubeCollector.ORDER_BY.MONTHLY); 29 | doTest(MostProfitableCubeCollector.ORDER_BY.YEARLY); 30 | } 31 | 32 | @Test 33 | public void testNull() { 34 | MostProfitableCubeCollector collector = 35 | new MostProfitableCubeCollector(null, null 36 | , null, 1); 37 | Assert.assertNotNull(collector.getMarket()); 38 | Assert.assertNotNull(collector.getOrder_by()); 39 | } 40 | 41 | 42 | @Test(expected = IllegalArgumentException.class) 43 | public void testIllegal() { 44 | new MostProfitableCubeCollector(null, MostProfitableCubeCollector.Market.CN 45 | , MostProfitableCubeCollector.ORDER_BY.DAILY, -500); 46 | } 47 | 48 | @Test 49 | public void testTopKMaxSize() { 50 | 51 | MostProfitableCubeCollector collector = 52 | new MostProfitableCubeCollector(null, MostProfitableCubeCollector.Market.CN 53 | , MostProfitableCubeCollector.ORDER_BY.DAILY, 1000); 54 | List cubes = collector.get(); 55 | Assert.assertTrue(cubes.size() <= MostProfitableCubeCollector.CUBE_SIZE_SHRESHOLD); 56 | 57 | } 58 | 59 | @Test 60 | public void testTopKSize() throws Exception { 61 | int orderSize = 10; 62 | MostProfitableCubeCollector collector = 63 | new MostProfitableCubeCollector(null, MostProfitableCubeCollector.Market.CN 64 | , MostProfitableCubeCollector.ORDER_BY.DAILY, orderSize); 65 | List cubes = collector.get(); 66 | Assert.assertTrue(cubes.size() == orderSize); 67 | } 68 | private void doTest(MostProfitableCubeCollector.Market market) { 69 | MostProfitableCubeCollector collector = new MostProfitableCubeCollector(market); 70 | List cubes = collector.get(); 71 | Assert.assertFalse(cubes.isEmpty()); 72 | } 73 | 74 | private void doTest(MostProfitableCubeCollector.ORDER_BY order_by) { 75 | MostProfitableCubeCollector collector = new MostProfitableCubeCollector( 76 | MostProfitableCubeCollector.Market.CN, 77 | order_by); 78 | List cubes = collector.get(); 79 | Assert.assertFalse(cubes.isEmpty()); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/stockFirst/StockToCapitalFlowEntryMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.stockFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.CapitalFlow; 5 | import org.decaywood.entity.Entry; 6 | import org.decaywood.entity.Stock; 7 | import org.decaywood.mapper.AbstractMapper; 8 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 9 | import org.decaywood.utils.EmptyObject; 10 | import org.decaywood.utils.RequestParaBuilder; 11 | import org.decaywood.utils.URLMapper; 12 | 13 | import java.net.URL; 14 | import java.rmi.RemoteException; 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | /** 19 | * @author: decaywood 20 | * @date: 2015/12/2 10:08 21 | */ 22 | 23 | /** 24 | * 股票资金流向装配器 25 | */ 26 | public class StockToCapitalFlowEntryMapper extends AbstractMapper> { 27 | 28 | public StockToCapitalFlowEntryMapper() { 29 | this(null); 30 | } 31 | 32 | 33 | /** 34 | * @param strategy 超时等待策略（null则设置为默认等待策略） 35 | */ 36 | public StockToCapitalFlowEntryMapper(TimeWaitingStrategy strategy) { 37 | super(strategy, URLMapper.NETEASE_MAIN_PAGE.toString()); 38 | } 39 | 40 | 41 | 42 | @Override 43 | public Entry mapLogic(Stock stock) throws Exception { 44 | if(stock == null || stock == EmptyObject.emptyStock) 45 | return new Entry<>(EmptyObject.emptyStock, EmptyObject.emptyCapitalFlow); 46 | 47 | String no = stock.getStockNo().substring(2); 48 | 49 | String target = URLMapper.STOCK_CAPITAL_FLOW.toString(); 50 | RequestParaBuilder builder = new RequestParaBuilder(target) 51 | .addParameter("symbol", no); 52 | URL url = new URL(builder.build()); 53 | 54 | String json = request(url); 55 | JsonNode node = mapper.readTree(json); 56 | return processNode(stock, node); 57 | 58 | } 59 | 60 | private Entry processNode(Stock stock, JsonNode node) { 61 | 62 | String capitalInflow = node.get("jlr_shishi").asText(); 63 | JsonNode subNode = node.get("jlr_json"); 64 | String largeQuantity = subNode.get(0).get("value").asText(); 65 | String midQuantity = subNode.get(1).get("value").asText(); 66 | String smallQuantity = subNode.get(2).get("value").asText(); 67 | String largeQuantBuy = node.get("dd_buy").asText(); 68 | String largeQuantSell = node.get("dd_sell").asText(); 69 | String largeQuantDealProp = node.get("percent").asText().replace("%", ""); 70 | String fiveDayInflow = node.get("jlr_5day_total").asText(); 71 | JsonNode fiveDayInflow_json = node.get("jlr_5day_json"); 72 | List fiveDayInflows = new ArrayList<>(); 73 | for (JsonNode jsonNode : fiveDayInflow_json) { 74 | Double val = jsonNode.get("value").asDouble(); 75 | fiveDayInflows.add(val); 76 | } 77 | CapitalFlow capitalFlow = new CapitalFlow( 78 | capitalInflow, largeQuantity, midQuantity, smallQuantity, largeQuantBuy, 79 | largeQuantSell, largeQuantDealProp, fiveDayInflow, fiveDayInflows 80 | ); 81 | return new Entry<>(stock, capitalFlow); 82 | } 83 | 84 | 85 | } 86 | -------------------------------------------------------------------------------- /info/RMI.md: -------------------------------------------------------------------------------- 1 | 2020.10.07 保证框架精简，此功能全部移除 2 | 3 | # RMI 4 | 5 | RMI特性用于组件的分布式调用，能够有效减少目标网站反爬虫机制触发几率，由于在顶层有效的封装， 6 | 模块贡献者无需额外开发并可以获得此特性，只需继承AbstractRemoteService或者其子类即可 7 | 唯一需要注意的是，所有模块内部参数必须实现Serializable接口，本框架entity顶层接口或者 8 | 顶层抽象类皆已继承Serializable接口。 9 | 10 | # 使用方法 11 | 12 | ## 配置文件config.rmi 13 | 14 | 配置文件中可以添加RMI相应配置,“##”为注释符，代表此行被忽略， 15 | 16 | ### master_req_ip 17 | 18 | master_req_ip代表master请求slave服务的slave地址，格式为ip:port（例如：192.168.1.100:7779） 19 | 在master端，master_req_ip可以配置多个，以减轻请求负载，前提是slave为不同的公网IP。 20 | 程序框架中默认随机选取其中一个slave为master进行服务，当然，后续会拓展更多策略， 21 | 如果你想贡献更高效率的slave调度策略，可以实现SlaveChooser接口。 22 | 23 | ### slave_rcv_ip 24 | 25 | slave_rcv_ip代表slave部署机器的本地ip地址，格式为ip:port（例如：192.168.1.100:7779） 26 | 在linux机器上，为了稳妥的部署slave集群，最好将hostname改为本机公网IP。 27 | 28 | ## 部署服务 29 | 30 | 由于程序顶层类做了大部分工作，服务的部署尤其简单，以按行业分类获取所有股票为例子 31 | 32 | ###普通版本： 33 | 34 | ```java 35 | 36 | //按行业分类获取所有股票 37 | public void IndustryStockInfo() throws RemoteException { 38 | 39 | CommissionIndustryCollector collector = new CommissionIndustryCollector(); 40 | IndustryToStocksMapper mapper = new IndustryToStocksMapper(); 41 | Map> res = collector.get() 42 | .parallelStream() 43 | .map(mapper) 44 | .flatMap(Collection::stream) 45 | .collect(Collectors.groupingBy(Stock::getIndustry)); 46 | 47 | for (Map.Entry> entry : res.entrySet()) { 48 | for (Stock stock : entry.getValue()) { 49 | System.out.println(entry.getKey().getIndustryName() + " -> " + stock.getStockName()); 50 | } 51 | } 52 | 53 | } 54 | 55 | ``` 56 | 57 | ###分布式版本： 58 | 59 | * slave集群配置（先配置好config.rmi，以及hostname） 60 | 61 | ```java 62 | 63 | @Test 64 | public void IndustryStockInfoSlave() throws RemoteException { 65 | CommissionIndustryCollector collector = new CommissionIndustryCollector(); 66 | IndustryToStocksMapper mapper = new IndustryToStocksMapper(); 67 | collector.asRMISlave();//切换为slave模式 68 | mapper.asRMISlave();//切换为slave模式 69 | } 70 | 71 | ``` 72 | 73 | * master配置 74 | 75 | ```java 76 | 77 | @Test 78 | public void IndustryStockInfoMaster() throws RemoteException { 79 | 80 | CommissionIndustryCollector collector = new CommissionIndustryCollector(); 81 | IndustryToStocksMapper mapper = new IndustryToStocksMapper(); 82 | collector.asRMIMaster(); 83 | collector.setSlaveChooser(new RandomSlaveChooser());//设置slave调度策略 84 | collector.setRMIOnly(true); //仅通过远程调用计算结果 85 | mapper.asRMIMaster(); 86 | mapper.setRMIOnly(true); //仅通过远程调用计算结果 87 | 88 | Map> res = collector.get() 89 | .parallelStream() 90 | .map(mapper) 91 | .flatMap(Collection::stream) 92 | .collect(Collectors.groupingBy(Stock::getIndustry)); 93 | 94 | for (Map.Entry> entry : res.entrySet()) { 95 | for (Stock stock : entry.getValue()) { 96 | System.out.println(entry.getKey().getIndustryName() + " -> " + stock.getStockName()); 97 | } 98 | } 99 | 100 | } 101 | 102 | ``` 103 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/stockFirst/StockToStockWithAttributeMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.stockFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.mapper.AbstractMapper; 6 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 7 | import org.decaywood.utils.EmptyObject; 8 | import org.decaywood.utils.RequestParaBuilder; 9 | import org.decaywood.utils.URLMapper; 10 | 11 | import java.net.URL; 12 | import java.rmi.RemoteException; 13 | 14 | /** 15 | * @author: decaywood 16 | * @date: 2015/11/23 21:25 17 | */ 18 | 19 | /** 20 | * 股票属性装配器 21 | */ 22 | public class StockToStockWithAttributeMapper extends AbstractMapper { 23 | 24 | 25 | /** 26 | * @param strategy 超时等待策略（null则设置为默认等待策略） 27 | */ 28 | public StockToStockWithAttributeMapper(TimeWaitingStrategy strategy) { 29 | super(strategy); 30 | } 31 | 32 | public StockToStockWithAttributeMapper() { 33 | this(null); 34 | } 35 | 36 | 37 | @Override 38 | public Stock mapLogic(Stock stock) throws Exception { 39 | 40 | if(stock == null || stock == EmptyObject.emptyStock) return EmptyObject.emptyStock; 41 | 42 | String target = URLMapper.STOCK_JSON.toString(); 43 | RequestParaBuilder builder = new RequestParaBuilder(target) 44 | .addParameter("code", stock.getStockNo()); 45 | URL url = new URL(builder.build()); 46 | String json = request(url); 47 | JsonNode node = mapper.readTree(json); 48 | node = node.get(stock.getStockNo()); 49 | 50 | stock.setTime(node.get("time").asText()); 51 | stock.setCurrency_unit(node.get("currency_unit").asText()); 52 | stock.setCurrent(node.get("current").asText()); 53 | stock.setVolume(node.get("volume").asText()); 54 | stock.setPercentage(node.get("percentage").asText()); 55 | stock.setChange(node.get("change").asText()); 56 | stock.setOpen(node.get("open").asText()); 57 | stock.setHigh(node.get("high").asText()); 58 | stock.setLow(node.get("low").asText()); 59 | stock.setAmplitude(node.get("amplitude").asText()); 60 | stock.setFall_stop(node.get("fall_stop").asText()); 61 | stock.setRise_stop(node.get("rise_stop").asText()); 62 | stock.setClose(node.get("close").asText()); 63 | stock.setLast_close(node.get("last_close").asText()); 64 | stock.setHigh52Week(node.get("high52week").asText()); 65 | stock.setLow52week(node.get("low52week").asText()); 66 | stock.setMarketCapital(node.get("marketCapital").asText()); 67 | stock.setFloat_market_capital(node.get("float_market_capital").asText()); 68 | stock.setFloat_shares(node.get("float_shares").asText()); 69 | stock.setTotalShares(node.get("totalShares").asText()); 70 | stock.setEps(node.get("eps").asText()); 71 | stock.setNet_assets(node.get("net_assets").asText()); 72 | stock.setPe_ttm(node.get("pe_ttm").asText()); 73 | stock.setPe_lyr(node.get("pe_lyr").asText()); 74 | stock.setDividend(node.get("dividend").asText()); 75 | stock.setPsr(node.get("psr").asText()); 76 | stock.setTurnover_rate(node.get("turnover_rate").asText()); 77 | stock.setAmount(node.get("amount").asText()); 78 | return stock; 79 | 80 | } 81 | 82 | 83 | 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/stockFirst/StockToLongHuBangMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.stockFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.LongHuBangInfo; 5 | import org.decaywood.entity.Stock; 6 | import org.decaywood.mapper.AbstractMapper; 7 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 8 | import org.decaywood.utils.DateParser; 9 | import org.decaywood.utils.EmptyObject; 10 | import org.decaywood.utils.RequestParaBuilder; 11 | import org.decaywood.utils.URLMapper; 12 | 13 | import java.net.URL; 14 | import java.rmi.RemoteException; 15 | import java.util.*; 16 | 17 | /** 18 | * @author: decaywood 19 | * @date: 2015/11/27 14:40 20 | */ 21 | 22 | /** 23 | * 股票 -> 龙虎榜数据 映射器 24 | */ 25 | public class StockToLongHuBangMapper extends AbstractMapper { 26 | 27 | 28 | public StockToLongHuBangMapper() { 29 | this(null); 30 | } 31 | 32 | 33 | /** 34 | * @param strategy 超时等待策略（null则设置为默认等待策略） 35 | */ 36 | public StockToLongHuBangMapper(TimeWaitingStrategy strategy) { 37 | super(strategy); 38 | } 39 | 40 | @Override 41 | public LongHuBangInfo mapLogic(Stock stock) throws Exception { 42 | 43 | if(stock == null || stock == EmptyObject.emptyStock) return EmptyObject.emptyLongHuBangInfo; 44 | 45 | Date date = stock.getStockQueryDate(); 46 | 47 | if (date == EmptyObject.emptyDate) 48 | throw new IllegalArgumentException("lost parameter: stockQueryDate"); 49 | 50 | String dateParam = DateParser.getTimePrefix(date, false); 51 | 52 | String target = URLMapper.LONGHUBANG_JSON.toString(); 53 | RequestParaBuilder builder = new RequestParaBuilder(target) 54 | .addParameter("date", dateParam) 55 | .addParameter("symbol", stock.getStockNo()); 56 | URL url = new URL(builder.build()); 57 | 58 | String json = request(url); 59 | JsonNode node = mapper.readTree(json); 60 | return processNode(stock, node); 61 | 62 | } 63 | 64 | private LongHuBangInfo processNode(Stock stock, JsonNode node) { 65 | 66 | JsonNode detail = node.get("detail"); 67 | JsonNode buyListNode = detail.get("tqQtBizunittrdinfoBuyList"); 68 | JsonNode saleListNode = detail.get("tqQtBizunittrdinfoSaleList"); 69 | 70 | Set buyList = new HashSet<>(); 71 | Set saleList = new HashSet<>(); 72 | 73 | for (JsonNode jsonNode : buyListNode) { 74 | LongHuBangInfo.BizsunitInfo info = composeInfo(jsonNode); 75 | buyList.add(info); 76 | } 77 | 78 | for (JsonNode jsonNode : saleListNode) { 79 | LongHuBangInfo.BizsunitInfo info = composeInfo(jsonNode); 80 | saleList.add(info); 81 | } 82 | 83 | return new LongHuBangInfo(stock, stock.getStockQueryDate(), buyList, saleList); 84 | } 85 | 86 | private LongHuBangInfo.BizsunitInfo composeInfo(JsonNode jsonNode) { 87 | 88 | String bizsunitcode = jsonNode.get("bizsunitcode").asText(); 89 | String bizsunitname = jsonNode.get("bizsunitname").asText(); 90 | String buyamt = jsonNode.get("buyamt").asText(); 91 | String saleamt = jsonNode.get("saleamt").asText(); 92 | String tradedate = jsonNode.get("tradedate").asText(); 93 | 94 | return new LongHuBangInfo.BizsunitInfo(bizsunitcode, bizsunitname, buyamt, saleamt, tradedate); 95 | } 96 | 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/CapitalFlow.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * @author: decaywood 8 | * @date: 2015/12/2 12:44 9 | */ 10 | 11 | 12 | /** 13 | * 资金流向 14 | */ 15 | public class CapitalFlow implements DeepCopy { 16 | 17 | private final String capitalInflow;//资金流向（万） 18 | private final String largeQuantity;//大单（万） 19 | private final String midQuantity;//中单（万） 20 | private final String smallQuantity;//小单（万） 21 | 22 | private final String largeQuantBuy;//大单主动买入（手） 23 | private final String largeQuantSell;//大单主动卖出（手） 24 | 25 | private final String largeQuantDealProp;//大单成交占比（%） 26 | 27 | private final String fiveDayInflow;//五天净流入总量 28 | private final List fiveDayInflows;//五天净流入详情 29 | 30 | 31 | /** 32 | * 33 | * @param capitalInflow 资金流向 34 | * @param largeQuantity 大单 35 | * @param midQuantity 中单 36 | * @param smallQuantity 小单 37 | * @param largeQuantBuy 大单主动买入 38 | * @param largeQuantSell 大单主动卖出 39 | * @param largeQuantDealProp 大单成交占比 40 | * @param fiveDayInflow 五天净流入总量 41 | * @param fiveDayInflows 五天净流入详情 42 | */ 43 | public CapitalFlow(String capitalInflow, 44 | String largeQuantity, 45 | String midQuantity, 46 | String smallQuantity, 47 | String largeQuantBuy, 48 | String largeQuantSell, 49 | String largeQuantDealProp, 50 | String fiveDayInflow, 51 | List fiveDayInflows) { 52 | 53 | this.capitalInflow = capitalInflow; 54 | this.largeQuantity = largeQuantity; 55 | this.midQuantity = midQuantity; 56 | this.smallQuantity = smallQuantity; 57 | this.largeQuantBuy = largeQuantBuy; 58 | this.largeQuantSell = largeQuantSell; 59 | this.largeQuantDealProp = largeQuantDealProp; 60 | this.fiveDayInflow = fiveDayInflow; 61 | this.fiveDayInflows = fiveDayInflows; 62 | } 63 | 64 | 65 | 66 | public String getCapitalInflow() { 67 | return capitalInflow; 68 | } 69 | 70 | public String getLargeQuantity() { 71 | return largeQuantity; 72 | } 73 | 74 | public String getMidQuantity() { 75 | return midQuantity; 76 | } 77 | 78 | public String getSmallQuantity() { 79 | return smallQuantity; 80 | } 81 | 82 | public String getLargeQuantBuy() { 83 | return largeQuantBuy; 84 | } 85 | 86 | public String getLargeQuantSell() { 87 | return largeQuantSell; 88 | } 89 | 90 | public String getLargeQuantDealProp() { 91 | return largeQuantDealProp; 92 | } 93 | 94 | public String getFiveDayInflow() { 95 | return fiveDayInflow; 96 | } 97 | 98 | public List getFiveDayInflows() { 99 | return fiveDayInflows; 100 | } 101 | 102 | @Override 103 | public CapitalFlow copy() { 104 | return new CapitalFlow( 105 | capitalInflow, 106 | largeQuantity, 107 | midQuantity, 108 | smallQuantity, 109 | largeQuantBuy, 110 | largeQuantSell, 111 | largeQuantDealProp, 112 | fiveDayInflow, 113 | new ArrayList<>(fiveDayInflows)); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/utils/DateParser.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.utils; 2 | 3 | import java.text.DateFormat; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Calendar; 6 | import java.util.Date; 7 | import java.util.Locale; 8 | 9 | /** 10 | * @author: decaywood 11 | * @date: 2015/11/24 16:04 12 | */ 13 | public abstract class DateParser { 14 | 15 | 16 | private static final Date quarter1; 17 | private static final Date quarter2; 18 | private static final Date quarter3; 19 | private static final Date quarter4; 20 | 21 | 22 | 23 | static { 24 | Calendar calendar = Calendar.getInstance(); 25 | int year = calendar.get(Calendar.YEAR); 26 | calendar.set(year, Calendar.MARCH, 31, 23, 59, 59); 27 | quarter1 = calendar.getTime(); 28 | calendar.set(year, Calendar.JUNE, 30, 23, 59, 59); 29 | quarter2 = calendar.getTime(); 30 | calendar.set(year, Calendar.SEPTEMBER, 30, 23, 59, 59); 31 | quarter3 = calendar.getTime(); 32 | calendar.set(year, Calendar.DECEMBER, 31, 23, 59, 59); 33 | quarter4 = calendar.getTime(); 34 | } 35 | 36 | public static Date parseToDate(String time) { 37 | DateFormat dateFormat = 38 | new SimpleDateFormat("EEE MMM dd hh:mm:ss zzz yyyy", Locale.ENGLISH); 39 | try { 40 | return dateFormat.parse(time); 41 | } catch (Exception e) { 42 | e.printStackTrace(); 43 | } 44 | return null; 45 | } 46 | 47 | public static String getTimePrefix(boolean quarterScope) { 48 | return getTimePrefix(new Date(), quarterScope); 49 | } 50 | 51 | public static String getTimePrefix(Date date, boolean quarterScope) { 52 | String time_prefix; 53 | Calendar calendar = Calendar.getInstance(); 54 | calendar.setTime(date); 55 | int year = calendar.get(Calendar.YEAR); 56 | int diff = 1 - Calendar.JANUARY; 57 | int month = calendar.get(Calendar.MONTH) + diff; 58 | int day = calendar.get(Calendar.DATE); 59 | 60 | if(quarterScope) { 61 | year = calendar.get(Calendar.YEAR); 62 | if (date.after(quarter3)) calendar.setTime(quarter3); 63 | else if(date.after(quarter2)) calendar.setTime(quarter2); 64 | else if(date.after(quarter1)) calendar.setTime(quarter1); 65 | else { 66 | year--; 67 | calendar.setTime(quarter4); 68 | } 69 | 70 | month = calendar.get(Calendar.MONTH) + diff; 71 | day = calendar.get(Calendar.DATE); 72 | } 73 | 74 | time_prefix = String.valueOf(year) + String.format("%02d", month) + String.format("%02d", day); 75 | return time_prefix; 76 | } 77 | 78 | 79 | public static Date parseDate(String yyyymmdd) { 80 | 81 | if(yyyymmdd == null || yyyymmdd.length() != 8 || !isdigit(yyyymmdd)) 82 | throw new IllegalArgumentException(); 83 | 84 | Calendar calendar = Calendar.getInstance(); 85 | int diff = Calendar.JANUARY - 1; 86 | int year = Integer.parseInt(yyyymmdd.substring(0, 4)); 87 | int month = Integer.parseInt(yyyymmdd.substring(4, 6)); 88 | int day = Integer.parseInt(yyyymmdd.substring(6, 8)); 89 | calendar.set(year, month + diff, day); 90 | return calendar.getTime(); 91 | 92 | } 93 | 94 | private static boolean isdigit(String str) { 95 | for (char c : str.toCharArray()) if(!Character.isDigit(c)) return false; 96 | return true; 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/PostInfo.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | import org.springframework.util.CollectionUtils; 4 | 5 | import java.util.HashSet; 6 | import java.util.List; 7 | import java.util.Set; 8 | 9 | /** 10 | * 评论模型 11 | * @author decaywood 12 | * @date 2020/10/7 16:56 13 | */ 14 | public class PostInfo implements User.UserSetter, Comment.CommentSetter { 15 | /** 描述 */ 16 | private String description; 17 | /** 正文 */ 18 | private String text; 19 | /** 帖子id */ 20 | private String id; 21 | /** userId/commentId */ 22 | private String target; 23 | /** 标题 */ 24 | private String title; 25 | /** 用户id */ 26 | private String user_id; 27 | /** 用户信息 填充次信息需要用到 UserSetMapper */ 28 | private User user; 29 | /** 点赞数 */ 30 | private String like_count; 31 | /** 已阅读次数 */ 32 | private String view_count; 33 | /** 回复数量 */ 34 | private String reply_count; 35 | 36 | private Set comments = new HashSet<>(); 37 | 38 | public String getDescription() { 39 | return description; 40 | } 41 | 42 | public void setDescription(String description) { 43 | this.description = description; 44 | } 45 | 46 | public String getText() { 47 | return text; 48 | } 49 | 50 | public void setText(String text) { 51 | this.text = text; 52 | } 53 | 54 | public String getTarget() { 55 | return target; 56 | } 57 | 58 | public void setTarget(String target) { 59 | this.target = target; 60 | } 61 | 62 | public String getTitle() { 63 | return title; 64 | } 65 | 66 | public void setTitle(String title) { 67 | this.title = title; 68 | } 69 | 70 | public String getUser_id() { 71 | return user_id; 72 | } 73 | 74 | public void setUser_id(String user_id) { 75 | this.user_id = user_id; 76 | } 77 | 78 | public String getId() { 79 | return id; 80 | } 81 | 82 | public void setId(String id) { 83 | this.id = id; 84 | } 85 | 86 | public String getLike_count() { 87 | return like_count; 88 | } 89 | 90 | public void setLike_count(String like_count) { 91 | this.like_count = like_count; 92 | } 93 | 94 | public String getView_count() { 95 | return view_count; 96 | } 97 | 98 | public void setView_count(String view_count) { 99 | this.view_count = view_count; 100 | } 101 | 102 | public String getReply_count() { 103 | return reply_count; 104 | } 105 | 106 | public void setReply_count(String reply_count) { 107 | this.reply_count = reply_count; 108 | } 109 | 110 | public User getUser() { 111 | return user; 112 | } 113 | 114 | public Set getComments() { 115 | return comments; 116 | } 117 | 118 | public void setComments(Set comments) { 119 | this.comments = comments; 120 | } 121 | 122 | @Override 123 | public void addComments(List comments) { 124 | if (CollectionUtils.isEmpty(comments)) { 125 | return; 126 | } 127 | this.comments = new HashSet<>(this.comments); 128 | this.comments.addAll(comments); 129 | } 130 | 131 | @Override 132 | public String getPostId() { 133 | return getId(); 134 | } 135 | 136 | @Override 137 | public int getReplyCnt() { 138 | return Integer.parseInt(reply_count); 139 | } 140 | 141 | @Override 142 | public void setUser(User user) { 143 | this.user = user; 144 | } 145 | 146 | @Override 147 | public String getUserId() { 148 | return this.user_id; 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/StockScopeHotRankCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 6 | import org.decaywood.utils.RequestParaBuilder; 7 | import org.decaywood.utils.URLMapper; 8 | 9 | import java.net.URL; 10 | import java.rmi.RemoteException; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | /** 15 | * @author: decaywood 16 | * @date: 2015/11/25 9:58 17 | */ 18 | 19 | /** 20 | * 热股榜收集器 21 | */ 22 | public class StockScopeHotRankCollector extends AbstractCollector> { 23 | 24 | /** 25 | * 热股关注范围 26 | */ 27 | public enum Scope { 28 | 29 | GLOBAL_WITHIN_1_HOUR("10", "10"),//全球1小时内 30 | GLOBAL_WITHIN_24_HOUR("10", "20"),//全球24小时内 31 | US_WITHIN_1_HOUR("11", "11"),//美股1小时内 32 | US_WITHIN_24_HOUR("11", "21"),//美股24小时内 33 | SH_SZ_WITHIN_1_HOUR("12", "12"),//沪深1小时内 34 | SH_SZ_WITHIN_24_HOUR("12", "22"),//沪深24小时内 35 | HK_WITHIN_1_HOUR("13", "13"),//港股1小时内 36 | HK_WITHIN_24_HOUR("13", "23");//港股24小时内 37 | 38 | 39 | private String scope; 40 | private String time; 41 | 42 | Scope(String scope, String time) { 43 | this.scope = scope; 44 | this.time = time; 45 | } 46 | 47 | public String getScope() { 48 | return scope; 49 | } 50 | 51 | public String getTime() { 52 | return time; 53 | } 54 | } 55 | 56 | //排名前K阈值 57 | public static final int PAGE_SIZE_SHRESHOLD = 20; 58 | private Scope scope; 59 | private int topK; 60 | 61 | public StockScopeHotRankCollector() { 62 | this(PAGE_SIZE_SHRESHOLD); 63 | } 64 | 65 | public StockScopeHotRankCollector(Scope scope) { 66 | this(null, scope, PAGE_SIZE_SHRESHOLD); 67 | } 68 | 69 | public StockScopeHotRankCollector(int topK) { 70 | this(null, Scope.SH_SZ_WITHIN_1_HOUR, topK); 71 | } 72 | 73 | 74 | /** 75 | * 76 | * @param strategy 超时等待策略（null则设置为默认等待策略） 77 | * @param scope 热股关注范围 78 | * @param topK 排名前K个股 79 | */ 80 | public StockScopeHotRankCollector(TimeWaitingStrategy strategy, Scope scope, int topK) { 81 | super(strategy); 82 | if(topK <= 0) throw new IllegalArgumentException(); 83 | this.topK = Math.min(topK, PAGE_SIZE_SHRESHOLD); 84 | this.scope = scope == null ? Scope.SH_SZ_WITHIN_24_HOUR : scope; 85 | } 86 | 87 | 88 | @Override 89 | public List collectLogic() throws Exception { 90 | 91 | String target = URLMapper.SCOPE_STOCK_RANK_JSON.toString(); 92 | RequestParaBuilder builder = new RequestParaBuilder(target) 93 | .addParameter("size", topK) 94 | .addParameter("_type", scope.getScope()) 95 | .addParameter("type", scope.getTime()); 96 | URL url = new URL(builder.build()); 97 | String json = request(url); 98 | JsonNode node = mapper.readTree(json); 99 | return processNode(node); 100 | 101 | } 102 | 103 | private List processNode(JsonNode node) { 104 | List stocks = new ArrayList<>(); 105 | JsonNode rankNode = node.get("ranks"); 106 | for (JsonNode jsonNode : rankNode) { 107 | String symbol = jsonNode.get("symbol").asText(); 108 | String name = jsonNode.get("name").asText(); 109 | Stock stock = new Stock(name, symbol); 110 | stocks.add(stock); 111 | } 112 | return stocks; 113 | } 114 | 115 | public Scope getScope() { 116 | return scope; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/test/java/mapperTest/StockToStockWithAttributeMapperTest.java: -------------------------------------------------------------------------------- 1 | package mapperTest; 2 | 3 | import org.decaywood.entity.Stock; 4 | import org.decaywood.mapper.stockFirst.StockToStockWithAttributeMapper; 5 | import org.decaywood.utils.EmptyObject; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import java.rmi.RemoteException; 10 | import java.util.List; 11 | 12 | /** 13 | * @author: decaywood 14 | * @date: 2015/11/25 19:25 15 | */ 16 | public class StockToStockWithAttributeMapperTest { 17 | 18 | @Test 19 | public void testFunction() { 20 | List industries = TestCaseGenerator.generateStocks(); 21 | StockToStockWithAttributeMapper mapper = new StockToStockWithAttributeMapper(); 22 | industries.stream().map(mapper) 23 | .forEach(StockToStockWithAttributeMapperTest::checkNotNull); 24 | } 25 | 26 | @Test 27 | public void testNull() { 28 | StockToStockWithAttributeMapper mapper = new StockToStockWithAttributeMapper(); 29 | Assert.assertEquals(EmptyObject.emptyStock, mapper.apply(null)); 30 | Assert.assertEquals(EmptyObject.emptyStock, mapper.apply(EmptyObject.emptyStock)); 31 | } 32 | 33 | private static void checkNotNull(Stock stock) { 34 | 35 | Assert.assertNotEquals(stock.getTime(), EmptyObject.emptyString); 36 | Assert.assertNotEquals(stock.getPsr(), EmptyObject.emptyString); 37 | Assert.assertNotEquals(stock.getAmount(), EmptyObject.emptyString); 38 | Assert.assertNotEquals(stock.getDividend(), EmptyObject.emptyString); 39 | Assert.assertNotEquals(stock.getStockPageSite(), EmptyObject.emptyString); 40 | Assert.assertNotEquals(stock.getPe_lyr(), EmptyObject.emptyString); 41 | Assert.assertNotEquals(stock.getStockTrend(), EmptyObject.emptyString); 42 | Assert.assertNotEquals(stock.getFloat_market_capital(), EmptyObject.emptyString); 43 | Assert.assertNotEquals(stock.getHigh52Week(), EmptyObject.emptyString); 44 | Assert.assertNotEquals(stock.getTotalShares(), EmptyObject.emptyString); 45 | Assert.assertNotEquals(stock.getNet_assets(), EmptyObject.emptyString); 46 | Assert.assertNotEquals(stock.getRise_stop(), EmptyObject.emptyString); 47 | Assert.assertNotEquals(stock.getPe_ttm(), EmptyObject.emptyString); 48 | Assert.assertNotEquals(stock.getHigh(), EmptyObject.emptyString); 49 | Assert.assertNotEquals(stock.getStockNo(), EmptyObject.emptyString); 50 | Assert.assertNotEquals(stock.getVolume(), EmptyObject.emptyString); 51 | Assert.assertNotEquals(stock.getLow52week(), EmptyObject.emptyString); 52 | Assert.assertNotEquals(stock.getStockName(), EmptyObject.emptyString); 53 | Assert.assertNotEquals(stock.getCurrency_unit(), EmptyObject.emptyString); 54 | Assert.assertNotEquals(stock.getChange(), EmptyObject.emptyString); 55 | Assert.assertNotEquals(stock.getClose(), EmptyObject.emptyString); 56 | Assert.assertNotEquals(stock.getCurrent(), EmptyObject.emptyString); 57 | Assert.assertNotEquals(stock.getOpen(), EmptyObject.emptyString); 58 | Assert.assertNotEquals(stock.getPercentage(), EmptyObject.emptyString); 59 | Assert.assertNotEquals(stock.getEps(), EmptyObject.emptyString); 60 | Assert.assertNotEquals(stock.getLow(), EmptyObject.emptyString); 61 | Assert.assertNotEquals(stock.getTurnover_rate(), EmptyObject.emptyString); 62 | Assert.assertNotEquals(stock.getFall_stop(), EmptyObject.emptyString); 63 | Assert.assertNotEquals(stock.getMarketCapital(), EmptyObject.emptyString); 64 | Assert.assertNotEquals(stock.getFloat_shares(), EmptyObject.emptyString); 65 | Assert.assertNotEquals(stock.getAmplitude(), EmptyObject.emptyString); 66 | Assert.assertNotEquals(stock.getLast_close(), EmptyObject.emptyString); 67 | 68 | } 69 | 70 | 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/Cube.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | import org.decaywood.entity.trend.CubeTrend; 4 | import org.decaywood.entity.trend.MarketIndexTrend; 5 | import org.decaywood.entity.trend.Rebalancing; 6 | import org.decaywood.utils.EmptyObject; 7 | 8 | /** 9 | * @author: decaywood 10 | * @date: 2015/11/25 21:46. 11 | */ 12 | public class Cube implements DeepCopy { 13 | 14 | private final String cubeID; 15 | private final String name; 16 | private final String symbol; 17 | 18 | private String daily_gain = EmptyObject.emptyString; //日收益 19 | private String monthly_gain = EmptyObject.emptyString;//月收益 20 | private String annualized_gain_rate = EmptyObject.emptyString;//年收益 21 | private String total_gain = EmptyObject.emptyString;//总收益 22 | 23 | 24 | private CubeTrend cubeTrend = EmptyObject.emptyCubeTrend;//组合收益历史走线 25 | private MarketIndexTrend marketIndexTrend = EmptyObject.emptyMarketIndexTrend;//大盘历史走线 26 | private Rebalancing rebalancing = EmptyObject.emptyRebalancing;//调仓记录 27 | 28 | 29 | /** 30 | * 31 | * @param cubeID 组合代码 32 | * @param name 组合名称 33 | * @param symbol 组合代码 34 | */ 35 | public Cube(String cubeID, String name, String symbol) { 36 | this.cubeID = cubeID; 37 | this.name = name; 38 | this.symbol = symbol; 39 | } 40 | 41 | 42 | @Override 43 | public Cube copy() { 44 | Cube cube = new Cube(cubeID, name, symbol); 45 | 46 | cube.setDaily_gain(daily_gain); 47 | cube.setMonthly_gain(monthly_gain); 48 | cube.setAnnualized_gain_rate(annualized_gain_rate); 49 | cube.setTotal_gain(total_gain); 50 | 51 | cube.setCubeTrend(cubeTrend.copy()); 52 | cube.setMarketIndexTrend(marketIndexTrend.copy()); 53 | cube.setRebalancing(rebalancing); 54 | 55 | return cube; 56 | } 57 | 58 | 59 | 60 | //日收益 61 | public void setDaily_gain(String daily_gain) { 62 | this.daily_gain = daily_gain; 63 | } 64 | //月收益 65 | public void setMonthly_gain(String monthly_gain) { 66 | this.monthly_gain = monthly_gain; 67 | } 68 | //总收益 69 | public void setTotal_gain(String total_gain) { 70 | this.total_gain = total_gain; 71 | } 72 | //组合收益历史走线 73 | public void setCubeTrend(CubeTrend cubeTrend) { 74 | this.cubeTrend = cubeTrend; 75 | } 76 | //大盘历史走线 77 | public void setMarketIndexTrend(MarketIndexTrend marketIndexTrend) { 78 | this.marketIndexTrend = marketIndexTrend; 79 | } 80 | //调仓记录 81 | public void setRebalancing(Rebalancing rebalancing) { 82 | this.rebalancing = rebalancing; 83 | } 84 | //年收益 85 | public void setAnnualized_gain_rate(String annualized_gain_rate) { 86 | this.annualized_gain_rate = annualized_gain_rate; 87 | } 88 | 89 | 90 | 91 | public String getDaily_gain() { 92 | return daily_gain; 93 | } 94 | 95 | public String getMonthly_gain() { 96 | return monthly_gain; 97 | } 98 | 99 | public String getAnnualized_gain_rate() { 100 | return annualized_gain_rate; 101 | } 102 | 103 | public String getTotal_gain() { 104 | return total_gain; 105 | } 106 | 107 | 108 | public String getCubeID() { 109 | return cubeID; 110 | } 111 | 112 | public String getName() { 113 | return name; 114 | } 115 | 116 | public String getSymbol() { 117 | return symbol; 118 | } 119 | 120 | public Rebalancing getRebalancing() { 121 | return rebalancing; 122 | } 123 | 124 | public CubeTrend getCubeTrend() { 125 | return cubeTrend; 126 | } 127 | 128 | public MarketIndexTrend getMarketIndexTrend() { 129 | return marketIndexTrend; 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/cubeFirst/CubeToCubeWithLastBalancingMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.cubeFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Cube; 5 | import org.decaywood.entity.trend.Rebalancing; 6 | import org.decaywood.mapper.AbstractMapper; 7 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 8 | import org.decaywood.utils.EmptyObject; 9 | import org.decaywood.utils.RequestParaBuilder; 10 | import org.decaywood.utils.URLMapper; 11 | 12 | import java.net.URL; 13 | import java.rmi.RemoteException; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | 17 | /** 18 | * @author: decaywood 19 | * @date: 2015/11/26 15:56 20 | */ 21 | 22 | 23 | /** 24 | * 股票组合最近持仓装配器 25 | */ 26 | public class CubeToCubeWithLastBalancingMapper extends AbstractMapper { 27 | 28 | 29 | 30 | private static final int COUNT_THRESHOLD = 50; 31 | 32 | private final int count; 33 | 34 | public CubeToCubeWithLastBalancingMapper() { 35 | this(null, 10); 36 | } 37 | 38 | public CubeToCubeWithLastBalancingMapper(int i) { 39 | this(null, i); 40 | } 41 | 42 | /** 43 | * 44 | * @param strategy 超时等待策略（null则设置为默认等待策略） 45 | * @param count 调仓记录数 46 | */ 47 | public CubeToCubeWithLastBalancingMapper(TimeWaitingStrategy strategy, int count) { 48 | super(strategy); 49 | if(count <= 0) throw new IllegalArgumentException(); 50 | this.count = Math.min(COUNT_THRESHOLD, count); 51 | } 52 | 53 | @Override 54 | public Cube mapLogic(Cube cube) throws Exception { 55 | 56 | if(cube == null || cube == EmptyObject.emptyCube) return EmptyObject.emptyCube; 57 | 58 | String target = URLMapper.CUBE_REBALANCING_JSON.toString(); 59 | RequestParaBuilder builder = new RequestParaBuilder(target) 60 | .addParameter("cube_symbol", cube.getSymbol()) 61 | .addParameter("count", count) 62 | .addParameter("page", 1); 63 | URL url = new URL(builder.build()); 64 | String json = request(url); 65 | 66 | JsonNode node = mapper.readTree(json); 67 | processCube(cube, node); 68 | return cube; 69 | 70 | } 71 | 72 | private void processCube(Cube cube, JsonNode node) { 73 | JsonNode list = node.get("list"); 74 | 75 | List history = new ArrayList<>(); 76 | for (JsonNode jsonNode : list) { 77 | for (JsonNode jn : jsonNode.get("rebalancing_histories")) { 78 | String stock_name = jn.get("stock_name").asText(); 79 | String stock_symbol = jn.get("stock_symbol").asText(); 80 | String created_at = jn.get("created_at").asText(); 81 | String prev_price = jn.get("prev_price").asText(); 82 | String price = jn.get("price").asText(); 83 | String prev_weight = jn.get("prev_weight").asText(); 84 | String target_weight = jn.get("target_weight").asText(); 85 | String weight = jn.get("weight").asText(); 86 | String rebalancing_id = jn.get("rebalancing_id").asText(); 87 | Rebalancing.TrendBlock trendBlock = new Rebalancing.TrendBlock( 88 | stock_name, 89 | stock_symbol, 90 | created_at, 91 | prev_price, 92 | price, 93 | prev_weight, 94 | target_weight, 95 | weight, 96 | rebalancing_id); 97 | history.add(trendBlock); 98 | } 99 | 100 | } 101 | Rebalancing rebalancing = new Rebalancing(history); 102 | cube.setRebalancing(rebalancing); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/stockFirst/StockToStockWithShareHolderTrendMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.stockFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.entity.trend.ShareHoldersTrend; 6 | import org.decaywood.mapper.AbstractMapper; 7 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 8 | import org.decaywood.utils.DateParser; 9 | import org.decaywood.utils.EmptyObject; 10 | import org.decaywood.utils.RequestParaBuilder; 11 | import org.decaywood.utils.URLMapper; 12 | 13 | import java.net.URL; 14 | import java.rmi.RemoteException; 15 | import java.util.ArrayList; 16 | import java.util.Date; 17 | import java.util.List; 18 | 19 | /** 20 | * @author: decaywood 21 | * @date: 2015/11/29 22:01. 22 | */ 23 | 24 | /** 25 | * 上司公司股东变动信息装配器 26 | */ 27 | public class StockToStockWithShareHolderTrendMapper extends AbstractMapper { 28 | 29 | private Date from; 30 | private Date to; 31 | 32 | public StockToStockWithShareHolderTrendMapper() { 33 | this(new Date(0), new Date()); 34 | } 35 | 36 | public StockToStockWithShareHolderTrendMapper(Date from, Date to) { 37 | this(null, from, to); 38 | } 39 | 40 | 41 | /** 42 | * 43 | * @param strategy 超时等待策略（null则设置为默认等待策略） 44 | * @param from 查询起始时间 45 | * @param to 查询结束时间 46 | */ 47 | public StockToStockWithShareHolderTrendMapper(TimeWaitingStrategy strategy, Date from, Date to) { 48 | super(strategy); 49 | if(from.after(to)) throw new IllegalArgumentException(); 50 | this.from = from; 51 | this.to = to; 52 | 53 | } 54 | 55 | 56 | @Override 57 | public Stock mapLogic(Stock stock) throws Exception { 58 | if(stock == null || stock == EmptyObject.emptyStock) return EmptyObject.emptyStock; 59 | 60 | String target = URLMapper.STOCK_SHAREHOLDERS_JSON.toString(); 61 | RequestParaBuilder builder = new RequestParaBuilder(target) 62 | .addParameter("symbol", stock.getStockNo()) 63 | .addParameter("page", 1) 64 | .addParameter("size", 500); 65 | 66 | URL url = new URL(builder.build()); 67 | String json = request(url); 68 | 69 | JsonNode node = mapper.readTree(json).get("list"); 70 | processStock(stock, node); 71 | return stock; 72 | 73 | } 74 | 75 | private void processStock(Stock stock, JsonNode node) { 76 | 77 | List history = new ArrayList<>(); 78 | 79 | for (JsonNode jsonNode : node) { 80 | 81 | String enddate = jsonNode.get("enddate").asText(); 82 | Date date = DateParser.parseDate(enddate); 83 | 84 | if(from.after(date) || date.after(to)) continue; 85 | 86 | String totalshamt = jsonNode.get("totalshamt").asText(); 87 | String holdproportionpacc = jsonNode.get("holdproportionpacc").asText(); 88 | String totalshrto = jsonNode.get("totalshrto").asText(); 89 | String proportionchg = jsonNode.get("proportionchg").asText(); 90 | String proportiongrhalfyear = jsonNode.get("proportiongrhalfyear").asText(); 91 | String proportiongrq = jsonNode.get("proportiongrq").asText(); 92 | String avgholdsumgrhalfyear = jsonNode.get("avgholdsumgrhalfyear").asText(); 93 | String avgholdsumgrq = jsonNode.get("avgholdsumgrq").asText(); 94 | 95 | ShareHoldersTrend.TrendBlock block = new ShareHoldersTrend.TrendBlock(enddate, 96 | totalshamt, holdproportionpacc, totalshrto, proportionchg, proportiongrhalfyear, 97 | proportiongrq, avgholdsumgrhalfyear, avgholdsumgrq 98 | ); 99 | 100 | history.add(block); 101 | 102 | } 103 | 104 | ShareHoldersTrend trend = history.isEmpty() ? EmptyObject.emptyShareHoldersTrend 105 | : new ShareHoldersTrend(history); 106 | 107 | stock.setShareHoldersTrend(trend); 108 | } 109 | 110 | 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/GlobalSystemConfigLoader.java: -------------------------------------------------------------------------------- 1 | package org.decaywood; 2 | 3 | import org.decaywood.entity.Entry; 4 | import org.decaywood.utils.FileLoader; 5 | 6 | import java.io.BufferedReader; 7 | import java.io.File; 8 | import java.io.FileReader; 9 | import java.io.IOException; 10 | import java.util.ArrayList; 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.function.Predicate; 15 | 16 | /** 17 | * @author: decaywood 18 | * @date: 2015/12/11 10:17 19 | */ 20 | 21 | 22 | public abstract class GlobalSystemConfigLoader { 23 | 24 | /** 25 | * 只读，无需同步 26 | */ 27 | private static final Map RMIConfig = new HashMap<>(); 28 | 29 | private static final List> addresses = new ArrayList<>(); 30 | 31 | private static final List> filters; 32 | 33 | static { 34 | filters = new ArrayList<>(); 35 | filters.add(x -> x == null || x.trim().contains("##") || x.length() == 0); 36 | filters.add(x -> { 37 | if (x.contains("master_req_ip")) { 38 | String[] kv = x.split("=")[1].split(":"); 39 | String address = kv[0].trim(); 40 | Integer port = Integer.parseInt(kv[1].trim()); 41 | addresses.add(new Entry<>(address, port)); 42 | return true; 43 | } else return false; 44 | }); 45 | } 46 | 47 | /** 48 | * 避免多线程情况重复加载 49 | */ 50 | private static volatile boolean loaded = false; 51 | 52 | public static void loadConfig() { 53 | 54 | if(loaded) return; 55 | 56 | loaded = true; 57 | 58 | loadSystemConfig(); 59 | loadRMIConfig(); 60 | 61 | } 62 | 63 | public static Entry getAddress(int index) { 64 | if(index < 0 || index >= addresses.size()) throw new ArrayIndexOutOfBoundsException(); 65 | return addresses.get(index); 66 | } 67 | 68 | 69 | public static int getAddressSize() { 70 | return addresses.size(); 71 | } 72 | 73 | 74 | /** 75 | * 加载系统配置 76 | */ 77 | private static void loadSystemConfig() { 78 | 79 | File file = FileLoader.loadFile("config.sys"); 80 | try (BufferedReader reader = new BufferedReader(new FileReader(file))) { 81 | final String[] text = new String[1]; 82 | while ((text[0] = reader.readLine()) != null) { 83 | if (filters.stream().noneMatch(x -> x.test(text[0]))) { 84 | int index = text[0].indexOf("="); 85 | String key = text[0].substring(0, index); 86 | String value = text[0].substring(index + 1); 87 | System.setProperty(key.trim(), value.trim()); 88 | } 89 | } 90 | } catch (IOException e) { 91 | System.out.println("GlobalSystemConfigLoader -> 配置文件 -- config.sys 格式错误!"); 92 | } 93 | 94 | } 95 | 96 | 97 | /** 98 | * 加载RMI配置 99 | */ 100 | private static void loadRMIConfig() { 101 | 102 | File file = FileLoader.loadFile("config.rmi"); 103 | try (BufferedReader reader = new BufferedReader(new FileReader(file))) { 104 | final String[] text = new String[1]; 105 | while ((text[0] = reader.readLine()) != null) { 106 | if (filters.stream().noneMatch(x -> x.test(text[0]))) { 107 | String[] kv = text[0].split("="); 108 | String key = kv[0]; 109 | String value = kv[1]; 110 | RMIConfig.put(key.trim(), value.trim()); 111 | } 112 | } 113 | } catch (IOException e) { 114 | System.out.println("GlobalSystemConfigLoader -> 配置文件 -- config.rmi 格式错误!"); 115 | } 116 | } 117 | 118 | /** 119 | * 120 | * @param key RMI 属性名字 121 | * @return RMI 对应配置 122 | */ 123 | public static String getRMIConfig(String key) { 124 | return RMIConfig.get(key); 125 | } 126 | 127 | } 128 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/trend/ShareHoldersTrend.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.trend; 2 | 3 | import org.decaywood.utils.StringUtils; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | /** 9 | * @author: decaywood 10 | * @date: 2015/11/29 21:19. 11 | */ 12 | 13 | /** 14 | * 股票股东情况 15 | */ 16 | public class ShareHoldersTrend extends Trend { 17 | 18 | public ShareHoldersTrend(List history) { 19 | super(history); 20 | } 21 | 22 | public static class TrendBlock implements ITrendBlock { 23 | 24 | private final String enddate;//统计日期 25 | private final String totalshamt;//股东总户数 26 | private final String holdproportionpacc;//户均持股比例 27 | private final String totalshrto;//股东总户数较上期增减 28 | private final String proportionchg;//户均持股比例环比变化 29 | private final String proportiongrhalfyear;//户均持股比例半年增长率 30 | private final String proportiongrq;//户均持股比例季度增长率 31 | private final String avgholdsumgrhalfyear;//A股户均持股数半年增长率 32 | private final String avgholdsumgrq;//A股户均持股数季度增长率 33 | 34 | 35 | /** 36 | * 37 | * @param enddate 统计日期 38 | * @param totalshamt 股东总户数 39 | * @param holdproportionpacc 户均持股比例 40 | * @param totalshrto 股东总户数较上期增减 41 | * @param proportionchg 户均持股比例环比变化 42 | * @param proportiongrhalfyear 户均持股比例半年增长率 43 | * @param proportiongrq 户均持股比例季度增长率 44 | * @param avgholdsumgrhalfyear A股户均持股数半年增长率 45 | * @param avgholdsumgrq A股户均持股数季度增长率 46 | */ 47 | public TrendBlock(String enddate, 48 | String totalshamt, 49 | String holdproportionpacc, 50 | String totalshrto, 51 | String proportionchg, 52 | String proportiongrhalfyear, 53 | String proportiongrq, 54 | String avgholdsumgrhalfyear, 55 | String avgholdsumgrq) { 56 | 57 | if(StringUtils.nullOrEmpty( 58 | enddate, 59 | totalshamt, 60 | holdproportionpacc, 61 | totalshrto, 62 | proportionchg, 63 | proportiongrhalfyear, 64 | proportiongrq, 65 | avgholdsumgrhalfyear, 66 | avgholdsumgrq)) throw new IllegalArgumentException(); 67 | 68 | this.enddate = enddate; 69 | this.totalshamt = totalshamt; 70 | this.holdproportionpacc = holdproportionpacc; 71 | this.totalshrto = totalshrto; 72 | this.proportionchg = proportionchg; 73 | this.proportiongrhalfyear = proportiongrhalfyear; 74 | this.proportiongrq = proportiongrq; 75 | this.avgholdsumgrhalfyear = avgholdsumgrhalfyear; 76 | this.avgholdsumgrq = avgholdsumgrq; 77 | } 78 | 79 | public String getEnddate() { 80 | return enddate; 81 | } 82 | 83 | public String getTotalshamt() { 84 | return totalshamt; 85 | } 86 | 87 | public String getHoldproportionpacc() { 88 | return holdproportionpacc; 89 | } 90 | 91 | public String getTotalshrto() { 92 | return totalshrto; 93 | } 94 | 95 | public String getProportionchg() { 96 | return proportionchg; 97 | } 98 | 99 | public String getProportiongrhalfyear() { 100 | return proportiongrhalfyear; 101 | } 102 | 103 | public String getProportiongrq() { 104 | return proportiongrq; 105 | } 106 | 107 | public String getAvgholdsumgrhalfyear() { 108 | return avgholdsumgrhalfyear; 109 | } 110 | 111 | public String getAvgholdsumgrq() { 112 | return avgholdsumgrq; 113 | } 114 | } 115 | 116 | 117 | 118 | @Override 119 | public ShareHoldersTrend copy() { 120 | return new ShareHoldersTrend(new ArrayList<>(history)); 121 | } 122 | 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/utils/FileLoader.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.utils; 2 | 3 | import java.io.*; 4 | import java.util.Map; 5 | import java.util.concurrent.ConcurrentHashMap; 6 | 7 | /** 8 | * @author: decaywood 9 | * @date: 2015/11/23 18:40 10 | */ 11 | public abstract class FileLoader { 12 | 13 | private static final String COOKIE_FLUSH_PATH = "cookie/index.txt"; 14 | private static final String ROOT_PATH = FileLoader.class.getResource("").getPath(); 15 | 16 | private static final Map cookie = new ConcurrentHashMap<>();//多线程写状态，存在并发 17 | 18 | 19 | 20 | /** 21 | * 加载最新cookie 22 | * @param key 关键字 23 | * @return cookie 24 | */ 25 | public static String loadCookie(String key) { 26 | String cookies = System.getProperty("cookies"); 27 | if (StringUtils.isNotNull(cookies)) { 28 | return cookies; 29 | } 30 | if(cookie.containsKey(key)) return cookie.get(key); 31 | return EmptyObject.emptyString; 32 | } 33 | 34 | 35 | 36 | /** 37 | * 更新cookie 38 | * @param cookie cookie内容 39 | * @param key 关键字 40 | */ 41 | public static void updateCookie(String cookie, String key) { 42 | FileLoader.cookie.put(key, cookie); 43 | String replacedKey = key.contains(".com") ? key.substring(7, key.indexOf(".com")) : key; 44 | updateCookie(COOKIE_FLUSH_PATH, cookie, replacedKey); 45 | } 46 | 47 | 48 | public static void updateCookie(String rawPath, String text, String key) { 49 | updateCookie(rawPath, text, key, new StringBuilder(), true); 50 | } 51 | 52 | 53 | /** 54 | * 若文件不存在则创建文件 55 | * @param rawPath 文件相对路径 56 | * @param text 更新内容 57 | * @param key 文件名字 58 | * @param append 是否追加 59 | */ 60 | public static void updateCookie(String rawPath, String text, String key, StringBuilder builder, boolean append) { 61 | String path = ROOT_PATH + rawPath; 62 | File cookie = new File(path); 63 | String p; 64 | if(!cookie.exists()) updateCookie(builder.append("../").toString() + rawPath, text, key, builder, append); 65 | else { 66 | 67 | try { 68 | p = path.replace("index", key); 69 | 70 | File file = new File(p); 71 | boolean success = true; 72 | if(!file.exists()) success = file.createNewFile(); 73 | if(!success) throw new Exception(); 74 | 75 | try (BufferedWriter writer = new BufferedWriter(new FileWriter(p, append))) { 76 | writer.write(text); 77 | } 78 | 79 | } catch (Exception e) { 80 | System.out.println("FileLoader -> 写入失败!"); 81 | } 82 | } 83 | } 84 | 85 | /** 86 | * 加载文件内容（文件必须存在） 87 | * @param rawPath 文件相对位置 88 | * @return File 89 | */ 90 | private static File loadFile(String rawPath, StringBuilder builder) { 91 | String path = ROOT_PATH + rawPath; 92 | File file = new File(path); 93 | 94 | if(!file.exists()) return loadFile(builder.append("../").toString() + rawPath, builder); 95 | else return file; 96 | } 97 | 98 | /** 99 | * 加载文件内容（文件必须存在） 100 | * @param rawPath 文件相对位置 101 | * @return 文件内容 102 | */ 103 | private static String loadFileContent(String rawPath, StringBuilder builder) { 104 | File file = loadFile(rawPath, builder); 105 | StringBuilder content = new StringBuilder(); 106 | try (BufferedReader reader = new BufferedReader(new FileReader(file))) { 107 | String text; 108 | while ((text = reader.readLine()) != null) { 109 | content.append(text); 110 | } 111 | } catch (IOException e) { 112 | System.out.println("FileLoader -> 读取失败!"); 113 | } 114 | return content.toString(); 115 | } 116 | 117 | public static File loadFile(String rawPath) { 118 | return loadFile(rawPath, new StringBuilder()); 119 | } 120 | 121 | public static String loadFileContent(String rawPath) { 122 | return loadFileContent(rawPath, new StringBuilder()); 123 | } 124 | 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/stockFirst/StockToVIPFollowerCountEntryMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.stockFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Entry; 5 | import org.decaywood.entity.Stock; 6 | import org.decaywood.mapper.AbstractMapper; 7 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 8 | import org.decaywood.utils.EmptyObject; 9 | import org.decaywood.utils.URLMapper; 10 | import org.jsoup.Jsoup; 11 | import org.jsoup.nodes.Document; 12 | 13 | import java.io.IOException; 14 | import java.net.URL; 15 | import java.rmi.RemoteException; 16 | 17 | /** 18 | * @author: decaywood 19 | * @date: 2015/11/30 16:39 20 | */ 21 | 22 | /** 23 | * 股票 -> 股票+雪球大V数量 映射器 24 | * 速度很慢，慎用 25 | */ 26 | public class StockToVIPFollowerCountEntryMapper extends AbstractMapper > { 27 | 28 | private static final String REQUEST_PREFIX = URLMapper.MAIN_PAGE + "/S/"; 29 | private static final String REQUEST_SUFFIX = "/follows?page="; 30 | 31 | 32 | private int VIPFriendsCountShreshold; 33 | private int latestK_NewFollowers; 34 | 35 | public StockToVIPFollowerCountEntryMapper() { 36 | this(10000, 5); 37 | } 38 | 39 | public StockToVIPFollowerCountEntryMapper(int VIPFriendsCountShreshold, int latestK_NewFollowers) { 40 | this(null, VIPFriendsCountShreshold, latestK_NewFollowers); 41 | } 42 | 43 | 44 | /** 45 | * 46 | * @param strategy 超时等待策略（null则设置为默认等待策略） 47 | * @param VIPFriendsCountShreshold 是否为大V的粉丝阈值（超过这个阈值视为大V） 48 | * @param latestK_NewFollowers 只将最近K个新增用户纳入计算范围 49 | */ 50 | public StockToVIPFollowerCountEntryMapper(TimeWaitingStrategy strategy, 51 | int VIPFriendsCountShreshold, 52 | int latestK_NewFollowers) { 53 | super(strategy); 54 | if(VIPFriendsCountShreshold < 0 || latestK_NewFollowers < 0) throw new IllegalArgumentException(); 55 | this.VIPFriendsCountShreshold = VIPFriendsCountShreshold; 56 | this.latestK_NewFollowers = latestK_NewFollowers; 57 | } 58 | 59 | @Override 60 | public Entry mapLogic(Stock stock) throws Exception { 61 | 62 | if(stock == null || stock == EmptyObject.emptyStock) 63 | return new Entry<>(EmptyObject.emptyStock, 0); 64 | 65 | String stockNo = stock.getStockNo(); 66 | 67 | int count = 0; 68 | 69 | for (int i = 1; i < latestK_NewFollowers; i++) { 70 | 71 | String reqUrl = REQUEST_PREFIX + stockNo + REQUEST_SUFFIX + i; 72 | URL url = new URL(reqUrl); 73 | 74 | String content; 75 | while (true) { 76 | try { 77 | 78 | content = request(url); 79 | break; 80 | 81 | } catch (Exception e) { 82 | System.out.println("Mapper: Network busy Retrying"); 83 | } 84 | } 85 | 86 | JsonNode node = parseHtmlToJsonNode(content).get("followers"); 87 | 88 | if(node.size() == 0) break; 89 | 90 | for (JsonNode jsonNode : node) { 91 | int followersCount = jsonNode.get("followers_count").asInt(); 92 | if(followersCount > VIPFriendsCountShreshold) count++; 93 | } 94 | 95 | } 96 | 97 | return new Entry<>(stock, count); 98 | } 99 | 100 | 101 | private JsonNode parseHtmlToJsonNode(String content) throws IOException { 102 | 103 | Document doc = Jsoup.parse(content); 104 | String indexer1 = "follows="; 105 | String indexer2 = ";seajs.use"; 106 | StringBuilder builder = new StringBuilder( 107 | doc.getElementsByTag("script") 108 | .get(15) 109 | .dataNodes() 110 | .get(0) 111 | .attr("data")); 112 | int index = builder.indexOf(indexer1); 113 | builder.delete(0, index + indexer1.length()); 114 | index = builder.indexOf(indexer2); 115 | builder.delete(index, builder.length()); 116 | return mapper.readTree(builder.toString()); 117 | 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/test/java/collectTest/MarketQuotationsRankCollectorTest.java: -------------------------------------------------------------------------------- 1 | package collectTest; 2 | 3 | import org.decaywood.collector.MarketQuotationsRankCollector; 4 | import org.decaywood.collector.MarketQuotationsRankCollector.StockType; 5 | import org.decaywood.entity.Stock; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import java.rmi.RemoteException; 10 | import java.util.List; 11 | 12 | /** 13 | * @author: decaywood 14 | * @date: 2015/11/25 14:55 15 | */ 16 | public class MarketQuotationsRankCollectorTest { 17 | 18 | 19 | @Test(expected = IllegalArgumentException.class) 20 | public void testNull() { 21 | 22 | MarketQuotationsRankCollector collector = 23 | new MarketQuotationsRankCollector(null, 24 | MarketQuotationsRankCollector.ORDER_BY_TURNOVER_RATE, 25 | -5); 26 | Assert.assertNotNull(collector.getStockType()); 27 | MarketQuotationsRankCollector collector1 = 28 | new MarketQuotationsRankCollector(StockType.GROWTH_ENTERPRISE_BOARD, 29 | null, 30 | -5); 31 | Assert.assertNotNull(collector1.getOrderPattern()); 32 | } 33 | 34 | 35 | @Test(expected = IllegalArgumentException.class) 36 | public void testOverFlowTopK() { 37 | MarketQuotationsRankCollector collector = 38 | new MarketQuotationsRankCollector(StockType.GROWTH_ENTERPRISE_BOARD, 39 | MarketQuotationsRankCollector.ORDER_BY_TURNOVER_RATE, 40 | -5); 41 | } 42 | 43 | 44 | @Test 45 | public void testTopKMaxSize() throws Exception { 46 | MarketQuotationsRankCollector collector = 47 | new MarketQuotationsRankCollector(StockType.GROWTH_ENTERPRISE_BOARD, 48 | MarketQuotationsRankCollector.ORDER_BY_TURNOVER_RATE, 49 | 100); 50 | List stocks = collector.get(); 51 | Assert.assertTrue(stocks.size() <= MarketQuotationsRankCollector.TOPK_MAX_SHRESHOLD); 52 | } 53 | 54 | @Test 55 | public void testTopKSize() throws Exception { 56 | int orderSize = 3; 57 | MarketQuotationsRankCollector collector = 58 | new MarketQuotationsRankCollector(StockType.GROWTH_ENTERPRISE_BOARD, 59 | MarketQuotationsRankCollector.ORDER_BY_TURNOVER_RATE, 60 | orderSize); 61 | List stocks = collector.get(); 62 | Assert.assertTrue(stocks.size() == orderSize); 63 | } 64 | 65 | @Test 66 | public void testStockType() { 67 | doTestStockType(StockType.GROWTH_ENTERPRISE_BOARD); 68 | doTestStockType(StockType.HK); 69 | doTestStockType(StockType.SH_A); 70 | doTestStockType(StockType.SH_B); 71 | doTestStockType(StockType.SMALL_MEDIUM_ENTERPRISE_BOARD); 72 | doTestStockType(StockType.SZ_A); 73 | doTestStockType(StockType.SZ_B); 74 | doTestStockType(StockType.US); 75 | } 76 | 77 | private void doTestStockType(StockType type) { 78 | MarketQuotationsRankCollector collector = 79 | new MarketQuotationsRankCollector(type, 80 | MarketQuotationsRankCollector.ORDER_BY_TURNOVER_RATE); 81 | List stocks = collector.get(); 82 | Assert.assertTrue(stocks.size() > 0); 83 | } 84 | 85 | @Test(expected = IllegalArgumentException.class) 86 | public void testWrongOrderBy() { 87 | doTestOrderBy("wrong"); 88 | } 89 | 90 | 91 | @Test 92 | public void testOrderBy() { 93 | doTestOrderBy(MarketQuotationsRankCollector.ORDER_BY_AMOUNT); 94 | doTestOrderBy(MarketQuotationsRankCollector.ORDER_BY_PERCENT); 95 | doTestOrderBy(MarketQuotationsRankCollector.ORDER_BY_TURNOVER_RATE); 96 | doTestOrderBy(MarketQuotationsRankCollector.ORDER_BY_VOLUME); 97 | } 98 | 99 | 100 | private void doTestOrderBy(String orderBy) { 101 | MarketQuotationsRankCollector collector = 102 | new MarketQuotationsRankCollector(StockType.HK, orderBy); 103 | List stocks = collector.get(); 104 | Assert.assertTrue(stocks.size() > 0); 105 | } 106 | 107 | 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/stockFirst/StockToStockWithCompanyInfoMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.stockFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.CompanyInfo; 5 | import org.decaywood.entity.Industry; 6 | import org.decaywood.entity.Stock; 7 | import org.decaywood.mapper.AbstractMapper; 8 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 9 | import org.decaywood.utils.EmptyObject; 10 | import org.decaywood.utils.RequestParaBuilder; 11 | import org.decaywood.utils.URLMapper; 12 | import org.jsoup.Jsoup; 13 | import org.jsoup.nodes.Document; 14 | import org.jsoup.nodes.Element; 15 | import org.jsoup.select.Elements; 16 | 17 | import java.net.URL; 18 | import java.rmi.RemoteException; 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | /** 25 | * @author: decaywood 26 | * @date: 2015/11/30 16:13 27 | */ 28 | 29 | /** 30 | * 股票公司信息装配器 31 | */ 32 | public class StockToStockWithCompanyInfoMapper extends AbstractMapper { 33 | 34 | private Map industryMap; 35 | private volatile boolean initializing; 36 | 37 | public StockToStockWithCompanyInfoMapper() { 38 | this(null); 39 | } 40 | 41 | /** 42 | * @param strategy 超时等待策略（null则设置为默认等待策略） 43 | */ 44 | public StockToStockWithCompanyInfoMapper(TimeWaitingStrategy strategy) { 45 | super(strategy); 46 | } 47 | 48 | private void initMap() throws Exception { 49 | 50 | industryMap = new HashMap<>(); 51 | 52 | String target = URLMapper.COMPREHENSIVE_PAGE.toString(); 53 | 54 | String content = request(new URL(target)); 55 | Document doc = Jsoup.parse(content); 56 | Elements element = doc.getElementsByClass("second-nav") 57 | .get(1).children() 58 | .get(3).children() 59 | .get(3).children() 60 | .select("a"); 61 | StringBuilder builder = new StringBuilder(); 62 | for (Element ele : element) { 63 | if (!ele.hasAttr("title") || !ele.hasAttr("href")) continue; 64 | builder.append(ele.attr("href")); 65 | industryMap.put(ele.attr("title"), new Industry(ele.attr("title"), builder.toString())); 66 | builder.delete(0, builder.length()); 67 | } 68 | } 69 | 70 | @Override 71 | public Stock mapLogic(Stock stock) throws Exception { 72 | 73 | if(stock == null || stock == EmptyObject.emptyStock) return EmptyObject.emptyStock; 74 | 75 | if(industryMap == null && !initializing) { 76 | initializing = true; 77 | initMap(); 78 | } else while (industryMap == null) { 79 | Thread.sleep(50); 80 | } 81 | 82 | String target = URLMapper.STOCK_INDUSTRY_JSON.toString(); 83 | RequestParaBuilder builder = new RequestParaBuilder(target) 84 | .addParameter("symbol", stock.getStockNo()); 85 | 86 | URL url = new URL(builder.build()); 87 | String json = request(url); 88 | 89 | JsonNode node = mapper.readTree(json).get("tqCompInfo"); 90 | processStock(stock, node); 91 | return stock; 92 | 93 | } 94 | 95 | private void processStock(Stock stock, JsonNode node) { 96 | String compsname = node.get("compsname").asText(); 97 | String orgtype = node.get("orgtype").asText(); 98 | String founddate = node.get("founddate").asText(); 99 | String bizscope = node.get("bizscope").asText(); 100 | String majorbiz = node.get("majorbiz").asText(); 101 | String region = node.get("region").asText(); 102 | 103 | List industries = new ArrayList<>(); 104 | JsonNode subNode = node.get("tqCompIndustryList"); 105 | 106 | for (JsonNode jsonNode : subNode) { 107 | String industryName = jsonNode.get("level2name").asText(); 108 | industries.add(industryMap.get(industryName).copy()); 109 | } 110 | 111 | CompanyInfo companyInfo = new CompanyInfo( 112 | compsname, orgtype, founddate, bizscope, majorbiz, region, industries); 113 | 114 | stock.setCompanyInfo(companyInfo); 115 | 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/HuShenNewsRefCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 5 | import org.decaywood.utils.RequestParaBuilder; 6 | import org.decaywood.utils.URLMapper; 7 | 8 | import java.io.IOException; 9 | import java.net.MalformedURLException; 10 | import java.net.URL; 11 | import java.rmi.RemoteException; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | /** 16 | * @author: decaywood 17 | * @date: 2015/12/2 21:37. 18 | */ 19 | 20 | 21 | /** 22 | * 收集沪深板块热点新闻URL 23 | */ 24 | public class HuShenNewsRefCollector extends AbstractCollector> { 25 | 26 | //新闻内容阈值 27 | private static final int MAX_PAGE_SIZE = 5; 28 | 29 | //新闻主题 30 | public enum Topic { 31 | 32 | TOTAL("5"),//全部新闻 33 | MARKET("6"),//市场动态 34 | ANALISIS("7"),//投资分析 35 | IDEA("8");//投资理念 36 | 37 | private String topic; 38 | Topic(String str) { 39 | this.topic = str; 40 | } 41 | 42 | public String getVal() { 43 | return this.topic; 44 | } 45 | 46 | } 47 | 48 | private Topic topicType = Topic.TOTAL; 49 | private int pageEndTo = 1; 50 | 51 | public HuShenNewsRefCollector() { 52 | this(Topic.TOTAL, 1); 53 | } 54 | 55 | public HuShenNewsRefCollector(Topic topicType, int pageEndTo) { 56 | this(null, topicType, pageEndTo); 57 | } 58 | 59 | /** 60 | * 61 | * @param strategy 超时等待策略（null则设置为默认等待策略） 62 | * @param topicType 主题类型 63 | * @param pageEndTo 搜索页面数 从1到pageEndTo 64 | */ 65 | public HuShenNewsRefCollector(TimeWaitingStrategy strategy, Topic topicType, int pageEndTo) { 66 | super(strategy); 67 | if(pageEndTo < 1) 68 | throw new IllegalArgumentException(); 69 | 70 | this.pageEndTo = Math.min(pageEndTo, MAX_PAGE_SIZE); 71 | this.topicType = topicType; 72 | 73 | } 74 | 75 | @Override 76 | public List collectLogic() throws Exception { 77 | 78 | String target = URLMapper.HU_SHEN_NEWS_REF_JSON.toString(); 79 | 80 | 81 | List nodeList = new ArrayList<>(); 82 | 83 | for (int i = 1; i <= pageEndTo; i++) { 84 | JsonNode node; 85 | try { 86 | int loopTime = 1; 87 | while (strategy.retryTimes() > loopTime) { 88 | try { 89 | RequestParaBuilder builder = new RequestParaBuilder(target) 90 | .addParameter("simple_user", "1") 91 | .addParameter("topicType", topicType.getVal()) 92 | .addParameter("page", i); 93 | 94 | URL url = new URL(builder.build()); 95 | String json = request(url); 96 | node = mapper.readTree(json); 97 | nodeList.add(node); 98 | break; 99 | } catch (Exception e) { 100 | if(!(e instanceof IOException)) throw e; 101 | System.out.println("Collector: Network busy Retrying -> " + loopTime + " times"); 102 | updateCookie(webSite); 103 | this.strategy.waiting(loopTime++); 104 | } 105 | } 106 | } catch (Exception e) { 107 | e.printStackTrace(); 108 | } 109 | } 110 | 111 | return processNode(nodeList); 112 | } 113 | 114 | private List processNode(List nodeList) { 115 | 116 | List res = new ArrayList<>(); 117 | 118 | for (JsonNode node : nodeList) { 119 | try { 120 | for (JsonNode jsonNode : node) { 121 | String suffix = jsonNode.get("target").asText(); 122 | String path = URLMapper.MAIN_PAGE.toString() + suffix; 123 | res.add(new URL(path)); 124 | } 125 | } catch (MalformedURLException e) { 126 | e.printStackTrace(); 127 | } 128 | } 129 | 130 | return res; 131 | 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/trend/Rebalancing.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.trend; 2 | 3 | import org.decaywood.utils.StringUtils; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * @author: decaywood 9 | * @date: 2015/11/26 17:15 10 | */ 11 | 12 | /** 13 | * 股票组合调仓记录 14 | */ 15 | public class Rebalancing extends Trend { 16 | 17 | 18 | /** 19 | * @param history 调仓历史 20 | */ 21 | public Rebalancing(List history) { 22 | super(history); 23 | } 24 | 25 | 26 | /** 27 | * 历史节点 28 | */ 29 | public static class TrendBlock implements ITrendBlock { 30 | private final String stock_name; 31 | private final String stock_symbol; 32 | private final String created_at;//time 33 | private final String prev_price; 34 | private final String price; 35 | private final String prev_weight; 36 | private final String target_weight; 37 | private final String weight; 38 | private final String rebalancing_id; 39 | 40 | /** 41 | * 42 | * @param stock_name 股票名称 43 | * @param stock_symbol 股票代码 44 | * @param created_at 调仓时间 45 | * @param prev_price 上一次调仓价格 46 | * @param price 当前价格 47 | * @param prev_weight 上一次持仓比例 48 | * @param target_weight 期望持仓比例 49 | * @param weight 实际持仓比例 50 | * @param rebalancing_id 调仓节点ID 51 | */ 52 | public TrendBlock(String stock_name, 53 | String stock_symbol, 54 | String created_at, 55 | String prev_price, 56 | String price, 57 | String prev_weight, 58 | String target_weight, 59 | String weight, 60 | String rebalancing_id 61 | ) { 62 | 63 | if(StringUtils.nullOrEmpty(stock_name, stock_symbol, created_at 64 | , prev_price, price, prev_weight, target_weight, weight, rebalancing_id)) 65 | throw new IllegalArgumentException(); 66 | 67 | this.stock_name = stock_name; 68 | this.stock_symbol = stock_symbol; 69 | this.created_at = created_at; 70 | this.prev_price = prev_price; 71 | this.price = price; 72 | this.prev_weight = prev_weight; 73 | this.target_weight = target_weight; 74 | this.weight = weight; 75 | this.rebalancing_id = rebalancing_id; 76 | } 77 | 78 | public String getStock_name() { 79 | return stock_name; 80 | } 81 | 82 | public String getStock_symbol() { 83 | return stock_symbol; 84 | } 85 | 86 | public String getCreated_at() { 87 | return created_at; 88 | } 89 | 90 | public String getPrev_price() { 91 | return prev_price; 92 | } 93 | 94 | public String getPrice() { 95 | return price; 96 | } 97 | 98 | public String getPrev_weight() { 99 | return prev_weight; 100 | } 101 | 102 | public String getTarget_weight() { 103 | return target_weight; 104 | } 105 | 106 | public String getWeight() { 107 | return weight; 108 | } 109 | 110 | public String getRebalancing_id() { 111 | return rebalancing_id; 112 | } 113 | 114 | @Override 115 | public String toString() { 116 | return "TrendBlock{" + 117 | "stock_name='" + stock_name + '\'' + 118 | ", stock_symbol='" + stock_symbol + '\'' + 119 | ", created_at='" + created_at + '\'' + 120 | ", prev_price='" + prev_price + '\'' + 121 | ", price='" + price + '\'' + 122 | ", prev_weight='" + prev_weight + '\'' + 123 | ", target_weight='" + target_weight + '\'' + 124 | ", weight='" + weight + '\'' + 125 | ", rebalancing_id='" + rebalancing_id + '\'' + 126 | '}'; 127 | } 128 | } 129 | 130 | 131 | 132 | @Override 133 | public Rebalancing copy() { 134 | return new Rebalancing(history); 135 | } 136 | 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/LongHuBangInfo.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity; 2 | 3 | import org.decaywood.utils.StringUtils; 4 | 5 | import java.io.Serializable; 6 | import java.util.Date; 7 | import java.util.HashSet; 8 | import java.util.Set; 9 | 10 | /** 11 | * @author: decaywood 12 | * @date: 2015/11/27 0:29 13 | */ 14 | public class LongHuBangInfo implements DeepCopy { 15 | 16 | private final Stock stock; 17 | private final Date date; 18 | 19 | private final Set topBuyList; 20 | private final Set topSaleList; 21 | 22 | public static class BizsunitInfo implements Serializable { 23 | private final String bizsunitcode; //营业部编号 24 | private final String bizsunitname; //营业部名称 25 | private final String buyamt; // 买入额度 26 | private final String saleamt; // 卖出额度 27 | private final String tradedate; //交易日期 yyyymmdd 28 | 29 | public BizsunitInfo(String bizsunitcode, String bizsunitname, String buyamt, String saleamt, String tradedate) { 30 | 31 | if(StringUtils.nullOrEmpty(bizsunitcode, bizsunitname, buyamt, saleamt, tradedate)) 32 | throw new IllegalArgumentException(); 33 | 34 | this.bizsunitcode = bizsunitcode; 35 | this.bizsunitname = bizsunitname; 36 | this.buyamt = buyamt; 37 | this.saleamt = saleamt; 38 | this.tradedate = tradedate; 39 | } 40 | 41 | public String getBizsunitcode() { 42 | return bizsunitcode; 43 | } 44 | 45 | public String getBizsunitname() { 46 | return bizsunitname; 47 | } 48 | 49 | public String getBuyamt() { 50 | return buyamt; 51 | } 52 | 53 | public String getSaleamt() { 54 | return saleamt; 55 | } 56 | 57 | public String getTradedate() { 58 | return tradedate; 59 | } 60 | 61 | @Override 62 | public boolean equals(Object o) { 63 | if (this == o) return true; 64 | if (o == null || getClass() != o.getClass()) return false; 65 | 66 | BizsunitInfo info = (BizsunitInfo) o; 67 | 68 | return bizsunitname.equals(info.bizsunitname); 69 | 70 | } 71 | 72 | @Override 73 | public int hashCode() { 74 | return bizsunitname.hashCode(); 75 | } 76 | } 77 | 78 | public LongHuBangInfo(Stock stock, Date date, Set topBuyList, Set topSaleList) { 79 | this.stock = stock; 80 | this.date = date; 81 | this.topBuyList = topBuyList; 82 | this.topSaleList = topSaleList; 83 | } 84 | 85 | 86 | 87 | public Stock getStock() { 88 | return stock; 89 | } 90 | 91 | public Date getDate() { 92 | return date; 93 | } 94 | 95 | //龙虎榜买入是否有该营业部出现 96 | public boolean bizsunitInBuyList(String name) { 97 | return bizsunitInBuyList(name, false); 98 | } 99 | 100 | //龙虎榜买入是否有该营业部出现 101 | public boolean bizsunitInBuyList(String name, boolean partlySearch) { 102 | if (partlySearch) { 103 | contains(topBuyList, name); 104 | } 105 | return topBuyList.contains(new BizsunitInfo("xx", name, "xx", "xx", "xx")); 106 | } 107 | 108 | 109 | //龙虎榜卖出是否有该营业部出现 110 | public boolean bizsunitInSaleList(String name) { 111 | return bizsunitInSaleList(name, false); 112 | } 113 | 114 | //龙虎榜卖出是否有该营业部出现 115 | public boolean bizsunitInSaleList(String name, boolean partlySearch) { 116 | if (partlySearch) { 117 | contains(topSaleList, name); 118 | } 119 | return topSaleList.contains(new BizsunitInfo("xx", name, "xx", "xx", "xx")); 120 | } 121 | 122 | private boolean contains(Set set, String name) { 123 | for (BizsunitInfo info : set) { 124 | if (info.getBizsunitname().contains(name)) return true; 125 | } 126 | return false; 127 | } 128 | 129 | public Set getTopBuyList() { 130 | return topBuyList; 131 | } 132 | 133 | public Set getTopSaleList() { 134 | return topSaleList; 135 | } 136 | 137 | @Override 138 | public LongHuBangInfo copy() { 139 | return new LongHuBangInfo(stock.copy(), date, new HashSet<>(topBuyList), new HashSet<>(topBuyList)); 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/MostProfitableCubeCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Cube; 5 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 6 | import org.decaywood.utils.RequestParaBuilder; 7 | import org.decaywood.utils.URLMapper; 8 | 9 | import java.net.URL; 10 | import java.rmi.RemoteException; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | /** 15 | * @author: decaywood 16 | * @date: 2015/11/25 21:45. 17 | */ 18 | 19 | /** 20 | * 雪球最赚钱组合（Cube）收集器 21 | */ 22 | public class MostProfitableCubeCollector extends AbstractCollector> { 23 | 24 | /** 25 | * 组合所在股市 26 | */ 27 | public enum Market { 28 | CN("cn"),//沪深组合 29 | US("us"),//美股组合 30 | HK("hk");//港股组合 31 | 32 | private String val; 33 | 34 | Market(String val) { 35 | this.val = val; 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | return val; 41 | } 42 | } 43 | 44 | /** 45 | * 收益排序规则 46 | */ 47 | public enum ORDER_BY { 48 | DAILY("daily_gain"),//按日收益排序 49 | MONTHLY("monthly_gain"),//按月收益排序 50 | YEARLY("annualized_gain_rate");//按年收益 51 | 52 | private String val; 53 | 54 | ORDER_BY(String val) { 55 | this.val = val; 56 | } 57 | 58 | @Override 59 | public String toString() { 60 | return val; 61 | } 62 | } 63 | 64 | 65 | public static final int CUBE_SIZE_SHRESHOLD = 400; //topK阈值 66 | private Market market; 67 | private ORDER_BY order_by; 68 | private int topK; 69 | 70 | public MostProfitableCubeCollector() { 71 | this(Market.CN); 72 | } 73 | 74 | public MostProfitableCubeCollector(Market market) { 75 | this(market, ORDER_BY.MONTHLY); 76 | } 77 | 78 | public MostProfitableCubeCollector(Market market, ORDER_BY order_by) { 79 | this(null, market, order_by, 10); 80 | } 81 | 82 | 83 | /** 84 | * 85 | * @param strategy 超时等待策略（null则设置为默认等待策略） 86 | * @param market 组合所在市场 87 | * @param order_by 收益排序规则 88 | * @param topK 排名前K的组合 89 | */ 90 | public MostProfitableCubeCollector(TimeWaitingStrategy strategy, Market market, ORDER_BY order_by, int topK) { 91 | super(strategy); 92 | 93 | this.market = market == null ? Market.CN : market; 94 | this.order_by = order_by == null ? ORDER_BY.MONTHLY : order_by; 95 | 96 | if(topK <= 0) throw new IllegalArgumentException(); 97 | this.topK = Math.min(topK, CUBE_SIZE_SHRESHOLD); 98 | 99 | } 100 | 101 | @Override 102 | public List collectLogic() throws Exception { 103 | String target = URLMapper.CUBES_RANK_JSON.toString(); 104 | RequestParaBuilder builder = new RequestParaBuilder(target) 105 | .addParameter("category", 12) 106 | .addParameter("count", topK) 107 | .addParameter("market", market.toString()) 108 | .addParameter("profit", order_by.toString()); 109 | 110 | URL url = new URL(builder.build()); 111 | String json = request(url); 112 | JsonNode node = mapper.readTree(json); 113 | return processNode(node); 114 | } 115 | 116 | 117 | private List processNode(JsonNode node) { 118 | 119 | JsonNode list = node.get("list"); 120 | List cubes = new ArrayList<>(); 121 | for (JsonNode jsonNode : list) { 122 | 123 | String id = jsonNode.get("id").asText(); 124 | String name = jsonNode.get("name").asText(); 125 | String symbol = jsonNode.get("symbol").asText(); 126 | String daily_gain = jsonNode.get("daily_gain").asText(); 127 | String monthly_gain = jsonNode.get("monthly_gain").asText(); 128 | String annualized_gain_rate = jsonNode.get("annualized_gain_rate").asText(); 129 | String total_gain = jsonNode.get("total_gain").asText(); 130 | Cube cube = new Cube(id, name, symbol); 131 | cube.setDaily_gain(daily_gain); 132 | cube.setMonthly_gain(monthly_gain); 133 | cube.setAnnualized_gain_rate(annualized_gain_rate); 134 | cube.setTotal_gain(total_gain); 135 | cubes.add(cube); 136 | 137 | } 138 | return cubes; 139 | 140 | } 141 | 142 | public Market getMarket() { 143 | return market; 144 | } 145 | 146 | public ORDER_BY getOrder_by() { 147 | return order_by; 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/stockFirst/StockToStockWithStockTrendMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.stockFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.entity.trend.StockTrend; 6 | import org.decaywood.entity.trend.StockTrend.Period; 7 | import org.decaywood.entity.trend.StockTrend.TrendBlock; 8 | import org.decaywood.mapper.AbstractMapper; 9 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 10 | import org.decaywood.utils.EmptyObject; 11 | import org.decaywood.utils.RequestParaBuilder; 12 | import org.decaywood.utils.URLMapper; 13 | 14 | import java.net.URL; 15 | import java.rmi.RemoteException; 16 | import java.util.ArrayList; 17 | import java.util.Calendar; 18 | import java.util.Date; 19 | import java.util.List; 20 | 21 | /** 22 | * @author: decaywood 23 | * @date: 2015/11/24 15:23 24 | */ 25 | public class StockToStockWithStockTrendMapper extends AbstractMapper { 26 | 27 | 28 | private Period period; 29 | private Date from; 30 | private Date to; 31 | 32 | public StockToStockWithStockTrendMapper() { 33 | this(Period.DAY, null, null); 34 | } 35 | 36 | 37 | public StockToStockWithStockTrendMapper(Date from, Date to) { 38 | this(Period.DAY, from, to); 39 | } 40 | 41 | public StockToStockWithStockTrendMapper(Period period, Date from, Date to) { 42 | this(null, period, from, to); 43 | } 44 | 45 | public StockToStockWithStockTrendMapper(TimeWaitingStrategy strategy, 46 | Period period, 47 | Date from, 48 | Date to) { 49 | super(strategy); 50 | if (from == null || to == null) { 51 | Calendar calendar = Calendar.getInstance(); 52 | this.to = new Date(); 53 | calendar.setTime(this.to); 54 | calendar.set(Calendar.DATE, calendar.get(Calendar.DATE) - 5); 55 | this.from = calendar.getTime(); 56 | } else { 57 | this.from = from; 58 | this.to = to; 59 | } 60 | if(this.to.before(this.from)) throw new IllegalArgumentException(); 61 | this.period = period; 62 | } 63 | 64 | @Override 65 | public Stock mapLogic(Stock stock) throws Exception { 66 | 67 | if(stock == null || stock == EmptyObject.emptyStock) return EmptyObject.emptyStock; 68 | 69 | String target = URLMapper.STOCK_TREND_JSON.toString(); 70 | RequestParaBuilder builder = new RequestParaBuilder(target) 71 | .addParameter("symbol", stock.getStockNo()) 72 | .addParameter("period", period.toString()) 73 | .addParameter("type", "normal") 74 | .addParameter("begin", from.getTime()) 75 | .addParameter("end", to.getTime()); 76 | 77 | URL url = new URL(builder.build()); 78 | 79 | String json = request(url); 80 | JsonNode node = mapper.readTree(json).get("chartlist"); 81 | processStock(stock, node); 82 | return stock; 83 | 84 | } 85 | 86 | 87 | private void processStock(Stock stock, JsonNode node) { 88 | 89 | List history = new ArrayList<>(); 90 | 91 | for (JsonNode jsonNode : node) { 92 | 93 | String volume = jsonNode.get("volume").asText(); 94 | String open = jsonNode.get("open").asText(); 95 | String high = jsonNode.get("high").asText(); 96 | String close = jsonNode.get("close").asText(); 97 | String low = jsonNode.get("low").asText(); 98 | String chg = jsonNode.get("chg").asText(); 99 | String percent = jsonNode.get("percent").asText(); 100 | String turnrate = jsonNode.get("turnrate").asText(); 101 | String ma5 = jsonNode.get("ma5").asText(); 102 | String ma10 = jsonNode.get("ma10").asText(); 103 | String ma20 = jsonNode.get("ma20").asText(); 104 | String ma30 = jsonNode.get("ma30").asText(); 105 | String dif = jsonNode.get("dif").asText(); 106 | String dea = jsonNode.get("dea").asText(); 107 | String macd = jsonNode.get("macd").asText(); 108 | String time = jsonNode.get("time").asText(); 109 | 110 | TrendBlock block = new TrendBlock( 111 | volume, open, high, close, low, chg, percent, turnrate, 112 | ma5, ma10, ma20, ma30, dif, dea, macd, time); 113 | history.add(block); 114 | 115 | } 116 | 117 | StockTrend trend = history.isEmpty() ? EmptyObject.emptyStockTrend 118 | : new StockTrend(stock.getStockNo(), period, history); 119 | stock.setStockTrend(trend); 120 | } 121 | 122 | 123 | 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/entity/selectorQuota/MarketQuotationsQuota.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.entity.selectorQuota; 2 | 3 | /** 4 | * @author: decaywood 5 | * @date: 2015/11/28 16:37 6 | */ 7 | 8 | /** 9 | * 市场指标节点 10 | */ 11 | public class MarketQuotationsQuota extends AbstractQuotaNode { 12 | 13 | private String current = "ALL"; //当前价 14 | private String pct = "ALL"; //本日涨跌幅(%) 15 | private String pct5 = "ALL"; //前5日涨跌幅(%) 16 | private String pct10 = "ALL"; //前10日涨跌幅(%) 17 | private String pct20 = "ALL"; //前20日涨跌幅(%) 18 | private String pct1m = "ALL"; //近1月涨跌幅(%) 19 | private String chgpct = "ALL"; //本日振幅(%) 20 | private String chgpct5 = "ALL"; //前5日振幅(%) 21 | private String chgpct10 = "ALL"; //前10日振幅(%) 22 | private String chgpct20 = "ALL"; //前20日振幅(%) 23 | private String chgpct1m = "ALL"; //近1月振幅(%) 24 | private String volume = "ALL"; //本日成交量(万) 25 | private String volavg30 = "ALL"; //30日均量(万) 26 | private String amount = "ALL"; //成交额(万) 27 | private String tr = "ALL"; // 本日换手率(%) 28 | private String tr5 = "ALL"; //前5日换手率(%) 29 | private String tr10 = "ALL"; // 前10日换手率(%) 30 | private String tr20 = "ALL"; //前20日换手率(%) 31 | private String tr1m = "ALL"; //近1月换手率(%) 32 | 33 | //设置当前价范围 34 | public void setCurrent(double from, double to) { 35 | this.current = from + "_" + to; 36 | } 37 | 38 | //设置本日涨跌幅范围 39 | public void setPct(double from, double to) { 40 | this.pct = from + "_" + to; 41 | } 42 | 43 | //设置前5日涨跌幅范围 44 | public void setPct5(double from, double to) { 45 | this.pct5 = from + "_" + to; 46 | } 47 | 48 | //设置前10日涨跌幅范围 49 | public void setPct10(double from, double to) { 50 | this.pct10 = from + "_" + to; 51 | } 52 | 53 | //设置前20日涨跌幅范围 54 | public void setPct20(double from, double to) { 55 | this.pct20 = from + "_" + to; 56 | } 57 | 58 | //设置近1月涨跌幅范围 59 | public void setPct1m(double from, double to) { 60 | this.pct1m = from + "_" + to; 61 | } 62 | 63 | //设置本日振幅范围 64 | public void setChgpct(double from, double to) { 65 | this.chgpct = from + "_" + to; 66 | } 67 | 68 | //设置前5日振幅范围 69 | public void setChgpct5(double from, double to) { 70 | this.chgpct5 = from + "_" + to; 71 | } 72 | 73 | //设置前10日振幅范围 74 | public void setChgpct10(double from, double to) { 75 | this.chgpct10 = from + "_" + to; 76 | } 77 | 78 | //设置前20日振幅范围 79 | public void setChgpct20(double from, double to) { 80 | this.chgpct20 = from + "_" + to; 81 | } 82 | 83 | //设置前1月振幅范围 84 | public void setChgpct1m(double from, double to) { 85 | this.chgpct1m = from + "_" + to; 86 | } 87 | 88 | //设置本日成交量范围 89 | public void setVolume(double from, double to) { 90 | this.volume = from + "_" + to; 91 | } 92 | 93 | //设置前30日均量范围 94 | public void setVolavg30(double from, double to) { 95 | this.volavg30 = from + "_" + to; 96 | } 97 | 98 | //设置成交额范围 99 | public void setAmount(double from, double to) { 100 | this.amount = from + "_" + to; 101 | } 102 | 103 | //设置本日换手率范围 104 | public void setTr(double from, double to) { 105 | this.tr = from + "_" + to; 106 | } 107 | 108 | //设置前5日换手率范围 109 | public void setTr5(double from, double to) { 110 | this.tr5 = from + "_" + to; 111 | } 112 | 113 | //设置前10日换手率范围 114 | public void setTr10(double from, double to) { 115 | this.tr10 = from + "_" + to; 116 | } 117 | 118 | //设置前20日换手率范围 119 | public void setTr20(double from, double to) { 120 | this.tr20 = from + "_" + to; 121 | } 122 | 123 | //设置前1月换手率范围 124 | public void setTr1m(double from, double to) { 125 | this.tr1m = from + "_" + to; 126 | } 127 | 128 | 129 | 130 | @Override 131 | StringBuilder builderSelf() { 132 | StringBuilder builder = new StringBuilder(); 133 | append(builder, "current", current); 134 | append(builder, "pct", pct); 135 | append(builder, "pct5", pct5); 136 | append(builder, "pct10", pct10); 137 | append(builder, "pct20", pct20); 138 | append(builder, "pct1m", pct1m); 139 | append(builder, "chgpct", chgpct); 140 | append(builder, "chgpct5", chgpct5); 141 | append(builder, "chgpct10", chgpct10); 142 | append(builder, "chgpct20", chgpct20); 143 | append(builder, "chgpct1m", chgpct1m); 144 | append(builder, "volume", volume); 145 | append(builder, "volavg30", volavg30); 146 | append(builder, "amount", amount); 147 | append(builder, "tr", tr); 148 | append(builder, "tr5", tr5); 149 | append(builder, "tr10", tr10); 150 | append(builder, "tr20", tr20); 151 | append(builder, "tr1m", tr1m); 152 | return builder; 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /src/test/java/collectTest/StockScopeHotRankCollectorTest.java: -------------------------------------------------------------------------------- 1 | package collectTest; 2 | 3 | import org.decaywood.collector.StockScopeHotRankCollector; 4 | import org.decaywood.entity.Stock; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import java.rmi.RemoteException; 9 | import java.util.List; 10 | 11 | /** 12 | * @author: decaywood 13 | * @date: 2015/11/25 10:49 14 | */ 15 | public class StockScopeHotRankCollectorTest { 16 | 17 | 18 | @Test 19 | public void testNull() { 20 | StockScopeHotRankCollector collector = new StockScopeHotRankCollector(null); 21 | Assert.assertNotNull(collector.getScope()); 22 | } 23 | 24 | @Test 25 | public void testGlobal() { 26 | StockScopeHotRankCollector collector1 = new StockScopeHotRankCollector(StockScopeHotRankCollector.Scope.GLOBAL_WITHIN_1_HOUR); 27 | StockScopeHotRankCollector collector24 = new StockScopeHotRankCollector(StockScopeHotRankCollector.Scope.GLOBAL_WITHIN_24_HOUR); 28 | List stocks1 = collector1.get(); 29 | List stocks24 = collector24.get(); 30 | Assert.assertTrue(stocks1.size() > 0); 31 | Assert.assertTrue(stocks24.size() > 0); 32 | for (Stock stock : stocks1) { 33 | Assert.assertTrue(stock.getStockNo().length() > 0); 34 | Assert.assertTrue(stock.getStockName().length() > 0); 35 | } 36 | for (Stock stock : stocks24) { 37 | Assert.assertTrue(stock.getStockNo().length() > 0); 38 | Assert.assertTrue(stock.getStockName().length() > 0); 39 | } 40 | } 41 | 42 | @Test 43 | public void test_SH_SZ() { 44 | StockScopeHotRankCollector collector1 = new StockScopeHotRankCollector(StockScopeHotRankCollector.Scope.SH_SZ_WITHIN_1_HOUR); 45 | StockScopeHotRankCollector collector24; 46 | collector24 = new StockScopeHotRankCollector(StockScopeHotRankCollector.Scope.SH_SZ_WITHIN_24_HOUR); 47 | List stocks1 = collector1.get(); 48 | List stocks24 = collector24.get(); 49 | Assert.assertTrue(stocks1.size() > 0); 50 | Assert.assertTrue(stocks24.size() > 0); 51 | for (Stock stock : stocks1) { 52 | Assert.assertTrue(stock.getStockNo().length() > 0); 53 | Assert.assertTrue(stock.getStockName().length() > 0); 54 | } 55 | for (Stock stock : stocks24) { 56 | Assert.assertTrue(stock.getStockNo().length() > 0); 57 | Assert.assertTrue(stock.getStockName().length() > 0); 58 | } 59 | } 60 | 61 | 62 | 63 | 64 | @Test 65 | public void test_HK() { 66 | StockScopeHotRankCollector collector1 = new StockScopeHotRankCollector(StockScopeHotRankCollector.Scope.HK_WITHIN_1_HOUR); 67 | StockScopeHotRankCollector collector24 = new StockScopeHotRankCollector(StockScopeHotRankCollector.Scope.HK_WITHIN_24_HOUR); 68 | List stocks1 = collector1.get(); 69 | List stocks24 = collector24.get(); 70 | Assert.assertTrue(stocks1.size() > 0); 71 | Assert.assertTrue(stocks24.size() > 0); 72 | for (Stock stock : stocks1) { 73 | Assert.assertTrue(stock.getStockNo().length() > 0); 74 | Assert.assertTrue(stock.getStockName().length() > 0); 75 | } 76 | for (Stock stock : stocks24) { 77 | Assert.assertTrue(stock.getStockNo().length() > 0); 78 | Assert.assertTrue(stock.getStockName().length() > 0); 79 | } 80 | } 81 | 82 | @Test 83 | public void test_US() { 84 | StockScopeHotRankCollector collector1 = new StockScopeHotRankCollector(StockScopeHotRankCollector.Scope.US_WITHIN_1_HOUR); 85 | StockScopeHotRankCollector collector24 = new StockScopeHotRankCollector(StockScopeHotRankCollector.Scope.US_WITHIN_24_HOUR); 86 | List stocks1 = collector1.get(); 87 | List stocks24 = collector24.get(); 88 | Assert.assertTrue(stocks1.size() > 0); 89 | Assert.assertTrue(stocks24.size() > 0); 90 | for (Stock stock : stocks1) { 91 | Assert.assertTrue(stock.getStockNo().length() > 0); 92 | Assert.assertTrue(stock.getStockName().length() > 0); 93 | } 94 | for (Stock stock : stocks24) { 95 | Assert.assertTrue(stock.getStockNo().length() > 0); 96 | Assert.assertTrue(stock.getStockName().length() > 0); 97 | } 98 | } 99 | 100 | 101 | 102 | @Test(expected = IllegalArgumentException.class) 103 | public void testPageSize() { 104 | new StockScopeHotRankCollector(-5); 105 | } 106 | 107 | 108 | @Test 109 | public void testOverFlowPageSize() { 110 | StockScopeHotRankCollector collector = new StockScopeHotRankCollector(30); 111 | List stocks = collector.get(); 112 | Assert.assertTrue(stocks.size() == StockScopeHotRankCollector.PAGE_SIZE_SHRESHOLD); 113 | } 114 | 115 | 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/collector/MarketQuotationsRankCollector.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.collector; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Stock; 5 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 6 | import org.decaywood.utils.RequestParaBuilder; 7 | import org.decaywood.utils.URLMapper; 8 | 9 | import java.net.URL; 10 | import java.rmi.RemoteException; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | /** 15 | * @author: decaywood 16 | * @date: 2015/11/25 13:05 17 | */ 18 | 19 | /** 20 | * 当日热点股票排行榜 21 | */ 22 | public class MarketQuotationsRankCollector extends AbstractCollector> { 23 | 24 | // 排序规则，待更多拓展.... 25 | public static final String ORDER_BY_PERCENT = "percent";//按涨幅排序 26 | public static final String ORDER_BY_VOLUME = "volume";//按成交量排序 27 | public static final String ORDER_BY_AMOUNT = "amount";//成交额 28 | public static final String ORDER_BY_TURNOVER_RATE = "turnover_rate";//按换手排序 29 | 30 | 31 | public static final int TOPK_MAX_SHRESHOLD = 500; 32 | 33 | 34 | /** 35 | * 股票类型 36 | */ 37 | public enum StockType { 38 | SH_A("sha"),//沪市A 39 | SH_B("shb"),//沪市B 40 | SZ_A("sza"),//深市A 41 | SZ_B("szb"),//深市B 42 | GROWTH_ENTERPRISE_BOARD("cyb"),//创业板 43 | SMALL_MEDIUM_ENTERPRISE_BOARD("zxb"),//中小板 44 | HK("hk"),//港股 45 | US("us");//美股 46 | 47 | private String val; 48 | 49 | StockType(String val) { 50 | this.val = val; 51 | } 52 | 53 | } 54 | 55 | private final StockType stockType; 56 | private final String orderPattern; 57 | private boolean asc; 58 | private final int topK; 59 | 60 | public MarketQuotationsRankCollector(StockType stockType, String orderPattern) { 61 | this(stockType, orderPattern, 10); 62 | } 63 | 64 | public MarketQuotationsRankCollector(StockType stockType, String orderPattern, int topK) { 65 | this(null, stockType, orderPattern, topK); 66 | } 67 | 68 | 69 | /** 70 | * 71 | * @param strategy 超时等待策略（null则设置为默认等待策略） 72 | * @param stockType 股票类型 73 | * @param orderPattern 排序规则 74 | * @param topK 取排名前K 75 | */ 76 | public MarketQuotationsRankCollector( 77 | TimeWaitingStrategy strategy, 78 | StockType stockType, 79 | String orderPattern, 80 | int topK 81 | ) { 82 | 83 | super(strategy); 84 | 85 | orderPattern = orderPattern == null ? "" : orderPattern; 86 | 87 | if(!isLegal(orderPattern) || topK <= 0) 88 | throw new IllegalArgumentException("Not legal or not support yet exception"); 89 | 90 | this.stockType = stockType == null ? StockType.SH_A : stockType; 91 | this.orderPattern = orderPattern; 92 | this.topK = Math.min(topK, TOPK_MAX_SHRESHOLD); 93 | 94 | } 95 | 96 | private boolean isLegal(String orderPattern) { 97 | 98 | return orderPattern.equals(ORDER_BY_AMOUNT) || 99 | orderPattern.equals(ORDER_BY_PERCENT) || 100 | orderPattern.equals(ORDER_BY_TURNOVER_RATE) || 101 | orderPattern.equals(ORDER_BY_VOLUME); 102 | 103 | } 104 | 105 | public MarketQuotationsRankCollector ascend() { 106 | this.asc = true; 107 | return this; 108 | } 109 | 110 | public MarketQuotationsRankCollector descend() { 111 | this.asc = false; 112 | return this; 113 | } 114 | 115 | @Override 116 | public List collectLogic() throws Exception { 117 | String target = URLMapper.MARKET_QUOTATIONS_RANK_JSON.toString(); 118 | RequestParaBuilder builder = new RequestParaBuilder(target) 119 | .addParameter("stockType", stockType.val) 120 | .addParameter("order", asc ? "asc" : "desc") 121 | .addParameter("orderBy", orderPattern) 122 | .addParameter("size", topK) 123 | .addParameter("page", 1) 124 | .addParameter("column", "symbol%2Cname"); 125 | URL url = new URL(builder.build()); 126 | 127 | String json = request(url); 128 | JsonNode node = mapper.readTree(json); 129 | return processNode(node); 130 | } 131 | 132 | private List processNode(JsonNode node) { 133 | 134 | List stocks = new ArrayList<>(); 135 | JsonNode data = node.get("data"); 136 | for (JsonNode jsonNode : data) { 137 | String symbol = jsonNode.get(0).asText(); 138 | String name = jsonNode.get(1).asText(); 139 | Stock stock = new Stock(name, symbol); 140 | stocks.add(stock); 141 | } 142 | return stocks; 143 | 144 | } 145 | 146 | public StockType getStockType() { 147 | return stockType; 148 | } 149 | 150 | public String getOrderPattern() { 151 | return orderPattern; 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/main/java/org/decaywood/mapper/cubeFirst/CubeToCubeWithTrendMapper.java: -------------------------------------------------------------------------------- 1 | package org.decaywood.mapper.cubeFirst; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.decaywood.entity.Cube; 5 | import org.decaywood.entity.trend.CubeTrend; 6 | import org.decaywood.entity.trend.MarketIndexTrend; 7 | import org.decaywood.mapper.AbstractMapper; 8 | import org.decaywood.timeWaitingStrategy.TimeWaitingStrategy; 9 | import org.decaywood.utils.EmptyObject; 10 | import org.decaywood.utils.RequestParaBuilder; 11 | import org.decaywood.utils.URLMapper; 12 | 13 | import java.net.URL; 14 | import java.rmi.RemoteException; 15 | import java.util.ArrayList; 16 | import java.util.Date; 17 | import java.util.List; 18 | 19 | /** 20 | * @author: decaywood 21 | * @date: 2015/11/26 9:48 22 | */ 23 | 24 | /** 25 | * 股票组合历史走势装配器 26 | */ 27 | public class CubeToCubeWithTrendMapper extends AbstractMapper { 28 | 29 | private final long since; 30 | private final long until; 31 | 32 | public CubeToCubeWithTrendMapper(Date since, Date until) { 33 | this(null, since, until); 34 | } 35 | 36 | 37 | /** 38 | * 39 | * @param strategy 超时等待策略（null则设置为默认等待策略） 40 | * @param since 走线计算起始时间 41 | * @param until 走线计算结束时间 42 | */ 43 | public CubeToCubeWithTrendMapper( 44 | TimeWaitingStrategy strategy, 45 | Date since, 46 | Date until) { 47 | 48 | super(strategy); 49 | if(since == null || until == null || since.after(until)) 50 | throw new IllegalArgumentException("Null Pointer"); 51 | 52 | this.since = since.getTime(); 53 | this.until = until.getTime(); 54 | 55 | } 56 | 57 | 58 | @Override 59 | public Cube mapLogic(Cube cube) throws Exception { 60 | if(cube == null || cube == EmptyObject.emptyCube) return EmptyObject.emptyCube; 61 | 62 | String target = URLMapper.CUBE_TREND_JSON.toString(); 63 | RequestParaBuilder builder = new RequestParaBuilder(target) 64 | .addParameter("cube_symbol", cube.getSymbol()) 65 | .addParameter("since", since) 66 | .addParameter("until", until); 67 | 68 | URL url = new URL(builder.build()); 69 | 70 | String json = request(url); 71 | 72 | JsonNode node = mapper.readTree(json); 73 | processCube(cube, node); 74 | return cube; 75 | 76 | } 77 | 78 | private void processCube(Cube cube, JsonNode node) { 79 | 80 | JsonNode cubeNode = node.get(0); 81 | JsonNode SH300Node = node.get(1); 82 | 83 | CubeTrend cubeTrend = processCubeNode(cubeNode); 84 | MarketIndexTrend marketIndexTrend = processSH00Node(SH300Node); 85 | cube.setCubeTrend(cubeTrend); 86 | cube.setMarketIndexTrend(marketIndexTrend); 87 | 88 | } 89 | 90 | private CubeTrend processCubeNode(JsonNode node) { 91 | 92 | JsonNode trendNode = node.get("list"); 93 | List blocks = new ArrayList<>(); 94 | 95 | for (JsonNode jsonNode : trendNode) { 96 | String time = jsonNode.get("time").asText(); 97 | String date = jsonNode.get("date").asText(); 98 | String value = jsonNode.get("value").asText(); 99 | String percent = jsonNode.get("percent").asText(); 100 | CubeTrend.TrendBlock trendBlock = new CubeTrend.TrendBlock( 101 | time, 102 | date, 103 | value, 104 | percent); 105 | blocks.add(trendBlock); 106 | } 107 | 108 | if(blocks.isEmpty()) return EmptyObject.emptyCubeTrend; 109 | 110 | return new CubeTrend( 111 | node.get("symbol").asText(), 112 | node.get("name").asText(), 113 | blocks.get(0).getTime(), 114 | blocks.get(blocks.size() - 1).getTime(), 115 | blocks); 116 | 117 | } 118 | 119 | private MarketIndexTrend processSH00Node(JsonNode node) { 120 | 121 | JsonNode trendNode = node.get("list"); 122 | List blocks = new ArrayList<>(); 123 | 124 | for (JsonNode jsonNode : trendNode) { 125 | String time = jsonNode.get("time").asText(); 126 | String date = jsonNode.get("date").asText(); 127 | String value = jsonNode.get("value").asText(); 128 | String percent = jsonNode.get("percent").asText(); 129 | MarketIndexTrend.TrendBlock trendBlock = new MarketIndexTrend.TrendBlock( 130 | time, 131 | date, 132 | value, 133 | percent); 134 | blocks.add(trendBlock); 135 | } 136 | 137 | if(blocks.isEmpty()) return EmptyObject.emptyMarketIndexTrend; 138 | 139 | return new MarketIndexTrend( 140 | node.get("symbol").asText(), 141 | node.get("name").asText(), 142 | blocks.get(0).getTime(), 143 | blocks.get(blocks.size() - 1).getTime(), 144 | blocks); 145 | } 146 | 147 | 148 | 149 | } 150 | --------------------------------------------------------------------------------