├── .gitignore ├── .travis.yml ├── CNAME ├── LICENSE ├── README.md ├── pom.xml └── src ├── main ├── assembly │ └── release.xml ├── java │ └── tech │ │ └── spiro │ │ └── addrparser │ │ ├── common │ │ ├── ContainPointJudge.java │ │ ├── ContainPointJudgeFactory.java │ │ ├── NotImplementedException.java │ │ ├── Point.java │ │ ├── RegionConverter.java │ │ ├── RegionDTO.java │ │ ├── RegionInfo.java │ │ └── RegionLevel.java │ │ ├── crawler │ │ ├── DataResp.java │ │ ├── GetRegionException.java │ │ ├── RegionDataCrawler.java │ │ ├── RegionResp.java │ │ └── RestClient.java │ │ ├── io │ │ ├── IOPipeline.java │ │ ├── RegionDataInput.java │ │ ├── RegionDataOutput.java │ │ ├── RegionDataReport.java │ │ ├── file │ │ │ ├── BinaryFileRegionDataOutput.java │ │ │ ├── JSONFileRegionDataInput.java │ │ │ └── JSONFileRegionDataOutput.java │ │ └── rdbms │ │ │ ├── RdbmsRegionDTOWrapper.java │ │ │ ├── RdbmsRegionDataInput.java │ │ │ ├── RdbmsRegionDataOutput.java │ │ │ └── RdbmsSQL.java │ │ ├── parser │ │ ├── DefaultContainPointJudge.java │ │ ├── Location.java │ │ ├── LocationParserEngine.java │ │ └── ParserEngineException.java │ │ └── tool │ │ ├── CrawlerServer.java │ │ ├── JSONFile2MySQL.java │ │ ├── MapTools.java │ │ └── MySQLTableInit.java └── script │ ├── crawler.bat │ ├── crawler.sh │ ├── jsonfile2mysql.bat │ └── jsonfile2mysql.sh └── test └── java └── tech └── spiro └── addrparser └── io └── rdbms └── RdbmsSQLTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.ear 17 | *.zip 18 | *.tar.gz 19 | *.rar 20 | 21 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 22 | hs_err_pid* 23 | 24 | target 25 | .idea 26 | *.iml 27 | 28 | !china-region.zip 29 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk8 4 | -------------------------------------------------------------------------------- /CNAME: -------------------------------------------------------------------------------- 1 | addrparser.spiro.tech -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://api.travis-ci.org/hsp8712/addrparser.svg?branch=master)](https://travis-ci.org/hsp8712/addrparser) 2 | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/tech.spiro/addrparser/badge.svg)](https://maven-badges.herokuapp.com/maven-central/tech.spiro/addrparser/) 3 | [![GitHub release](https://img.shields.io/github/release/hsp8712/addrparser.svg)](https://github.com/hsp8712/addrparser/releases) 4 | [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) 5 | 6 | # addrparser 7 | 离线高效的解析中国范围内的经纬度为省市区信息，省市区信息包括: 行政区划编码、行政区划中文名称、行政区域的中心点经纬度，行政区域的边界点经纬度集合。 8 | 9 | # 依赖安装 10 | ***依赖 jdk1.8*** 11 | ## Maven 12 | ```xml 13 | 14 | tech.spiro 15 | addrparser 16 | 1.1 17 | 18 | ``` 19 | ## Jar文件 20 | [addrparser-1.1](http://repo1.maven.org/maven2/tech/spiro/addrparser/1.1/addrparser-1.1.jar) 21 | 22 | # 使用说明-解析地址 23 | ## 下载数据文件 24 | [china-region-20190902.zip](https://github.com/hsp8712/addrparser/releases/download/addrparser-1.0.1/china-region-20190902.zip) 25 | 解压后得到文件：china-region.json，文件中包括了所有的省市区行政区域信息，包括编码、名称、中心点、边界点集合。 26 | 27 | **经纬度数据基于GCJ-02坐标系** 28 | 29 | ## 方式一 （文件） 30 | 直接使用china-region.json文件作为基础数据。 31 | 32 | 示例代码： 33 | ```java 34 | 35 | // china-region.json文件作为基础数据 36 | RegionDataInput regionDataInput = new JSONFileRegionDataInput("path/china-region.json"); 37 | 38 | // 创建并初始化位置解析引擎，一般配置为全局单例 39 | LocationParserEngine engine = new LocationParserEngine(regionDataInput); 40 | // 初始化，加载数据，比较耗时 41 | engine.init(); 42 | 43 | // 执行解析操作 44 | Location location = engine.parse(118.750934,32.038634); 45 | 46 | // 获取省市区信息 47 | RegionInfo provInfo = location.getProv(); 48 | RegionInfo cityInfo = location.getCity(); 49 | RegionInfo districtInfo = location.getDistrict(); 50 | ``` 51 | 52 | ## 方式二 （MySQL） 53 | 将数据导入mysql数据库，使用mysql中数据作为基础数据。 54 | 55 | 下载数据导入工具： 56 | [addrparser-1.1.tar.gz](https://github.com/hsp8712/addrparser/releases/download/addrparser-1.1/addrparser-1.1.tar.gz) 57 | 58 | 或通过源码编译工具： `mvn clean package -P tool` 59 | 60 | 61 | 解压后，进入addrparser/bin目录，执行jsonfile2mysql.sh (Linux) 或 jsonfile2mysql.bat (Windows) 62 | 63 | ``` 64 | usage: JSONFile2MySQL 65 | -a,--password MySQL password 66 | -d,--db MySQL database 67 | -f,--file         Json region data file path  上述china-region.json文件的路径 68 | -h,--host         MySQL host 69 | -i,--init Init table schema 70 | -p,--port MySQL port, default 3306 71 | -t,--table-name Default: 'region_data', Region data table name 72 | -u,--user MySQL user 73 | ``` 74 | 75 | 如果不指定-t，使用默认表名region_data 76 | 77 | 示例代码： 78 | ```java 79 | // javax.sql.DataSource 数据源 80 | DataSource ds = ...; 81 | 82 | // 关系型数据库区域数据输入 83 | RegionDataInput regionDataInput = new RdbmsRegionDataInput(ds); 84 | 85 | // 如果上述命令如果指定了定制的表名，则该处也要指定表名。 86 | // RegionDataInput regionDataInput = new RdbmsRegionDataInput(ds, customTableName); 87 | 88 | // 创建并初始化位置解析引擎，一般配置为全局单例 89 | LocationParserEngine engine = new LocationParserEngine(regionDataInput); 90 | // 初始化，加载数据，比较耗时 91 | engine.init(); 92 | 93 | // 执行解析操作 94 | Location location = engine.parse(118.750934,32.038634); 95 | 96 | // 获取省市区信息 97 | RegionInfo provInfo = location.getProv(); 98 | RegionInfo cityInfo = location.getCity(); 99 | RegionInfo districtInfo = location.getDistrict(); 100 | ``` 101 | 102 | # 使用说明-地图工具 103 | `tech.spiro.addrparser.tool.MapTools` 类中包含了常用的地图工具方法： 104 | 105 | ## getPoint2PointDistance 106 | 获取点到点的距离 107 | 108 | ## getPoint2LineDistance 109 | 获取点到直线的距离 110 | 111 | ## inCircleArea 112 | 判断点是否在圆形范围内 113 | 114 | ## inRectangleArea 115 | 判断点是否在矩形范围内 116 | 117 | ## inPolygonArea 118 | 判断点是否在多边形范围内 119 | 120 | ## yawing 121 | 判断点是否偏离一个轨迹 122 | 123 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | tech.spiro 4 | addrparser 5 | 1.2-SNAPSHOT 6 | ${project.groupId}:${project.artifactId} 7 | Tool for parsing longitude/latitude to region info in china. 8 | https://github.com/hsp8712/addrparser 9 | 10 | 11 | The Apache Software License, Version 2.0 12 | http://www.apache.org/licenses/LICENSE-2.0.txt 13 | 14 | 15 | 16 | 17 | Spiro Huang 18 | hsp8712@qq.com 19 | 20 | 21 | 22 | scm:git:git@github.com:hsp8712/addrparser.git 23 | scm:git:git@github.com:hsp8712/addrparser.git 24 | https://github.com/hsp8712/addrparser/tree/master 25 | HEAD 26 | 27 | 28 | 29 | ossrh 30 | https://oss.sonatype.org/content/repositories/snapshots 31 | 32 | 33 | ossrh 34 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 35 | 36 | 37 | 38 | 39 | UTF-8 40 | 41 | 42 | 43 | 44 | junit 45 | junit 46 | 4.13.1 47 | test 48 | 49 | 50 | org.slf4j 51 | slf4j-api 52 | 1.7.25 53 | 54 | 55 | com.alibaba 56 | fastjson 57 | 1.2.83 58 | 59 | 60 | org.apache.httpcomponents 61 | httpclient 62 | 4.5.13 63 | provided 64 | 65 | 66 | commons-cli 67 | commons-cli 68 | 1.4 69 | provided 70 | 71 | 72 | mysql 73 | mysql-connector-java 74 | 8.0.28 75 | provided 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | org.apache.maven.plugins 84 | maven-compiler-plugin 85 | 3.3 86 | 87 | 1.8 88 | 1.8 89 | UTF-8 90 | 91 | 92 | 93 | org.apache.maven.plugins 94 | maven-resources-plugin 95 | 2.7 96 | 97 | UTF-8 98 | 99 | 100 | 101 | org.apache.maven.plugins 102 | maven-jar-plugin 103 | 2.5 104 | 105 | 106 | db 107 | 108 | 109 | 110 | 111 | org.apache.maven.plugins 112 | maven-release-plugin 113 | 2.5.3 114 | 115 | true 116 | false 117 | release 118 | deploy 119 | 120 | 121 | 122 | 123 | 124 | 125 | release 126 | 127 | 128 | 129 | org.apache.maven.plugins 130 | maven-source-plugin 131 | 2.2.1 132 | 133 | 134 | attach-sources 135 | 136 | jar-no-fork 137 | 138 | 139 | 140 | 141 | 142 | org.apache.maven.plugins 143 | maven-javadoc-plugin 144 | 2.9.1 145 | 146 | 147 | attach-javadocs 148 | 149 | jar 150 | 151 | 152 | 153 | 154 | 155 | org.apache.maven.plugins 156 | maven-gpg-plugin 157 | 1.5 158 | 159 | 160 | sign-artifacts 161 | verify 162 | 163 | sign 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | tool 173 | 174 | 175 | org.apache.httpcomponents 176 | httpclient 177 | 4.5.13 178 | 179 | 180 | commons-cli 181 | commons-cli 182 | 1.4 183 | 184 | 185 | mysql 186 | mysql-connector-java 187 | 8.0.28 188 | 189 | 190 | org.slf4j 191 | slf4j-simple 192 | 1.7.25 193 | 194 | 195 | 196 | 197 | 198 | org.apache.maven.plugins 199 | maven-assembly-plugin 200 | 2.4.1 201 | 202 | 203 | make-assembly-release 204 | package 205 | 206 | single 207 | 208 | 209 | 210 | src/main/assembly/release.xml 211 | 212 | addrparser 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | -------------------------------------------------------------------------------- /src/main/assembly/release.xml: -------------------------------------------------------------------------------- 1 | 4 | ${project.version} 5 | 6 | tar.gz 7 | 8 | true 9 | 10 | 11 | 12 | false 13 | lib 14 | runtime 15 | true 16 | true 17 | true 18 | 19 | 20 | 21 | 22 | 23 | ${project.basedir}/src/main/resources 24 | 25 | *.xml 26 | *.properties 27 | 28 | conf 29 | 30 | 31 | ${project.basedir}/src/main/script 32 | 33 | *.sh 34 | *.bat 35 | 36 | 755 37 | unix 38 | bin 39 | 40 | 41 | 42 | ${project.basedir}/target 43 | 44 | *.jar 45 | 46 | lib 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/common/ContainPointJudge.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.common; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Interface to judge whether a point in polygons 7 | * @author Spiro Huang 8 | * @since 1.0 9 | */ 10 | public interface ContainPointJudge { 11 | 12 | /** 13 | * Initialize polygons 14 | * @param polyline Multi polygons point list 15 | */ 16 | void initPolygons(List> polyline); 17 | 18 | boolean contain(Point point); 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/common/ContainPointJudgeFactory.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.common; 2 | 3 | import tech.spiro.addrparser.parser.DefaultContainPointJudge; 4 | 5 | /** 6 | * @author Spiro Huang 7 | * @since 1.0 8 | */ 9 | public class ContainPointJudgeFactory { 10 | 11 | public static ContainPointJudge create() { 12 | return new DefaultContainPointJudge(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/common/NotImplementedException.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.common; 2 | 3 | /** 4 | * Some code have not implemented already, throw this exception. 5 | * @author Spiro Huang 6 | * @since 1.0 7 | */ 8 | public class NotImplementedException extends RuntimeException { 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/common/Point.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.common; 2 | 3 | /** 4 | * Point with longitude/latitude pair. 5 | * @author Spiro Huang 6 | * @since 1.0 7 | */ 8 | public class Point 9 | { 10 | private final double lon; 11 | private final double lat; 12 | 13 | public Point(double lon, double lat) { 14 | this.lon = lon; 15 | this.lat = lat; 16 | } 17 | 18 | public double getLon() { 19 | return lon; 20 | } 21 | 22 | public double getLat() { 23 | return lat; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return "Point{" + 29 | "lon=" + lon + 30 | ", lat=" + lat + 31 | '}'; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/common/RegionConverter.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.common; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * Convert {@link RegionDTO} to {@link RegionInfo} 8 | * @author Spiro Huang 9 | * @since 1.0 10 | */ 11 | public class RegionConverter { 12 | 13 | public static RegionInfo convert(RegionDTO regionDTO) { 14 | RegionInfo.Builder builder = new RegionInfo.Builder(); 15 | 16 | // parentCode 17 | builder.parentCode(Integer.valueOf(regionDTO.getParentCode())); 18 | // code 19 | builder.code(Integer.valueOf(regionDTO.getCode())); 20 | // name 21 | builder.name(regionDTO.getName()); 22 | 23 | // level 24 | builder.level(regionDTO.getLevel()); 25 | 26 | // center 27 | String[] center = regionDTO.getCenter().split(","); 28 | Point centerPoint = new Point(Double.valueOf(center[0]), Double.valueOf(center[1])); 29 | builder.center(centerPoint); 30 | 31 | // polyline 32 | String polyline = regionDTO.getPolyline(); 33 | String[] blocks = polyline.split("\\|"); 34 | 35 | for (String block : blocks) { 36 | List blockObj = new ArrayList<>(); 37 | String[] points = block.split(";"); 38 | 39 | for (String point : points) { 40 | String[] lonLat = point.split(","); 41 | Point pointObj = new Point(Double.valueOf(lonLat[0]), Double.valueOf(lonLat[1])); 42 | blockObj.add(pointObj); 43 | } 44 | builder.addPolyline(blockObj); 45 | } 46 | 47 | return builder.build(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/common/RegionDTO.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.common; 2 | 3 | /** 4 | * Region data object for I/O. 5 | * @author Spiro Huang 6 | * @since 1.0 7 | */ 8 | public class RegionDTO { 9 | private int parentCode; 10 | private int code; 11 | private String name; 12 | 13 | /** 14 | * Available value: 1: "province", 2: "city", 3: "district", 4: "street" 15 | */ 16 | private RegionLevel level; 17 | 18 | /** 19 | * Format: "[longitude],[latitude]" 20 | */ 21 | private String center; 22 | 23 | /** 24 | * Maybe represent multi polygons separated by "|". 25 | * Format: "[longitude1],[latitude1];[longitude2],[latitude2];[longitude3],[latitude3]|[longitude4],[latitude4];..." 26 | */ 27 | private String polyline; 28 | 29 | public int getParentCode() { 30 | return parentCode; 31 | } 32 | 33 | public void setParentCode(int parentCode) { 34 | this.parentCode = parentCode; 35 | } 36 | 37 | public int getCode() { 38 | return code; 39 | } 40 | 41 | public void setCode(int code) { 42 | this.code = code; 43 | } 44 | 45 | public String getName() { 46 | return name; 47 | } 48 | 49 | public void setName(String name) { 50 | this.name = name; 51 | } 52 | 53 | public RegionLevel getLevel() { 54 | return level; 55 | } 56 | 57 | public void setLevel(RegionLevel level) { 58 | this.level = level; 59 | } 60 | 61 | public String getCenter() { 62 | return center; 63 | } 64 | 65 | public void setCenter(String center) { 66 | this.center = center; 67 | } 68 | 69 | public String getPolyline() { 70 | return polyline; 71 | } 72 | 73 | public void setPolyline(String polyline) { 74 | this.polyline = polyline; 75 | } 76 | 77 | @Override 78 | public String toString() { 79 | return "RegionDTO{" + 80 | "parentCode=" + parentCode + 81 | ", code=" + code + 82 | ", name='" + name + '\'' + 83 | ", level=" + level + 84 | ", center='" + center + '\'' + 85 | ", polyline='...'" + 86 | '}'; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/common/RegionInfo.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.common; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collections; 5 | import java.util.List; 6 | 7 | /** 8 | * Region data object for calculating. 9 | * @author Spiro Huang 10 | * @since 1.0 11 | */ 12 | public class RegionInfo { 13 | 14 | private final int parentCode; 15 | private final int code; 16 | private final String name; 17 | private final RegionLevel level; 18 | private final Point center; 19 | private final List> polyline; 20 | 21 | private final ContainPointJudge containPointJudge = ContainPointJudgeFactory.create(); 22 | 23 | private RegionInfo(Builder builder) { 24 | this.parentCode = builder.parentCode; 25 | this.code = builder.code; 26 | this.name = builder.name; 27 | this.level = builder.level; 28 | this.center = builder.center; 29 | this.polyline = Collections.unmodifiableList(builder.polyline); 30 | containPointJudge.initPolygons(this.polyline); 31 | } 32 | 33 | public int getParentCode() { 34 | return parentCode; 35 | } 36 | 37 | public int getCode() { 38 | return code; 39 | } 40 | 41 | public String getName() { 42 | return name; 43 | } 44 | 45 | public RegionLevel getLevel() { 46 | return level; 47 | } 48 | 49 | public Point getCenter() { 50 | return center; 51 | } 52 | 53 | public List> getPolyline() { 54 | return polyline; 55 | } 56 | 57 | public boolean contain(Point point) { 58 | return this.containPointJudge.contain(point); 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return "RegionInfo{" + 64 | "parentCode=" + parentCode + 65 | ", code=" + code + 66 | ", name='" + name + '\'' + 67 | ", level=" + level + 68 | '}'; 69 | } 70 | 71 | public static class Builder { 72 | private int parentCode; 73 | private int code; 74 | private String name; 75 | private RegionLevel level; 76 | private Point center; 77 | private List> polyline = new ArrayList<>(); 78 | 79 | public Builder() { 80 | } 81 | 82 | public Builder code(int code) { 83 | this.code = code; 84 | return this; 85 | } 86 | 87 | public Builder parentCode(int parentCode) { 88 | this.parentCode = parentCode; 89 | return this; 90 | } 91 | 92 | public Builder name(String name) { 93 | this.name = name; 94 | return this; 95 | } 96 | 97 | public Builder level(RegionLevel level) { 98 | this.level = level; 99 | return this; 100 | } 101 | 102 | public Builder center(Point center) { 103 | this.center = center; 104 | return this; 105 | } 106 | 107 | public Builder addPolyline(List polyline) { 108 | if (polyline != null && (!polyline.isEmpty())) { 109 | this.polyline.add(Collections.unmodifiableList(polyline)); 110 | } 111 | return this; 112 | } 113 | 114 | public RegionInfo build() { 115 | 116 | if (this.name == null) { 117 | throw new IllegalArgumentException(" cannot be null."); 118 | } 119 | 120 | if (this.level == null) { 121 | throw new IllegalArgumentException(" cannot be null."); 122 | } 123 | 124 | if (this.center == null) { 125 | throw new IllegalArgumentException("
cannot be null."); 126 | } 127 | 128 | return new RegionInfo(this); 129 | } 130 | 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/common/RegionLevel.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.common; 2 | 3 | /** 4 | * @author Spiro Huang 5 | * @since 1.0 6 | */ 7 | public enum RegionLevel { 8 | 9 | COUNTRY, 10 | PROVINCE, 11 | CITY, 12 | DISTRICT, 13 | STREET; 14 | 15 | public static RegionLevel valueOf(int ordinal) { 16 | if (ordinal < 0 || ordinal >= values().length) { 17 | throw new IndexOutOfBoundsException("Invalid ordinal"); 18 | } 19 | return values()[ordinal]; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/crawler/DataResp.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.crawler; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Response data object for rest request. 7 | * @author Spiro Huang 8 | * @since 1.0 9 | */ 10 | public class DataResp { 11 | 12 | private String status; 13 | private String info; 14 | private String infocode; 15 | private String count; 16 | private List districts; 17 | 18 | public String getStatus() { 19 | return status; 20 | } 21 | 22 | public void setStatus(String status) { 23 | this.status = status; 24 | } 25 | 26 | public String getInfo() { 27 | return info; 28 | } 29 | 30 | public void setInfo(String info) { 31 | this.info = info; 32 | } 33 | 34 | public String getInfocode() { 35 | return infocode; 36 | } 37 | 38 | public void setInfocode(String infocode) { 39 | this.infocode = infocode; 40 | } 41 | 42 | public String getCount() { 43 | return count; 44 | } 45 | 46 | public void setCount(String count) { 47 | this.count = count; 48 | } 49 | 50 | public List getDistricts() { 51 | return districts; 52 | } 53 | 54 | public void setDistricts(List districts) { 55 | this.districts = districts; 56 | } 57 | 58 | @Override 59 | public String toString() { 60 | return "DataResp{" + 61 | "status='" + status + '\'' + 62 | ", info='" + info + '\'' + 63 | ", infocode='" + infocode + '\'' + 64 | ", count='" + count + '\'' + 65 | ", districts=" + districts + 66 | '}'; 67 | } 68 | 69 | public boolean isSuccess() { 70 | return "1".equals(this.getStatus()); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/crawler/GetRegionException.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.crawler; 2 | 3 | /** 4 | * Get region exception by crawler. 5 | * @author Spiro Huang 6 | * @since 1.0 7 | */ 8 | public class GetRegionException extends Exception { 9 | public GetRegionException() { 10 | super(); 11 | } 12 | 13 | public GetRegionException(String message) { 14 | super(message); 15 | } 16 | 17 | public GetRegionException(String message, Throwable cause) { 18 | super(message, cause); 19 | } 20 | 21 | public GetRegionException(Throwable cause) { 22 | super(cause); 23 | } 24 | 25 | protected GetRegionException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { 26 | super(message, cause, enableSuppression, writableStackTrace); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/crawler/RegionDataCrawler.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.crawler; 2 | 3 | import tech.spiro.addrparser.common.RegionDTO; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import tech.spiro.addrparser.common.RegionLevel; 7 | import tech.spiro.addrparser.io.RegionDataOutput; 8 | 9 | import java.io.IOException; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.concurrent.atomic.AtomicInteger; 13 | 14 | /** 15 | * Region data crawler, use by invoke methods: 16 | *
    17 | *
  • {@code loadCountry()}
    • 18 | *
  • {@code loadProv(int provCode)}
    • 19 | *
  • {@code loadCity(int cityCode)}
    • 20 | *
21 | * @author Spiro Huang 22 | * @since 1.0 23 | */ 24 | public class RegionDataCrawler { 25 | 26 | private static final Logger LOG = LoggerFactory.getLogger(RegionDataCrawler.class); 27 | 28 | // country root code 29 | public static final int COUNTRY_CODE = 100000; 30 | 31 | private AtomicInteger invokerCount = new AtomicInteger(0); 32 | 33 | private final RegionDataOutput regionOutput; 34 | 35 | /** AMAP api key */ 36 | private final String amapKey; 37 | 38 | public RegionDataCrawler(RegionDataOutput regionOutput, String amapKey) throws IOException { 39 | this.regionOutput = regionOutput; 40 | this.regionOutput.init(); 41 | this.amapKey = amapKey; 42 | } 43 | 44 | private List getSubRegionDTOs(int code) throws GetRegionException { 45 | 46 | RestClient restClient = new RestClient(); 47 | restClient.setKeywords(Integer.toString(code)); 48 | restClient.setKey(this.amapKey); 49 | restClient.setExtensions("base"); 50 | restClient.setSubdistrict("1"); 51 | 52 | DataResp dataResponse = restClient.getDistrictResponse(); 53 | LOG.debug("Invoker.count=" + invokerCount.incrementAndGet()); 54 | 55 | if (!dataResponse.isSuccess()) { 56 | throw new GetRegionException("Get failed, infocode=" 57 | + dataResponse.getInfocode() + ", info=" + dataResponse.getInfo()); 58 | } 59 | 60 | List regionResps = dataResponse.getDistricts(); 61 | if (regionResps.isEmpty()) { 62 | LOG.warn("Have no regionResps return, keyword code={}", code); 63 | return null; 64 | } 65 | 66 | RegionResp curRegionResp = regionResps.get(0); 67 | List subRegionResps = curRegionResp.getDistricts(); 68 | 69 | List subRegionDTOs = new ArrayList<>(); 70 | 71 | if (subRegionResps == null || subRegionResps.isEmpty()) { 72 | LOG.warn("Have no sub regionResps, {}", curRegionResp); 73 | } else { 74 | for (RegionResp regionResp : subRegionResps) { 75 | RegionDTO regionDTO = new RegionDTO(); 76 | 77 | try { 78 | regionDTO.setCode(Integer.valueOf(regionResp.getAdcode())); 79 | regionDTO.setLevel(RegionLevel.valueOf(regionResp.getLevel().toUpperCase())); 80 | regionDTO.setCenter(regionResp.getCenter()); 81 | regionDTO.setParentCode(code); 82 | regionDTO.setName(regionResp.getName()); 83 | } catch (Exception e) { 84 | throw new GetRegionException(" cannot convert to RegionDTO, maybe regionResp invalid, regionResp:" 85 | + regionResp.toString(), e); 86 | } 87 | subRegionDTOs.add(regionDTO); 88 | } 89 | } 90 | 91 | // Fill boundary info 92 | for (RegionDTO subRegionDTO : subRegionDTOs) { 93 | RestClient subRestClient = new RestClient(); 94 | subRestClient.setExtensions("all"); 95 | subRestClient.setKey(this.amapKey); 96 | subRestClient.setKeywords(Integer.toString(subRegionDTO.getCode())); 97 | subRestClient.setSubdistrict("0"); 98 | 99 | DataResp subDataResp = subRestClient.getDistrictResponse(); 100 | LOG.debug("Invoker.count=" + invokerCount.incrementAndGet()); 101 | 102 | if (!subDataResp.isSuccess()) { 103 | throw new GetRegionException("Get sub district failed"); 104 | } 105 | 106 | List _subRegionResps = subDataResp.getDistricts(); 107 | if (_subRegionResps.isEmpty()) { 108 | throw new GetRegionException("Get sub district failed"); 109 | } 110 | 111 | RegionResp _subRegionResp = _subRegionResps.get(0); 112 | subRegionDTO.setPolyline(_subRegionResp.getPolyline()); 113 | } 114 | 115 | return subRegionDTOs; 116 | } 117 | 118 | private void regionBatchOutput(List regionDTOs) throws GetRegionException { 119 | try { 120 | for (RegionDTO regionDTO : regionDTOs) { 121 | this.regionOutput.write(regionDTO); 122 | } 123 | } catch (IOException e) { 124 | throw new GetRegionException(e.getMessage(), e); 125 | } 126 | } 127 | 128 | public void loadCountry() throws GetRegionException { 129 | if (regionOutput == null) { 130 | throw new GetRegionException(" is null."); 131 | } 132 | // Get provinces 133 | List provRegionDTOs = getSubRegionDTOs(COUNTRY_CODE); 134 | regionBatchOutput(provRegionDTOs); 135 | 136 | for (RegionDTO provRegionDTO : provRegionDTOs) { 137 | loadProv(provRegionDTO.getCode()); 138 | } 139 | } 140 | 141 | public void loadProv(int provCode) throws GetRegionException { 142 | if (regionOutput == null) { 143 | throw new GetRegionException(" is null."); 144 | } 145 | 146 | // Get cities 147 | List cityRegionDTOs = getSubRegionDTOs(provCode); 148 | regionBatchOutput(cityRegionDTOs); 149 | 150 | for (RegionDTO cityRegionDTO : cityRegionDTOs) { 151 | loadCity(cityRegionDTO.getCode()); 152 | } 153 | } 154 | 155 | public void loadCity(int cityCode) throws GetRegionException { 156 | if (regionOutput == null) { 157 | throw new GetRegionException(" is null."); 158 | } 159 | 160 | // Get districts 161 | List districtRegionDTOs = getSubRegionDTOs(cityCode); 162 | regionBatchOutput(districtRegionDTOs); 163 | } 164 | 165 | } 166 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/crawler/RegionResp.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.crawler; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 7 | * @author Spiro Huang 8 | * @since 1.0 9 | */ 10 | public class RegionResp { 11 | private String adcode; 12 | private String name; 13 | private String center; 14 | private String level; 15 | private String polyline; 16 | private List districts; 17 | 18 | public String getAdcode() { 19 | return adcode; 20 | } 21 | 22 | public void setAdcode(String adcode) { 23 | this.adcode = adcode; 24 | } 25 | 26 | public String getName() { 27 | return name; 28 | } 29 | 30 | public void setName(String name) { 31 | this.name = name; 32 | } 33 | 34 | public String getCenter() { 35 | return center; 36 | } 37 | 38 | public void setCenter(String center) { 39 | this.center = center; 40 | } 41 | 42 | public String getLevel() { 43 | return level; 44 | } 45 | 46 | public void setLevel(String level) { 47 | this.level = level; 48 | } 49 | 50 | public String getPolyline() { 51 | return polyline; 52 | } 53 | 54 | public void setPolyline(String polyline) { 55 | this.polyline = polyline; 56 | } 57 | 58 | public List getDistricts() { 59 | return districts; 60 | } 61 | 62 | public void setDistricts(List districts) { 63 | this.districts = districts; 64 | } 65 | 66 | @Override 67 | public String toString() { 68 | return "RegionResp{" + 69 | "adcode='" + adcode + '\'' + 70 | ", name='" + name + '\'' + 71 | ", center='" + center + '\'' + 72 | ", level='" + level + '\'' + 73 | ", polyline='" + polyline + '\'' + 74 | ", districts=" + districts + 75 | '}'; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/crawler/RestClient.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.crawler; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import org.apache.http.client.methods.CloseableHttpResponse; 5 | import org.apache.http.client.methods.HttpGet; 6 | import org.apache.http.client.utils.URIBuilder; 7 | import org.apache.http.impl.client.CloseableHttpClient; 8 | import org.apache.http.impl.client.HttpClients; 9 | import org.apache.http.util.EntityUtils; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.io.IOException; 14 | import java.net.URI; 15 | import java.net.URISyntaxException; 16 | 17 | /** 18 | * Send http restful request and get region data response. 19 | * @author Spiro Huang 20 | * @since 1.0 21 | */ 22 | public class RestClient { 23 | 24 | private static final Logger LOG = LoggerFactory.getLogger(RestClient.class); 25 | 26 | private String key; 27 | 28 | private String keywords; 29 | private String subdistrict; 30 | private String extensions; 31 | 32 | private CloseableHttpClient httpClient; 33 | 34 | 35 | public RestClient() { 36 | this.httpClient = HttpClients.createDefault(); 37 | } 38 | 39 | public DataResp getDistrictResponse() throws GetRegionException { 40 | URI uri = null; 41 | try { 42 | uri = new URIBuilder() 43 | .setScheme("http") 44 | .setHost("restapi.amap.com") 45 | .setPath("/v3/config/district") 46 | .setParameter("key", this.getKey()) 47 | .setParameter("keywords", this.getKeywords()) 48 | .setParameter("subdistrict", this.getSubdistrict()) 49 | .setParameter("extensions", this.getExtensions()) 50 | .build(); 51 | } catch (URISyntaxException e) { 52 | throw new GetRegionException("uri syntax error."); 53 | } 54 | 55 | HttpGet httpGet = new HttpGet(uri); 56 | 57 | CloseableHttpResponse response = null; 58 | try { 59 | response = this.httpClient.execute(httpGet); 60 | if (response.getStatusLine().getStatusCode() != 200) { 61 | throw new GetRegionException("Execute get request failed, http-code=" 62 | + response.getStatusLine().getStatusCode()); 63 | } 64 | 65 | String responseContent = EntityUtils.toString(response.getEntity()); 66 | return JSON.parseObject(responseContent, DataResp.class); 67 | } catch (IOException e) { 68 | throw new GetRegionException("Execute get request exception."); 69 | } finally { 70 | if (response != null) { 71 | try { 72 | response.close(); 73 | } catch (IOException e) { 74 | } 75 | } 76 | } 77 | } 78 | 79 | public String getSubdistrict() { 80 | return subdistrict; 81 | } 82 | 83 | public void setSubdistrict(String subdistrict) { 84 | this.subdistrict = subdistrict; 85 | } 86 | 87 | public String getKey() { 88 | return key; 89 | } 90 | 91 | public void setKey(String key) { 92 | this.key = key; 93 | } 94 | 95 | public String getKeywords() { 96 | return keywords; 97 | } 98 | 99 | public void setKeywords(String keywords) { 100 | this.keywords = keywords; 101 | } 102 | 103 | public String getExtensions() { 104 | return extensions; 105 | } 106 | 107 | public void setExtensions(String extensions) { 108 | this.extensions = extensions; 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/IOPipeline.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import tech.spiro.addrparser.common.RegionDTO; 6 | 7 | import java.io.IOException; 8 | 9 | /** 10 | * Connect input to output. 11 | * @author Spiro Huang 12 | * @since 1.0 13 | */ 14 | public class IOPipeline { 15 | 16 | private static final Logger LOG = LoggerFactory.getLogger(IOPipeline.class); 17 | 18 | private final RegionDataInput input; 19 | private final RegionDataOutput output; 20 | 21 | public IOPipeline(RegionDataInput input, RegionDataOutput output) { 22 | this.input = input; 23 | this.output = output; 24 | } 25 | 26 | public void start() throws IOException { 27 | input.init(); 28 | output.init(); 29 | 30 | try { 31 | while (true) { 32 | RegionDTO regionDTO = input.read(); 33 | if (regionDTO == null) { 34 | break; 35 | } 36 | output.write(regionDTO); 37 | } 38 | } finally { 39 | try { 40 | input.close(); 41 | } catch (Exception e) { 42 | LOG.error(e.getMessage(), e); 43 | } 44 | 45 | try { 46 | output.close(); 47 | } catch (Exception e) { 48 | LOG.error(e.getMessage(), e); 49 | } 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/RegionDataInput.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io; 2 | 3 | import tech.spiro.addrparser.common.RegionDTO; 4 | 5 | import java.io.Closeable; 6 | import java.io.IOException; 7 | 8 | /** 9 | * Input region data from external storage, just like local file or rdbms. 10 | * @author Spiro Huang 11 | * @since 1.0 12 | */ 13 | public interface RegionDataInput extends Closeable { 14 | void init() throws IOException; 15 | 16 | /** 17 | * Read region data 18 | * @return if null, implicit to the end. 19 | * @throws IOException if an I/O error occurs. 20 | */ 21 | RegionDTO read() throws IOException; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/RegionDataOutput.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io; 2 | 3 | import tech.spiro.addrparser.common.RegionDTO; 4 | 5 | import java.io.Closeable; 6 | import java.io.IOException; 7 | 8 | /** 9 | * Output region data to external storage, just like local file or rdbms. 10 | * @author Spiro Huang 11 | * @since 1.0 12 | */ 13 | public interface RegionDataOutput extends Closeable { 14 | void init() throws IOException; 15 | 16 | /** 17 | * Write region data 18 | * @param regionDTO region data to write. 19 | * @throws IOException if an I/O error occurs. 20 | */ 21 | void write(RegionDTO regionDTO) throws IOException; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/RegionDataReport.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io; 2 | 3 | import tech.spiro.addrparser.common.RegionDTO; 4 | import tech.spiro.addrparser.common.RegionLevel; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | /** 10 | * Region Data Output Report 11 | * @author Spiro Huang 12 | * @since 1.0 13 | */ 14 | public class RegionDataReport { 15 | 16 | private int totalCount = 0; 17 | 18 | private Map levelCountMap = new HashMap<>(); 19 | public void record(RegionDTO regionDTO) { 20 | 21 | if (regionDTO == null) { 22 | return; 23 | } 24 | 25 | totalCount++; 26 | 27 | // Update counter 28 | RegionLevel regionLevel = regionDTO.getLevel(); 29 | Integer count = this.levelCountMap.get(regionLevel); 30 | if (count == null) { 31 | this.levelCountMap.put(regionLevel, 1); 32 | } else { 33 | this.levelCountMap.put(regionLevel, ++count); 34 | } 35 | } 36 | 37 | public String report() { 38 | StringBuilder sb = new StringBuilder("Region Data Report => "); 39 | sb.append("\n"); 40 | sb.append("totalCount: " + this.totalCount); 41 | sb.append("\n"); 42 | for (Map.Entry entry : levelCountMap.entrySet()) { 43 | sb.append(entry.getKey().toString() + ": " + entry.getValue()); 44 | sb.append("\n"); 45 | } 46 | return sb.toString(); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/file/BinaryFileRegionDataOutput.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io.file; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import tech.spiro.addrparser.common.RegionDTO; 6 | import tech.spiro.addrparser.io.RegionDataOutput; 7 | import tech.spiro.addrparser.io.RegionDataReport; 8 | 9 | import java.io.BufferedOutputStream; 10 | import java.io.FileOutputStream; 11 | import java.io.IOException; 12 | import java.io.OutputStream; 13 | 14 | /** 15 | * Output region data to binary file. 16 | * @author Spiro Huang 17 | * @since 1.0 18 | */ 19 | public abstract class BinaryFileRegionDataOutput implements RegionDataOutput { 20 | 21 | private static final Logger LOG = LoggerFactory.getLogger(BinaryFileRegionDataOutput.class); 22 | 23 | protected OutputStream out; 24 | protected String filename; 25 | protected long writeBytes = 0; 26 | private boolean initialized = false; 27 | 28 | private RegionDataReport report = new RegionDataReport(); 29 | 30 | public BinaryFileRegionDataOutput(String filename) { 31 | this.filename = filename; 32 | } 33 | 34 | @Override 35 | public void init() throws IOException { 36 | if (initialized) { 37 | return; 38 | } 39 | synchronized (this) { 40 | if (initialized) { 41 | return; 42 | } 43 | this.out = new BufferedOutputStream(new FileOutputStream(this.filename)); 44 | this.initialized = true; 45 | } 46 | } 47 | 48 | @Override 49 | public void write(RegionDTO regionDTO) throws IOException { 50 | if (!initialized) { 51 | throw new IllegalStateException("Have not initialized already."); 52 | } 53 | if (regionDTO == null) { 54 | throw new IllegalArgumentException(" cannot be null."); 55 | } 56 | // write object bytes 57 | byte[] serializedBytes = serialize(regionDTO); 58 | this.out.write(serializedBytes); 59 | writeBytes += serializedBytes.length; 60 | 61 | report.record(regionDTO); 62 | } 63 | 64 | @Override 65 | public void close() throws IOException { 66 | if (!initialized) { 67 | throw new IllegalStateException("Have not initialized already."); 68 | } 69 | LOG.info("------------------ BinaryFile Report ----------------"); 70 | LOG.info("Write byte size: {}.", writeBytes); 71 | LOG.info(report.report()); 72 | LOG.info("-----------------------------------------------------"); 73 | this.out.close(); 74 | } 75 | 76 | 77 | protected abstract byte[] serialize(RegionDTO regionDTO); 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/file/JSONFileRegionDataInput.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io.file; 2 | 3 | import com.alibaba.fastjson.JSONReader; 4 | import com.alibaba.fastjson.serializer.SerializerFeature; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import tech.spiro.addrparser.common.RegionDTO; 8 | import tech.spiro.addrparser.io.RegionDataInput; 9 | import tech.spiro.addrparser.io.RegionDataReport; 10 | 11 | import java.io.FileReader; 12 | import java.io.IOException; 13 | import java.io.Reader; 14 | 15 | /** 16 | * Input region data from json file. 17 | * @author Spiro Huang 18 | * @since 1.0 19 | */ 20 | public class JSONFileRegionDataInput implements RegionDataInput { 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(JSONFileRegionDataInput.class); 23 | 24 | private Reader _reader; 25 | private String filename; 26 | private JSONReader reader; 27 | private RegionDataReport report = new RegionDataReport(); 28 | private boolean initialized = false; 29 | 30 | public JSONFileRegionDataInput(String filename) { 31 | this.filename = filename; 32 | if (this.filename == null) { 33 | throw new IllegalArgumentException(" cannot be null."); 34 | } 35 | } 36 | 37 | public JSONFileRegionDataInput(Reader _reader) { 38 | this._reader = _reader; 39 | } 40 | 41 | @Override 42 | public void init() throws IOException { 43 | if (initialized) { 44 | return; 45 | } 46 | synchronized (this) { 47 | if (initialized) { 48 | return; 49 | } 50 | if (this.filename == null) { 51 | this.reader = new JSONReader(this._reader); 52 | } else { 53 | this.reader = new JSONReader(new FileReader(this.filename)); 54 | } 55 | this.reader.startArray(); 56 | this.initialized = true; 57 | } 58 | } 59 | 60 | @Override 61 | public RegionDTO read() throws IOException { 62 | 63 | if (!initialized) { 64 | throw new IllegalStateException("Have not initialized already."); 65 | } 66 | 67 | if (reader.hasNext()) { 68 | RegionDTO regionDTO = reader.readObject(RegionDTO.class); 69 | this.report.record(regionDTO); 70 | return regionDTO; 71 | } 72 | return null; 73 | } 74 | 75 | @Override 76 | public void close() throws IOException { 77 | if (!initialized) { 78 | throw new IllegalStateException("Have not initialized already."); 79 | } 80 | LOG.info(this.report.report()); 81 | try { 82 | reader.endArray(); 83 | } finally { 84 | reader.close(); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/file/JSONFileRegionDataOutput.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io.file; 2 | 3 | import com.alibaba.fastjson.JSONWriter; 4 | import com.alibaba.fastjson.serializer.SerializerFeature; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import tech.spiro.addrparser.common.RegionDTO; 8 | import tech.spiro.addrparser.io.RegionDataOutput; 9 | import tech.spiro.addrparser.io.RegionDataReport; 10 | 11 | import java.io.FileWriter; 12 | import java.io.IOException; 13 | import java.io.Writer; 14 | 15 | /** 16 | * Output region data to json file. 17 | * @author Spiro Huang 18 | * @since 1.0 19 | */ 20 | public class JSONFileRegionDataOutput implements RegionDataOutput { 21 | 22 | private static final Logger LOG = LoggerFactory.getLogger(JSONFileRegionDataOutput.class); 23 | 24 | private Writer _writer; 25 | private String filename; 26 | private JSONWriter writer = null; 27 | private RegionDataReport report = new RegionDataReport(); 28 | 29 | private boolean initialized = false; 30 | 31 | public JSONFileRegionDataOutput(String filename) { 32 | this.filename = filename; 33 | if (this.filename == null) { 34 | throw new IllegalArgumentException(" cannot be null."); 35 | } 36 | } 37 | 38 | public JSONFileRegionDataOutput(Writer _writer) { 39 | this._writer = _writer; 40 | } 41 | 42 | @Override 43 | public void init() throws IOException { 44 | 45 | if (initialized) { 46 | return; 47 | } 48 | 49 | synchronized (this) { 50 | if (initialized) { 51 | return; 52 | } 53 | 54 | if (this.filename == null) { 55 | this.writer = new JSONWriter(_writer); 56 | } else { 57 | this.writer = new JSONWriter(new FileWriter(filename)); 58 | } 59 | 60 | this.writer.config(SerializerFeature.WriteEnumUsingName, false); 61 | this.writer.config(SerializerFeature.SortField, false); 62 | this.writer.startArray(); 63 | 64 | this.initialized = true; 65 | } 66 | } 67 | 68 | @Override 69 | public void write(RegionDTO regionDTO) throws IOException { 70 | 71 | if (!initialized) { 72 | throw new IllegalStateException("Have not initialized already."); 73 | } 74 | 75 | this.writer.writeValue(regionDTO); 76 | report.record(regionDTO); 77 | } 78 | 79 | @Override 80 | public void close() throws IOException { 81 | 82 | if (!initialized) { 83 | throw new IllegalStateException("Have not initialized already."); 84 | } 85 | 86 | LOG.info(this.report.report()); 87 | try { 88 | this.writer.endArray(); 89 | } finally { 90 | this.writer.close(); 91 | } 92 | 93 | 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/rdbms/RdbmsRegionDTOWrapper.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io.rdbms; 2 | 3 | import tech.spiro.addrparser.common.RegionDTO; 4 | import tech.spiro.addrparser.common.RegionLevel; 5 | 6 | import java.sql.PreparedStatement; 7 | import java.sql.ResultSet; 8 | import java.sql.SQLException; 9 | 10 | /** 11 | * Rdbms wrapper of {@link RegionDTO}, add support of read from and write to rdbms. 12 | * @author Spiro Huang 13 | * @since 1.0 14 | */ 15 | public class RdbmsRegionDTOWrapper extends RegionDTO { 16 | 17 | private RegionDTO regionDTO; 18 | 19 | public RdbmsRegionDTOWrapper(RegionDTO regionDTO) { 20 | this.regionDTO = regionDTO; 21 | } 22 | 23 | public void read(ResultSet rs) throws SQLException { 24 | this.setCode(rs.getInt("code")); 25 | this.setParentCode(rs.getInt("parent_code")); 26 | this.setName(rs.getString("name")); 27 | this.setLevel(RegionLevel.valueOf(rs.getInt("level"))); 28 | this.setCenter(rs.getString("center")); 29 | this.setPolyline(rs.getString("polyline")); 30 | } 31 | 32 | public void write(PreparedStatement stmt) throws SQLException { 33 | stmt.setInt( 1, this.getCode()); 34 | stmt.setInt( 2, this.getParentCode()); 35 | stmt.setString( 3, this.getName()); 36 | stmt.setInt( 4, this.getLevel().ordinal()); 37 | stmt.setString( 5, this.getCenter()); 38 | stmt.setString( 6, this.getPolyline()); 39 | } 40 | 41 | @Override 42 | public int getParentCode() { 43 | return this.regionDTO.getParentCode(); 44 | } 45 | 46 | @Override 47 | public void setParentCode(int parentCode) { 48 | this.regionDTO.setParentCode(parentCode); 49 | } 50 | 51 | @Override 52 | public int getCode() { 53 | return this.regionDTO.getCode(); 54 | } 55 | 56 | @Override 57 | public void setCode(int code) { 58 | this.regionDTO.setCode(code); 59 | } 60 | 61 | @Override 62 | public String getName() { 63 | return this.regionDTO.getName(); 64 | } 65 | 66 | @Override 67 | public void setName(String name) { 68 | this.regionDTO.setName(name); 69 | } 70 | 71 | @Override 72 | public RegionLevel getLevel() { 73 | return this.regionDTO.getLevel(); 74 | } 75 | 76 | @Override 77 | public void setLevel(RegionLevel level) { 78 | this.regionDTO.setLevel(level); 79 | } 80 | 81 | @Override 82 | public String getCenter() { 83 | return this.regionDTO.getCenter(); 84 | } 85 | 86 | @Override 87 | public void setCenter(String center) { 88 | this.regionDTO.setCenter(center); 89 | } 90 | 91 | @Override 92 | public String getPolyline() { 93 | return this.regionDTO.getPolyline(); 94 | } 95 | 96 | @Override 97 | public void setPolyline(String polyline) { 98 | this.regionDTO.setPolyline(polyline); 99 | } 100 | 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/rdbms/RdbmsRegionDataInput.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io.rdbms; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import tech.spiro.addrparser.common.RegionDTO; 6 | import tech.spiro.addrparser.io.RegionDataInput; 7 | import tech.spiro.addrparser.io.RegionDataReport; 8 | 9 | import javax.sql.DataSource; 10 | import java.io.IOException; 11 | import java.sql.Connection; 12 | import java.sql.PreparedStatement; 13 | import java.sql.ResultSet; 14 | import java.sql.SQLException; 15 | 16 | /** 17 | * Input region data from rdbms. 18 | * @author Spiro Huang 19 | * @since 1.0 20 | */ 21 | public class RdbmsRegionDataInput implements RegionDataInput { 22 | 23 | private static final Logger LOG = LoggerFactory.getLogger(RdbmsRegionDataInput.class); 24 | 25 | private DataSource ds = null; 26 | private Connection conn = null; 27 | private PreparedStatement stmt = null; 28 | private ResultSet rs = null; 29 | private RegionDataReport report = new RegionDataReport(); 30 | private boolean initialized = false; 31 | private final String sql; 32 | 33 | public RdbmsRegionDataInput(DataSource ds, String tableName) { 34 | if (ds == null) { 35 | throw new IllegalArgumentException("DataSource: is null."); 36 | } 37 | if (tableName == null) { 38 | throw new IllegalArgumentException(" is null."); 39 | } 40 | this.ds = ds; 41 | this.sql = RdbmsSQL.selectSQL(tableName); 42 | } 43 | 44 | public RdbmsRegionDataInput(DataSource ds) { 45 | if (ds == null) { 46 | throw new IllegalArgumentException("DataSource: is null."); 47 | } 48 | this.ds = ds; 49 | this.sql = RdbmsSQL.defaultSelectSQL(); 50 | } 51 | 52 | @Override 53 | public void init() throws IOException { 54 | 55 | if (initialized) { 56 | return; 57 | } 58 | 59 | synchronized (this) { 60 | if (initialized) { 61 | return; 62 | } 63 | 64 | LOG.debug("Initializing...sql: {}", this.sql); 65 | 66 | try { 67 | this.conn = this.ds.getConnection(); 68 | } catch (SQLException e) { 69 | throw new IOException(e.getMessage(), e); 70 | } 71 | 72 | LOG.debug("Initializing: Get connection completely."); 73 | 74 | try { 75 | this.stmt = this.conn.prepareStatement(this.sql); 76 | this.rs = this.stmt.executeQuery(); 77 | } catch (SQLException e) { 78 | try { 79 | this.conn.close(); 80 | } catch (SQLException e1) { 81 | } 82 | throw new IOException(e.getMessage(), e); 83 | } 84 | 85 | LOG.debug("Initializing: Get preparedStatement/resultSet completely."); 86 | this.initialized = true; 87 | LOG.debug("Initialized."); 88 | } 89 | 90 | } 91 | 92 | @Override 93 | public RegionDTO read() throws IOException { 94 | 95 | if (!initialized) { 96 | throw new IllegalStateException("Have not initialized already."); 97 | } 98 | 99 | try { 100 | if (this.rs.next()) { 101 | RegionDTO regionDTO = new RegionDTO(); 102 | RdbmsRegionDTOWrapper regionDTOWrapper 103 | = new RdbmsRegionDTOWrapper(regionDTO); 104 | regionDTOWrapper.read(this.rs); 105 | report.record(regionDTO); 106 | return regionDTO; 107 | } 108 | } catch (SQLException e) { 109 | throw new IOException(e.getMessage(), e); 110 | } 111 | return null; 112 | } 113 | 114 | @Override 115 | public void close() throws IOException { 116 | 117 | if (!initialized) { 118 | throw new IllegalStateException("Have not initialized already."); 119 | } 120 | 121 | LOG.info(this.report.report()); 122 | 123 | try { 124 | this.rs.close(); 125 | } catch (SQLException e) { 126 | } 127 | 128 | try { 129 | this.stmt.close(); 130 | } catch (SQLException e) { 131 | } 132 | 133 | try { 134 | this.conn.close(); 135 | } catch (SQLException e) { 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/rdbms/RdbmsRegionDataOutput.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io.rdbms; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import tech.spiro.addrparser.common.RegionDTO; 6 | import tech.spiro.addrparser.common.RegionLevel; 7 | import tech.spiro.addrparser.io.RegionDataOutput; 8 | import tech.spiro.addrparser.io.RegionDataReport; 9 | 10 | import javax.sql.DataSource; 11 | import java.io.IOException; 12 | import java.sql.Connection; 13 | import java.sql.PreparedStatement; 14 | import java.sql.ResultSet; 15 | import java.sql.SQLException; 16 | 17 | /** 18 | * Region data output to rdbms, write method argument must be {@link RdbmsRegionDTOWrapper}. 19 | * @author Spiro Huang 20 | * @since 1.0 21 | */ 22 | public class RdbmsRegionDataOutput implements RegionDataOutput { 23 | 24 | private static final Logger LOG = LoggerFactory.getLogger(RdbmsRegionDataOutput.class); 25 | 26 | private static final int BATCH_SIZE = 100; 27 | 28 | private DataSource ds = null; 29 | private Connection conn = null; 30 | private PreparedStatement stmt = null; 31 | private int batchCount = 0; 32 | private RegionDataReport report = new RegionDataReport(); 33 | private boolean initialized = false; 34 | private final String sql; 35 | 36 | public RdbmsRegionDataOutput(DataSource ds, String tableName) { 37 | if (ds == null) { 38 | throw new IllegalArgumentException("DataSource: is null."); 39 | } 40 | if (tableName == null) { 41 | throw new IllegalArgumentException(" is null."); 42 | } 43 | this.ds = ds; 44 | this.sql = RdbmsSQL.insertSQL(tableName); 45 | } 46 | 47 | public RdbmsRegionDataOutput(DataSource ds) { 48 | if (ds == null) { 49 | throw new IllegalArgumentException("DataSource: is null."); 50 | } 51 | this.ds = ds; 52 | this.sql = RdbmsSQL.defaultInsertSQL(); 53 | } 54 | 55 | @Override 56 | public void init() throws IOException { 57 | 58 | if (initialized) { 59 | return; 60 | } 61 | 62 | synchronized (this) { 63 | if (initialized) { 64 | return; 65 | } 66 | 67 | LOG.debug("Initializing...sql: {}", this.sql); 68 | 69 | try { 70 | this.conn = this.ds.getConnection(); 71 | this.conn.setAutoCommit(false); 72 | } catch (SQLException e) { 73 | throw new IOException(e.getMessage(), e); 74 | } 75 | 76 | LOG.debug("Initializing: Get connection completely."); 77 | 78 | try { 79 | this.stmt = this.conn.prepareStatement(this.sql); 80 | } catch (SQLException e) { 81 | try { 82 | this.conn.close(); 83 | } catch (SQLException e1) { 84 | } 85 | throw new IOException(e.getMessage(), e); 86 | } 87 | 88 | LOG.debug("Initializing: Get preparedStatement completely."); 89 | this.initialized = true; 90 | } 91 | 92 | 93 | } 94 | 95 | @Override 96 | public void write(RegionDTO regionDTO) throws IOException { 97 | 98 | if (!initialized) { 99 | throw new IllegalStateException("Have not initialized already."); 100 | } 101 | 102 | if (regionDTO == null) { 103 | return; 104 | } 105 | if (!(regionDTO instanceof RdbmsRegionDTOWrapper)) { 106 | regionDTO = new RdbmsRegionDTOWrapper(regionDTO); 107 | } 108 | 109 | RdbmsRegionDTOWrapper wrapper = (RdbmsRegionDTOWrapper) regionDTO; 110 | this.report.record(wrapper); 111 | try { 112 | wrapper.write(this.stmt); 113 | this.stmt.addBatch(); 114 | batchCount++; 115 | 116 | if (batchCount >= BATCH_SIZE) { 117 | flushBatch(); 118 | } 119 | } catch (SQLException e) { 120 | throw new IOException(e.getMessage(), e); 121 | } 122 | 123 | } 124 | 125 | private void flushBatch() throws SQLException { 126 | 127 | if (batchCount == 0) { 128 | return; 129 | } 130 | 131 | this.stmt.executeBatch(); 132 | this.conn.commit(); 133 | batchCount = 0; 134 | } 135 | 136 | @Override 137 | public void close() throws IOException { 138 | 139 | if (!initialized) { 140 | throw new IllegalStateException("Have not initialized already."); 141 | } 142 | 143 | LOG.info(this.report.report()); 144 | try { 145 | flushBatch(); 146 | } catch (SQLException e) { 147 | 148 | throw new IOException(e.getMessage(), e); 149 | } finally { 150 | try { 151 | this.stmt.close(); 152 | } catch (SQLException e) { 153 | } 154 | 155 | try { 156 | this.conn.close(); 157 | } catch (SQLException e) { 158 | } 159 | } 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/io/rdbms/RdbmsSQL.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io.rdbms; 2 | 3 | public class RdbmsSQL { 4 | 5 | public static final String DEFAULT_TABLE_NAME = "region_data"; 6 | private static final String SELECT_SQL_FORMAT 7 | = "SELECT code, parent_code, name, level, center, polyline FROM %s"; 8 | private static final String INSERT_SQL_FORMAT 9 | = "INSERT INTO %s(code, parent_code, name, level, center, polyline) VALUES (?,?,?,?,?,?)"; 10 | 11 | public static String selectSQL(String tableName) { 12 | return String.format(SELECT_SQL_FORMAT, tableName); 13 | } 14 | 15 | public static String insertSQL(String tableName) { 16 | return String.format(INSERT_SQL_FORMAT, tableName); 17 | } 18 | 19 | public static String defaultSelectSQL() { 20 | return selectSQL(DEFAULT_TABLE_NAME); 21 | } 22 | 23 | public static String defaultInsertSQL() { 24 | return insertSQL(DEFAULT_TABLE_NAME); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/parser/DefaultContainPointJudge.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.parser; 2 | 3 | import tech.spiro.addrparser.common.ContainPointJudge; 4 | import tech.spiro.addrparser.common.Point; 5 | 6 | import java.awt.*; 7 | import java.util.HashSet; 8 | import java.util.List; 9 | import java.util.Set; 10 | 11 | /** 12 | * Default implements of {@link ContainPointJudge}. 13 | * @author Spiro Huang 14 | * @since 1.0 15 | */ 16 | public class DefaultContainPointJudge implements ContainPointJudge { 17 | 18 | private Set polygonSet = new HashSet<>(); 19 | 20 | @Override 21 | public void initPolygons(List> polyline) { 22 | for (List points : polyline) { 23 | Polygon polygon = new Polygon(); 24 | for (tech.spiro.addrparser.common.Point point : points) { 25 | polygon.addPoint(double2Int(point.getLon()), double2Int(point.getLat())); 26 | } 27 | polygon.getBounds(); 28 | polygonSet.add(polygon); 29 | } 30 | } 31 | 32 | @Override 33 | public boolean contain(Point point) { 34 | 35 | int x = double2Int(point.getLon()); 36 | int y = double2Int(point.getLat()); 37 | 38 | for (Polygon polygon : polygonSet) { 39 | if (polygon.contains(x, y)) { 40 | return true; 41 | } 42 | } 43 | return false; 44 | } 45 | 46 | public int double2Int(double d) { 47 | return new Double((d * 1000000)).intValue(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/parser/Location.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.parser; 2 | 3 | import tech.spiro.addrparser.common.RegionInfo; 4 | 5 | /** 6 | * Location information as result of parsing. 7 | * @author Spiro Huang 8 | * @since 1.0 9 | */ 10 | public class Location { 11 | private final RegionInfo prov; 12 | private final RegionInfo city; 13 | private final RegionInfo district; 14 | 15 | public Location(RegionInfo prov, RegionInfo city, RegionInfo district) { 16 | this.prov = prov; 17 | this.city = city; 18 | this.district = district; 19 | } 20 | 21 | public RegionInfo getProv() { 22 | return prov; 23 | } 24 | 25 | public RegionInfo getCity() { 26 | return city; 27 | } 28 | 29 | public RegionInfo getDistrict() { 30 | return district; 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | return "Location{" + 36 | "prov=" + prov + 37 | ", city=" + city + 38 | ", district=" + district + 39 | '}'; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/parser/LocationParserEngine.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.parser; 2 | 3 | import java.io.*; 4 | import java.util.*; 5 | 6 | import tech.spiro.addrparser.common.*; 7 | import tech.spiro.addrparser.io.RegionDataInput; 8 | 9 | /** 10 | * The engine to parse {@link Point} or longitude/latitude pair to {@link Location}. 11 | * @author Spiro Huang 12 | * @since 1.0 13 | */ 14 | public class LocationParserEngine { 15 | 16 | private RegionDataInput regionDataInput; 17 | 18 | /** 19 | * Province region info set 20 | */ 21 | private Set provinceSet = new HashSet<>(); 22 | 23 | /** 24 | * Province and city set mapping 25 | * Key: province code Value: province provCitySetMap 26 | */ 27 | private Map> provCitySetMap = new HashMap<>(); 28 | 29 | /** 30 | * City and district or street set mapping 31 | * Key: city code Value: district or street 32 | */ 33 | private Map> cityDistrictSetMap = new HashMap<>(); 34 | 35 | public LocationParserEngine(RegionDataInput regionDataInput) { 36 | this.regionDataInput = regionDataInput; 37 | } 38 | 39 | public void init() throws ParserEngineException { 40 | try { 41 | regionDataInput.init(); 42 | } catch (IOException e) { 43 | throw new ParserEngineException(e.getMessage(), e); 44 | } 45 | 46 | try { 47 | while (true) { 48 | RegionDTO regionDTO = regionDataInput.read(); 49 | if (regionDTO == null) { 50 | break; 51 | } 52 | sortThrough(regionDTO); 53 | } 54 | } catch (IOException e) { 55 | throw new ParserEngineException(e.getMessage(), e); 56 | } finally { 57 | try { 58 | regionDataInput.close(); 59 | } catch (IOException e) { 60 | } 61 | } 62 | } 63 | 64 | /** 65 | * SortThrough region info. 66 | */ 67 | private void sortThrough(RegionDTO regionDTO) { 68 | RegionInfo regionInfo = RegionConverter.convert(regionDTO); 69 | RegionLevel regionLevel = regionInfo.getLevel(); 70 | 71 | if (regionLevel == RegionLevel.PROVINCE) { 72 | provinceSet.add(regionInfo); 73 | return; 74 | } 75 | 76 | if (regionLevel == RegionLevel.CITY) { 77 | Integer provCode = regionInfo.getParentCode(); 78 | Set provCitySet = provCitySetMap.get(provCode); 79 | if (provCitySet == null) { 80 | provCitySet = new HashSet<>(); 81 | provCitySetMap.put(provCode, provCitySet); 82 | } 83 | provCitySet.add(regionInfo); 84 | return; 85 | } 86 | 87 | if (regionLevel == RegionLevel.DISTRICT || regionLevel == RegionLevel.STREET) { 88 | Integer cityCode = regionInfo.getParentCode(); 89 | Set cityDistrictSet = cityDistrictSetMap.get(cityCode); 90 | if (cityDistrictSet == null) { 91 | cityDistrictSet = new HashSet<>(); 92 | cityDistrictSetMap.put(cityCode, cityDistrictSet); 93 | } 94 | cityDistrictSet.add(regionInfo); 95 | } 96 | } 97 | 98 | /** 99 | * Get location info by {@link Point} 100 | * @param point {@link Point} to parse. 101 | * @return location info 102 | */ 103 | public Location parse(Point point) { 104 | 105 | if (point == null) { 106 | throw new IllegalArgumentException(" is null."); 107 | } 108 | 109 | RegionInfo prov = null; 110 | RegionInfo city = null; 111 | RegionInfo district = null; 112 | 113 | for (RegionInfo regionInfo : provinceSet) { 114 | if (regionInfo.contain(point)) { 115 | prov = regionInfo; 116 | break; 117 | } 118 | } 119 | 120 | if (prov == null) { 121 | return null; 122 | } 123 | 124 | Integer provCode = prov.getCode(); 125 | Set citySet = provCitySetMap.get(provCode); 126 | 127 | if (citySet == null) { 128 | return new Location(prov, null, null); 129 | } 130 | 131 | for (RegionInfo regionInfo : citySet) { 132 | if (regionInfo.contain(point)) { 133 | city = regionInfo; 134 | break; 135 | } 136 | } 137 | 138 | if (city == null) { 139 | return new Location(prov, null, null); 140 | } 141 | 142 | Integer cityCode = city.getCode(); 143 | Set districtSet = cityDistrictSetMap.get(cityCode); 144 | 145 | if (districtSet == null) { 146 | return new Location(prov, city, null); 147 | } 148 | 149 | for (RegionInfo regionInfo : districtSet) { 150 | if (regionInfo.contain(point)) { 151 | district = regionInfo; 152 | break; 153 | } 154 | } 155 | 156 | return new Location(prov, city, district); 157 | } 158 | 159 | /** 160 | * Get location info by longitude/latitude 161 | * @param lon longitude 162 | * @param lat latitude 163 | * @return location info 164 | */ 165 | public Location parse(double lon, double lat) { 166 | Point point = new Point(lon, lat); 167 | return parse(point); 168 | } 169 | 170 | 171 | /** 172 | * Get region info by {@code region.code} 173 | * @param code region code 174 | * @return region info 175 | */ 176 | public RegionInfo getRegionInfo(int code) { 177 | throw new NotImplementedException(); 178 | } 179 | 180 | /** 181 | * Get region info by {@code region.name} 182 | * @param name region name 183 | * @return region info 184 | */ 185 | public RegionInfo getRegionInfo(String name) { 186 | throw new NotImplementedException(); 187 | } 188 | 189 | /** 190 | * Get sub region info list by {@code region.code} 191 | * @param code region code 192 | * @return region info 193 | */ 194 | public List getSubRegionInfoList(int code) { 195 | throw new NotImplementedException(); 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/parser/ParserEngineException.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.parser; 2 | 3 | /** 4 | * Exception when {@link LocationParserEngine} parsing. 5 | * @author Spiro Huang 6 | * @since 1.0 7 | */ 8 | public class ParserEngineException extends Exception { 9 | 10 | /** 11 | * 12 | */ 13 | private static final long serialVersionUID = 6162368331216310167L; 14 | 15 | public ParserEngineException() { 16 | super(); 17 | } 18 | 19 | public ParserEngineException(String message, Throwable cause, 20 | boolean enableSuppression, boolean writableStackTrace) { 21 | super(message, cause, enableSuppression, writableStackTrace); 22 | } 23 | 24 | public ParserEngineException(String message, Throwable cause) { 25 | super(message, cause); 26 | } 27 | 28 | public ParserEngineException(String message) { 29 | super(message); 30 | } 31 | 32 | public ParserEngineException(Throwable cause) { 33 | super(cause); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/tool/CrawlerServer.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.tool; 2 | 3 | import org.apache.commons.cli.*; 4 | import tech.spiro.addrparser.crawler.GetRegionException; 5 | import tech.spiro.addrparser.crawler.RegionDataCrawler; 6 | import tech.spiro.addrparser.io.RegionDataOutput; 7 | import tech.spiro.addrparser.io.file.JSONFileRegionDataOutput; 8 | 9 | import java.io.IOException; 10 | import java.util.Arrays; 11 | 12 | /** 13 | * A command-line tool to crawl region data. 14 | * @author Spiro Huang 15 | * @since 1.0 16 | */ 17 | public class CrawlerServer { 18 | 19 | private static Options options = new Options(); 20 | static { 21 | options.addRequiredOption("k", "key", true, "Amap enterprise dev key"); 22 | options.addRequiredOption("l", "level", true, "Root region level: 0-country, 1-province, 2-city"); 23 | options.addRequiredOption("c", "code", true, "Root region code"); 24 | options.addRequiredOption("o", "out", true, "Output file."); 25 | } 26 | 27 | private static void printHelp() { 28 | HelpFormatter formatter = new HelpFormatter(); 29 | formatter.printHelp("CrawlerServer", options ); 30 | } 31 | 32 | public static void main(String[] args) throws IOException, GetRegionException { 33 | 34 | CommandLineParser parser = new DefaultParser(); 35 | try { 36 | CommandLine cmd = parser.parse(options, args); 37 | String key = cmd.getOptionValue("k"); 38 | String level = cmd.getOptionValue('l'); 39 | String code = cmd.getOptionValue('c'); 40 | 41 | if (!Arrays.asList("0", "1", "2").contains(level)) { 42 | throw new ParseException("option:level invalid."); 43 | } 44 | 45 | String out = cmd.getOptionValue('o'); 46 | int _code = 0; 47 | try { 48 | _code = Integer.valueOf(code); 49 | } catch (NumberFormatException e) { 50 | throw new ParseException("code must be numeric."); 51 | } 52 | 53 | execute(key, level, _code, out); 54 | 55 | } catch (ParseException e) { 56 | System.out.println(e.getMessage()); 57 | printHelp(); 58 | System.exit(-1); 59 | } 60 | } 61 | 62 | private static void execute(String amapKey, String level, int code, String out) throws IOException, GetRegionException { 63 | try (RegionDataOutput regionOutput = new JSONFileRegionDataOutput(out)) { 64 | RegionDataCrawler infoLoader = new RegionDataCrawler(regionOutput, amapKey); 65 | 66 | if ("0".equals(level)) { 67 | infoLoader.loadCountry(); 68 | } else if ("1".equals(level)) { 69 | infoLoader.loadProv(code); 70 | } else if ("2".equals(level)) { 71 | infoLoader.loadCity(code); 72 | } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/tool/JSONFile2MySQL.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.tool; 2 | 3 | import com.mysql.cj.jdbc.MysqlDataSource; 4 | import org.apache.commons.cli.*; 5 | import tech.spiro.addrparser.crawler.GetRegionException; 6 | import tech.spiro.addrparser.io.IOPipeline; 7 | import tech.spiro.addrparser.io.RegionDataInput; 8 | import tech.spiro.addrparser.io.RegionDataOutput; 9 | import tech.spiro.addrparser.io.file.JSONFileRegionDataInput; 10 | import tech.spiro.addrparser.io.rdbms.RdbmsRegionDataOutput; 11 | import tech.spiro.addrparser.io.rdbms.RdbmsSQL; 12 | 13 | import javax.sql.DataSource; 14 | import java.io.IOException; 15 | import java.sql.Connection; 16 | import java.sql.SQLException; 17 | 18 | /** 19 | * A command-line tool to stream region data from json file to mysql. 20 | * @author Spiro Huang 21 | * @since 1.0 22 | */ 23 | public class JSONFile2MySQL { 24 | private static Options options = new Options(); 25 | static { 26 | options.addOption("f", "file", true, "Json region data file path"); 27 | 28 | options.addRequiredOption("h", "host", true, "MySQL host"); 29 | options.addOption("p", "port", true, "MySQL port, default 3306"); 30 | options.addRequiredOption("d", "db", true, "MySQL database"); 31 | options.addRequiredOption("u", "user", true, "MySQL user"); 32 | options.addRequiredOption("a", "password", true, "MySQL password"); 33 | 34 | options.addOption("t", "table-name", true, "Default: '" + RdbmsSQL.DEFAULT_TABLE_NAME + "', Region data table name"); 35 | options.addOption("i", "init", false, "Init table schema"); 36 | } 37 | 38 | private static void printHelp() { 39 | HelpFormatter formatter = new HelpFormatter(); 40 | formatter.printHelp("JSONFile2MySQL", options ); 41 | } 42 | 43 | public static void main(String[] args) throws IOException, GetRegionException { 44 | 45 | CommandLineParser parser = new DefaultParser(); 46 | try { 47 | CommandLine cmd = parser.parse(options, args); 48 | 49 | DataSource dataSource = buildDataSource(cmd); 50 | 51 | String tableName = cmd.getOptionValue('t'); 52 | if (cmd.hasOption('i')) { 53 | try { 54 | MySQLTableInit.init(dataSource, tableName); 55 | } catch (SQLException e) { 56 | e.printStackTrace(); 57 | System.exit(-1); 58 | } 59 | } 60 | 61 | String file = cmd.getOptionValue('f'); 62 | if (file != null) { 63 | RegionDataInput dataInput = new JSONFileRegionDataInput(file); 64 | RegionDataOutput dataOutput = null; 65 | if (tableName == null) { 66 | dataOutput = new RdbmsRegionDataOutput(dataSource); 67 | } else { 68 | dataOutput = new RdbmsRegionDataOutput(dataSource, tableName); 69 | } 70 | 71 | IOPipeline pipeline = new IOPipeline(dataInput, dataOutput); 72 | pipeline.start(); 73 | } 74 | 75 | } catch (ParseException e) { 76 | System.out.println(e.getMessage()); 77 | printHelp(); 78 | System.exit(-1); 79 | } 80 | } 81 | 82 | public static DataSource buildDataSource(CommandLine cmd) throws ParseException { 83 | String mysqlHost = cmd.getOptionValue('h'); 84 | int mysqlPort = 3306; 85 | 86 | if (cmd.hasOption('p')) { 87 | try { 88 | mysqlPort = Integer.valueOf(cmd.getOptionValue('p')); 89 | } catch (NumberFormatException e) { 90 | throw new ParseException(e.getMessage()); 91 | } 92 | } 93 | String mysqlDB = cmd.getOptionValue('d'); 94 | String mysqlUser = cmd.getOptionValue('u'); 95 | String mysqlPassword = cmd.getOptionValue('a'); 96 | 97 | MysqlDataSource dataSource = new MysqlDataSource(); 98 | dataSource.setServerName(mysqlHost); 99 | dataSource.setPort(mysqlPort); 100 | dataSource.setDatabaseName(mysqlDB); 101 | dataSource.setUser(mysqlUser); 102 | dataSource.setPassword(mysqlPassword); 103 | 104 | try { 105 | Connection connection = dataSource.getConnection(); 106 | } catch (SQLException e) { 107 | throw new ParseException("DataSource get connection failed:" + e.getMessage()); 108 | } 109 | return dataSource; 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/tool/MapTools.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.tool; 2 | 3 | import tech.spiro.addrparser.common.Point; 4 | 5 | import java.awt.Polygon; 6 | import java.awt.geom.Line2D; 7 | 8 | /** 9 | * Tools related with map calculating. 10 | * @author Spiro Huang 11 | * @since 1.0 12 | */ 13 | public class MapTools 14 | { 15 | private static double EARTH_RADIUS = 6378137; 16 | 17 | 18 | private static double rad(double d) 19 | { 20 | return d * Math.PI / 180.0; 21 | } 22 | 23 | /** 24 | * Calculate point to point distance (meter) 25 | * @param p1 one of 2 points 26 | * @param p2 one of 2 points 27 | * @return Distance of 2 points. 28 | */ 29 | public static double getPoint2PointDistance(Point p1, Point p2) 30 | { 31 | double radLat1 = rad( p1.getLat() ); 32 | double radLat2 = rad( p2.getLat() ); 33 | double a = radLat1 - radLat2; 34 | double b = rad( p1.getLon() ) - rad( p2.getLon() ); 35 | double s = 2 * Math.asin( Math.sqrt( Math.pow( Math.sin( a / 2 ), 2 ) + Math.cos( radLat1 ) * Math.cos( radLat2 ) 36 | * Math.pow( Math.sin( b / 2 ), 2 ) ) ); 37 | s = s * EARTH_RADIUS; 38 | return s; 39 | } 40 | 41 | /** 42 | * Calculate point to line distance (meter) 43 | * @param p point 44 | * @param lineStartPoint line start point 45 | * @param lineEndPoint line end point 46 | * @return distance of point to line. 47 | */ 48 | public static double getPoint2LineDistance(Point p, Point lineStartPoint, Point lineEndPoint) 49 | { 50 | double a = getPoint2PointDistance(lineStartPoint, lineEndPoint); 51 | double b = getPoint2PointDistance(p, lineStartPoint); 52 | double c = getPoint2PointDistance(p, lineEndPoint); 53 | 54 | double cosα = (Math.pow(a, 2) + Math.pow(b, 2) - Math.pow(c, 2)) / (2 * a * b); 55 | double cosβ = (Math.pow(a, 2) + Math.pow(c, 2) - Math.pow(b, 2)) / (2 * a * c); 56 | 57 | double distance; 58 | 59 | if (cosα <= 0) { 60 | distance = b; 61 | } else if (cosβ <= 0) { 62 | distance = c; 63 | } else { 64 | distance = (Math.sqrt((a + b + c) * (a + b - c) * (a + c - b) * (b + c - a))) / (2 * a); 65 | } 66 | 67 | return distance; 68 | } 69 | 70 | /** 71 | * To determine whether a point in a circle 72 | * @param p The point to determine 73 | * @param centerPoint Circle center point 74 | * @param radius Circle radius 75 | * @return true, point in circle, otherwise false. 76 | */ 77 | public static boolean inCircleArea(Point p, Point centerPoint, double radius) 78 | { 79 | double dis = getPoint2PointDistance( p, centerPoint ); 80 | if (dis < radius) 81 | { 82 | return true; 83 | } 84 | else 85 | { 86 | return false; 87 | } 88 | } 89 | 90 | /** 91 | * To determine whether a point in a rectangle 92 | * @param p The point to determine 93 | * @param upLeftPoint Rectangle up left point 94 | * @param downRightPoint Rectangle down right point 95 | * @return true, point in rectangle, otherwise false. 96 | */ 97 | public static boolean inRectangleArea(Point p, Point upLeftPoint, Point downRightPoint) 98 | { 99 | if ((p.getLon() > upLeftPoint.getLon() && p.getLon() < downRightPoint.getLon()) 100 | && (p.getLat() < upLeftPoint.getLat() && p.getLat() > downRightPoint.getLat())) 101 | { 102 | return true; 103 | } 104 | else 105 | { 106 | return false; 107 | } 108 | } 109 | 110 | /** 111 | * To determine whether a point in a polygon 112 | * @param p The point to determine 113 | * @param area Polygon 114 | * @return true, point in polygon, otherwise false. 115 | */ 116 | public static boolean inPolygonArea(Point p, Point area[]) 117 | { 118 | Polygon pol = new Polygon(); 119 | for ( int i = 0; i < area.length; i++ ) 120 | { 121 | pol.addPoint( (int) (area[i].getLon() * 100000), (int) (area[i].getLat() * 100000) ); 122 | } 123 | return pol.contains( (int) (p.getLon() * 100000), (int) (p.getLat() * 100000) ); 124 | } 125 | 126 | /** 127 | * To determine whether a point is yawing 128 | * @param p The point to determine 129 | * @param line Track points 130 | * @param distance Yawing distance 131 | * @return true, point yawing, otherwise false. 132 | */ 133 | public static boolean yawing(Point p, Point line[], double distance) 134 | { 135 | for ( int i = 0; i < line.length - 1; i++ ) 136 | { 137 | if (getPoint2LineDistance( p, line[i], line[i + 1] ) < distance) 138 | { 139 | return false; 140 | } 141 | } 142 | return true; 143 | } 144 | 145 | } -------------------------------------------------------------------------------- /src/main/java/tech/spiro/addrparser/tool/MySQLTableInit.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.tool; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import tech.spiro.addrparser.io.rdbms.RdbmsSQL; 6 | 7 | import javax.sql.DataSource; 8 | import java.sql.Connection; 9 | import java.sql.SQLException; 10 | import java.sql.Statement; 11 | 12 | public class MySQLTableInit { 13 | private static final Logger LOG = LoggerFactory.getLogger(MySQLTableInit.class); 14 | 15 | private static final String CREATE_SQL_FORMAT = "CREATE TABLE `%s` (" + 16 | " `code` int(11) NOT NULL," + 17 | " `parent_code` int(11) NOT NULL," + 18 | " `name` varchar(100) NOT NULL," + 19 | " `level` tinyint(4) NOT NULL," + 20 | " `center` varchar(30) DEFAULT NULL," + 21 | " `polyline` mediumtext NOT NULL" + 22 | ")"; 23 | 24 | public static void init(DataSource ds, String tableName) throws SQLException { 25 | String createSql = null; 26 | if (tableName == null) { 27 | createSql = String.format(CREATE_SQL_FORMAT, RdbmsSQL.DEFAULT_TABLE_NAME); 28 | } else { 29 | createSql = String.format(CREATE_SQL_FORMAT, tableName); 30 | } 31 | 32 | LOG.info("Initialize mysql table, DDL: {}", createSql); 33 | 34 | try (Connection conn = ds.getConnection(); 35 | Statement stmt = conn.createStatement()) { 36 | stmt.execute(createSql); 37 | } 38 | 39 | LOG.info("Initialize mysql table successfully."); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/script/crawler.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set "WORK_DIR=%cd%" 4 | cd /d %~dp0 5 | cd .. 6 | set "PRJ_HOME=%cd%" 7 | cd /d %WORK_DIR% 8 | 9 | setlocal EnableDelayedExpansion 10 | set CLASSPATH=%PRJ_HOME%\conf 11 | for /F %%j in ('dir /s/b %PRJ_HOME%\lib\*.jar') do set CLASSPATH=!CLASSPATH!;%%j 12 | 13 | java -Dfile.encoding=utf-8 -classpath %CLASSPATH% tech.spiro.addrparser.tool.CrawlerServer %* -------------------------------------------------------------------------------- /src/main/script/crawler.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bin=`dirname "${BASH_SOURCE-$0}"` 4 | bin=`cd $bin; cd ..; pwd` 5 | PRJ_HOME=$bin 6 | 7 | CLASSPATH=$PRJ_HOME/conf 8 | 9 | for f in $PRJ_HOME/lib/*.jar; do 10 | if [ "$CLASSPATH" ]; then 11 | CLASSPATH=$CLASSPATH:$f 12 | fi 13 | done 14 | 15 | java -Dfile.encoding=utf-8 -classpath $CLASSPATH tech.spiro.addrparser.tool.CrawlerServer $@ 16 | -------------------------------------------------------------------------------- /src/main/script/jsonfile2mysql.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set "WORK_DIR=%cd%" 4 | cd /d %~dp0 5 | cd .. 6 | set "PRJ_HOME=%cd%" 7 | cd /d %WORK_DIR% 8 | 9 | setlocal EnableDelayedExpansion 10 | set CLASSPATH=%PRJ_HOME%\conf 11 | for /F %%j in ('dir /s/b %PRJ_HOME%\lib\*.jar') do set CLASSPATH=!CLASSPATH!;%%j 12 | 13 | java -Dfile.encoding=utf-8 -classpath %CLASSPATH% tech.spiro.addrparser.tool.JSONFile2MySQL %* -------------------------------------------------------------------------------- /src/main/script/jsonfile2mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bin=`dirname "${BASH_SOURCE-$0}"` 4 | bin=`cd $bin; cd ..; pwd` 5 | PRJ_HOME=$bin 6 | 7 | CLASSPATH=$PRJ_HOME/conf 8 | 9 | for f in $PRJ_HOME/lib/*.jar; do 10 | if [ "$CLASSPATH" ]; then 11 | CLASSPATH=$CLASSPATH:$f 12 | fi 13 | done 14 | 15 | java -Dfile.encoding=utf-8 -classpath $CLASSPATH tech.spiro.addrparser.tool.JSONFile2MySQL $@ 16 | -------------------------------------------------------------------------------- /src/test/java/tech/spiro/addrparser/io/rdbms/RdbmsSQLTest.java: -------------------------------------------------------------------------------- 1 | package tech.spiro.addrparser.io.rdbms; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.*; 6 | 7 | public class RdbmsSQLTest { 8 | 9 | @Test 10 | public void selectSQL() { 11 | assertEquals("SELECT code, parent_code, name, level, center, polyline FROM test", 12 | RdbmsSQL.selectSQL("test")); 13 | } 14 | 15 | @Test 16 | public void insertSQL() { 17 | assertEquals("INSERT INTO test(code, parent_code, name, level, center, polyline) VALUES (?,?,?,?,?,?)", 18 | RdbmsSQL.insertSQL("test")); 19 | } 20 | 21 | @Test 22 | public void defaultSelectSQL() { 23 | assertEquals("SELECT code, parent_code, name, level, center, polyline FROM region_data", 24 | RdbmsSQL.defaultSelectSQL()); 25 | } 26 | 27 | @Test 28 | public void defaultInsertSQL() { 29 | assertEquals("INSERT INTO region_data(code, parent_code, name, level, center, polyline) VALUES (?,?,?,?,?,?)", 30 | RdbmsSQL.defaultInsertSQL()); 31 | } 32 | } --------------------------------------------------------------------------------