├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── example │ │ └── esbboss │ │ ├── EsBbossApplication.java │ │ ├── HttpClientBuilderCallbackDemo.java │ │ ├── agent │ │ ├── AgentInfo.java │ │ ├── AgentInfoBo.java │ │ ├── AgentLifeCycleBo.java │ │ ├── AgentLifeCycleState.java │ │ ├── AgentStatus.java │ │ ├── AutomaticBuffer.java │ │ ├── Buffer.java │ │ ├── BytesUtils.java │ │ ├── FixedBuffer.java │ │ ├── HBaseTables.java │ │ ├── JvmInfoBo.java │ │ ├── PinpointConstants.java │ │ ├── ServerMetaDataBo.java │ │ ├── ServiceInfoBo.java │ │ └── TimeUtils.java │ │ ├── controller │ │ ├── DataTranController.java │ │ ├── EsController.java │ │ └── ScheduleControlDataTranController.java │ │ ├── entity │ │ ├── Demo.java │ │ ├── DemoSearchResult.java │ │ ├── LoginModuleMetric.java │ │ ├── LoginUserMetric.java │ │ └── PositionUrl.java │ │ └── service │ │ ├── AutoschedulePauseDataTran.java │ │ ├── DataTran.java │ │ ├── DocumentCRUD.java │ │ ├── DocumentCRUD7.java │ │ ├── Es2Redis.java │ │ ├── SyncDataXXLJob.java │ │ ├── TestPropertiesInterceptor.java │ │ └── zjh │ │ ├── Db2EleasticsearchFullRunOncestore_order_detail_pos.java │ │ ├── Db2EleasticsearchFullRunOncestore_order_detail_pos1.java │ │ ├── Db2EleasticsearchFullRunOncestore_order_pos.java │ │ ├── Db2EleasticsearchFullRunOncestore_order_pos1.java │ │ ├── In.java │ │ └── JobStart.java └── resources │ ├── application.properties │ ├── dbbulktest.xml │ ├── esmapper │ ├── demo.xml │ ├── demo7.xml │ ├── sentence_paragrah.xml │ └── span_query.xml │ └── sqlFile.xml └── test └── java └── com └── example └── esbboss └── service ├── RestClientTest.java ├── RestClientTest7.java └── SpanQueryTest.java /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows thumbnail cache files 2 | Thumbs.db 3 | ehthumbs.db 4 | ehthumbs_vista.db 5 | 6 | # Folder config file 7 | Desktop.ini 8 | 9 | # Recycle Bin used on file shares 10 | $RECYCLE.BIN/ 11 | 12 | # Windows Installer files 13 | *.cab 14 | *.msi 15 | *.msm 16 | *.msp 17 | 18 | # Windows shortcuts 19 | *.lnk 20 | 21 | # ========================= 22 | # Operating System Files 23 | # ========================= 24 | /target/ 25 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.example 7 | springboot-elasticsearch-webservice 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | springboot-elasticsearch-webservice 12 | Demo project for Spring Boot 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 2.7.12 18 | 19 | 20 | 21 | 22 | UTF-8 23 | UTF-8 24 | 1.8 25 | 1.8 26 | 27 | 28 | 29 | 30 | 31 | 32 | org.springframework.boot 33 | spring-boot-starter-web 34 | 35 | 36 | slf4j-log4j12 37 | org.slf4j 38 | 39 | 40 | logback-classic 41 | ch.qos.logback 42 | 43 | 44 | log4j-to-slf4j 45 | org.apache.logging.log4j 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | org.springframework.boot 56 | spring-boot-starter-test 57 | 58 | 59 | slf4j-log4j12 60 | org.slf4j 61 | 62 | 63 | logback-classic 64 | ch.qos.logback 65 | 66 | 67 | log4j-to-slf4j 68 | org.apache.logging.log4j 69 | 70 | 71 | test 72 | 73 | 74 | 75 | com.bbossgroups.plugins 76 | bboss-elasticsearch-spring-boot-starter 77 | 7.3.8 78 | 79 | 80 | slf4j-log4j12 81 | org.slf4j 82 | 83 | 84 | 85 | 86 | 87 | 88 | com.bbossgroups.plugins 89 | bboss-datatran-jdbc 90 | 7.3.8 91 | 92 | 93 | slf4j-log4j12 94 | org.slf4j 95 | 96 | 97 | 98 | 99 | com.bbossgroups.plugins 100 | bboss-datatran-hbase 101 | 7.3.8 102 | 103 | 104 | slf4j-log4j12 105 | org.slf4j 106 | 107 | 108 | 109 | 110 | com.bbossgroups 111 | bboss-spring-boot-starter 112 | 6.3.8 113 | 114 | 115 | slf4j-log4j12 116 | org.slf4j 117 | 118 | 119 | logback-classic 120 | ch.qos.logback 121 | 122 | 123 | 124 | 125 | org.apache.hbase 126 | hbase-shaded-client 127 | 128 | 1.2.4 129 | 130 | 131 | slf4j-log4j12 132 | org.slf4j 133 | 134 | 135 | 136 | 137 | 138 | org.xerial 139 | sqlite-jdbc 140 | 3.42.0.0 141 | compile 142 | 143 | 144 | slf4j-log4j12 145 | org.slf4j 146 | 147 | 148 | 149 | 150 | 151 | com.mysql 152 | mysql-connector-j 153 | 154 | 8.2.0 155 | 156 | 157 | 158 | com.bbossgroups.plugins 159 | bboss-datatran-fileftp 160 | 7.3.8 161 | 162 | 163 | com.bbossgroups.plugins 164 | bboss-datatran-kafka2x 165 | 166 | 7.3.8 167 | 168 | 169 | 170 | org.apache.kafka 171 | kafka_2.12 172 | 1.1.0 173 | 174 | 175 | slf4j-log4j12 176 | org.slf4j 177 | 178 | 179 | log4j 180 | log4j 181 | 182 | 183 | kafka-clients 184 | org.apache.kafka 185 | 186 | 187 | kafka-clients 188 | org.apache.kafka 189 | 190 | 191 | kafka-clients 192 | org.apache.kafka 193 | 194 | 195 | 196 | 197 | 198 | org.apache.kafka 199 | kafka-tools 200 | 1.1.0 201 | 202 | 203 | slf4j-log4j12 204 | org.slf4j 205 | 206 | 207 | log4j 208 | log4j 209 | 210 | 211 | org.eclipse.jetty 212 | jetty-server 213 | 214 | 215 | org.eclipse.jetty 216 | jetty-servlets 217 | 218 | 219 | org.eclipse.jetty 220 | jetty-servlet 221 | 222 | 223 | org.glassfish.jersey.containers 224 | jersey-container-servlet 225 | 226 | 227 | 228 | 229 | org.apache.kafka 230 | kafka-clients 231 | 1.1.0 232 | 233 | 234 | slf4j-log4j12 235 | org.slf4j 236 | 237 | 238 | log4j 239 | log4j 240 | 241 | 242 | 243 | 244 | org.apache.kafka 245 | kafka-streams 246 | 1.1.0 247 | 248 | 249 | slf4j-log4j12 250 | org.slf4j 251 | 252 | 253 | log4j 254 | log4j 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | org.springframework.boot 268 | spring-boot-maven-plugin 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/EsBbossApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008-2019 bboss 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.example.esbboss; 17 | 18 | import org.springframework.boot.SpringApplication; 19 | import org.springframework.boot.autoconfigure.SpringBootApplication; 20 | /** 21 | * @author yinbp[yin-bp@163.com] 22 | */ 23 | @SpringBootApplication 24 | //@EnableAutoConfiguration(exclude = {ElasticsearchAutoConfiguration.class, ElasticsearchDataAutoConfiguration.class}) 25 | public class EsBbossApplication { 26 | 27 | public static void main(String[] args) { 28 | SpringApplication.run(EsBbossApplication.class, args); 29 | } 30 | } 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/HttpClientBuilderCallbackDemo.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss; 2 | /** 3 | * Copyright 2020 bboss 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import org.apache.http.impl.client.HttpClientBuilder; 19 | import org.frameworkset.spi.remote.http.ClientConfiguration; 20 | import org.frameworkset.spi.remote.http.callback.HttpClientBuilderCallback; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | /** 25 | *

Description:

26 | *

27 | *

Copyright (c) 2020

28 | * @Date 2020/5/14 10:27 29 | * @author biaoping.yin 30 | * @version 1.0 31 | */ 32 | public class HttpClientBuilderCallbackDemo implements HttpClientBuilderCallback { 33 | private static Logger logger = LoggerFactory.getLogger(HttpClientBuilderCallbackDemo.class); 34 | public HttpClientBuilder customizeHttpClient(HttpClientBuilder builder, ClientConfiguration clientConfiguration) { 35 | /** 36 | AWSCredentials credentials = new BasicAWSCredentials("", ""); 37 | AWS4Signer signer = new AWS4Signer(); 38 | AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(credentials); 39 | signer.setServiceName("es"); 40 | signer.setRegionName("us-east-1"); 41 | 42 | HttpRequestInterceptor interceptor = new AWSRequestSigningApacheInterceptor( 43 | "es", signer, awsCredentialsProvider); 44 | builder.addInterceptorLast(interceptor); 45 | */ 46 | logger.info("HttpClientBuilderCallbackDemo--------------------------------"); 47 | return builder; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/AgentInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | import java.util.Comparator; 20 | 21 | /** 22 | * @author HyunGil Jeong 23 | */ 24 | public class AgentInfo { 25 | 26 | public static final Comparator AGENT_NAME_ASC_COMPARATOR = new Comparator() { 27 | @Override 28 | public int compare(AgentInfo lhs, AgentInfo rhs) { 29 | final String lhsAgentId = lhs.agentId == null ? "" : lhs.agentId; 30 | final String rhsAgentId = rhs.agentId == null ? "" : rhs.agentId; 31 | return lhsAgentId.compareTo(rhsAgentId); 32 | } 33 | }; 34 | 35 | private String applicationName; 36 | private String agentId; 37 | private long startTimestamp; 38 | private String hostName; 39 | private String ip; 40 | private String ports; 41 | private short serviceTypeCode; 42 | private int pid; 43 | private String vmVersion; 44 | private String agentVersion; 45 | private ServerMetaDataBo serverMetaData; 46 | private JvmInfoBo jvmInfo; 47 | private long initialStartTimestamp; 48 | private AgentStatus status; 49 | 50 | public AgentInfo() { 51 | } 52 | 53 | public AgentInfo(AgentInfoBo agentInfoBo) { 54 | this.applicationName = agentInfoBo.getApplicationName(); 55 | this.agentId = agentInfoBo.getAgentId(); 56 | this.startTimestamp = agentInfoBo.getStartTime(); 57 | this.hostName = agentInfoBo.getHostName(); 58 | this.ip = agentInfoBo.getIp(); 59 | this.ports = agentInfoBo.getPorts(); 60 | this.serviceTypeCode = agentInfoBo.getServiceTypeCode(); 61 | this.pid = agentInfoBo.getPid(); 62 | this.vmVersion = agentInfoBo.getVmVersion(); 63 | this.agentVersion = agentInfoBo.getAgentVersion(); 64 | this.serverMetaData = agentInfoBo.getServerMetaData(); 65 | this.jvmInfo = agentInfoBo.getJvmInfo(); 66 | } 67 | 68 | public String getApplicationName() { 69 | return applicationName; 70 | } 71 | 72 | public void setApplicationName(String applicationName) { 73 | this.applicationName = applicationName; 74 | } 75 | 76 | public String getAgentId() { 77 | return agentId; 78 | } 79 | 80 | public void setAgentId(String agentId) { 81 | this.agentId = agentId; 82 | } 83 | 84 | public long getStartTimestamp() { 85 | return startTimestamp; 86 | } 87 | 88 | public void setStartTimestamp(long startTimestamp) { 89 | this.startTimestamp = startTimestamp; 90 | } 91 | 92 | public String getHostName() { 93 | return hostName; 94 | } 95 | 96 | public void setHostName(String hostName) { 97 | this.hostName = hostName; 98 | } 99 | 100 | public String getIp() { 101 | return ip; 102 | } 103 | 104 | public void setIp(String ip) { 105 | this.ip = ip; 106 | } 107 | 108 | public String getPorts() { 109 | return ports; 110 | } 111 | 112 | public void setPorts(String ports) { 113 | this.ports = ports; 114 | } 115 | 116 | public short getServiceTypeCode() { 117 | return serviceTypeCode; 118 | } 119 | 120 | public void setServiceTypeCode(short serviceTypeCode) { 121 | this.serviceTypeCode = serviceTypeCode; 122 | } 123 | 124 | public int getPid() { 125 | return pid; 126 | } 127 | 128 | public void setPid(int pid) { 129 | this.pid = pid; 130 | } 131 | 132 | public String getVmVersion() { 133 | return vmVersion; 134 | } 135 | 136 | public void setVmVersion(String vmVersion) { 137 | this.vmVersion = vmVersion; 138 | } 139 | 140 | public String getAgentVersion() { 141 | return agentVersion; 142 | } 143 | 144 | public void setAgentVersion(String agentVersion) { 145 | this.agentVersion = agentVersion; 146 | } 147 | 148 | public ServerMetaDataBo getServerMetaData() { 149 | return serverMetaData; 150 | } 151 | 152 | public void setServerMetaData(ServerMetaDataBo serverMetaData) { 153 | this.serverMetaData = serverMetaData; 154 | } 155 | 156 | public JvmInfoBo getJvmInfo() { 157 | return jvmInfo; 158 | } 159 | 160 | public void setJvmInfo(JvmInfoBo jvmInfo) { 161 | this.jvmInfo = jvmInfo; 162 | } 163 | 164 | public long getInitialStartTimestamp() { 165 | return initialStartTimestamp; 166 | } 167 | 168 | public void setInitialStartTimestamp(long initialStartTimestamp) { 169 | this.initialStartTimestamp = initialStartTimestamp; 170 | } 171 | 172 | public AgentStatus getStatus() { 173 | return status; 174 | } 175 | 176 | public void setStatus(AgentStatus status) { 177 | this.status = status; 178 | } 179 | 180 | @Override 181 | public boolean equals(Object o) { 182 | if (this == o) return true; 183 | if (o == null || getClass() != o.getClass()) return false; 184 | 185 | AgentInfo agentInfo = (AgentInfo) o; 186 | 187 | if (startTimestamp != agentInfo.startTimestamp) return false; 188 | return agentId != null ? agentId.equals(agentInfo.agentId) : agentInfo.agentId == null; 189 | 190 | } 191 | 192 | @Override 193 | public int hashCode() { 194 | int result = agentId != null ? agentId.hashCode() : 0; 195 | result = 31 * result + (int) (startTimestamp ^ (startTimestamp >>> 32)); 196 | return result; 197 | } 198 | 199 | @Override 200 | public String toString() { 201 | final StringBuilder sb = new StringBuilder("AgentInfo{"); 202 | sb.append("applicationName='").append(applicationName).append('\''); 203 | sb.append(", agentId='").append(agentId).append('\''); 204 | sb.append(", startTimestamp=").append(startTimestamp); 205 | sb.append(", hostName='").append(hostName).append('\''); 206 | sb.append(", ip='").append(ip).append('\''); 207 | sb.append(", ports='").append(ports).append('\''); 208 | sb.append(", serviceTypeCode=").append(serviceTypeCode); 209 | sb.append(", pid=").append(pid); 210 | sb.append(", vmVersion='").append(vmVersion).append('\''); 211 | sb.append(", agentVersion='").append(agentVersion).append('\''); 212 | sb.append(", serverMetaData=").append(serverMetaData); 213 | sb.append(", jvmInfo=").append(jvmInfo); 214 | sb.append(", initialStartTimestamp=").append(initialStartTimestamp); 215 | sb.append(", status=").append(status); 216 | sb.append('}'); 217 | return sb.toString(); 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/AgentInfoBo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | 20 | import java.util.Comparator; 21 | 22 | /** 23 | * @author emeroad 24 | * @author hyungil.jeong 25 | */ 26 | public class AgentInfoBo { 27 | 28 | public static final Comparator AGENT_NAME_ASC_COMPARATOR = new Comparator() { 29 | @Override 30 | public int compare(AgentInfoBo that, AgentInfoBo other) { 31 | final String thatAgentId = defaultString(that.agentId); 32 | final String otherAgentId = defaultString(other.agentId); 33 | return thatAgentId.compareTo(otherAgentId); 34 | } 35 | 36 | private String defaultString(String string) { 37 | return string == null ? "" : string; 38 | } 39 | }; 40 | 41 | private final String hostName; 42 | private final String ip; 43 | private final String ports; 44 | private final String agentId; 45 | private final String applicationName; 46 | private final short serviceTypeCode; 47 | private final int pid; 48 | private final String vmVersion; 49 | private final String agentVersion; 50 | 51 | private final long startTime; 52 | 53 | private final long endTimeStamp; 54 | private final int endStatus; 55 | 56 | // Should be serialized separately 57 | private final ServerMetaDataBo serverMetaData; 58 | private final JvmInfoBo jvmInfo; 59 | 60 | private AgentInfoBo(Builder builder) { 61 | this.hostName = builder.hostName; 62 | this.ip = builder.ip; 63 | this.ports = builder.ports; 64 | this.agentId = builder.agentId; 65 | this.applicationName = builder.applicationName; 66 | this.serviceTypeCode = builder.serviceTypeCode; 67 | this.pid = builder.pid; 68 | this.vmVersion = builder.vmVersion; 69 | this.agentVersion = builder.agentVersion; 70 | this.startTime = builder.startTime; 71 | this.endTimeStamp = builder.endTimeStamp; 72 | this.endStatus = builder.endStatus; 73 | this.serverMetaData = builder.serverMetaData; 74 | this.jvmInfo = builder.jvmInfo; 75 | } 76 | 77 | public String getIp() { 78 | return ip; 79 | } 80 | 81 | public String getHostName() { 82 | return hostName; 83 | } 84 | 85 | public String getPorts() { 86 | return ports; 87 | } 88 | 89 | public String getAgentId() { 90 | return agentId; 91 | } 92 | 93 | public String getApplicationName() { 94 | return applicationName; 95 | } 96 | 97 | public long getStartTime() { 98 | return startTime; 99 | } 100 | 101 | public long getEndTimeStamp() { 102 | return endTimeStamp; 103 | } 104 | 105 | public int getEndStatus() { 106 | return endStatus; 107 | } 108 | 109 | public int getPid() { 110 | return pid; 111 | } 112 | 113 | public short getServiceTypeCode() { 114 | return serviceTypeCode; 115 | } 116 | 117 | 118 | public String getVmVersion() { 119 | return vmVersion; 120 | } 121 | 122 | public String getAgentVersion() { 123 | return agentVersion; 124 | } 125 | 126 | public ServerMetaDataBo getServerMetaData() { 127 | return this.serverMetaData; 128 | } 129 | 130 | public JvmInfoBo getJvmInfo() { 131 | return this.jvmInfo; 132 | } 133 | 134 | public byte[] writeValue() { 135 | final Buffer buffer = new AutomaticBuffer(); 136 | buffer.putPrefixedString(this.getHostName()); 137 | buffer.putPrefixedString(this.getIp()); 138 | buffer.putPrefixedString(this.getPorts()); 139 | buffer.putPrefixedString(this.getApplicationName()); 140 | buffer.putShort(this.getServiceTypeCode()); 141 | buffer.putInt(this.getPid()); 142 | buffer.putPrefixedString(this.getAgentVersion()); 143 | 144 | buffer.putLong(this.getStartTime()); 145 | buffer.putLong(this.getEndTimeStamp()); 146 | buffer.putInt(this.getEndStatus()); 147 | 148 | buffer.putPrefixedString(this.getVmVersion()); 149 | 150 | return buffer.getBuffer(); 151 | } 152 | 153 | @Override 154 | public int hashCode() { 155 | final int prime = 31; 156 | int result = 1; 157 | result = prime * result + ((agentId == null) ? 0 : agentId.hashCode()); 158 | return result; 159 | } 160 | 161 | @Override 162 | public boolean equals(Object obj) { 163 | if (this == obj) 164 | return true; 165 | if (obj == null) 166 | return false; 167 | if (getClass() != obj.getClass()) 168 | return false; 169 | AgentInfoBo other = (AgentInfoBo)obj; 170 | if (agentId == null) { 171 | if (other.agentId != null) 172 | return false; 173 | } else if (!agentId.equals(other.agentId)) 174 | return false; 175 | return true; 176 | } 177 | 178 | @Override 179 | public String toString() { 180 | final StringBuilder sb = new StringBuilder("AgentInfoBo{"); 181 | sb.append("hostName='").append(hostName).append('\''); 182 | sb.append(", ip='").append(ip).append('\''); 183 | sb.append(", ports='").append(ports).append('\''); 184 | sb.append(", agentId='").append(agentId).append('\''); 185 | sb.append(", applicationName='").append(applicationName).append('\''); 186 | sb.append(", serviceTypeCode=").append(serviceTypeCode); 187 | sb.append(", pid=").append(pid); 188 | sb.append(", vmVersion=").append(vmVersion).append('\''); 189 | sb.append(", agentVersion='").append(agentVersion).append('\''); 190 | sb.append(", startTime=").append(startTime); 191 | sb.append(", endTimeStamp=").append(endTimeStamp); 192 | sb.append(", endStatus=").append(endStatus); 193 | sb.append('}'); 194 | return sb.toString(); 195 | } 196 | 197 | public static class Builder { 198 | private String hostName; 199 | private String ip; 200 | private String ports; 201 | private String agentId; 202 | private String applicationName; 203 | private short serviceTypeCode; 204 | private int pid; 205 | private String vmVersion; 206 | private String agentVersion; 207 | 208 | private long startTime; 209 | private long endTimeStamp; 210 | private int endStatus; 211 | 212 | // Should be serialized separately 213 | private ServerMetaDataBo serverMetaData; 214 | private JvmInfoBo jvmInfo; 215 | 216 | public Builder() { 217 | } 218 | 219 | public void setHostName(String hostName) { 220 | this.hostName = hostName; 221 | } 222 | 223 | public void setIp(String ip) { 224 | this.ip = ip; 225 | } 226 | 227 | public void setPorts(String ports) { 228 | this.ports = ports; 229 | } 230 | 231 | public void setAgentId(String agentId) { 232 | this.agentId = agentId; 233 | } 234 | 235 | public void setApplicationName(String applicationName) { 236 | this.applicationName = applicationName; 237 | } 238 | 239 | public void setServiceTypeCode(short serviceTypeCode) { 240 | this.serviceTypeCode = serviceTypeCode; 241 | } 242 | 243 | public short getServiceTypeCode() { 244 | return serviceTypeCode; 245 | } 246 | 247 | public void setPid(int pid) { 248 | this.pid = pid; 249 | } 250 | 251 | public void setVmVersion(String vmVersion) { 252 | this.vmVersion = vmVersion; 253 | } 254 | 255 | public void setAgentVersion(String agentVersion) { 256 | this.agentVersion = agentVersion; 257 | } 258 | 259 | public void setStartTime(long startTime) { 260 | this.startTime = startTime; 261 | } 262 | 263 | public void setEndTimeStamp(long endTimeStamp) { 264 | this.endTimeStamp = endTimeStamp; 265 | } 266 | 267 | public void setEndStatus(int endStatus) { 268 | this.endStatus = endStatus; 269 | } 270 | 271 | public void setServerMetaData(ServerMetaDataBo serverMetaData) { 272 | this.serverMetaData = serverMetaData; 273 | } 274 | 275 | public void setJvmInfo(JvmInfoBo jvmInfo) { 276 | this.jvmInfo = jvmInfo; 277 | } 278 | 279 | public AgentInfoBo build() { 280 | if (this.hostName == null) 281 | this.hostName = ""; 282 | if (this.ip == null) 283 | this.ip = ""; 284 | if (this.ports == null) 285 | this.ports = ""; 286 | if (this.agentId == null) 287 | this.agentId = ""; 288 | if (this.applicationName == null) 289 | this.applicationName = ""; 290 | if (this.vmVersion == null) 291 | this.vmVersion = ""; 292 | if (this.agentVersion == null) { 293 | this.agentVersion = ""; 294 | } 295 | return new AgentInfoBo(this); 296 | } 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/AgentLifeCycleBo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | /** 20 | * @author HyunGil Jeong 21 | */ 22 | public class AgentLifeCycleBo { 23 | 24 | public static final int CURRENT_VERSION = 0; 25 | 26 | private final byte version; 27 | private final String agentId; 28 | private final long startTimestamp; 29 | private final long eventTimestamp; 30 | private final long eventIdentifier; 31 | private final AgentLifeCycleState agentLifeCycleState; 32 | 33 | public AgentLifeCycleBo(String agentId, long startTimestamp, long eventTimestamp, long eventIdentifier, AgentLifeCycleState agentLifeCycleState) { 34 | this(CURRENT_VERSION, agentId, startTimestamp, eventTimestamp, eventIdentifier, agentLifeCycleState); 35 | } 36 | 37 | public AgentLifeCycleBo(int version, String agentId, long startTimestamp, long eventTimestamp, long eventIdentifier, AgentLifeCycleState agentLifeCycleState) { 38 | if (version < 0 || version > 255) { 39 | throw new IllegalArgumentException("version out of range (0~255)"); 40 | } 41 | if (agentId == null) { 42 | throw new IllegalArgumentException("agentId cannot be null"); 43 | } 44 | if (agentId.isEmpty()) { 45 | throw new IllegalArgumentException("agentId cannot be empty"); 46 | } 47 | if (startTimestamp < 0) { 48 | throw new IllegalArgumentException("startTimestamp cannot be less than 0"); 49 | } 50 | if (eventTimestamp < 0) { 51 | throw new IllegalArgumentException("eventTimestamp cannot be less than 0"); 52 | } 53 | if (eventIdentifier < 0) { 54 | throw new IllegalArgumentException("eventIdentifier cannot be less than 0"); 55 | } 56 | if (agentLifeCycleState == null) { 57 | throw new IllegalArgumentException("agentLifeCycleState cannot be null"); 58 | } 59 | this.version = (byte)(version & 0xFF); 60 | this.agentId = agentId; 61 | this.startTimestamp = startTimestamp; 62 | this.eventTimestamp = eventTimestamp; 63 | this.eventIdentifier = eventIdentifier; 64 | this.agentLifeCycleState = agentLifeCycleState; 65 | } 66 | 67 | public int getVersion() { 68 | return this.version & 0xFF; 69 | } 70 | 71 | public String getAgentId() { 72 | return agentId; 73 | } 74 | 75 | public long getStartTimestamp() { 76 | return startTimestamp; 77 | } 78 | 79 | public long getEventTimestamp() { 80 | return eventTimestamp; 81 | } 82 | 83 | public long getEventIdentifier() { 84 | return this.eventIdentifier; 85 | } 86 | 87 | public AgentLifeCycleState getAgentLifeCycleState() { 88 | return this.agentLifeCycleState; 89 | } 90 | 91 | @Override 92 | public String toString() { 93 | final StringBuilder sb = new StringBuilder("AgentLifeCycleBo{"); 94 | sb.append("version=").append(this.getVersion()); 95 | sb.append(", agentId='").append(this.getAgentId()).append('\''); 96 | sb.append(", startTimestamp=").append(this.getStartTimestamp()); 97 | sb.append(", eventTimestamp=").append(this.getEventTimestamp()); 98 | sb.append(", eventIdentifier=").append(this.eventIdentifier); 99 | sb.append(", state=").append(this.agentLifeCycleState.toString()); 100 | return sb.toString(); 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/AgentLifeCycleState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | public enum AgentLifeCycleState { 20 | RUNNING((short)100, "Running"), 21 | SHUTDOWN((short)200, "Shutdown"), 22 | UNEXPECTED_SHUTDOWN((short)201, "Unexpected Shutdown"), 23 | DISCONNECTED((short)300, "Disconnected"), 24 | UNKNOWN((short)-1, "Unknown"); 25 | 26 | private final short code; 27 | private final String desc; 28 | 29 | AgentLifeCycleState(short code, String desc) { 30 | this.code = code; 31 | this.desc = desc; 32 | } 33 | 34 | public short getCode() { 35 | return this.code; 36 | } 37 | 38 | public String getDesc() { 39 | return this.desc; 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return this.desc; 45 | } 46 | 47 | public static AgentLifeCycleState getStateByCode(short code) { 48 | for (AgentLifeCycleState state : AgentLifeCycleState.values()) { 49 | if (state.code == code) { 50 | return state; 51 | } 52 | } 53 | return UNKNOWN; 54 | } 55 | } -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/AgentStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | /** 20 | * 21 | * @author netspider 22 | * @author HyunGil Jeong 23 | */ 24 | public class AgentStatus { 25 | 26 | private final String agentId; 27 | 28 | private long eventTimestamp; 29 | 30 | private AgentLifeCycleState state = AgentLifeCycleState.UNKNOWN; 31 | 32 | public AgentStatus(String agentId) { 33 | this.agentId = agentId; 34 | } 35 | 36 | public AgentStatus(AgentLifeCycleBo agentLifeCycleBo) { 37 | this.agentId = agentLifeCycleBo.getAgentId(); 38 | this.eventTimestamp = agentLifeCycleBo.getEventTimestamp(); 39 | this.state = agentLifeCycleBo.getAgentLifeCycleState(); 40 | } 41 | 42 | public String getAgentId() { 43 | return agentId; 44 | } 45 | 46 | public long getEventTimestamp() { 47 | return eventTimestamp; 48 | } 49 | 50 | public void setEventTimestamp(long eventTimestamp) { 51 | this.eventTimestamp = eventTimestamp; 52 | } 53 | 54 | public AgentLifeCycleState getState() { 55 | return state; 56 | } 57 | 58 | public void setState(AgentLifeCycleState state) { 59 | this.state = state; 60 | } 61 | 62 | @Override 63 | public int hashCode() { 64 | final int prime = 31; 65 | int result = 1; 66 | result = prime * result + ((agentId == null) ? 0 : agentId.hashCode()); 67 | result = prime * result + (int)(eventTimestamp ^ (eventTimestamp >>> 32)); 68 | result = prime * result + ((state == null) ? 0 : state.hashCode()); 69 | return result; 70 | } 71 | 72 | @Override 73 | public boolean equals(Object obj) { 74 | if (this == obj) 75 | return true; 76 | if (obj == null) 77 | return false; 78 | if (getClass() != obj.getClass()) 79 | return false; 80 | AgentStatus other = (AgentStatus)obj; 81 | if (agentId == null) { 82 | if (other.agentId != null) 83 | return false; 84 | } else if (!agentId.equals(other.agentId)) 85 | return false; 86 | if (eventTimestamp != other.eventTimestamp) 87 | return false; 88 | if (state != other.state) 89 | return false; 90 | return true; 91 | } 92 | 93 | @Override 94 | public String toString() { 95 | final StringBuilder sb = new StringBuilder("AgentStatus{"); 96 | sb.append("agentId='").append(agentId).append('\''); 97 | sb.append(", eventTimestamp=").append(eventTimestamp); 98 | sb.append(", state=").append(state); 99 | sb.append('}'); 100 | return sb.toString(); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/AutomaticBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | /** 20 | * Buffer that can be expanded automatically 21 | * @author emeroad 22 | */ 23 | public class AutomaticBuffer extends FixedBuffer { 24 | 25 | public AutomaticBuffer() { 26 | super(32); 27 | } 28 | 29 | public AutomaticBuffer(final int size) { 30 | super(size); 31 | } 32 | 33 | public AutomaticBuffer(final byte[] buffer) { 34 | super(buffer); 35 | } 36 | 37 | 38 | protected void checkExpand(final int size) { 39 | final int remain = remaining(); 40 | if (remain >= size) { 41 | return; 42 | } 43 | int length = buffer.length; 44 | if (length == 0) { 45 | length = 1; 46 | } 47 | 48 | // after compute the buffer size, allocate it once for ado. 49 | final int expandedBufferSize = computeExpandedBufferSize(size, length, remain); 50 | // allocate buffer 51 | final byte[] expandedBuffer = new byte[expandedBufferSize]; 52 | System.arraycopy(buffer, 0, expandedBuffer, 0, buffer.length); 53 | buffer = expandedBuffer; 54 | } 55 | 56 | protected int computeExpandedBufferSize(final int size, int length, int remain) { 57 | int expandedBufferSize = 0; 58 | while (remain < size) { 59 | length <<= 1; 60 | expandedBufferSize = length; 61 | remain = expandedBufferSize - offset; 62 | } 63 | return expandedBufferSize; 64 | } 65 | 66 | @Override 67 | public void putPadBytes(byte[] bytes, int totalLength) { 68 | checkExpand(totalLength); 69 | super.putPadBytes(bytes, totalLength); 70 | } 71 | 72 | 73 | @Override 74 | public void putPrefixedBytes(final byte[] bytes) { 75 | if (bytes == null) { 76 | checkExpand(1); 77 | super.putSVInt(NULL); 78 | } else { 79 | checkExpand(bytes.length + BytesUtils.VINT_MAX_SIZE); 80 | super.putSVInt(bytes.length); 81 | super.putBytes(bytes); 82 | } 83 | } 84 | 85 | @Override 86 | public void put2PrefixedBytes(final byte[] bytes) { 87 | if (bytes == null) { 88 | checkExpand(BytesUtils.SHORT_BYTE_LENGTH); 89 | super.putShort((short)NULL); 90 | } else { 91 | if (bytes.length > Short.MAX_VALUE) { 92 | throw new IndexOutOfBoundsException("too large bytes length:" + bytes.length); 93 | } 94 | checkExpand(bytes.length + BytesUtils.SHORT_BYTE_LENGTH); 95 | super.putShort((short)bytes.length); 96 | super.putBytes(bytes); 97 | } 98 | } 99 | 100 | @Override 101 | public void put4PrefixedBytes(final byte[] bytes) { 102 | if (bytes == null) { 103 | checkExpand(BytesUtils.INT_BYTE_LENGTH); 104 | super.putInt(NULL); 105 | } else { 106 | checkExpand(bytes.length + BytesUtils.INT_BYTE_LENGTH); 107 | super.putInt(bytes.length); 108 | super.putBytes(bytes); 109 | } 110 | } 111 | 112 | @Override 113 | public void putPadString(String string, int totalLength) { 114 | checkExpand(totalLength); 115 | super.putPadString(string, totalLength); 116 | } 117 | 118 | 119 | @Override 120 | public void putPrefixedString(final String string) { 121 | byte[] bytes = BytesUtils.toBytes(string); 122 | this.putPrefixedBytes(bytes); 123 | } 124 | 125 | @Override 126 | public void put2PrefixedString(final String string) { 127 | byte[] bytes = BytesUtils.toBytes(string); 128 | this.put2PrefixedBytes(bytes); 129 | } 130 | 131 | @Override 132 | public void put4PrefixedString(final String string) { 133 | byte[] bytes = BytesUtils.toBytes(string); 134 | this.put4PrefixedBytes(bytes); 135 | } 136 | 137 | @Override 138 | public void putByte(final byte v) { 139 | checkExpand(1); 140 | super.putByte(v); 141 | } 142 | 143 | @Deprecated 144 | @Override 145 | public void put(final byte v){ 146 | putByte(v); 147 | } 148 | 149 | @Override 150 | public void putBoolean(final boolean v) { 151 | checkExpand(1); 152 | super.putBoolean(v); 153 | } 154 | 155 | @Deprecated 156 | @Override 157 | public void put(final boolean v) { 158 | putBoolean(v); 159 | } 160 | 161 | @Override 162 | public void putShort(final short v) { 163 | checkExpand(2); 164 | super.putShort(v); 165 | } 166 | 167 | @Deprecated 168 | @Override 169 | public void put(final short v) { 170 | putShort(v); 171 | } 172 | 173 | @Override 174 | public void putInt(final int v) { 175 | checkExpand(4); 176 | super.putInt(v); 177 | } 178 | 179 | @Deprecated 180 | @Override 181 | public void put(final int v) { 182 | putInt(v); 183 | } 184 | 185 | @Override 186 | public void putVInt(final int v) { 187 | checkExpand(BytesUtils.VLONG_MAX_SIZE); 188 | super.putVInt(v); 189 | } 190 | 191 | @Deprecated 192 | @Override 193 | public void putVar(final int v) { 194 | putVInt(v); 195 | } 196 | 197 | @Override 198 | public void putSVInt(final int v) { 199 | checkExpand(BytesUtils.VINT_MAX_SIZE); 200 | super.putSVInt(v); 201 | } 202 | 203 | @Deprecated 204 | @Override 205 | public void putSVar(final int v) { 206 | putSVInt(v); 207 | } 208 | 209 | 210 | @Override 211 | public void putVLong(final long v) { 212 | checkExpand(BytesUtils.VLONG_MAX_SIZE); 213 | super.putVLong(v); 214 | } 215 | 216 | @Deprecated 217 | @Override 218 | public void putVar(final long v) { 219 | putVLong(v); 220 | } 221 | 222 | @Override 223 | public void putSVLong(final long v) { 224 | checkExpand(BytesUtils.VLONG_MAX_SIZE); 225 | super.putSVLong(v); 226 | } 227 | 228 | @Deprecated 229 | @Override 230 | public void putSVar(final long v) { 231 | putSVLong(v); 232 | } 233 | 234 | @Override 235 | public void putLong(final long v) { 236 | checkExpand(8); 237 | super.putLong(v); 238 | } 239 | 240 | @Deprecated 241 | @Override 242 | public void put(final long v) { 243 | putLong(v); 244 | } 245 | 246 | 247 | @Override 248 | public void putBytes(final byte[] v) { 249 | if (v == null) { 250 | throw new NullPointerException("v must not be null"); 251 | } 252 | checkExpand(v.length); 253 | super.putBytes(v); 254 | } 255 | 256 | @Override 257 | public void put(final byte[] v) { 258 | putBytes(v); 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/Buffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | import java.nio.ByteBuffer; 20 | import java.nio.charset.Charset; 21 | 22 | /** 23 | * @author emeroad 24 | */ 25 | public interface Buffer { 26 | 27 | int BOOLEAN_FALSE = 0; 28 | int BOOLEAN_TRUE = 1; 29 | 30 | byte[] EMPTY = new byte[0]; 31 | 32 | String UTF8 = "UTF-8"; 33 | 34 | Charset UTF8_CHARSET = Charset.forName(UTF8); 35 | 36 | void putPadBytes(byte[] bytes, int totalLength); 37 | 38 | void putPrefixedBytes(byte[] bytes); 39 | 40 | void put2PrefixedBytes(byte[] bytes); 41 | 42 | void put4PrefixedBytes(byte[] bytes); 43 | 44 | void putPadString(String string, int totalLength); 45 | 46 | void putPrefixedString(String string); 47 | 48 | void put2PrefixedString(String string); 49 | 50 | void put4PrefixedString(String string); 51 | 52 | void putByte(byte v); 53 | 54 | /** 55 | * @deprecated Since 1.6.0. Use {@link Buffer#putByte(byte)} 56 | */ 57 | @Deprecated 58 | void put(byte v); 59 | 60 | void putBoolean(boolean v); 61 | 62 | /** 63 | * @deprecated Since 1.6.0. Use {@link Buffer#putBoolean(boolean)} 64 | */ 65 | @Deprecated 66 | void put(boolean v); 67 | 68 | void putInt(int v); 69 | 70 | /** 71 | * @deprecated Since 1.6.0. Use {@link Buffer#putInt(int)} 72 | */ 73 | @Deprecated 74 | void put(int v); 75 | 76 | /** 77 | * put value using the variable-length encoding especially for constants 78 | * the size using variable-length encoding is bigger than using fixed-length int when v is negative. 79 | * if there are a lot of negative value in a buffer, it's very inefficient. 80 | * instead use putSVar in that case. 81 | * putVar compared to putSVar has a little benefit to use a less cpu due to no zigzag operation. 82 | * it has more benefit to use putSVar whenever v is negative. 83 | * consume 1~10 bytes ( integer's max value consumes 5 bytes, integer's min value consumes 10 bytes) 84 | * @param v 85 | */ 86 | void putVInt(int v); 87 | 88 | /** 89 | * @deprecated Since 1.6.0. Use {@link Buffer#putVInt(int)} 90 | */ 91 | @Deprecated 92 | void putVar(int v); 93 | 94 | /** 95 | * put value using variable-length encoding 96 | * useful for same distribution of constants and negatives value 97 | * consume 1~5 bytes ( integer's max value consumes 5 bytes, integer's min value consumes 5 bytes) 98 | 99 | * @param v 100 | */ 101 | void putSVInt(int v); 102 | 103 | /** 104 | * @deprecated Since 1.6.0. Use {@link Buffer#putSVInt(int)} 105 | */ 106 | @Deprecated 107 | void putSVar(int v); 108 | 109 | void putShort(short v); 110 | 111 | /** 112 | * @deprecated Since 1.6.0. Use {@link Buffer#putShort(short)} 113 | */ 114 | @Deprecated 115 | void put(short v); 116 | 117 | void putLong(long v); 118 | 119 | /** 120 | * @deprecated Since 1.6.0. Use {@link Buffer#putLong(long)} 121 | */ 122 | @Deprecated 123 | void put(long v); 124 | 125 | /** 126 | * put value using the variable-length encoding especially for constants 127 | * the size using variable-length encoding is bigger than using fixed-length int when v is negative. 128 | * if there are a lot of negative value in a buffer, it's very inefficient. 129 | * instead use putSVar in that case. 130 | * @param v 131 | */ 132 | void putVLong(long v); 133 | 134 | /** 135 | * @deprecated Since 1.6.0. Use {@link Buffer#putVLong(long)} 136 | */ 137 | @Deprecated 138 | void putVar(long v); 139 | 140 | /** 141 | * put value using variable-length encoding 142 | * useful for same distribution of constants and negatives value 143 | * @param v 144 | */ 145 | void putSVLong(long v); 146 | 147 | /** 148 | * @deprecated Since 1.6.0. Use {@link Buffer#putSVLong(long)} 149 | */ 150 | @Deprecated 151 | void putSVar(long v); 152 | 153 | void putDouble(double v); 154 | 155 | /** 156 | * @deprecated Since 1.6.0. Use {@link Buffer#putDouble(double)} 157 | */ 158 | @Deprecated 159 | void put(double v); 160 | 161 | /** 162 | * put value using the variable-length encoding especially for constants 163 | * the size using variable-length encoding is bigger than using fixed-length int when v is negative. 164 | * if there are a lot of negative value in a buffer, it's very inefficient. 165 | * instead use putSVar in that case. 166 | * @param v 167 | */ 168 | void putVDouble(double v); 169 | 170 | /** 171 | * @deprecated Since 1.6.0. Use {@link Buffer#putVDouble(double)} 172 | */ 173 | @Deprecated 174 | void putVar(double v); 175 | 176 | /** 177 | * put value using variable-length encoding 178 | * useful for same distribution of constants and negatives value 179 | * @param v 180 | */ 181 | void putSVDouble(double v); 182 | 183 | /** 184 | * @deprecated Since 1.6.0. Use {@link Buffer#putSVDouble(double)} 185 | */ 186 | @Deprecated 187 | void putSVar(double v); 188 | 189 | void putBytes(byte[] v); 190 | 191 | /** 192 | * @deprecated Since 1.6.0. Use {@link Buffer#putBytes(byte[])} 193 | */ 194 | @Deprecated 195 | void put(byte[] v); 196 | 197 | byte getByte(int index); 198 | 199 | byte readByte(); 200 | 201 | int readUnsignedByte(); 202 | 203 | boolean readBoolean(); 204 | 205 | int readInt(); 206 | 207 | int readVInt(); 208 | 209 | /** 210 | * @deprecated Since 1.6.0. Use {@link Buffer#readVInt()} 211 | */ 212 | @Deprecated 213 | int readVarInt(); 214 | 215 | 216 | int readSVInt(); 217 | 218 | /** 219 | * @deprecated Since 1.6.0. Use {@link Buffer#readSVInt()} 220 | */ 221 | @Deprecated 222 | int readSVarInt(); 223 | 224 | 225 | short readShort(); 226 | 227 | long readLong(); 228 | 229 | long readVLong(); 230 | 231 | /** 232 | * @deprecated Since 1.6.0. Use {@link Buffer#readVLong()} 233 | */ 234 | @Deprecated 235 | long readVarLong(); 236 | 237 | long readSVLong(); 238 | 239 | /** 240 | * @deprecated Since 1.6.0. Use {@link Buffer#readSVLong()} 241 | */ 242 | @Deprecated 243 | long readSVarLong(); 244 | 245 | double readDouble(); 246 | 247 | double readVDouble(); 248 | 249 | /** 250 | * @deprecated Since 1.6.0. Use {@link Buffer#readVDouble()} 251 | */ 252 | @Deprecated 253 | double readVarDouble(); 254 | 255 | double readSVDouble(); 256 | 257 | /** 258 | * @deprecated Since 1.6.0. Use {@link Buffer#readSVDouble()} 259 | */ 260 | @Deprecated 261 | double readSVarDouble(); 262 | 263 | byte[] readPadBytes(int totalLength); 264 | 265 | String readPadString(int totalLength); 266 | 267 | String readPadStringAndRightTrim(int totalLength); 268 | 269 | byte[] readPrefixedBytes(); 270 | 271 | byte[] read2PrefixedBytes(); 272 | 273 | byte[] read4PrefixedBytes(); 274 | 275 | String readPrefixedString(); 276 | 277 | String read2PrefixedString(); 278 | 279 | String read4PrefixedString(); 280 | 281 | byte[] getBuffer(); 282 | 283 | byte[] copyBuffer(); 284 | 285 | byte[] getInternalBuffer(); 286 | 287 | ByteBuffer wrapByteBuffer(); 288 | 289 | void setOffset(int offset); 290 | 291 | int getOffset(); 292 | 293 | /** 294 | * @deprecated Since 1.6.0. Use {@link Buffer#remaining()} 295 | */ 296 | @Deprecated 297 | int limit(); 298 | 299 | int remaining(); 300 | 301 | boolean hasRemaining(); 302 | 303 | } 304 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/HBaseTables.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | import org.apache.hadoop.hbase.TableName; 20 | import org.apache.hadoop.hbase.util.Bytes; 21 | 22 | /** 23 | * @author emeroad 24 | */ 25 | public final class HBaseTables { 26 | 27 | public static final int APPLICATION_NAME_MAX_LEN = PinpointConstants.APPLICATION_NAME_MAX_LEN; 28 | public static final int AGENT_NAME_MAX_LEN = PinpointConstants.AGENT_NAME_MAX_LEN; 29 | 30 | // Time delta (in milliseconds) we can store in each row of AgentStatV2 31 | public static final int AGENT_STAT_TIMESPAN_MS = 5 * 60 * 1000; 32 | 33 | public static final TableName APPLICATION_TRACE_INDEX = TableName.valueOf("ApplicationTraceIndex"); 34 | public static final byte[] APPLICATION_TRACE_INDEX_CF_TRACE = Bytes.toBytes("I"); // applicationIndex 35 | public static final int APPLICATION_TRACE_INDEX_ROW_DISTRIBUTE_SIZE = 1; // applicationIndex hash size 36 | 37 | @Deprecated public static final TableName AGENT_STAT = TableName.valueOf("AgentStat"); 38 | public static final TableName AGENT_STAT_VER2 = TableName.valueOf("AgentStatV2"); 39 | 40 | public static final byte[] AGENT_STAT_CF_STATISTICS = Bytes.toBytes("S"); // agent statistics column family 41 | // FIXME (2014.08) Legacy column for storing serialzied TAgentStat Thrift DTO. 42 | @Deprecated public static final byte[] AGENT_STAT_CF_STATISTICS_V1 = Bytes.toBytes("V1"); // qualifier 43 | // FIXME (2015.10) Legacy column for storing serialzied Bos separately. 44 | @Deprecated public static final byte[] AGENT_STAT_CF_STATISTICS_MEMORY_GC = Bytes.toBytes("Gc"); // qualifier for Heap Memory/Gc statistics 45 | @Deprecated public static final byte[] AGENT_STAT_CF_STATISTICS_CPU_LOAD = Bytes.toBytes("Cpu"); // qualifier for CPU load statistics 46 | // FIXME (2016.06) Legacy column for storing stat data directly to columns 47 | @Deprecated public static final byte[] AGENT_STAT_COL_INTERVAL = Bytes.toBytes("int"); // qualifier for collection interval 48 | @Deprecated public static final byte[] AGENT_STAT_COL_GC_TYPE = Bytes.toBytes("gcT"); // qualifier for GC type 49 | @Deprecated public static final byte[] AGENT_STAT_COL_GC_OLD_COUNT = Bytes.toBytes("gcOldC"); // qualifier for GC old count 50 | @Deprecated public static final byte[] AGENT_STAT_COL_GC_OLD_TIME = Bytes.toBytes("gcOldT"); // qualifier for GC old time 51 | @Deprecated public static final byte[] AGENT_STAT_COL_HEAP_USED = Bytes.toBytes("hpU"); // gualifier for heap used 52 | @Deprecated public static final byte[] AGENT_STAT_COL_HEAP_MAX = Bytes.toBytes("hpM"); // qualifier for heap max 53 | @Deprecated public static final byte[] AGENT_STAT_COL_NON_HEAP_USED = Bytes.toBytes("nHpU"); // qualifier for non-heap used 54 | @Deprecated public static final byte[] AGENT_STAT_COL_NON_HEAP_MAX = Bytes.toBytes("nHpM"); // qualifier for non-heap max 55 | @Deprecated public static final byte[] AGENT_STAT_COL_JVM_CPU = Bytes.toBytes("jvmCpu"); // qualifier for JVM CPU usage 56 | @Deprecated public static final byte[] AGENT_STAT_COL_SYS_CPU = Bytes.toBytes("sysCpu"); // qualifier for system CPU usage 57 | @Deprecated public static final byte[] AGENT_STAT_COL_TRANSACTION_SAMPLED_NEW = Bytes.toBytes("tSN"); // qualifier for sampled new count 58 | @Deprecated public static final byte[] AGENT_STAT_COL_TRANSACTION_SAMPLED_CONTINUATION = Bytes.toBytes("tSC"); // qualifier for sampled continuation count 59 | @Deprecated public static final byte[] AGENT_STAT_COL_TRANSACTION_UNSAMPLED_NEW = Bytes.toBytes("tUnSN"); // qualifier for unsampled new count 60 | @Deprecated public static final byte[] AGENT_STAT_COL_TRANSACTION_UNSAMPLED_CONTINUATION = Bytes.toBytes("tUnSC"); // qualifier for unsampled continuation count 61 | @Deprecated public static final byte[] AGENT_STAT_COL_ACTIVE_TRACE_HISTOGRAM = Bytes.toBytes("aH"); // qualifier for active trace histogram 62 | 63 | @Deprecated 64 | public static final TableName TRACES = TableName.valueOf("Traces"); 65 | @Deprecated 66 | public static final byte[] TRACES_CF_SPAN = Bytes.toBytes("S"); //Span 67 | @Deprecated 68 | public static final byte[] TRACES_CF_ANNOTATION = Bytes.toBytes("A"); //Annotation 69 | @Deprecated 70 | public static final byte[] TRACES_CF_TERMINALSPAN = Bytes.toBytes("T"); //SpanEvent 71 | 72 | public static final TableName TRACE_V2 = TableName.valueOf("TraceV2"); 73 | public static final byte[] TRACE_V2_CF_SPAN = Bytes.toBytes("S"); //Span 74 | 75 | public static final TableName APPLICATION_INDEX = TableName.valueOf("ApplicationIndex"); 76 | public static final byte[] APPLICATION_INDEX_CF_AGENTS = Bytes.toBytes("Agents"); 77 | 78 | public static final TableName AGENTINFO = TableName.valueOf("AgentInfo"); 79 | public static final byte[] AGENTINFO_CF_INFO = Bytes.toBytes("Info"); 80 | public static final byte[] AGENTINFO_CF_INFO_IDENTIFIER = Bytes.toBytes("i"); 81 | public static final byte[] AGENTINFO_CF_INFO_SERVER_META_DATA = Bytes.toBytes("m"); 82 | public static final byte[] AGENTINFO_CF_INFO_JVM = Bytes.toBytes("j"); 83 | 84 | public static final TableName AGENT_LIFECYCLE = TableName.valueOf("AgentLifeCycle"); 85 | public static final byte[] AGENT_LIFECYCLE_CF_STATUS = Bytes.toBytes("S"); // agent lifecycle column family 86 | public static final byte[] AGENT_LIFECYCLE_CF_STATUS_QUALI_STATES = Bytes.toBytes("states"); // qualifier for agent lifecycle states 87 | 88 | public static final TableName AGENT_EVENT = TableName.valueOf("AgentEvent"); 89 | public static final byte[] AGENT_EVENT_CF_EVENTS = Bytes.toBytes("E"); // agent events column family 90 | 91 | @Deprecated 92 | public static final TableName AGENTID_APPLICATION_INDEX = TableName.valueOf("AgentIdApplicationIndex"); 93 | @Deprecated 94 | public static final byte[] AGENTID_APPLICATION_INDEX_CF_APPLICATION = Bytes.toBytes("Application"); 95 | 96 | 97 | public static final TableName SQL_METADATA_VER2 = TableName.valueOf("SqlMetaData_Ver2"); 98 | public static final byte[] SQL_METADATA_VER2_CF_SQL = Bytes.toBytes("Sql"); 99 | public static final byte[] SQL_METADATA_VER2_CF_SQL_QUALI_SQLSTATEMENT = Bytes.toBytes("P_sql_statement"); 100 | 101 | public static final TableName STRING_METADATA = TableName.valueOf("StringMetaData"); 102 | public static final byte[] STRING_METADATA_CF_STR = Bytes.toBytes("Str"); 103 | public static final byte[] STRING_METADATA_CF_STR_QUALI_STRING = Bytes.toBytes("P_string"); 104 | 105 | public static final TableName API_METADATA = TableName.valueOf("ApiMetaData"); 106 | public static final byte[] API_METADATA_CF_API = Bytes.toBytes("Api"); 107 | public static final byte[] API_METADATA_CF_API_QUALI_SIGNATURE = Bytes.toBytes("P_api_signature"); 108 | 109 | public static final TableName MAP_STATISTICS_CALLER_VER2 = TableName.valueOf("ApplicationMapStatisticsCaller_Ver2"); 110 | public static final byte[] MAP_STATISTICS_CALLER_VER2_CF_COUNTER = Bytes.toBytes("C"); 111 | 112 | public static final TableName MAP_STATISTICS_CALLEE_VER2 = TableName.valueOf("ApplicationMapStatisticsCallee_Ver2"); 113 | public static final byte[] MAP_STATISTICS_CALLEE_VER2_CF_COUNTER = Bytes.toBytes("C"); 114 | 115 | public static final TableName MAP_STATISTICS_SELF_VER2 = TableName.valueOf("ApplicationMapStatisticsSelf_Ver2"); 116 | public static final byte[] MAP_STATISTICS_SELF_VER2_CF_COUNTER = Bytes.toBytes("C"); 117 | 118 | public static final TableName HOST_APPLICATION_MAP_VER2 = TableName.valueOf("HostApplicationMap_Ver2"); 119 | public static final byte[] HOST_APPLICATION_MAP_VER2_CF_MAP = Bytes.toBytes("M"); 120 | 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/JvmInfoBo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | /** 20 | * @author HyunGil Jeong 21 | */ 22 | public class JvmInfoBo { 23 | 24 | private final byte version; 25 | private String jvmVersion; 26 | private String gcTypeName; 27 | 28 | public JvmInfoBo(int version) { 29 | if (version < 0 || version > 255) { 30 | throw new IllegalArgumentException("version out of range (0~255)"); 31 | } 32 | this.version = (byte) (version & 0xFF); 33 | } 34 | 35 | public JvmInfoBo(byte[] serializedJvmInfoBo) { 36 | final Buffer buffer = new FixedBuffer(serializedJvmInfoBo); 37 | this.version = buffer.readByte(); 38 | int version = this.version & 0xFF; 39 | switch (version) { 40 | case 0: 41 | this.jvmVersion = buffer.readPrefixedString(); 42 | this.gcTypeName = buffer.readPrefixedString(); 43 | break; 44 | default: 45 | this.jvmVersion = ""; 46 | this.gcTypeName = ""; 47 | break; 48 | } 49 | } 50 | 51 | public int getVersion() { 52 | return version & 0xFF; 53 | } 54 | 55 | public String getJvmVersion() { 56 | return jvmVersion; 57 | } 58 | 59 | public void setJvmVersion(String jvmVersion) { 60 | this.jvmVersion = jvmVersion; 61 | } 62 | 63 | public String getGcTypeName() { 64 | return gcTypeName; 65 | } 66 | 67 | public void setGcTypeName(String gcTypeName) { 68 | this.gcTypeName = gcTypeName; 69 | } 70 | 71 | public byte[] writeValue() { 72 | final Buffer buffer = new AutomaticBuffer(); 73 | buffer.putByte(this.version); 74 | int version = this.version & 0xFF; 75 | switch (version) { 76 | case 0: 77 | buffer.putPrefixedString(this.jvmVersion); 78 | buffer.putPrefixedString(this.gcTypeName); 79 | break; 80 | default: 81 | break; 82 | } 83 | return buffer.getBuffer(); 84 | } 85 | 86 | @Override 87 | public String toString() { 88 | return "JvmInfoBo{" + 89 | "version=" + version + 90 | ", jvmVersion='" + jvmVersion + '\'' + 91 | ", gcTypeName='" + gcTypeName + '\'' + 92 | '}'; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/PinpointConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | /** 20 | * @author emeroad 21 | */ 22 | public final class PinpointConstants { 23 | 24 | public static final int APPLICATION_NAME_MAX_LEN = 24; 25 | 26 | public static final int AGENT_NAME_MAX_LEN = 24; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/ServerMetaDataBo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | 20 | import java.util.ArrayList; 21 | import java.util.Collections; 22 | import java.util.List; 23 | 24 | /** 25 | * @author hyungil.jeong 26 | */ 27 | public class ServerMetaDataBo { 28 | 29 | private final String serverInfo; 30 | private final List vmArgs; 31 | private final List serviceInfos; 32 | 33 | private ServerMetaDataBo(Builder builder) { 34 | this.serverInfo = builder.serverInfo; 35 | this.vmArgs = builder.vmArgs; 36 | this.serviceInfos = builder.serviceInfos; 37 | } 38 | 39 | public String getServerInfo() { 40 | return this.serverInfo; 41 | } 42 | 43 | public List getVmArgs() { 44 | return this.vmArgs; 45 | } 46 | 47 | public List getServiceInfos() { 48 | return this.serviceInfos; 49 | } 50 | 51 | public byte[] writeValue() { 52 | final Buffer buffer = new AutomaticBuffer(); 53 | buffer.put2PrefixedString(this.serverInfo); 54 | final int numVmArgs = this.vmArgs == null ? 0 : this.vmArgs.size(); 55 | buffer.putVInt(numVmArgs); 56 | if (this.vmArgs != null) { 57 | for (String vmArg : this.vmArgs) { 58 | buffer.put2PrefixedString(vmArg); 59 | } 60 | } 61 | final int numServiceInfos = this.serviceInfos == null ? 0 : this.serviceInfos.size(); 62 | buffer.putVInt(numServiceInfos); 63 | if (this.serviceInfos != null) { 64 | for (ServiceInfoBo serviceInfo : this.serviceInfos) { 65 | buffer.putPrefixedBytes(serviceInfo.writeValue()); 66 | } 67 | } 68 | return buffer.getBuffer(); 69 | } 70 | 71 | @Override 72 | public String toString() { 73 | StringBuilder sb = new StringBuilder("ServerMetaDataBo{"); 74 | sb.append("serverInfo='").append(this.serverInfo).append('\''); 75 | sb.append(", vmArgs=").append(this.vmArgs); 76 | sb.append(", serviceInfos=").append(this.serviceInfos.toString()); 77 | return sb.toString(); 78 | } 79 | 80 | @Override 81 | public int hashCode() { 82 | final int prime = 31; 83 | int result = 1; 84 | result = prime * result + ((serverInfo == null) ? 0 : serverInfo.hashCode()); 85 | result = prime * result + ((serviceInfos == null) ? 0 : serviceInfos.hashCode()); 86 | result = prime * result + ((vmArgs == null) ? 0 : vmArgs.hashCode()); 87 | return result; 88 | } 89 | 90 | @Override 91 | public boolean equals(Object obj) { 92 | if (this == obj) 93 | return true; 94 | if (obj == null) 95 | return false; 96 | if (getClass() != obj.getClass()) 97 | return false; 98 | ServerMetaDataBo other = (ServerMetaDataBo)obj; 99 | if (serverInfo == null) { 100 | if (other.serverInfo != null) 101 | return false; 102 | } else if (!serverInfo.equals(other.serverInfo)) 103 | return false; 104 | if (serviceInfos == null) { 105 | if (other.serviceInfos != null) 106 | return false; 107 | } else if (!serviceInfos.equals(other.serviceInfos)) 108 | return false; 109 | if (vmArgs == null) { 110 | if (other.vmArgs != null) 111 | return false; 112 | } else if (!vmArgs.equals(other.vmArgs)) 113 | return false; 114 | return true; 115 | } 116 | 117 | public static class Builder { 118 | private String serverInfo; 119 | private List vmArgs; 120 | private List serviceInfos; 121 | 122 | public Builder() { 123 | } 124 | 125 | public Builder(final byte[] value) { 126 | final Buffer buffer = new FixedBuffer(value); 127 | this.serverInfo = buffer.read2PrefixedString(); 128 | final int numVmArgs = buffer.readVInt(); 129 | this.vmArgs = new ArrayList(numVmArgs); 130 | for (int i = 0; i < numVmArgs; ++i) { 131 | this.vmArgs.add(buffer.read2PrefixedString()); 132 | } 133 | final int numServiceInfos = buffer.readVInt(); 134 | this.serviceInfos = new ArrayList(numServiceInfos); 135 | for (int i = 0; i < numServiceInfos; ++i) { 136 | ServiceInfoBo serviceInfoBo = new ServiceInfoBo.Builder(buffer.readPrefixedBytes()).build(); 137 | this.serviceInfos.add(serviceInfoBo); 138 | } 139 | } 140 | 141 | public void serverInfo(String serverInfo) { 142 | this.serverInfo = serverInfo; 143 | } 144 | 145 | public void vmArgs(List vmArgs) { 146 | this.vmArgs = vmArgs; 147 | } 148 | 149 | public void serviceInfos(List serviceInfos) { 150 | this.serviceInfos = serviceInfos; 151 | } 152 | 153 | public ServerMetaDataBo build() { 154 | if (this.serverInfo == null) { 155 | this.serverInfo = ""; 156 | } 157 | if (this.vmArgs == null) { 158 | this.vmArgs = Collections.emptyList(); 159 | } 160 | if (this.serviceInfos == null) { 161 | this.serviceInfos = Collections.emptyList(); 162 | } 163 | return new ServerMetaDataBo(this); 164 | } 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/ServiceInfoBo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Collections; 21 | import java.util.List; 22 | 23 | /** 24 | * @author hyungil.jeong 25 | */ 26 | public class ServiceInfoBo { 27 | 28 | private final String serviceName; 29 | private final List serviceLibs; 30 | 31 | private ServiceInfoBo(Builder builder) { 32 | this.serviceName = builder.serviceName; 33 | this.serviceLibs = builder.serviceLibs; 34 | } 35 | 36 | public String getServiceName() { 37 | return this.serviceName; 38 | } 39 | 40 | public List getServiceLibs() { 41 | return this.serviceLibs; 42 | } 43 | 44 | public byte[] writeValue() { 45 | final Buffer buffer = new AutomaticBuffer(); 46 | buffer.put2PrefixedString(this.serviceName); 47 | int numServiceLibs = this.serviceLibs == null ? 0 : this.serviceLibs.size(); 48 | buffer.putVInt(numServiceLibs); 49 | for (int i = 0; i < numServiceLibs; ++i) { 50 | buffer.put2PrefixedString(this.serviceLibs.get(i)); 51 | } 52 | return buffer.getBuffer(); 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | StringBuilder sb = new StringBuilder("ServiceInfoBo{"); 58 | sb.append("serviceName='").append(this.serviceName).append('\''); 59 | sb.append(", serviceLibs=").append(this.serviceLibs).append('}'); 60 | return sb.toString(); 61 | } 62 | 63 | @Override 64 | public int hashCode() { 65 | final int prime = 31; 66 | int result = 1; 67 | result = prime * result + ((serviceLibs == null) ? 0 : serviceLibs.hashCode()); 68 | result = prime * result + ((serviceName == null) ? 0 : serviceName.hashCode()); 69 | return result; 70 | } 71 | 72 | @Override 73 | public boolean equals(Object obj) { 74 | if (this == obj) 75 | return true; 76 | if (obj == null) 77 | return false; 78 | if (getClass() != obj.getClass()) 79 | return false; 80 | ServiceInfoBo other = (ServiceInfoBo)obj; 81 | if (serviceLibs == null) { 82 | if (other.serviceLibs != null) 83 | return false; 84 | } else if (!serviceLibs.equals(other.serviceLibs)) 85 | return false; 86 | if (serviceName == null) { 87 | if (other.serviceName != null) 88 | return false; 89 | } else if (!serviceName.equals(other.serviceName)) 90 | return false; 91 | return true; 92 | } 93 | 94 | public static class Builder { 95 | private String serviceName; 96 | private List serviceLibs; 97 | 98 | public Builder() {} 99 | 100 | public Builder(final byte[] value) { 101 | final Buffer buffer = new FixedBuffer(value); 102 | this.serviceName = buffer.read2PrefixedString(); 103 | final int numServiceLibs = buffer.readVInt(); 104 | this.serviceLibs = new ArrayList(numServiceLibs); 105 | for (int i = 0; i < numServiceLibs; ++i) { 106 | this.serviceLibs.add(buffer.read2PrefixedString()); 107 | } 108 | } 109 | 110 | public Builder serviceName(String serviceName) { 111 | this.serviceName = serviceName; 112 | return this; 113 | } 114 | 115 | public Builder serviceLibs(List serviceLibs) { 116 | this.serviceLibs = serviceLibs; 117 | return this; 118 | } 119 | 120 | public ServiceInfoBo build() { 121 | if (this.serviceName == null) { 122 | this.serviceName = ""; 123 | } 124 | if (this.serviceLibs == null) { 125 | this.serviceLibs = Collections.emptyList(); 126 | } 127 | return new ServiceInfoBo(this); 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/agent/TimeUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.example.esbboss.agent; 18 | 19 | /** 20 | * @author emeroad 21 | */ 22 | public final class TimeUtils { 23 | private TimeUtils() { 24 | } 25 | 26 | public static long reverseTimeMillis(long currentTimeMillis) { 27 | return Long.MAX_VALUE - currentTimeMillis; 28 | } 29 | 30 | public static long reverseCurrentTimeMillis() { 31 | return reverseTimeMillis(System.currentTimeMillis()); 32 | } 33 | 34 | public static long recoveryTimeMillis(long reverseCurrentTimeMillis) { 35 | return Long.MAX_VALUE - reverseCurrentTimeMillis; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/controller/DataTranController.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.controller; 2 | /** 3 | * Copyright 2008 biaoping.yin 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.example.esbboss.service.DataTran; 19 | import org.springframework.beans.factory.annotation.Autowired; 20 | import org.springframework.web.bind.annotation.RequestBody; 21 | import org.springframework.web.bind.annotation.RequestMapping; 22 | import org.springframework.web.bind.annotation.ResponseBody; 23 | import org.springframework.web.bind.annotation.RestController; 24 | 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | /** 30 | *

Description:

31 | *

32 | *

Copyright (c) 2018

33 | * @Date 2020/1/5 12:14 34 | * @author biaoping.yin 35 | * @version 1.0 36 | */ 37 | @RestController 38 | public class DataTranController { 39 | @Autowired 40 | private DataTran dataTran; 41 | 42 | /** 43 | * 启动db-kafka同步作业 44 | * @return 45 | */ 46 | @RequestMapping("/scheduleDB2KafkaJob") 47 | public @ResponseBody 48 | String scheduleDB2KafkaJob(){ 49 | return dataTran.scheduleDB2KafkaJob(); 50 | } 51 | 52 | /** 53 | * 停止db-kafka同步作业 54 | * @return 55 | */ 56 | @RequestMapping("/stopDB2kafkaJob") 57 | public @ResponseBody String stopDB2kafkaJob(){ 58 | return dataTran.stopDB2kafkaJob(); 59 | } 60 | 61 | /** 62 | * 启动kafka-es同步作业 63 | * @return 64 | */ 65 | @RequestMapping("/scheduleKafka2esJob") 66 | public @ResponseBody 67 | String scheduleKafka2esJob(){ 68 | return dataTran.scheduleKafka2esJob(); 69 | } 70 | 71 | /** 72 | * 停止kafka-es同步作业 73 | * @return 74 | */ 75 | @RequestMapping("/stopKafka2esJob") 76 | public @ResponseBody String stopKafka2esJob(){ 77 | return dataTran.stopKafka2esJob(); 78 | } 79 | 80 | 81 | /** 82 | * 启动db-es同步作业 83 | * @return 84 | */ 85 | @RequestMapping("/scheduleDB2ESJob") 86 | public @ResponseBody 87 | String scheduleDB2ESJob(){ 88 | return dataTran.scheduleDB2ESJob(); 89 | } 90 | 91 | /** 92 | * 停止db-es同步作业 93 | * @return 94 | */ 95 | @RequestMapping("/stopDB2ESJob") 96 | public @ResponseBody String stopDB2ESJob(){ 97 | return dataTran.stopDB2ESJob(); 98 | } 99 | 100 | /** 101 | * 启动hbase-es同步作业 102 | * @return 103 | */ 104 | @RequestMapping("/scheduleHBase2ESJob") 105 | public @ResponseBody 106 | String scheduleHBase2ESJob(){ 107 | return dataTran.scheduleHBase2ESJob(); 108 | } 109 | 110 | 111 | /** 112 | * 启动hbase-es同步作业 113 | * @return 114 | */ 115 | @RequestMapping("/stopDb2EleasticsearchMetrics") 116 | public @ResponseBody 117 | String stopDb2EleasticsearchMetricsDemo(){ 118 | return dataTran.stopDb2EleasticsearchMetricsDemo(); 119 | } 120 | 121 | /** 122 | * 启动hbase-es同步作业 123 | * @return 124 | */ 125 | @RequestMapping("/scheduleDb2EleasticsearchMetrics") 126 | public @ResponseBody 127 | String scheduleDb2EleasticsearchMetricsDemo(){ 128 | return dataTran.scheduleDb2EleasticsearchMetricsDemo(); 129 | } 130 | 131 | /** 132 | * 启动hbase-es同步作业 133 | * @return 134 | */ 135 | @RequestMapping("/getData") 136 | public @ResponseBody 137 | List getData(@RequestBody Map parmas){ 138 | return new ArrayList<>(); 139 | } 140 | 141 | /** 142 | * 停止作业 143 | * @return 144 | */ 145 | @RequestMapping("/stopHBase2ESJob") 146 | public @ResponseBody String stopHBase2ESJob(){ 147 | return dataTran.stopHBase2ESJob(); 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/controller/ScheduleControlDataTranController.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.controller; 2 | /** 3 | * Copyright 2008 biaoping.yin 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.example.esbboss.service.AutoschedulePauseDataTran; 19 | import org.springframework.beans.factory.annotation.Autowired; 20 | import org.springframework.beans.factory.annotation.Qualifier; 21 | import org.springframework.web.bind.annotation.RequestMapping; 22 | import org.springframework.web.bind.annotation.ResponseBody; 23 | import org.springframework.web.bind.annotation.RestController; 24 | 25 | /** 26 | *

Description:

27 | *

28 | *

Copyright (c) 2018

29 | * @Date 2020/1/5 12:14 30 | * @author biaoping.yin 31 | * @version 1.0 32 | */ 33 | @RestController 34 | public class ScheduleControlDataTranController { 35 | @Autowired 36 | @Qualifier("autoschedulePauseDataTran") 37 | private AutoschedulePauseDataTran dataTran; 38 | 39 | /** 40 | * 启动db-es同步作业 41 | * @return 42 | */ 43 | @RequestMapping("/schedulecontrol/scheduleDB2ESJob") 44 | public @ResponseBody 45 | String scheduleDB2ESJob(boolean autoPause){ 46 | return dataTran.scheduleDB2ESJob(autoPause); 47 | } 48 | 49 | /** 50 | * 停止db-es同步作业 51 | * @return 52 | */ 53 | @RequestMapping("/schedulecontrol/stopDB2ESJob") 54 | public @ResponseBody String stopDB2ESJob(){ 55 | return dataTran.stopDB2ESJob(); 56 | } 57 | 58 | /** 59 | * 暂停调度db-es同步作业 60 | * @return 61 | */ 62 | @RequestMapping("/schedulecontrol/pauseScheduleDB2ESJob") 63 | public @ResponseBody String pauseScheduleDB2ESJob(){ 64 | return dataTran.pauseScheduleDB2ESJob(); 65 | } 66 | 67 | /** 68 | * 继续调度db-es同步作业 69 | * @return 70 | */ 71 | @RequestMapping("/schedulecontrol/resumeScheduleDB2ESJob") 72 | public @ResponseBody String resumeScheduleDB2ESJob(){ 73 | return dataTran.resumeScheduleDB2ESJob(); 74 | } 75 | 76 | /** 77 | * 启动hbase-es同步作业 78 | * @return 79 | */ 80 | @RequestMapping("/schedulecontrol/scheduleHBase2ESJob") 81 | public @ResponseBody 82 | String scheduleHBase2ESJob(){ 83 | return dataTran.scheduleHBase2ESJob(); 84 | } 85 | 86 | /** 87 | * 停止作业 88 | * @return 89 | */ 90 | @RequestMapping("/schedulecontrol/stopHBase2ESJob") 91 | public @ResponseBody String stopHBase2ESJob(){ 92 | return dataTran.stopHBase2ESJob(); 93 | } 94 | 95 | /** 96 | * 暂停调度hbase-es同步作业 97 | * @return 98 | */ 99 | @RequestMapping("/schedulecontrol/pauseScheduleHBase2ESJob") 100 | public @ResponseBody String pauseScheduleHBase2ESJob(){ 101 | return dataTran.pauseScheduleHBase2ESJob(); 102 | } 103 | 104 | /** 105 | * 继续调度hbase-es同步作业 106 | * @return 107 | */ 108 | @RequestMapping("/schedulecontrol/resumeScheduleHBase2ESJob") 109 | public @ResponseBody String resumeScheduleHBase2ESJob(){ 110 | return dataTran.resumeScheduleHBase2ESJob(); 111 | } 112 | 113 | 114 | /** 115 | * 启动file-es同步作业 116 | * @return 117 | */ 118 | @RequestMapping("/schedulecontrol/startfile2es") 119 | public @ResponseBody 120 | String startfile2es(boolean autoPause){ 121 | return dataTran.startfile2es( autoPause); 122 | } 123 | 124 | /** 125 | * 停止作业 126 | * @return 127 | */ 128 | @RequestMapping("/schedulecontrol/stopfile2es") 129 | public @ResponseBody String stopfile2es(){ 130 | return dataTran.stopfile2es(); 131 | } 132 | 133 | /** 134 | * 暂停调度hbase-es同步作业 135 | * @return 136 | */ 137 | @RequestMapping("/schedulecontrol/pauseFile2es") 138 | public @ResponseBody String pauseFile2es(){ 139 | return dataTran.pauseFile2es(); 140 | } 141 | 142 | /** 143 | * 继续调度hbase-es同步作业 144 | * @return 145 | */ 146 | @RequestMapping("/schedulecontrol/resumeFile2es") 147 | public @ResponseBody String resumeFile2es(){ 148 | return dataTran.resumeFile2es(); 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/entity/Demo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008-2019 bboss 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.example.esbboss.entity; 17 | 18 | import com.fasterxml.jackson.annotation.JsonFormat; 19 | import com.frameworkset.orm.annotation.Column; 20 | import com.frameworkset.orm.annotation.ESId; 21 | import org.frameworkset.elasticsearch.entity.ESBaseData; 22 | 23 | import java.util.Date; 24 | 25 | /** 26 | * Test entity, which can inherit the meta attribute from the ESBaseData object. 27 | * The meta attribute of the document will be set to the object instance during retrieval 28 | * @author yinbp[yin-bp@163.com] 29 | */ 30 | public class Demo extends ESBaseData { 31 | private Object dynamicPriceTemplate; 32 | //Set the document identity field 33 | @ESId(readSet = true,persistent = false) 34 | private Long demoId; 35 | private String contentbody; 36 | /** When the date format is specified in the mapping definition, 37 | * the following two annotations need to be specified, for example: 38 | * 39 | "agentStarttime": { 40 | "type": "date", 41 | ###Specify multiple date formats 42 | "format":"yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd'T'HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss||epoch_millis" 43 | } 44 | @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS") 45 | @Column(dataformat = "yyyy-MM-dd HH:mm:ss.SSS") 46 | */ 47 | 48 | protected Date agentStarttime; 49 | 50 | 51 | 52 | public Date getAgentStarttimezh() { 53 | return agentStarttimezh; 54 | } 55 | 56 | public void setAgentStarttimezh(Date agentStarttimezh) { 57 | this.agentStarttimezh = agentStarttimezh; 58 | } 59 | 60 | @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") 61 | @Column(dataformat = "yyyy-MM-dd HH:mm:ss") 62 | private Date agentStarttimezh; 63 | private String applicationName; 64 | private String orderId; 65 | // @JsonProperty(value="contrast_status",access= JsonProperty.Access.WRITE_ONLY) 66 | private int contrastStatus; 67 | 68 | public String getName() { 69 | return name; 70 | } 71 | 72 | public void setName(String name) { 73 | this.name = name; 74 | } 75 | 76 | private String name; 77 | 78 | public String getContentbody() { 79 | return contentbody; 80 | } 81 | 82 | public void setContentbody(String contentbody) { 83 | this.contentbody = contentbody; 84 | } 85 | 86 | public Date getAgentStarttime() { 87 | return agentStarttime; 88 | } 89 | 90 | public void setAgentStarttime(Date agentStarttime) { 91 | this.agentStarttime = agentStarttime; 92 | } 93 | 94 | public String getApplicationName() { 95 | return applicationName; 96 | } 97 | 98 | public void setApplicationName(String applicationName) { 99 | this.applicationName = applicationName; 100 | } 101 | 102 | public Long getDemoId() { 103 | return demoId; 104 | } 105 | 106 | public void setDemoId(Long demoId) { 107 | this.demoId = demoId; 108 | } 109 | 110 | public Object getDynamicPriceTemplate() { 111 | return dynamicPriceTemplate; 112 | } 113 | 114 | public void setDynamicPriceTemplate(Object dynamicPriceTemplate) { 115 | this.dynamicPriceTemplate = dynamicPriceTemplate; 116 | } 117 | 118 | public String getOrderId() { 119 | return orderId; 120 | } 121 | 122 | public void setOrderId(String orderId) { 123 | this.orderId = orderId; 124 | } 125 | 126 | public int getContrastStatus() { 127 | return contrastStatus; 128 | } 129 | 130 | public void setContrastStatus(int contrastStatus) { 131 | this.contrastStatus = contrastStatus; 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/entity/DemoSearchResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008-2019 bboss 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.example.esbboss.entity; 17 | 18 | 19 | import java.util.List; 20 | 21 | /** 22 | * @author yinbp[yin-bp@163.com] 23 | */ 24 | public class DemoSearchResult { 25 | private List demos; 26 | private long totalSize; 27 | 28 | public List getDemos() { 29 | return demos; 30 | } 31 | 32 | public void setDemos(List demos) { 33 | this.demos = demos; 34 | } 35 | 36 | public long getTotalSize() { 37 | return totalSize; 38 | } 39 | 40 | public void setTotalSize(long totalSize) { 41 | this.totalSize = totalSize; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/entity/LoginModuleMetric.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.entity; 2 | /** 3 | * Copyright 2023 bboss 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import org.frameworkset.tran.CommonRecord; 19 | import org.frameworkset.tran.metrics.entity.MapData; 20 | import org.frameworkset.tran.metrics.entity.TimeMetric; 21 | 22 | /** 23 | *

Description:

24 | *

25 | *

Copyright (c) 2023

26 | * @Date 2023/2/15 27 | * @author biaoping.yin 28 | * @version 1.0 29 | */ 30 | public class LoginModuleMetric extends TimeMetric { 31 | private String operModule ; 32 | @Override 33 | public void init(MapData firstData) { 34 | CommonRecord data = (CommonRecord) firstData.getData(); 35 | operModule = (String) data.getData("operModule"); 36 | } 37 | 38 | @Override 39 | public void incr(MapData data) { 40 | count ++; 41 | } 42 | 43 | public String getOperModule() { 44 | return operModule; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/entity/LoginUserMetric.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.entity; 2 | /** 3 | * Copyright 2023 bboss 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import org.frameworkset.tran.CommonRecord; 19 | import org.frameworkset.tran.metrics.entity.MapData; 20 | import org.frameworkset.tran.metrics.entity.TimeMetric; 21 | 22 | /** 23 | *

Description:

24 | *

25 | *

Copyright (c) 2023

26 | * @Date 2023/2/15 27 | * @author biaoping.yin 28 | * @version 1.0 29 | */ 30 | public class LoginUserMetric extends TimeMetric { 31 | private String logUser; 32 | @Override 33 | public void init(MapData firstData) { 34 | CommonRecord data = (CommonRecord) firstData.getData(); 35 | logUser = (String) data.getData("logOperuser"); 36 | } 37 | 38 | @Override 39 | public void incr(MapData data) { 40 | count ++; 41 | } 42 | 43 | public String getLogUser() { 44 | return logUser; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/entity/PositionUrl.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.entity; 2 | /** 3 | * Copyright 2008-2010 biaoping.yin 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | import com.frameworkset.orm.annotation.PrimaryKey; 20 | 21 | import java.sql.Timestamp; 22 | 23 | /** 24 | *

Description:

25 | * 26 | * @author jun.xie 27 | * @version 1.0 28 | * @Date 2022/1/12 16:21 29 | */ 30 | public class PositionUrl implements java.io.Serializable { 31 | /** 32 | * 主键 33 | */ 34 | @PrimaryKey 35 | private String id; 36 | /** 37 | * 触点URL 38 | */ 39 | private String positionUrl; 40 | /** 41 | * 触点名称 42 | */ 43 | private String positionName; 44 | /** 45 | * 创建时间 46 | */ 47 | private Timestamp createtime; 48 | public PositionUrl() { 49 | } 50 | 51 | public String getId() { 52 | return id; 53 | } 54 | 55 | public void setId(String id) { 56 | this.id = id; 57 | } 58 | 59 | public String getPositionUrl() { 60 | return positionUrl; 61 | } 62 | 63 | public void setPositionUrl(String positionUrl) { 64 | this.positionUrl = positionUrl; 65 | } 66 | 67 | public String getPositionName() { 68 | return positionName; 69 | } 70 | 71 | public void setPositionName(String positionName) { 72 | this.positionName = positionName; 73 | } 74 | 75 | public Timestamp getCreatetime() { 76 | return createtime; 77 | } 78 | 79 | public void setCreatetime(Timestamp createtime) { 80 | this.createtime = createtime; 81 | } 82 | } -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/DocumentCRUD.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008-2019 bboss 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.example.esbboss.service; 17 | 18 | import com.example.esbboss.entity.Demo; 19 | import com.example.esbboss.entity.DemoSearchResult; 20 | import org.frameworkset.elasticsearch.ElasticSearchException; 21 | import org.frameworkset.elasticsearch.boot.BBossESStarter; 22 | import org.frameworkset.elasticsearch.client.ClientInterface; 23 | import org.frameworkset.elasticsearch.entity.ESDatas; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | import org.springframework.beans.factory.annotation.Autowired; 27 | import org.springframework.stereotype.Service; 28 | 29 | import java.text.DateFormat; 30 | import java.text.ParseException; 31 | import java.text.SimpleDateFormat; 32 | import java.util.Date; 33 | import java.util.HashMap; 34 | import java.util.List; 35 | import java.util.Map; 36 | 37 | /** 38 | * @author yinbp[yin-bp@163.com] 39 | */ 40 | @Service 41 | public class DocumentCRUD { 42 | private Logger logger = LoggerFactory.getLogger(DocumentCRUD.class); 43 | @Autowired 44 | private BBossESStarter bbossESStarter; 45 | //DSL config file path 46 | private String mappath = "esmapper/demo.xml"; 47 | 48 | 49 | public void dropAndCreateAndGetIndice(){ 50 | //Create a client tool to load configuration files, single instance multithreaded security 51 | ClientInterface clientUtil = bbossESStarter.getConfigRestClient(mappath); 52 | try { 53 | //To determine whether the indice demo exists, it returns true if it exists and false if it does not 54 | boolean exist = clientUtil.existIndice("demo"); 55 | 56 | //Delete mapping if the indice demo already exists 57 | if(exist) { 58 | String r = clientUtil.dropIndice("demo"); 59 | logger.debug("clientUtil.dropIndice(\"demo\") response:"+r); 60 | 61 | } 62 | //Create index demo 63 | clientUtil.createIndiceMapping("demo",//The indice name 64 | "createDemoIndice");//Index mapping DSL script name, defined createDemoIndice in esmapper/demo.xml 65 | 66 | String demoIndice = clientUtil.getIndice("demo");//Gets the newly created indice structure 67 | logger.info("after createIndiceMapping clientUtil.getIndice(\"demo\") response:"+demoIndice); 68 | } catch (ElasticSearchException e) { 69 | // TODO Auto-generated catch block 70 | e.printStackTrace(); 71 | } 72 | 73 | } 74 | 75 | 76 | 77 | public void addAndUpdateDocument() { 78 | //Build a create/modify/get/delete document client object, single instance multi-thread security 79 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 80 | //Build an object as index document 81 | Demo demo = new Demo(); 82 | demo.setDemoId(2l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 83 | demo.setAgentStarttime(new Date()); 84 | demo.setAgentStarttimezh(new Date()); 85 | demo.setApplicationName("blackcatdemo2"); 86 | demo.setContentbody("this is content body2"); 87 | demo.setName("liudehua"); 88 | demo.setOrderId("NFZF15045871807281445364228"); 89 | demo.setContrastStatus(2); 90 | 91 | 92 | //Add the document and force refresh 93 | String response = clientUtil.addDocument("demo",//indice name 94 | "demo",//idnex type 95 | demo,"refresh=true"); 96 | 97 | 98 | 99 | logger.debug("Print the result：addDocument-------------------------"); 100 | logger.debug(response); 101 | 102 | demo = new Demo(); 103 | demo.setDemoId(3l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 104 | demo.setAgentStarttime(new Date()); 105 | demo.setApplicationName("blackcatdemo3"); 106 | demo.setContentbody("this is content body3"); 107 | demo.setName("zhangxueyou"); 108 | demo.setOrderId("NFZF15045871807281445364228"); 109 | demo.setContrastStatus(3); 110 | demo.setAgentStarttime(new Date()); 111 | demo.setAgentStarttimezh(new Date()); 112 | 113 | //Add the document and force refresh 114 | response = clientUtil.addDocument("demo",//indice name 115 | "demo",//idnex type 116 | demo,"refresh=true"); 117 | 118 | //Get the document object according to the document id, and return the Demo object 119 | demo = clientUtil.getDocument("demo",//indice name 120 | "demo",//idnex type 121 | "2",//document id 122 | Demo.class); 123 | 124 | //update document 125 | demo = new Demo(); 126 | demo.setDemoId(2l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 127 | demo.setAgentStarttime(new Date()); 128 | demo.setApplicationName("blackcatdemo2"); 129 | demo.setContentbody("this is modify content body2"); 130 | demo.setName("刘德华modify\t"); 131 | demo.setOrderId("NFZF15045871807281445364228"); 132 | demo.setContrastStatus(2); 133 | demo.setAgentStarttimezh(new Date()); 134 | //Execute update and force refresh 135 | response = clientUtil.addDocument("demo",//index name 136 | "demo",//idnex type 137 | demo,"refresh=true"); 138 | 139 | 140 | //Get the modified document object according to the document id and return the json message string 141 | response = clientUtil.getDocument("demo",//indice name 142 | "demo",//idnex type 143 | "2");//document id 144 | logger.debug("Print the modified result:getDocument-------------------------"); 145 | logger.debug(response); 146 | 147 | 148 | 149 | 150 | logger.debug("Print the modified result：getDocument-------------------------"); 151 | logger.debug(response); 152 | 153 | 154 | } 155 | 156 | public void deleteDocuments(){ 157 | //Build a create/modify/get/delete document client object, single instance multi-thread security 158 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 159 | //Batch delete documents 160 | clientUtil.deleteDocuments("demo",//indice name 161 | "demo",//idnex type 162 | new String[]{"2","3"});//Batch delete document ids 163 | } 164 | 165 | /** 166 | * Use slice parallel scoll query all documents of indice demo by 2 thread tasks. DEFAULT_FETCHSIZE is 5000 167 | */ 168 | public void searchAllPararrel(){ 169 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 170 | ESDatas esDatas = clientUtil.searchAllParallel("demo", Demo.class,2); 171 | } 172 | 173 | 174 | 175 | /** 176 | * Search the documents 177 | */ 178 | public DemoSearchResult search() { 179 | //Create a load DSL file client instance to retrieve documents, single instance multithread security 180 | ClientInterface clientUtil = bbossESStarter.getConfigRestClient(mappath); 181 | //Set query conditions, pass variable parameter values via map,key for variable names in DSL 182 | //There are four variables in the DSL: 183 | // applicationName1 184 | // applicationName2 185 | // startTime 186 | // endTime 187 | Map params = new HashMap(); 188 | //Set the values of applicationName1 and applicationName2 variables 189 | params.put("applicationName1","blackcatdemo2"); 190 | params.put("applicationName2","blackcatdemo3"); 191 | DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 192 | //Set the time range, and accept the long value as the time parameter 193 | try { 194 | params.put("startTime",dateFormat.parse("2017-09-02 00:00:00").getTime()); 195 | } catch (ParseException e) { 196 | e.printStackTrace(); 197 | } 198 | params.put("endTime",new Date().getTime()); 199 | 200 | 201 | //Execute the query 202 | ESDatas esDatas = //ESDatas contains a collection of currently retrieved records, up to 1000 records, specified by the size attribute in the DSL 203 | clientUtil.searchList("demo/_search",//demo as the indice, _search as the search action 204 | "searchDatas",//DSL statement name defined in esmapper/demo.xml 205 | params,//Query parameters 206 | Demo.class);//Data object type Demo returned 207 | 208 | 209 | //Gets a list of result objects and returns max up to 1000 records (specified in DSL) 210 | List demos = esDatas.getDatas(); 211 | 212 | // String json = clientUtil.executeRequest("demo/_search",//demo as the index table, _search as the search action 213 | // "searchDatas",//DSL statement name defined in esmapper/demo.xml 214 | // params);//Query parameters 215 | 216 | // String json = com.frameworkset.util.SimpleStringUtil.object2json(demos); 217 | //Gets the total number of records 218 | long totalSize = esDatas.getTotalSize(); 219 | DemoSearchResult demoSearchResult = new DemoSearchResult(); 220 | demoSearchResult.setDemos(demos); 221 | demoSearchResult.setTotalSize(totalSize); 222 | return demoSearchResult; 223 | } 224 | 225 | 226 | } 227 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/DocumentCRUD7.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2008-2019 bboss 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.example.esbboss.service; 17 | 18 | import com.example.esbboss.entity.Demo; 19 | import com.example.esbboss.entity.DemoSearchResult; 20 | import org.frameworkset.elasticsearch.ElasticSearchException; 21 | import org.frameworkset.elasticsearch.boot.BBossESStarter; 22 | import org.frameworkset.elasticsearch.client.ClientInterface; 23 | import org.frameworkset.elasticsearch.entity.ESDatas; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | import org.springframework.beans.factory.annotation.Autowired; 27 | import org.springframework.stereotype.Service; 28 | 29 | import java.text.DateFormat; 30 | import java.text.ParseException; 31 | import java.text.SimpleDateFormat; 32 | import java.util.Date; 33 | import java.util.HashMap; 34 | import java.util.List; 35 | import java.util.Map; 36 | 37 | /** 38 | * @author yinbp[yin-bp@163.com] 39 | */ 40 | @Service 41 | public class DocumentCRUD7 { 42 | private Logger logger = LoggerFactory.getLogger(DocumentCRUD7.class); 43 | @Autowired 44 | private BBossESStarter bbossESStarter; 45 | //DSL config file path 46 | private String mappath = "esmapper/demo7.xml"; 47 | 48 | 49 | public void dropAndCreateAndGetIndice(){ 50 | //Create a client tool to load configuration files, single instance multithreaded security 51 | 52 | ClientInterface clientUtil = bbossESStarter.getConfigRestClient(mappath); 53 | clientUtil.getIndexesWithCluster("default"); 54 | clientUtil.getIndexMappingFieldsWithCluster("default","indiceName"); 55 | try { 56 | //To determine whether the indice demo exists, it returns true if it exists and false if it does not 57 | boolean exist = clientUtil.existIndice("demo"); 58 | 59 | //Delete mapping if the indice demo already exists 60 | if(exist) { 61 | String r = clientUtil.dropIndice("demo"); 62 | logger.debug("clientUtil.dropIndice(\"demo\") response:"+r); 63 | 64 | } 65 | //Create index demo 66 | clientUtil.createIndiceMapping("demo",//The indice name 67 | "createDemoIndice");//Index mapping DSL script name, defined createDemoIndice in esmapper/demo.xml 68 | 69 | String demoIndice = clientUtil.getIndice("demo");//Gets the newly created indice structure 70 | logger.info("after createIndiceMapping clientUtil.getIndice(\"demo\") response:"+demoIndice); 71 | } catch (ElasticSearchException e) { 72 | // TODO Auto-generated catch block 73 | e.printStackTrace(); 74 | } 75 | 76 | } 77 | 78 | 79 | 80 | public void addAndUpdateDocument() { 81 | //Build a create/modify/get/delete document client object, single instance multi-thread security 82 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 83 | //Build an object as index document 84 | Demo demo = new Demo(); 85 | demo.setDemoId(2l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 86 | demo.setAgentStarttime(new Date()); 87 | demo.setAgentStarttimezh(new Date()); 88 | demo.setApplicationName("blackcatdemo2"); 89 | demo.setContentbody("this is content body2"); 90 | demo.setName("liudehua"); 91 | demo.setOrderId("NFZF15045871807281445364228"); 92 | demo.setContrastStatus(2); 93 | 94 | 95 | //Add the document and force refresh 96 | String response = clientUtil.addDocument("demo",//indice name 97 | demo,"refresh=true"); 98 | 99 | 100 | 101 | logger.debug("Print the result：addDocument-------------------------"); 102 | logger.debug(response); 103 | 104 | demo = new Demo(); 105 | demo.setDemoId(3l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 106 | demo.setAgentStarttime(new Date()); 107 | demo.setApplicationName("blackcatdemo3"); 108 | demo.setContentbody("this is content body3"); 109 | demo.setName("zhangxueyou"); 110 | demo.setOrderId("NFZF15045871807281445364228"); 111 | demo.setContrastStatus(3); 112 | demo.setAgentStarttime(new Date()); 113 | demo.setAgentStarttimezh(new Date()); 114 | 115 | //Add the document and force refresh 116 | response = clientUtil.addDocument("demo",//indice name 117 | demo,"refresh=true"); 118 | 119 | //Get the document object according to the document id, and return the Demo object 120 | demo = clientUtil.getDocument("demo",//indice name 121 | "2",//document id 122 | Demo.class); 123 | 124 | //update document 125 | demo = new Demo(); 126 | demo.setDemoId(2l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 127 | demo.setAgentStarttime(new Date()); 128 | demo.setApplicationName("blackcatdemo2"); 129 | demo.setContentbody("this is modify content body2"); 130 | demo.setName("刘德华modify\t"); 131 | demo.setOrderId("NFZF15045871807281445364228"); 132 | demo.setContrastStatus(2); 133 | demo.setAgentStarttimezh(new Date()); 134 | //Execute update and force refresh 135 | response = clientUtil.addDocument("demo",//index name 136 | demo,"refresh=true"); 137 | 138 | 139 | //Get the modified document object according to the document id and return the json message string 140 | response = clientUtil.getDocument("demo",//indice name 141 | "2");//document id 142 | logger.debug("Print the modified result:getDocument-------------------------"); 143 | logger.debug(response); 144 | 145 | 146 | 147 | 148 | logger.debug("Print the modified result：getDocument-------------------------"); 149 | logger.debug(response); 150 | 151 | 152 | } 153 | 154 | public void deleteDocuments(){ 155 | //Build a create/modify/get/delete document client object, single instance multi-thread security 156 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 157 | //Batch delete documents 158 | clientUtil.deleteDocuments("demo",//indice name 159 | new String[]{"2","3"});//Batch delete document ids 160 | } 161 | 162 | /** 163 | * Use slice parallel scoll query all documents of indice demo by 2 thread tasks. DEFAULT_FETCHSIZE is 5000 164 | */ 165 | public void searchAllPararrel(){ 166 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 167 | ESDatas esDatas = clientUtil.searchAllParallel("demo", Demo.class,2); 168 | } 169 | 170 | 171 | 172 | /** 173 | * Search the documents 174 | */ 175 | public DemoSearchResult search() { 176 | //Create a load DSL file client instance to retrieve documents, single instance multithread security 177 | ClientInterface clientUtil = bbossESStarter.getConfigRestClient(mappath); 178 | //Set query conditions, pass variable parameter values via map,key for variable names in DSL 179 | //There are four variables in the DSL: 180 | // applicationName1 181 | // applicationName2 182 | // startTime 183 | // endTime 184 | Map params = new HashMap(); 185 | //Set the values of applicationName1 and applicationName2 variables 186 | params.put("applicationName1","blackcatdemo2"); 187 | params.put("applicationName2","blackcatdemo3"); 188 | DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 189 | //Set the time range, and accept the long value as the time parameter 190 | try { 191 | params.put("startTime",dateFormat.parse("2017-09-02 00:00:00").getTime()); 192 | } catch (ParseException e) { 193 | e.printStackTrace(); 194 | } 195 | params.put("endTime",new Date().getTime()); 196 | 197 | 198 | //Execute the query 199 | ESDatas esDatas = //ESDatas contains a collection of currently retrieved records, up to 1000 records, specified by the size attribute in the DSL 200 | clientUtil.searchList("demo/_search",//demo as the indice, _search as the search action 201 | "searchDatas",//DSL statement name defined in esmapper/demo.xml 202 | params,//Query parameters 203 | Demo.class);//Data object type Demo returned 204 | 205 | 206 | //Gets a list of result objects and returns max up to 1000 records (specified in DSL) 207 | List demos = esDatas.getDatas(); 208 | 209 | // String json = clientUtil.executeRequest("demo/_search",//demo as the index table, _search as the search action 210 | // "searchDatas",//DSL statement name defined in esmapper/demo.xml 211 | // params);//Query parameters 212 | 213 | // String json = com.frameworkset.util.SimpleStringUtil.object2json(demos); 214 | //Gets the total number of records 215 | long totalSize = esDatas.getTotalSize(); 216 | DemoSearchResult demoSearchResult = new DemoSearchResult(); 217 | demoSearchResult.setDemos(demos); 218 | demoSearchResult.setTotalSize(totalSize); 219 | return demoSearchResult; 220 | } 221 | 222 | 223 | } 224 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/SyncDataXXLJob.java: -------------------------------------------------------------------------------- 1 | //package com.example.esbboss.service; 2 | // 3 | //import com.xxl.job.core.biz.model.ReturnT; 4 | //import com.xxl.job.core.handler.IJobHandler; 5 | //import com.xxl.job.core.handler.annotation.JobHandler; 6 | //import com.xxl.job.core.util.ShardingUtil; 7 | //import org.frameworkset.elasticsearch.ElasticSearchHelper; 8 | //import org.frameworkset.tran.config.ImportBuilder; 9 | //import org.frameworkset.tran.plugin.db.input.DBInputConfig; 10 | //import org.frameworkset.tran.plugin.es.output.ElasticsearchOutputConfig; 11 | //import org.frameworkset.tran.schedule.ExternalScheduler; 12 | //import org.frameworkset.tran.schedule.ImportIncreamentConfig; 13 | //import org.slf4j.Logger; 14 | //import org.slf4j.LoggerFactory; 15 | //import org.springframework.stereotype.Component; 16 | // 17 | //import java.util.concurrent.locks.Lock; 18 | //import java.util.concurrent.locks.ReentrantLock; 19 | // 20 | ///** 21 | // * 一个xxl-job调度的elasticsearch到database数据同步的作业案例 22 | // * @author bboss 23 | // */ 24 | //@Component 25 | //@JobHandler(value = "SyncDataXXLJob") 26 | //public class SyncDataXXLJob extends IJobHandler { 27 | //// @Resource 28 | //// private ICustomerService customerService; 29 | //// @Resource 30 | //// private IContactService contactService; 31 | //// @Resource 32 | //// private IContractService contractService; 33 | // 34 | // private static Logger logger = LoggerFactory.getLogger(SyncDataXXLJob.class); 35 | // 36 | // 37 | // protected ExternalScheduler externalScheduler; 38 | // 39 | // private Lock lock = new ReentrantLock(); 40 | // public void init(){ 41 | // externalScheduler = new ExternalScheduler(); 42 | // externalScheduler.dataStream((Object params)->{ 43 | // ShardingUtil.ShardingVO shardingVO = ShardingUtil.getShardingVo(); 44 | // int shardIndex = 0; 45 | // if(shardingVO != null) { 46 | // shardIndex = shardingVO.getIndex(); 47 | // logger.info("index:>>>>>>>>>>>>>>>>>>>" + shardingVO.getIndex()); 48 | // logger.info("total:>>>>>>>>>>>>>>>>>>>" + shardingVO.getTotal()); 49 | // } 50 | // logger.info("params:>>>>>>>>>>>>>>>>>>>" + params); 51 | // ImportBuilder importBuilder = new ImportBuilder(); 52 | // //增量定时任务不要删表，但是可以通过删表来做初始化操作 53 | //// if(dropIndice) { 54 | // try { 55 | // //清除测试表,导入的时候回重建表，测试的时候加上为了看测试效果，实际线上环境不要删表 56 | // String repsonse = ElasticSearchHelper.getRestClientUtil().dropIndice("quartz"); 57 | // System.out.println(repsonse); 58 | // } catch (Exception e) { 59 | // } 60 | //// } 61 | // 62 | // 63 | // //指定导入数据的sql语句，必填项，可以设置自己的提取逻辑， 64 | // // 设置增量变量log_id，增量变量名称#[log_id]可以多次出现在sql语句的不同位置中，例如： 65 | // // select * from td_sm_log where log_id > #[log_id] and parent_id = #[log_id] 66 | // // 需要设置setLastValueColumn信息log_id， 67 | // // 通过setLastValueType方法告诉工具增量字段的类型，默认是数字类型 68 | // DBInputConfig dbInputConfig = new DBInputConfig(); 69 | // dbInputConfig.setSql("select * from td_sm_log where log_id > #[log_id]"); 70 | // importBuilder.setInputConfig(dbInputConfig) 71 | // .addIgnoreFieldMapping("remark1"); 72 | //// importBuilder.setSql("select * from td_sm_log "); 73 | // /** 74 | // * es相关配置 75 | // */ 76 | // ElasticsearchOutputConfig elasticsearchOutputConfig = new ElasticsearchOutputConfig(); 77 | // elasticsearchOutputConfig 78 | // .setIndex("quartz") ;//必填项 79 | //// .setIndexType("quartz") //es 7以后的版本不需要设置indexType，es7以前的版本必需设置indexType 80 | //// .setRefreshOption("refresh")//可选项，null表示不实时刷新，importBuilder.setRefreshOption("refresh");表示实时刷新 81 | // elasticsearchOutputConfig.setEsIdField("log_id");//设置文档主键，不设置，则自动产生文档id 82 | // 83 | // elasticsearchOutputConfig.setDebugResponse(false);//设置是否将每次处理的reponse打印到日志文件中，默认false 84 | // elasticsearchOutputConfig.setDiscardBulkResponse(true);//设置是否需要批量处理的响应报文，不需要设置为false，true为需要，默认false 85 | // /** 86 | // importBuilder.setEsIdGenerator(new EsIdGenerator() { 87 | // //如果指定EsIdGenerator，则根据下面的方法生成文档id， 88 | // // 否则根据setEsIdField方法设置的字段值作为文档id， 89 | // // 如果默认没有配置EsIdField和如果指定EsIdGenerator，则由es自动生成文档id 90 | // 91 | // @Override 92 | // public Object genId(Context context) throws Exception { 93 | // return SimpleStringUtil.getUUID();//返回null，则由es自动生成文档id 94 | // } 95 | // }); 96 | // */ 97 | // importBuilder.setOutputConfig(elasticsearchOutputConfig) 98 | // .setUseJavaName(false) //可选项,将数据库字段名称转换为java驼峰规范的名称，true转换，false不转换，默认false，例如:doc_id -> docId 99 | // .setUseLowcase(false) //可选项，true 列名称转小写，false列名称不转换小写，默认false，只要在UseJavaName为false的情况下，配置才起作用 100 | // .setPrintTaskLog(true) //可选项，true 打印任务执行日志（耗时，处理记录数） false 不打印，默认值false 101 | // .setBatchSize(10); //可选项,批量导入es的记录数，默认为-1，逐条处理，> 0时批量处理 102 | // 103 | // //定时任务配置， 104 | // //采用内部定时任务 105 | //// importBuilder.setFixedRate(false)//参考jdk timer task文档对fixedRate的说明 106 | ////// .setScheduleDate(date) //指定任务开始执行时间：日期 107 | //// .setDeyLay(1000L) // 任务延迟执行deylay毫秒后执行 108 | //// .setPeriod(10000L); //每隔period毫秒执行，如果不设置，只执行一次 109 | // //采用外部定时任务 110 | // importBuilder.setExternalTimer(true); 111 | // //定时任务配置结束 112 | //// 113 | //// //设置任务执行拦截器，可以添加多个 114 | //// importBuilder.addCallInterceptor(new CallInterceptor() { 115 | //// @Override 116 | //// public void preCall(TaskContext taskContext) { 117 | //// System.out.println("preCall"); 118 | //// } 119 | //// 120 | //// @Override 121 | //// public void afterCall(TaskContext taskContext) { 122 | //// System.out.println("afterCall"); 123 | //// } 124 | //// 125 | //// @Override 126 | //// public void throwException(TaskContext taskContext, Throwable e) { 127 | //// System.out.println("throwException"); 128 | //// } 129 | //// }).addCallInterceptor(new CallInterceptor() { 130 | //// @Override 131 | //// public void preCall(TaskContext taskContext) { 132 | //// System.out.println("preCall 1"); 133 | //// } 134 | //// 135 | //// @Override 136 | //// public void afterCall(TaskContext taskContext) { 137 | //// System.out.println("afterCall 1"); 138 | //// } 139 | //// 140 | //// @Override 141 | //// public void throwException(TaskContext taskContext, Throwable e) { 142 | //// System.out.println("throwException 1"); 143 | //// } 144 | //// }); 145 | //// //设置任务执行拦截器结束，可以添加多个 146 | // //增量配置开始 147 | // importBuilder.setLastValueColumn("log_id");//指定数字增量查询字段 148 | // importBuilder.setFromFirst(true);//setFromfirst(false)，如果作业停了，作业重启后从上次截止位置开始采集数据， 149 | // //setFromfirst(true) 如果作业停了，作业重启后，重新开始采集数据 150 | // importBuilder.setLastValueStorePath("logtable_import");//记录上次采集的增量字段值的文件路径，作为下次增量（或者重启后）采集数据的起点，不同的任务这个路径要不一样 151 | // importBuilder.setLastValueStoreTableName("logs"+shardIndex);//记录上次采集的增量字段值的表，可以不指定，采用默认表名increament_tab，增量状态表会自动创建。如果xxl-job是shard分片模式运行， 152 | // // 需要独立的表来记录每个分片增量同步状态， 153 | // // 并且采用xxl-job等分布式任务调度引擎时，同步状态表必须存放于db.config=test指定的数据源，不能采用本地sqlite数据库 154 | // importBuilder.setLastValueType(ImportIncreamentConfig.NUMBER_TYPE);//如果没有指定增量查询字段名称，则需要指定字段类型：ImportIncreamentConfig.NUMBER_TYPE 数字类型 155 | // // 或者ImportIncreamentConfig.TIMESTAMP_TYPE 日期类型 156 | // //增量配置结束 157 | // 158 | // //映射和转换配置开始 159 | //// /** 160 | //// * db-es mapping 表字段名称到es 文档字段的映射：比如document_id -> docId 161 | //// * 可以配置mapping，也可以不配置，默认基于java 驼峰规则进行db field-es field的映射和转换 162 | //// */ 163 | //// importBuilder.addFieldMapping("document_id","docId") 164 | //// .addFieldMapping("docwtime","docwTime") 165 | //// .addIgnoreFieldMapping("channel_id");//添加忽略字段 166 | //// 167 | //// 168 | //// /** 169 | //// * 为每条记录添加额外的字段和值 170 | //// * 可以为基本数据类型，也可以是复杂的对象 171 | //// */ 172 | //// importBuilder.addFieldValue("testF1","f1value"); 173 | //// importBuilder.addFieldValue("testInt",0); 174 | //// importBuilder.addFieldValue("testDate",new Date()); 175 | //// importBuilder.addFieldValue("testFormateDate","yyyy-MM-dd HH",new Date()); 176 | //// TestObject testObject = new TestObject(); 177 | //// testObject.setId("testid"); 178 | //// testObject.setName("jackson"); 179 | //// importBuilder.addFieldValue("testObject",testObject); 180 | //// 181 | //// /** 182 | //// * 重新设置es数据结构 183 | //// */ 184 | //// importBuilder.setDataRefactor(new DataRefactor() { 185 | //// public void refactor(Context context) throws Exception { 186 | //// CustomObject customObject = new CustomObject(); 187 | //// customObject.setAuthor((String)context.getValue("author")); 188 | //// customObject.setTitle((String)context.getValue("title")); 189 | //// customObject.setSubtitle((String)context.getValue("subtitle")); 190 | //// customObject.setIds(new int[]{1,2,3}); 191 | //// context.addFieldValue("docInfo",customObject);//如果还需要构建更多的内部对象，可以继续构建 192 | //// 193 | //// //上述三个属性已经放置到docInfo中，如果无需再放置到索引文档中，可以忽略掉这些属性 194 | //// context.addIgnoreFieldMapping("author"); 195 | //// context.addIgnoreFieldMapping("title"); 196 | //// context.addIgnoreFieldMapping("subtitle"); 197 | //// } 198 | //// }); 199 | // //映射和转换配置结束 200 | // 201 | // /** 202 | // * 内置线程池配置，实现多线程并行数据导入功能，作业完成退出时自动关闭该线程池 203 | // */ 204 | // importBuilder.setParallel(true);//设置为多线程并行批量导入,false串行 205 | // importBuilder.setQueue(10);//设置批量导入线程池等待队列长度 206 | // importBuilder.setThreadCount(5);//设置批量导入线程池工作线程数量 207 | // importBuilder.setContinueOnError(true);//任务出现异常，是否继续执行作业：true（默认值）继续执行 false 中断作业执行 208 | // 209 | // /** 210 | // * 执行数据库表数据导入es操作 211 | // */ 212 | // return importBuilder; 213 | // }); 214 | // 215 | // } 216 | // public ReturnT execute(String param){ 217 | // try { 218 | // lock.lock(); 219 | // externalScheduler.execute( param); 220 | // return SUCCESS; 221 | // } 222 | // finally { 223 | // lock.unlock(); 224 | // } 225 | // } 226 | // 227 | // public void destroy(){ 228 | // if(externalScheduler != null){ 229 | // externalScheduler.destroy(); 230 | // } 231 | // } 232 | // 233 | //} -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/TestPropertiesInterceptor.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.service; 2 | /** 3 | * Copyright 2023 bboss 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import org.frameworkset.spi.assemble.PropertiesInterceptor; 19 | import org.frameworkset.spi.assemble.PropertyContext; 20 | 21 | /** 22 | *

Description:

23 | *

24 | *

Copyright (c) 2023

25 | * @Date 2023/9/11 26 | * @author biaoping.yin 27 | * @version 1.0 28 | */ 29 | public class TestPropertiesInterceptor implements PropertiesInterceptor { 30 | @Override 31 | public Object convert(PropertyContext propertyContext) { 32 | if(propertyContext.getProperty() == null){ 33 | return propertyContext.getValue(); 34 | } 35 | //对加密口令进行解密处理，根据实际加密算法采用特定的解密算法即可 36 | else if(propertyContext.getProperty().equals("elasticPassword") 37 | || propertyContext.getProperty().equals("http.authPassword")) { 38 | // BasicTextEncryptor passwordEncrypt = new BasicTextEncryptor(); 39 | // // 设置salt值，可随意定义 40 | // passwordEncrypt.setPassword("dvvm"); 41 | // return passwordEncrypt.decrypt(String.valueOf(propertyContext.getValue())); 42 | return propertyContext.getValue(); 43 | } 44 | else if(propertyContext.getProperty().equals("http.authAccount") 45 | || propertyContext.getProperty().equals("elasticUser")) { 46 | return propertyContext.getValue(); 47 | } 48 | 49 | return propertyContext.getValue(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/zjh/Db2EleasticsearchFullRunOncestore_order_detail_pos.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.service.zjh; 2 | /** 3 | * Copyright 2008 biaoping.yin 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.frameworkset.util.SimpleStringUtil; 19 | import org.frameworkset.spi.geoip.IpInfo; 20 | import org.frameworkset.tran.DataRefactor; 21 | import org.frameworkset.tran.DataStream; 22 | import org.frameworkset.tran.ExportResultHandler; 23 | import org.frameworkset.tran.config.ImportBuilder; 24 | import org.frameworkset.tran.context.Context; 25 | import org.frameworkset.tran.metrics.TaskMetrics; 26 | import org.frameworkset.tran.plugin.db.input.DBInputConfig; 27 | import org.frameworkset.tran.plugin.es.output.ElasticsearchOutputConfig; 28 | import org.frameworkset.tran.task.TaskCommand; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import java.util.Date; 33 | 34 | /** 35 | *

Description: 基于数字类型db-es增量同步案例，同步处理程序，如需调试同步功能，直接运行main方法即可 36 | *

37 | *

Copyright (c) 2018

38 | * 39 | * @author biaoping.yin 40 | * @version 1.0 41 | * @Date 2018/9/27 20:38 42 | */ 43 | public class Db2EleasticsearchFullRunOncestore_order_detail_pos { 44 | private static Logger logger = LoggerFactory.getLogger(Db2EleasticsearchFullRunOncestore_order_detail_pos.class); 45 | 46 | public static void main(String args[]) { 47 | Db2EleasticsearchFullRunOncestore_order_detail_pos db2EleasticsearchDemo = new Db2EleasticsearchFullRunOncestore_order_detail_pos(); 48 | db2EleasticsearchDemo.importDataRunOnce(false); 49 | } 50 | 51 | /** 52 | * elasticsearch地址和数据库地址都从外部配置文件application.properties中获取，加载数据源配置和es配置 53 | * 从配置文件application.properties中获取参数值方法 54 | * boolean dropIndice = PropertiesUtil.getPropertiesContainer().getBooleanSystemEnvProperty("dropIndice",true); 55 | * int threadCount = PropertiesUtil.getPropertiesContainer().getIntSystemEnvProperty("log.threadCount",2); 56 | */ 57 | public void importDataRunOnce(boolean dropIndice) { 58 | 59 | ImportBuilder importBuilder = new ImportBuilder(); 60 | //在任务数据抽取之前做一些初始化处理，例如：通过删表来做初始化操作 61 | 62 | 63 | String jdbcUrl = "jdbc:mysql://192.168.88.10:3306/middle_platform?rewriteBatchedStatements=true&useServerPrepStmts=false&useCompression=true&useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&serverTimezone=Asia/Shanghai"; 64 | 65 | DBInputConfig dbInputConfig = new DBInputConfig(); 66 | //指定导入数据的sql语句，必填项，可以设置自己的提取逻辑， 67 | // 设置增量变量log_id，增量变量名称#[log_id]可以多次出现在sql语句的不同位置中，例如： 68 | // select * from td_sm_log where log_id > #[log_id] and parent_id = #[log_id] 69 | // 需要设置setLastValueColumn信息log_id， 70 | // 通过setLastValueType方法告诉工具增量字段的类型，默认是数字类型 71 | 72 | // importBuilder.setSql("select * from td_sm_log where LOG_OPERTIME > #[LOG_OPERTIME]"); 73 | dbInputConfig.setSql("select * from store_order_detail_pos") 74 | .setDbName("middle_platform"); 75 | 76 | importBuilder.setInputConfig(dbInputConfig); 77 | importBuilder.setIncreamentEndOffset(100);//create_time__endTime,当前时间往前推100秒 78 | // importBuilder.addFieldMapping("LOG_CONTENT","message"); 79 | // importBuilder.addIgnoreFieldMapping("remark1"); 80 | // importBuilder.setSql("select * from td_sm_log "); 81 | ElasticsearchOutputConfig elasticsearchOutputConfig = new ElasticsearchOutputConfig(); 82 | elasticsearchOutputConfig 83 | .setTargetElasticsearch("hemiao_es") 84 | .setIndex("es_store_order_detail_pos") 85 | .setEsIdField("id")//设置文档主键，不设置，则自动产生文档id 86 | .setDebugResponse(false)//设置是否将每次处理的reponse打印到日志文件中，默认false 87 | .setDiscardBulkResponse(false);//设置是否需要批量处理的响应报文，不需要设置为false，true为需要，默认false 88 | /** 89 | elasticsearchOutputConfig.setEsIdGenerator(new EsIdGenerator() { 90 | //如果指定EsIdGenerator，则根据下面的方法生成文档id， 91 | // 否则根据setEsIdField方法设置的字段值作为文档id， 92 | // 如果默认没有配置EsIdField和如果指定EsIdGenerator，则由es自动生成文档id 93 | 94 | @Override public Object genId(Context context) throws Exception { 95 | return SimpleStringUtil.getUUID();//返回null，则由es自动生成文档id 96 | } 97 | }); 98 | */ 99 | // .setIndexType("dbdemo") ;//es 7以后的版本不需要设置indexType，es7以前的版本必需设置indexType; 100 | // .setRefreshOption("refresh")//可选项，null表示不实时刷新，importBuilder.setRefreshOption("refresh");表示实时刷新 101 | /** 102 | * es相关配置 103 | */ 104 | // elasticsearchOutputConfig.setTargetElasticsearch("default,test");//同步数据到两个es集群 105 | 106 | importBuilder.setOutputConfig(elasticsearchOutputConfig); 107 | 108 | /** 109 | * 设置IP地址信息库 110 | */ 111 | importBuilder.setGeoipDatabase("/opt/ip_data/GeoLite2-City.mmdb"); 112 | importBuilder.setGeoipAsnDatabase("/opt/ip_data/GeoLite2-ASN.mmdb"); 113 | importBuilder.setGeoip2regionDatabase("/opt/ip_data/ip2region.db"); 114 | 115 | importBuilder 116 | // 117 | .setUseJavaName(true) //可选项,将数据库字段名称转换为java驼峰规范的名称，true转换，false不转换，默认false，例如:doc_id -> docId 118 | .setPrintTaskLog(true) //可选项，true 打印任务执行日志（耗时，处理记录数） false 不打印，默认值false 119 | .setBatchSize(10); //可选项,批量导入es的记录数，默认为-1，逐条处理，> 0时批量处理 120 | 121 | // //定时任务配置， 122 | // importBuilder.setFixedRate(false)//参考jdk timer task文档对fixedRate的说明 123 | //// .setScheduleDate(date) //指定任务开始执行时间：日期 124 | // .setDeyLay(1000L) // 任务延迟执行deylay毫秒后执行 125 | // .setPeriod(5000L); //每隔period毫秒执行，如果不设置，只执行一次 126 | // //定时任务配置结束 127 | // 128 | // //设置任务执行拦截器，可以添加多个，定时任务每次执行的拦截器 129 | 130 | // //设置任务执行拦截器结束，可以添加多个 131 | //增量配置开始 132 | // importBuilder.setStatusDbname("test");//设置增量状态数据源名称 133 | 134 | // 或者ImportIncreamentConfig.TIMESTAMP_TYPE 日期类型 135 | //增量配置结束 136 | 137 | //映射和转换配置开始 138 | // /** 139 | // * db-es mapping 表字段名称到es 文档字段的映射：比如document_id -> docId 140 | // * 可以配置mapping，也可以不配置，默认基于java 驼峰规则进行db field-es field的映射和转换 141 | // */ 142 | // importBuilder.addFieldMapping("document_id","docId") 143 | // .addFieldMapping("docwtime","docwTime") 144 | // .addIgnoreFieldMapping("channel_id");//添加忽略字段 145 | // 146 | // 147 | // /** 148 | // * 为每条记录添加额外的字段和值 149 | // * 可以为基本数据类型，也可以是复杂的对象 150 | // */ 151 | // importBuilder.addFieldValue("testF1","f1value"); 152 | // importBuilder.addFieldValue("testInt",0); 153 | // importBuilder.addFieldValue("testDate",new Date()); 154 | // importBuilder.addFieldValue("testFormateDate","yyyy-MM-dd HH",new Date()); 155 | // TestObject testObject = new TestObject(); 156 | // testObject.setId("testid"); 157 | // testObject.setName("jackson"); 158 | // importBuilder.addFieldValue("testObject",testObject); 159 | // 160 | /** 161 | * 重新设置es数据结构 162 | */ 163 | importBuilder.setDataRefactor(new DataRefactor() { 164 | public void refactor(Context context) throws Exception { 165 | // Date date = context.getDateValue("LOG_OPERTIME"); 166 | context.addFieldValue("collecttime", new Date()); 167 | IpInfo ipInfo = context.getIpInfoByIp("219.133.80.136"); 168 | if (ipInfo != null) 169 | context.addFieldValue("ipInfo", SimpleStringUtil.object2json(ipInfo)); 170 | } 171 | }); 172 | //映射和转换配置结束 173 | 174 | /** 175 | * 内置线程池配置，实现多线程并行数据导入功能，作业完成退出时自动关闭该线程池 176 | */ 177 | importBuilder.setParallel(true);//设置为多线程并行批量导入,false串行 178 | importBuilder.setQueue(10);//设置批量导入线程池等待队列长度 179 | importBuilder.setThreadCount(50);//设置批量导入线程池工作线程数量 180 | importBuilder.setContinueOnError(true);//任务出现异常，是否继续执行作业：true（默认值）继续执行 false 中断作业执行 181 | 182 | importBuilder.setExportResultHandler(new ExportResultHandler() { 183 | @Override 184 | public void success(TaskCommand taskCommand, String result) { 185 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 186 | logger.info(taskMetrics.toString()); 187 | logger.debug(result); 188 | } 189 | 190 | @Override 191 | public void error(TaskCommand taskCommand, String result) { 192 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 193 | logger.info(taskMetrics.toString()); 194 | logger.debug(result); 195 | } 196 | 197 | @Override 198 | public void exception(TaskCommand taskCommand, Throwable exception) { 199 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 200 | logger.debug(taskMetrics.toString()); 201 | } 202 | 203 | 204 | }); 205 | 206 | 207 | /** 208 | * 执行数据库表数据导入es操作 209 | */ 210 | DataStream dataStream = importBuilder.builder(); 211 | dataStream.execute();//执行导入操作 212 | 213 | 214 | } 215 | 216 | 217 | } 218 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/zjh/Db2EleasticsearchFullRunOncestore_order_detail_pos1.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.service.zjh; 2 | /** 3 | * Copyright 2008 biaoping.yin 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.frameworkset.util.SimpleStringUtil; 19 | import org.frameworkset.spi.geoip.IpInfo; 20 | import org.frameworkset.tran.DataRefactor; 21 | import org.frameworkset.tran.DataStream; 22 | import org.frameworkset.tran.ExportResultHandler; 23 | import org.frameworkset.tran.config.ImportBuilder; 24 | import org.frameworkset.tran.context.Context; 25 | import org.frameworkset.tran.metrics.TaskMetrics; 26 | import org.frameworkset.tran.plugin.db.input.DBInputConfig; 27 | import org.frameworkset.tran.plugin.es.output.ElasticsearchOutputConfig; 28 | import org.frameworkset.tran.task.TaskCommand; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import java.util.Date; 33 | 34 | /** 35 | *

Description: 基于数字类型db-es增量同步案例，同步处理程序，如需调试同步功能，直接运行main方法即可 36 | *

37 | *

Copyright (c) 2018

38 | * 39 | * @author biaoping.yin 40 | * @version 1.0 41 | * @Date 2018/9/27 20:38 42 | */ 43 | public class Db2EleasticsearchFullRunOncestore_order_detail_pos1 { 44 | private static Logger logger = LoggerFactory.getLogger(Db2EleasticsearchFullRunOncestore_order_detail_pos1.class); 45 | 46 | public static void main(String args[]) { 47 | Db2EleasticsearchFullRunOncestore_order_detail_pos1 db2EleasticsearchDemo = new Db2EleasticsearchFullRunOncestore_order_detail_pos1(); 48 | db2EleasticsearchDemo.importDataRunOnce(false); 49 | } 50 | 51 | /** 52 | * elasticsearch地址和数据库地址都从外部配置文件application.properties中获取，加载数据源配置和es配置 53 | * 从配置文件application.properties中获取参数值方法 54 | * boolean dropIndice = PropertiesUtil.getPropertiesContainer().getBooleanSystemEnvProperty("dropIndice",true); 55 | * int threadCount = PropertiesUtil.getPropertiesContainer().getIntSystemEnvProperty("log.threadCount",2); 56 | */ 57 | public void importDataRunOnce(boolean dropIndice) { 58 | 59 | ImportBuilder importBuilder = new ImportBuilder(); 60 | //在任务数据抽取之前做一些初始化处理，例如：通过删表来做初始化操作 61 | 62 | 63 | String jdbcUrl = "jdbc:mysql://192.168.88.10:3306/middle_platform?rewriteBatchedStatements=true&useServerPrepStmts=false&useCompression=true&useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&serverTimezone=Asia/Shanghai"; 64 | 65 | DBInputConfig dbInputConfig = new DBInputConfig(); 66 | //指定导入数据的sql语句，必填项，可以设置自己的提取逻辑， 67 | // 设置增量变量log_id，增量变量名称#[log_id]可以多次出现在sql语句的不同位置中，例如： 68 | // select * from td_sm_log where log_id > #[log_id] and parent_id = #[log_id] 69 | // 需要设置setLastValueColumn信息log_id， 70 | // 通过setLastValueType方法告诉工具增量字段的类型，默认是数字类型 71 | 72 | // importBuilder.setSql("select * from td_sm_log where LOG_OPERTIME > #[LOG_OPERTIME]"); 73 | dbInputConfig.setSql("select * from store_order_detail_pos") 74 | .setDbName("middle_platform") 75 | .setDbDriver("com.mysql.cj.jdbc.Driver") //数据库驱动程序，必须导入相关数据库的驱动jar包 76 | 77 | 78 | .setDbUrl(jdbcUrl) //通过useCursorFetch=true启用mysql的游标fetch机制，否则会有严重的性能隐患，useCursorFetch必须和jdbcFetchSize参数配合使用，否则不会生效 79 | .setJdbcFetchSize(-2147483648) 80 | .setDbUser("sync") 81 | .setDbPassword("zFfBu21vvfuUzkEE") 82 | .setValidateSQL("select 1") 83 | .setUsePool(false) 84 | // .setDbInitSize(5) 85 | // .setDbMinIdleSize(5) 86 | // .setDbMaxSize(10) 87 | .setShowSql(true);//是否使用连接池; 88 | importBuilder.setInputConfig(dbInputConfig); 89 | importBuilder.setIncreamentEndOffset(100);//create_time__endTime,当前时间往前推100秒 90 | // importBuilder.addFieldMapping("LOG_CONTENT","message"); 91 | // importBuilder.addIgnoreFieldMapping("remark1"); 92 | // importBuilder.setSql("select * from td_sm_log "); 93 | ElasticsearchOutputConfig elasticsearchOutputConfig = new ElasticsearchOutputConfig(); 94 | elasticsearchOutputConfig 95 | .addTargetElasticsearch("elasticsearch.serverNames", "hemiao_es") 96 | .addElasticsearchProperty("hemiao_es.elasticsearch.rest.hostNames", "es-cn-x0r3bkhai000dw7t4.elasticsearch.aliyuncs.com:9200") 97 | .addElasticsearchProperty("hemiao_es.elasticsearch.showTemplate", "true") 98 | .addElasticsearchProperty("hemiao_es.elasticUser", "elastic") 99 | .addElasticsearchProperty("hemiao_es.elasticPassword", "axsMFaGASJwDTOh3") 100 | .addElasticsearchProperty("hemiao_es.elasticsearch.failAllContinue", "true") 101 | .addElasticsearchProperty("hemiao_es.http.timeoutSocket", "60000") 102 | .addElasticsearchProperty("hemiao_es.http.timeoutConnection", "40000") 103 | .addElasticsearchProperty("hemiao_es.http.connectionRequestTimeout", "70000") 104 | .addElasticsearchProperty("hemiao_es.http.maxTotal", "200") 105 | .addElasticsearchProperty("hemiao_es.http.defaultMaxPerRoute", "100") 106 | .setIndex("es_store_order_detail_pos") 107 | .setEsIdField("id")//设置文档主键，不设置，则自动产生文档id 108 | .setDebugResponse(false)//设置是否将每次处理的reponse打印到日志文件中，默认false 109 | .setDiscardBulkResponse(false);//设置是否需要批量处理的响应报文，不需要设置为false，true为需要，默认false 110 | /** 111 | elasticsearchOutputConfig.setEsIdGenerator(new EsIdGenerator() { 112 | //如果指定EsIdGenerator，则根据下面的方法生成文档id， 113 | // 否则根据setEsIdField方法设置的字段值作为文档id， 114 | // 如果默认没有配置EsIdField和如果指定EsIdGenerator，则由es自动生成文档id 115 | 116 | @Override public Object genId(Context context) throws Exception { 117 | return SimpleStringUtil.getUUID();//返回null，则由es自动生成文档id 118 | } 119 | }); 120 | */ 121 | // .setIndexType("dbdemo") ;//es 7以后的版本不需要设置indexType，es7以前的版本必需设置indexType; 122 | // .setRefreshOption("refresh")//可选项，null表示不实时刷新，importBuilder.setRefreshOption("refresh");表示实时刷新 123 | /** 124 | * es相关配置 125 | */ 126 | // elasticsearchOutputConfig.setTargetElasticsearch("default,test");//同步数据到两个es集群 127 | 128 | importBuilder.setOutputConfig(elasticsearchOutputConfig); 129 | 130 | /** 131 | * 设置IP地址信息库 132 | */ 133 | importBuilder.setGeoipDatabase("/opt/ip_data/GeoLite2-City.mmdb"); 134 | importBuilder.setGeoipAsnDatabase("/opt/ip_data/GeoLite2-ASN.mmdb"); 135 | importBuilder.setGeoip2regionDatabase("/opt/ip_data/ip2region.db"); 136 | 137 | importBuilder 138 | // 139 | .setUseJavaName(true) //可选项,将数据库字段名称转换为java驼峰规范的名称，true转换，false不转换，默认false，例如:doc_id -> docId 140 | .setPrintTaskLog(true) //可选项，true 打印任务执行日志（耗时，处理记录数） false 不打印，默认值false 141 | .setBatchSize(10); //可选项,批量导入es的记录数，默认为-1，逐条处理，> 0时批量处理 142 | 143 | // //定时任务配置， 144 | // importBuilder.setFixedRate(false)//参考jdk timer task文档对fixedRate的说明 145 | //// .setScheduleDate(date) //指定任务开始执行时间：日期 146 | // .setDeyLay(1000L) // 任务延迟执行deylay毫秒后执行 147 | // .setPeriod(5000L); //每隔period毫秒执行，如果不设置，只执行一次 148 | // //定时任务配置结束 149 | // 150 | // //设置任务执行拦截器，可以添加多个，定时任务每次执行的拦截器 151 | 152 | // //设置任务执行拦截器结束，可以添加多个 153 | //增量配置开始 154 | // importBuilder.setStatusDbname("test");//设置增量状态数据源名称 155 | 156 | // 或者ImportIncreamentConfig.TIMESTAMP_TYPE 日期类型 157 | //增量配置结束 158 | 159 | //映射和转换配置开始 160 | // /** 161 | // * db-es mapping 表字段名称到es 文档字段的映射：比如document_id -> docId 162 | // * 可以配置mapping，也可以不配置，默认基于java 驼峰规则进行db field-es field的映射和转换 163 | // */ 164 | // importBuilder.addFieldMapping("document_id","docId") 165 | // .addFieldMapping("docwtime","docwTime") 166 | // .addIgnoreFieldMapping("channel_id");//添加忽略字段 167 | // 168 | // 169 | // /** 170 | // * 为每条记录添加额外的字段和值 171 | // * 可以为基本数据类型，也可以是复杂的对象 172 | // */ 173 | // importBuilder.addFieldValue("testF1","f1value"); 174 | // importBuilder.addFieldValue("testInt",0); 175 | // importBuilder.addFieldValue("testDate",new Date()); 176 | // importBuilder.addFieldValue("testFormateDate","yyyy-MM-dd HH",new Date()); 177 | // TestObject testObject = new TestObject(); 178 | // testObject.setId("testid"); 179 | // testObject.setName("jackson"); 180 | // importBuilder.addFieldValue("testObject",testObject); 181 | // 182 | /** 183 | * 重新设置es数据结构 184 | */ 185 | importBuilder.setDataRefactor(new DataRefactor() { 186 | public void refactor(Context context) throws Exception { 187 | // Date date = context.getDateValue("LOG_OPERTIME"); 188 | context.addFieldValue("collecttime", new Date()); 189 | IpInfo ipInfo = context.getIpInfoByIp("219.133.80.136"); 190 | if (ipInfo != null) 191 | context.addFieldValue("ipInfo", SimpleStringUtil.object2json(ipInfo)); 192 | } 193 | }); 194 | //映射和转换配置结束 195 | 196 | /** 197 | * 内置线程池配置，实现多线程并行数据导入功能，作业完成退出时自动关闭该线程池 198 | */ 199 | importBuilder.setParallel(true);//设置为多线程并行批量导入,false串行 200 | importBuilder.setQueue(10);//设置批量导入线程池等待队列长度 201 | importBuilder.setThreadCount(50);//设置批量导入线程池工作线程数量 202 | importBuilder.setContinueOnError(true);//任务出现异常，是否继续执行作业：true（默认值）继续执行 false 中断作业执行 203 | 204 | importBuilder.setExportResultHandler(new ExportResultHandler() { 205 | @Override 206 | public void success(TaskCommand taskCommand, String result) { 207 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 208 | logger.info(taskMetrics.toString()); 209 | logger.debug(result); 210 | } 211 | 212 | @Override 213 | public void error(TaskCommand taskCommand, String result) { 214 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 215 | logger.info(taskMetrics.toString()); 216 | logger.debug(result); 217 | } 218 | 219 | @Override 220 | public void exception(TaskCommand taskCommand, Throwable exception) { 221 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 222 | logger.debug(taskMetrics.toString()); 223 | } 224 | 225 | 226 | }); 227 | 228 | 229 | /** 230 | * 执行数据库表数据导入es操作 231 | */ 232 | DataStream dataStream = importBuilder.builder(); 233 | dataStream.execute();//执行导入操作 234 | 235 | 236 | } 237 | 238 | 239 | } 240 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/zjh/Db2EleasticsearchFullRunOncestore_order_pos.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.service.zjh; 2 | /** 3 | * Copyright 2008 biaoping.yin 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.frameworkset.util.SimpleStringUtil; 19 | import org.frameworkset.spi.geoip.IpInfo; 20 | import org.frameworkset.tran.DataRefactor; 21 | import org.frameworkset.tran.DataStream; 22 | import org.frameworkset.tran.ExportResultHandler; 23 | import org.frameworkset.tran.config.ImportBuilder; 24 | import org.frameworkset.tran.context.Context; 25 | import org.frameworkset.tran.metrics.TaskMetrics; 26 | import org.frameworkset.tran.plugin.db.input.DBInputConfig; 27 | import org.frameworkset.tran.plugin.es.output.ElasticsearchOutputConfig; 28 | import org.frameworkset.tran.task.TaskCommand; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import java.util.Date; 33 | 34 | /** 35 | *

Description: 基于数字类型db-es增量同步案例，同步处理程序，如需调试同步功能，直接运行main方法即可 36 | *

37 | *

Copyright (c) 2018

38 | * 39 | * @author biaoping.yin 40 | * @version 1.0 41 | * @Date 2018/9/27 20:38 42 | */ 43 | public class Db2EleasticsearchFullRunOncestore_order_pos { 44 | private static Logger logger = LoggerFactory.getLogger(Db2EleasticsearchFullRunOncestore_order_pos.class); 45 | 46 | public static void main(String args[]) { 47 | Db2EleasticsearchFullRunOncestore_order_pos db2EleasticsearchDemo = new Db2EleasticsearchFullRunOncestore_order_pos(); 48 | db2EleasticsearchDemo.importDataRunOnce(false); 49 | } 50 | 51 | /** 52 | * elasticsearch地址和数据库地址都从外部配置文件application.properties中获取，加载数据源配置和es配置 53 | * 从配置文件application.properties中获取参数值方法 54 | * boolean dropIndice = PropertiesUtil.getPropertiesContainer().getBooleanSystemEnvProperty("dropIndice",true); 55 | * int threadCount = PropertiesUtil.getPropertiesContainer().getIntSystemEnvProperty("log.threadCount",2); 56 | */ 57 | public void importDataRunOnce(boolean dropIndice) { 58 | 59 | ImportBuilder importBuilder = new ImportBuilder(); 60 | //在任务数据抽取之前做一些初始化处理，例如：通过删表来做初始化操作 61 | 62 | String jdbcUrl = "jdbc:mysql://192.168.88.10:3306/middle_platform?rewriteBatchedStatements=true&useServerPrepStmts=false&useCompression=true&useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&serverTimezone=Asia/Shanghai"; 63 | 64 | DBInputConfig dbInputConfig = new DBInputConfig(); 65 | //指定导入数据的sql语句，必填项，可以设置自己的提取逻辑， 66 | // 设置增量变量log_id，增量变量名称#[log_id]可以多次出现在sql语句的不同位置中，例如： 67 | // select * from td_sm_log where log_id > #[log_id] and parent_id = #[log_id] 68 | // 需要设置setLastValueColumn信息log_id， 69 | // 通过setLastValueType方法告诉工具增量字段的类型，默认是数字类型 70 | 71 | // importBuilder.setSql("select * from td_sm_log where LOG_OPERTIME > #[LOG_OPERTIME]"); 72 | dbInputConfig.setSql("select * from store_order_pos where create_time") 73 | .setDbName("middle_platform"); 74 | importBuilder.setInputConfig(dbInputConfig); 75 | 76 | // importBuilder.addFieldMapping("LOG_CONTENT","message"); 77 | // importBuilder.addIgnoreFieldMapping("remark1"); 78 | // importBuilder.setSql("select * from td_sm_log "); 79 | ElasticsearchOutputConfig elasticsearchOutputConfig = new ElasticsearchOutputConfig(); 80 | elasticsearchOutputConfig 81 | .setTargetElasticsearch("hemiao_es") 82 | .setIndex("es_store_order_pos") 83 | .setEsIdField("order_offline_id")//设置文档主键，不设置，则自动产生文档id 84 | .setDebugResponse(false)//设置是否将每次处理的reponse打印到日志文件中，默认false 85 | .setDiscardBulkResponse(false);//设置是否需要批量处理的响应报文，不需要设置为false，true为需要，默认false 86 | /** 87 | elasticsearchOutputConfig.setEsIdGenerator(new EsIdGenerator() { 88 | //如果指定EsIdGenerator，则根据下面的方法生成文档id， 89 | // 否则根据setEsIdField方法设置的字段值作为文档id， 90 | // 如果默认没有配置EsIdField和如果指定EsIdGenerator，则由es自动生成文档id 91 | 92 | @Override public Object genId(Context context) throws Exception { 93 | return SimpleStringUtil.getUUID();//返回null，则由es自动生成文档id 94 | } 95 | }); 96 | */ 97 | // .setIndexType("dbdemo") ;//es 7以后的版本不需要设置indexType，es7以前的版本必需设置indexType; 98 | // .setRefreshOption("refresh")//可选项，null表示不实时刷新，importBuilder.setRefreshOption("refresh");表示实时刷新 99 | /** 100 | * es相关配置 101 | */ 102 | // elasticsearchOutputConfig.setTargetElasticsearch("default,test");//同步数据到两个es集群 103 | 104 | importBuilder.setOutputConfig(elasticsearchOutputConfig); 105 | 106 | /** 107 | * 设置IP地址信息库 108 | */ 109 | importBuilder.setGeoipDatabase("/opt/ip_data/GeoLite2-City.mmdb"); 110 | importBuilder.setGeoipAsnDatabase("/opt/ip_data/GeoLite2-ASN.mmdb"); 111 | importBuilder.setGeoip2regionDatabase("/opt/ip_data/ip2region.db"); 112 | 113 | importBuilder 114 | // 115 | .setUseJavaName(true) //可选项,将数据库字段名称转换为java驼峰规范的名称，true转换，false不转换，默认false，例如:doc_id -> docId 116 | .setPrintTaskLog(true) //可选项，true 打印任务执行日志（耗时，处理记录数） false 不打印，默认值false 117 | .setBatchSize(10); //可选项,批量导入es的记录数，默认为-1，逐条处理，> 0时批量处理 118 | 119 | 120 | 121 | //映射和转换配置开始 122 | // /** 123 | // * db-es mapping 表字段名称到es 文档字段的映射：比如document_id -> docId 124 | // * 可以配置mapping，也可以不配置，默认基于java 驼峰规则进行db field-es field的映射和转换 125 | // */ 126 | // importBuilder.addFieldMapping("document_id","docId") 127 | // .addFieldMapping("docwtime","docwTime") 128 | // .addIgnoreFieldMapping("channel_id");//添加忽略字段 129 | // 130 | // 131 | // /** 132 | // * 为每条记录添加额外的字段和值 133 | // * 可以为基本数据类型，也可以是复杂的对象 134 | // */ 135 | // importBuilder.addFieldValue("testF1","f1value"); 136 | // importBuilder.addFieldValue("testInt",0); 137 | // importBuilder.addFieldValue("testDate",new Date()); 138 | // importBuilder.addFieldValue("testFormateDate","yyyy-MM-dd HH",new Date()); 139 | // TestObject testObject = new TestObject(); 140 | // testObject.setId("testid"); 141 | // testObject.setName("jackson"); 142 | // importBuilder.addFieldValue("testObject",testObject); 143 | // 144 | /** 145 | * 重新设置es数据结构 146 | */ 147 | importBuilder.setDataRefactor(new DataRefactor() { 148 | public void refactor(Context context) throws Exception { 149 | // Date date = context.getDateValue("LOG_OPERTIME"); 150 | context.addFieldValue("collecttime", new Date()); 151 | IpInfo ipInfo = context.getIpInfoByIp("219.133.80.136"); 152 | if (ipInfo != null) 153 | context.addFieldValue("ipInfo", SimpleStringUtil.object2json(ipInfo)); 154 | } 155 | }); 156 | //映射和转换配置结束 157 | 158 | /** 159 | * 内置线程池配置，实现多线程并行数据导入功能，作业完成退出时自动关闭该线程池 160 | */ 161 | importBuilder.setParallel(true);//设置为多线程并行批量导入,false串行 162 | importBuilder.setQueue(10);//设置批量导入线程池等待队列长度 163 | importBuilder.setThreadCount(50);//设置批量导入线程池工作线程数量 164 | importBuilder.setContinueOnError(true);//任务出现异常，是否继续执行作业：true（默认值）继续执行 false 中断作业执行 165 | 166 | importBuilder.setExportResultHandler(new ExportResultHandler() { 167 | @Override 168 | public void success(TaskCommand taskCommand, String result) { 169 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 170 | logger.info(taskMetrics.toString()); 171 | logger.debug(result); 172 | } 173 | 174 | @Override 175 | public void error(TaskCommand taskCommand, String result) { 176 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 177 | logger.info(taskMetrics.toString()); 178 | logger.debug(result); 179 | } 180 | 181 | @Override 182 | public void exception(TaskCommand taskCommand, Throwable exception) { 183 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 184 | logger.debug(taskMetrics.toString()); 185 | } 186 | 187 | 188 | }); 189 | 190 | 191 | /** 192 | * 执行数据库表数据导入es操作 193 | */ 194 | DataStream dataStream = importBuilder.builder(); 195 | dataStream.execute();//执行导入操作 196 | // dataStream.destroy();//释放资源 197 | 198 | 199 | } 200 | 201 | 202 | } 203 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/zjh/Db2EleasticsearchFullRunOncestore_order_pos1.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.service.zjh; 2 | /** 3 | * Copyright 2008 biaoping.yin 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.frameworkset.util.SimpleStringUtil; 19 | import org.frameworkset.spi.geoip.IpInfo; 20 | import org.frameworkset.tran.DataRefactor; 21 | import org.frameworkset.tran.DataStream; 22 | import org.frameworkset.tran.ExportResultHandler; 23 | import org.frameworkset.tran.config.ImportBuilder; 24 | import org.frameworkset.tran.context.Context; 25 | import org.frameworkset.tran.metrics.TaskMetrics; 26 | import org.frameworkset.tran.plugin.db.input.DBInputConfig; 27 | import org.frameworkset.tran.plugin.es.output.ElasticsearchOutputConfig; 28 | import org.frameworkset.tran.task.TaskCommand; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import java.util.Date; 33 | 34 | /** 35 | *

Description: 基于数字类型db-es增量同步案例，同步处理程序，如需调试同步功能，直接运行main方法即可 36 | *

37 | *

Copyright (c) 2018

38 | * 39 | * @author biaoping.yin 40 | * @version 1.0 41 | * @Date 2018/9/27 20:38 42 | */ 43 | public class Db2EleasticsearchFullRunOncestore_order_pos1 { 44 | private static Logger logger = LoggerFactory.getLogger(Db2EleasticsearchFullRunOncestore_order_pos1.class); 45 | 46 | public static void main(String args[]) { 47 | Db2EleasticsearchFullRunOncestore_order_pos1 db2EleasticsearchDemo = new Db2EleasticsearchFullRunOncestore_order_pos1(); 48 | db2EleasticsearchDemo.importDataRunOnce(false); 49 | } 50 | 51 | /** 52 | * elasticsearch地址和数据库地址都从外部配置文件application.properties中获取，加载数据源配置和es配置 53 | * 从配置文件application.properties中获取参数值方法 54 | * boolean dropIndice = PropertiesUtil.getPropertiesContainer().getBooleanSystemEnvProperty("dropIndice",true); 55 | * int threadCount = PropertiesUtil.getPropertiesContainer().getIntSystemEnvProperty("log.threadCount",2); 56 | */ 57 | public void importDataRunOnce(boolean dropIndice) { 58 | 59 | ImportBuilder importBuilder = new ImportBuilder(); 60 | //在任务数据抽取之前做一些初始化处理，例如：通过删表来做初始化操作 61 | 62 | String jdbcUrl = "jdbc:mysql://192.168.88.10:3306/middle_platform?rewriteBatchedStatements=true&useServerPrepStmts=false&useCompression=true&useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&serverTimezone=Asia/Shanghai"; 63 | 64 | DBInputConfig dbInputConfig = new DBInputConfig(); 65 | //指定导入数据的sql语句，必填项，可以设置自己的提取逻辑， 66 | // 设置增量变量log_id，增量变量名称#[log_id]可以多次出现在sql语句的不同位置中，例如： 67 | // select * from td_sm_log where log_id > #[log_id] and parent_id = #[log_id] 68 | // 需要设置setLastValueColumn信息log_id， 69 | // 通过setLastValueType方法告诉工具增量字段的类型，默认是数字类型 70 | 71 | // importBuilder.setSql("select * from td_sm_log where LOG_OPERTIME > #[LOG_OPERTIME]"); 72 | dbInputConfig.setSql("select * from store_order_pos where create_time") 73 | .setDbName("middle_platformsecond") 74 | .setDbDriver("com.mysql.cj.jdbc.Driver") //数据库驱动程序，必须导入相关数据库的驱动jar包 75 | 76 | 77 | .setDbUrl(jdbcUrl) //通过useCursorFetch=true启用mysql的游标fetch机制，否则会有严重的性能隐患，useCursorFetch必须和jdbcFetchSize参数配合使用，否则不会生效 78 | .setJdbcFetchSize(-2147483648) 79 | .setDbUser("sync") 80 | .setDbPassword("zFfBu21vvfuUzkEE") 81 | .setValidateSQL("select 1") 82 | .setUsePool(false) 83 | .setDbInitSize(5) 84 | .setDbMinIdleSize(5) 85 | .setDbMaxSize(10) 86 | .setShowSql(true);//是否使用连接池; 87 | importBuilder.setInputConfig(dbInputConfig); 88 | 89 | // importBuilder.addFieldMapping("LOG_CONTENT","message"); 90 | // importBuilder.addIgnoreFieldMapping("remark1"); 91 | // importBuilder.setSql("select * from td_sm_log "); 92 | ElasticsearchOutputConfig elasticsearchOutputConfig = new ElasticsearchOutputConfig(); 93 | elasticsearchOutputConfig 94 | .addTargetElasticsearch("elasticsearch.serverNames", "hemiao_essecond") 95 | .addElasticsearchProperty("hemiao_essecond.elasticsearch.rest.hostNames", "es-cn-x0r3bkhai000dw7t4.elasticsearch.aliyuncs.com:9200") 96 | .addElasticsearchProperty("hemiao_essecond.elasticsearch.showTemplate", "true") 97 | .addElasticsearchProperty("hemiao_essecond.elasticUser", "elastic") 98 | .addElasticsearchProperty("hemiao_essecond.elasticPassword", "axsMFaGASJwDTOh3") 99 | .addElasticsearchProperty("hemiao_essecond.elasticsearch.failAllContinue", "true") 100 | .addElasticsearchProperty("hemiao_essecond.http.timeoutSocket", "60000") 101 | .addElasticsearchProperty("hemiao_essecond.http.timeoutConnection", "40000") 102 | .addElasticsearchProperty("hemiao_essecond.http.connectionRequestTimeout", "70000") 103 | .addElasticsearchProperty("hemiao_essecond.http.maxTotal", "200") 104 | .addElasticsearchProperty("hemiao_essecond.http.defaultMaxPerRoute", "100") 105 | .setIndex("es_store_order_pos") 106 | .setEsIdField("order_offline_id")//设置文档主键，不设置，则自动产生文档id 107 | .setDebugResponse(false)//设置是否将每次处理的reponse打印到日志文件中，默认false 108 | .setDiscardBulkResponse(false);//设置是否需要批量处理的响应报文，不需要设置为false，true为需要，默认false 109 | /** 110 | elasticsearchOutputConfig.setEsIdGenerator(new EsIdGenerator() { 111 | //如果指定EsIdGenerator，则根据下面的方法生成文档id， 112 | // 否则根据setEsIdField方法设置的字段值作为文档id， 113 | // 如果默认没有配置EsIdField和如果指定EsIdGenerator，则由es自动生成文档id 114 | 115 | @Override public Object genId(Context context) throws Exception { 116 | return SimpleStringUtil.getUUID();//返回null，则由es自动生成文档id 117 | } 118 | }); 119 | */ 120 | // .setIndexType("dbdemo") ;//es 7以后的版本不需要设置indexType，es7以前的版本必需设置indexType; 121 | // .setRefreshOption("refresh")//可选项，null表示不实时刷新，importBuilder.setRefreshOption("refresh");表示实时刷新 122 | /** 123 | * es相关配置 124 | */ 125 | // elasticsearchOutputConfig.setTargetElasticsearch("default,test");//同步数据到两个es集群 126 | 127 | importBuilder.setOutputConfig(elasticsearchOutputConfig); 128 | 129 | /** 130 | * 设置IP地址信息库 131 | */ 132 | importBuilder.setGeoipDatabase("/opt/ip_data/GeoLite2-City.mmdb"); 133 | importBuilder.setGeoipAsnDatabase("/opt/ip_data/GeoLite2-ASN.mmdb"); 134 | importBuilder.setGeoip2regionDatabase("/opt/ip_data/ip2region.db"); 135 | 136 | importBuilder 137 | // 138 | .setUseJavaName(true) //可选项,将数据库字段名称转换为java驼峰规范的名称，true转换，false不转换，默认false，例如:doc_id -> docId 139 | .setPrintTaskLog(true) //可选项，true 打印任务执行日志（耗时，处理记录数） false 不打印，默认值false 140 | .setBatchSize(10); //可选项,批量导入es的记录数，默认为-1，逐条处理，> 0时批量处理 141 | 142 | 143 | 144 | //映射和转换配置开始 145 | // /** 146 | // * db-es mapping 表字段名称到es 文档字段的映射：比如document_id -> docId 147 | // * 可以配置mapping，也可以不配置，默认基于java 驼峰规则进行db field-es field的映射和转换 148 | // */ 149 | // importBuilder.addFieldMapping("document_id","docId") 150 | // .addFieldMapping("docwtime","docwTime") 151 | // .addIgnoreFieldMapping("channel_id");//添加忽略字段 152 | // 153 | // 154 | // /** 155 | // * 为每条记录添加额外的字段和值 156 | // * 可以为基本数据类型，也可以是复杂的对象 157 | // */ 158 | // importBuilder.addFieldValue("testF1","f1value"); 159 | // importBuilder.addFieldValue("testInt",0); 160 | // importBuilder.addFieldValue("testDate",new Date()); 161 | // importBuilder.addFieldValue("testFormateDate","yyyy-MM-dd HH",new Date()); 162 | // TestObject testObject = new TestObject(); 163 | // testObject.setId("testid"); 164 | // testObject.setName("jackson"); 165 | // importBuilder.addFieldValue("testObject",testObject); 166 | // 167 | /** 168 | * 重新设置es数据结构 169 | */ 170 | importBuilder.setDataRefactor(new DataRefactor() { 171 | public void refactor(Context context) throws Exception { 172 | // Date date = context.getDateValue("LOG_OPERTIME"); 173 | context.addFieldValue("collecttime", new Date()); 174 | IpInfo ipInfo = context.getIpInfoByIp("219.133.80.136"); 175 | if (ipInfo != null) 176 | context.addFieldValue("ipInfo", SimpleStringUtil.object2json(ipInfo)); 177 | } 178 | }); 179 | //映射和转换配置结束 180 | 181 | /** 182 | * 内置线程池配置，实现多线程并行数据导入功能，作业完成退出时自动关闭该线程池 183 | */ 184 | importBuilder.setParallel(true);//设置为多线程并行批量导入,false串行 185 | importBuilder.setQueue(10);//设置批量导入线程池等待队列长度 186 | importBuilder.setThreadCount(50);//设置批量导入线程池工作线程数量 187 | importBuilder.setContinueOnError(true);//任务出现异常，是否继续执行作业：true（默认值）继续执行 false 中断作业执行 188 | 189 | importBuilder.setExportResultHandler(new ExportResultHandler() { 190 | @Override 191 | public void success(TaskCommand taskCommand, String result) { 192 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 193 | logger.info(taskMetrics.toString()); 194 | logger.debug(result); 195 | } 196 | 197 | @Override 198 | public void error(TaskCommand taskCommand, String result) { 199 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 200 | logger.info(taskMetrics.toString()); 201 | logger.debug(result); 202 | } 203 | 204 | @Override 205 | public void exception(TaskCommand taskCommand, Throwable exception) { 206 | TaskMetrics taskMetrics = taskCommand.getTaskMetrics(); 207 | logger.debug(taskMetrics.toString()); 208 | } 209 | 210 | 211 | }); 212 | 213 | 214 | /** 215 | * 执行数据库表数据导入es操作 216 | */ 217 | DataStream dataStream = importBuilder.builder(); 218 | dataStream.execute();//执行导入操作 219 | // dataStream.destroy();//释放资源 220 | 221 | 222 | } 223 | 224 | 225 | } 226 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/zjh/In.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.service.zjh; 2 | /** 3 | * Copyright 2023 bboss 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /** 19 | *

Description:

20 | *

21 | *

Copyright (c) 2023

22 | * @Date 2023/7/28 23 | * @author biaoping.yin 24 | * @version 1.0 25 | */ 26 | public class In { 27 | // @Override 28 | // public void refreshClusterInfo(MdElasticsearchClusterInfoModel model) { 29 | //// List clusterList = mdElasticsearchClusterInfoService.list(new LambdaQueryWrapper().eq(MdElasticsearchClusterInfo::getDelFlag, NORMAL)); 30 | // if (CollUtil.isNotEmpty(clusterList)) { 31 | // ClientInterface clientInterfaceDefault = bbossESStarterDefault.getRestClient(); 32 | // for (int i = 0; i < clusterList.size(); i++) { 33 | // List indexes = clientInterfaceDefault.getIndexesWithCluster("esdatasourcename");//mdElasticsearchClusterInfo.getDatasourceName(); 34 | // if(CollUtil.isNotEmpty(indexes)){ 35 | // for (int j = 0; j < indexes.size(); j++) { 36 | // List indexMappingFields = clientInterfaceDefault.getIndexMappingFieldsWithCluster("esdatasourcename",indexes.get(j).getIndex()); 37 | //// List indexMappingFields = clientInterfaceDefault.getIndexMappingFields(indexes.get(j).getIndex()); 38 | // log.info(""); 39 | // } 40 | // } 41 | // 42 | // } 43 | // } 44 | // log.info(""); 45 | // } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/example/esbboss/service/zjh/JobStart.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.service.zjh; 2 | /** 3 | * Copyright 2023 bboss 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.frameworkset.common.poolman.util.DBConf; 19 | import com.frameworkset.common.poolman.util.SQLManager; 20 | import org.frameworkset.elasticsearch.boot.ElasticSearchBoot; 21 | 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | /** 26 | *

Description:

27 | *

28 | *

Copyright (c) 2023

29 | * @Date 2023/7/28 30 | * @author biaoping.yin 31 | * @version 1.0 32 | */ 33 | public class JobStart { 34 | public static void main(String[] args){ 35 | //初始化Elasticsearch数据源： 36 | Map properties = new HashMap(); 37 | /** 38 | * 这里只设置必须的配置项，其他的属性参考配置文件：resources/application.properties 39 | * 40 | */ 41 | ////认证账号和口令配置，如果启用了安全认证才需要，支持xpack和searchguard 42 | // properties.put("elasticUser","elastic"); 43 | // properties.put("elasticPassword","changeme"); 44 | ////es服务器地址和端口，多个用逗号分隔 45 | // properties.put("elasticsearch.rest.hostNames","127.0.0.1:9200"); 46 | ////是否在控制台打印dsl语句，log4j组件日志级别为INFO或者DEBUG 47 | // properties.put("elasticsearch.showTemplate","true"); 48 | ////集群节点自动发现 49 | // properties.put("elasticsearch.discoverHost","true"); 50 | // properties.put("http.timeoutSocket","60000"); 51 | // properties.put("http.timeoutConnection","40000"); 52 | // properties.put("http.connectionRequestTimeout","70000"); 53 | 54 | properties.put("elasticsearch.serverNames", "hemiao_es"); 55 | properties.put("hemiao_es.elasticsearch.rest.hostNames", "es-cn-x0r3bkhai000dw7t4.elasticsearch.aliyuncs.com:9200"); 56 | properties.put("hemiao_es.elasticsearch.showTemplate", "true"); 57 | properties.put("hemiao_es.elasticUser", "elastic"); 58 | properties.put("hemiao_es.elasticPassword", "axsMFaGASJwDTOh3"); 59 | properties.put("hemiao_es.elasticsearch.failAllContinue", "true"); 60 | properties.put("hemiao_es.http.timeoutSocket", "60000"); 61 | properties.put("hemiao_es.http.timeoutConnection", "40000"); 62 | properties.put("hemiao_es.http.connectionRequestTimeout", "70000"); 63 | properties.put("hemiao_es.http.maxTotal", "200"); 64 | properties.put("hemiao_es.http.defaultMaxPerRoute", "100"); 65 | ElasticSearchBoot.boot(properties); 66 | 67 | 68 | /** 69 | //从配置文件加载配置：初始化数据库数据源 70 | PropertiesContainer propertiesContainer = PropertiesUtil.getPropertiesContainer("application.properties"); 71 | String dbName = propertiesContainer.getProperty("db.name"); 72 | String dbUser = propertiesContainer.getProperty("db.user"); 73 | String dbPassword = propertiesContainer.getProperty("db.password"); 74 | String dbDriver = propertiesContainer.getProperty("db.driver"); 75 | String dbUrl = propertiesContainer.getProperty("db.url"); 76 | 77 | String showsql = propertiesContainer.getProperty("db.showsql"); 78 | String validateSQL = propertiesContainer.getProperty("db.validateSQL"); 79 | String dbInfoEncryptClass = propertiesContainer.getProperty("db.dbInfoEncryptClass"); 80 | 81 | DBConf tempConf = new DBConf(); 82 | tempConf.setPoolname(dbName); 83 | tempConf.setDriver(dbDriver); 84 | tempConf.setJdbcurl(dbUrl); 85 | tempConf.setUsername(dbUser); 86 | tempConf.setPassword(dbPassword); 87 | tempConf.setValidationQuery(validateSQL); 88 | tempConf.setShowsql(showsql != null && showsql.equals("true")); 89 | //tempConf.setTxIsolationLevel("READ_COMMITTED"); 90 | tempConf.setJndiName("jndi-"+dbName); 91 | tempConf.setDbInfoEncryptClass(dbInfoEncryptClass); 92 | String initialConnections = propertiesContainer.getProperty("db.initialSize"); 93 | int _initialConnections = 10; 94 | if(initialConnections != null && !initialConnections.equals("")){ 95 | _initialConnections = Integer.parseInt(initialConnections); 96 | } 97 | String minimumSize = propertiesContainer.getProperty("db.minimumSize"); 98 | int _minimumSize = 10; 99 | if(minimumSize != null && !minimumSize.equals("")){ 100 | _minimumSize = Integer.parseInt(minimumSize); 101 | } 102 | String maximumSize = propertiesContainer.getProperty("db.maximumSize"); 103 | int _maximumSize = 20; 104 | if(maximumSize != null && !maximumSize.equals("")){ 105 | _maximumSize = Integer.parseInt(maximumSize); 106 | } 107 | tempConf.setInitialConnections(_initialConnections); 108 | tempConf.setMinimumSize(_minimumSize); 109 | tempConf.setMaximumSize(_maximumSize); 110 | tempConf.setUsepool(false); 111 | tempConf.setExternal(false); 112 | tempConf.setEncryptdbinfo(false); 113 | if(showsql != null && showsql.equalsIgnoreCase("true")) 114 | tempConf.setShowsql(true); 115 | else{ 116 | tempConf.setShowsql(false); 117 | } 118 | //# 控制map中的列名采用小写，默认为大写 119 | tempConf.setColumnLableUpperCase(false); 120 | //启动数据源 121 | SQLManager.startPool(tempConf); 122 | */ 123 | 124 | // .setDbName("middle_platformsecond") 125 | // .setDbDriver("com.mysql.cj.jdbc.Driver") //数据库驱动程序，必须导入相关数据库的驱动jar包 126 | // 127 | // 128 | // .setDbUrl(jdbcUrl) //通过useCursorFetch=true启用mysql的游标fetch机制，否则会有严重的性能隐患，useCursorFetch必须和jdbcFetchSize参数配合使用，否则不会生效 129 | // .setJdbcFetchSize(-2147483648) 130 | // .setDbUser("sync") 131 | // .setDbPassword("zFfBu21vvfuUzkEE") 132 | // .setValidateSQL("select 1") 133 | // .setUsePool(false) 134 | // .setDbInitSize(5) 135 | // .setDbMinIdleSize(5) 136 | // .setDbMaxSize(10) 137 | // .setShowSql(true);//是否使用连接池; 138 | String jdbcUrl = "jdbc:mysql://192.168.88.10:3306/middle_platform?rewriteBatchedStatements=true&useServerPrepStmts=false&useCompression=true&useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&allowMultiQueries=true&serverTimezone=Asia/Shanghai"; 139 | 140 | DBConf tempConf = new DBConf(); 141 | tempConf.setPoolname("middle_platform"); 142 | tempConf.setDriver("com.mysql.cj.jdbc.Driver"); 143 | tempConf.setJdbcurl(jdbcUrl); 144 | tempConf.setUsername("sync"); 145 | tempConf.setPassword("zFfBu21vvfuUzkEE"); 146 | tempConf.setValidationQuery("select 1"); 147 | tempConf.setShowsql(true); 148 | //tempConf.setTxIsolationLevel("READ_COMMITTED"); 149 | tempConf.setJndiName("jndi-middle_platform"); 150 | // String initialConnections = propertiesContainer.getProperty("db.initialSize"); 151 | // int _initialConnections = 10; 152 | // if(initialConnections != null && !initialConnections.equals("")){ 153 | // _initialConnections = Integer.parseInt(initialConnections); 154 | // } 155 | // String minimumSize = propertiesContainer.getProperty("db.minimumSize"); 156 | // int _minimumSize = 10; 157 | // if(minimumSize != null && !minimumSize.equals("")){ 158 | // _minimumSize = Integer.parseInt(minimumSize); 159 | // } 160 | // String maximumSize = propertiesContainer.getProperty("db.maximumSize"); 161 | // int _maximumSize = 20; 162 | // if(maximumSize != null && !maximumSize.equals("")){ 163 | // _maximumSize = Integer.parseInt(maximumSize); 164 | // } 165 | // tempConf.setInitialConnections(10); 166 | // tempConf.setMinimumSize(20); 167 | // tempConf.setMaximumSize(20); 168 | tempConf.setUsepool(false); 169 | 170 | //# 控制map中的列名采用小写，默认为大写 171 | tempConf.setColumnLableUpperCase(false); 172 | //启动数据源 173 | SQLManager.startPool(tempConf); 174 | 175 | //运行作业，可以不同的作业在不同的线程里面运行 176 | Db2EleasticsearchFullRunOncestore_order_detail_pos.main(args); 177 | Db2EleasticsearchFullRunOncestore_order_pos.main(args); 178 | 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=808 2 | logging.level.org.bboss=INFO 3 | logging.level.bboss=INFO 4 | logging.level.com.frameworkset=INFO 5 | logging.level.org.frameworkset=INFO 6 | 7 | logging.level.org.apache=INFO 8 | logging.level.com.example=INFO 9 | 10 | spring.elasticsearch.bboss.propertiesInterceptor=com.example.esbboss.service.TestPropertiesInterceptor 11 | # DSL configuration file hot load scan interval, in milliseconds, 5 seconds scan by default, turn off scan mechanism when <= 0 12 | spring.elasticsearch.bboss.dslfile.refreshInterval = -1 13 | ##ES authentication configuration, support for x-pack and searchguard 14 | spring.elasticsearch.bboss.elasticUser=elastic 15 | spring.elasticsearch.bboss.elasticPassword=changeme 16 | 17 | spring.elasticsearch.bboss.elasticsearch.includeTypeName = false 18 | spring.elasticsearch.bboss.elasticsearch.rest.hostNames=192.168.137.1:9200 19 | #spring.elasticsearch.bboss.elasticsearch.rest.hostNames=10.180.211.27:9280,10.180.211.27:9281,10.180.211.27:9282 20 | ##HTTPS configuration, add the https:// protocol header 21 | #spring.elasticsearch.bboss.default.elasticsearch.rest.hostNames=https://10.180.211.27:9280,https://10.180.211.27:9281,https://10.180.211.27:9282 22 | spring.elasticsearch.bboss.elasticsearch.dateFormat=yyyy.MM.dd 23 | spring.elasticsearch.bboss.elasticsearch.timeZone=Asia/Shanghai 24 | #Debug switch to output DSL statement on console: showTemplate,false off, true on, log4j at least info level 25 | spring.elasticsearch.bboss.elasticsearch.showTemplate=false 26 | #spring.elasticsearch.bboss.elasticsearch.logDslCallback=org.frameworkset.elasticsearch.client.LoggerDslCallback 27 | 28 | spring.elasticsearch.bboss.elasticsearch.discoverHost=false 29 | 30 | spring.elasticsearch.bboss.elasticsearch.sliceScrollThreadCount=20 31 | 32 | spring.elasticsearch.bboss.elasticsearch.sliceScrollThreadQueue=20 33 | spring.elasticsearch.bboss.elasticsearch.scrollThreadCount=10 34 | 35 | spring.elasticsearch.bboss.elasticsearch.scrollThreadQueue=10 36 | #spring.elasticsearch.bboss.elasticsearch.slowDslThreshold = 1000 37 | #spring.elasticsearch.bboss.elasticsearch.slowDslCallback=org.bboss.elasticsearchtest.crud.TestSlowDslCallback 38 | ##Elasticsearch restclient HTTP connection pool configuration 39 | spring.elasticsearch.bboss.http.timeoutConnection = 50000 40 | spring.elasticsearch.bboss.http.timeoutSocket = 50000 41 | spring.elasticsearch.bboss.http.connectionRequestTimeout=50000 42 | spring.elasticsearch.bboss.http.retryTime = 1 43 | spring.elasticsearch.bboss.http.maxLineLength = -1 44 | spring.elasticsearch.bboss.http.maxHeaderCount = 200 45 | spring.elasticsearch.bboss.http.maxTotal = 200 46 | spring.elasticsearch.bboss.http.defaultMaxPerRoute = 100 47 | spring.elasticsearch.bboss.http.soReuseAddress = false 48 | spring.elasticsearch.bboss.http.soKeepAlive = false 49 | 50 | spring.elasticsearch.bboss.http.keepAlive = 3600000 51 | spring.elasticsearch.bboss.http.keystore = 52 | spring.elasticsearch.bboss.http.keyPassword = 53 | spring.elasticsearch.bboss.http.backoffAuth=true 54 | # SSL host name validation, whether default configuration is used, 55 | # If the specified as the default, use DefaultHostnameVerifier, otherwise use SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER 56 | spring.elasticsearch.bboss.http.hostnameVerifier = 57 | 58 | spring.elasticsearch.bboss.http.evictExpiredConnections=false 59 | spring.elasticsearch.bboss.http.timeToLive = 3600000 60 | #The idle connection is verified every few milliseconds, 61 | # and the invalid connection is automatically released, with <=0 unchecked 62 | 63 | spring.elasticsearch.bboss.http.validateAfterInactivity=2000 64 | # There is a performance overhead to validate connections every time a connection is acquired, 65 | # true：check, false：no check 66 | 67 | spring.elasticsearch.bboss.http.staleConnectionCheckEnabled=false 68 | #* Custom retry control interface. Interface methods must be implemented 69 | #* public interface CustomHttpRequestRetryHandler { 70 | #* public boolean retryRequest(IOException exception, int executionCount, HttpContext context,ClientConfiguration configuration); 71 | #* } 72 | #* Method returns true and retries false without retrying 73 | spring.elasticsearch.bboss.http.customHttpRequestRetryHandler=org.frameworkset.spi.remote.http.ConnectionResetHttpRequestRetryHandler 74 | spring.elasticsearch.bboss.http.httpClientBuilderCallback=com.example.esbboss.HttpClientBuilderCallbackDemo 75 | 76 | # 演示数据库数据导入elasticsearch源配置 77 | # ip地址信息库配置 78 | spring.elasticsearch.bboss.ip.cachesize = 2000 79 | # 库下载地址https://dev.maxmind.com/geoip/geoip2/geolite2/ 80 | spring.elasticsearch.bboss.ip.database = d:/geolite2/GeoLite2-City.mmdb 81 | spring.elasticsearch.bboss.ip.asnDatabase = d:/geolite2/GeoLite2-ASN.mmdb 82 | 83 | 84 | # 演示数据库数据导入elasticsearch源配置 85 | spring.elasticsearch.bboss.db.name = test 86 | spring.elasticsearch.bboss.db.user = root 87 | spring.elasticsearch.bboss.db.password = 123456 88 | spring.elasticsearch.bboss.db.driver = com.mysql.cj.jdbc.Driver 89 | #db.url = jdbc:mysql://192.168.137.1:3306/bboss?useCursorFetch=true&useUnicode=true&characterEncoding=utf-8&useSSL=false 90 | ## mysql 5 jdbc配置 91 | #spring.elasticsearch.bboss.db.url = jdbc:mysql://192.168.137.1:3306/bboss?useUnicode=true&characterEncoding=utf-8&useSSL=false&rewriteBatchedStatements=true 92 | ## mysql jdbc 8配置 93 | spring.elasticsearch.bboss.db.url = jdbc:mysql://192.168.137.1:3306/bboss?allowPublicKeyRetrieval=true&useUnicode=true&characterEncoding=utf-8&useSSL=false&rewriteBatchedStatements=true&serverTimezone=Asia/Shanghai 94 | spring.elasticsearch.bboss.db.usePool = true 95 | 96 | spring.elasticsearch.bboss.db.initSize=5 97 | spring.elasticsearch.bboss.db.minIdleSize=5 98 | spring.elasticsearch.bboss.db.maxSize=5 99 | 100 | 101 | spring.elasticsearch.bboss.db.validateSQL = select 1 102 | spring.elasticsearch.bboss.db.showsql = true 103 | 104 | ## 同步状态status管理数据源配置，在同步作业中通过db.name引用并设置default为status表对应的数据库： 105 | ## importBuilder.setStatusDbname("default"); 106 | #spring.bboss.db.name = default 107 | #spring.bboss.db.user = root 108 | #spring.bboss.db.password = 123456 109 | #spring.bboss.db.driver = com.mysql.cj.jdbc.Driver 110 | #spring.bboss.db.url = jdbc:mysql://localhost:3306/mysql 111 | #spring.bboss.db.usePool = true 112 | #spring.bboss.db.validateSQL = select 1 113 | 114 | -------------------------------------------------------------------------------- /src/main/resources/dbbulktest.xml: -------------------------------------------------------------------------------- 1 | 2 | 28 | 29 | 30 | 31 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/main/resources/esmapper/demo.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 54 | 55 | 56 | 68 | 69 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /src/main/resources/esmapper/sentence_paragrah.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | java python javascript

oracle mysql sqlserver

"], 8 | "analyzer": "simple1_mainContent" 9 | }]]> 10 | 11 | 12 | 13 | java python javascript

oracle mysql sqlserver

"], 15 | "analyzer": "simple2_mainContent" 16 | }]]> 17 | 18 | 19 | 20 | => \u0020paragraph\u0020""", 40 | @""" => \u0020sentence\u0020paragraph\u0020""", 41 | @"""

=> \u0020paragraph\u0020""", 42 | @"""

=> \u0020sentence\u0020paragraph\u0020""", 43 | @"""

=> \u0020paragraph\u0020""", 44 | @"""

=> \u0020sentence\u0020paragraph\u0020""", 45 | @"""! => \u0020sentence\u0020""", 46 | @"""? => \u0020sentence\u0020""", 47 | @"""。=> \u0020sentence\u0020""", 48 | @"""？=> \u0020sentence\u0020""", 49 | @"""！=> \u0020sentence\u0020""" 50 | ] 51 | } 52 | } 53 | } 54 | }, 55 | "mappings": { 56 | "properties": { 57 | "mainContent": { 58 | "type": "text", 59 | "analyzer": "simple1_mainContent", 60 | "search_analyzer": "ik_smart" 61 | } 62 | } 63 | } 64 | }]]> 65 | 66 | 67 | 68 | 69 | java python javascript

oracle mysql sqlserver

"} 72 | ]]> 73 | 74 | 75 | 76 | 77 | 112 | 113 | 114 | 115 | 116 | => \u0020paragraph\u0020""", 135 | @""" => \u0020sentence\u0020paragraph\u0020""", 136 | @"""

=> \u0020paragraph\u0020""", 137 | @"""

=> \u0020sentence\u0020paragraph\u0020""", 138 | @"""

=> \u0020paragraph\u0020""", 139 | @"""

=> \u0020sentence\u0020paragraph\u0020""", 140 | @"""! => \u0020sentence\u0020""", 141 | @"""? => \u0020sentence\u0020""", 142 | @"""。=> \u0020sentence\u0020""", 143 | @"""？=> \u0020sentence\u0020""", 144 | @"""\n => \u0020sentence\u0020paragraph\u0020""", 145 | @"""！=> \u0020sentence\u0020""" 146 | ] 147 | } 148 | } 149 | } 150 | }, 151 | "mappings": { 152 | "properties": { 153 | "mainContent": { 154 | "type": "text", 155 | "analyzer": "simple2_mainContent", 156 | "search_analyzer": "ik_smart" 157 | } 158 | } 159 | } 160 | }]]> 161 | 162 | 163 | 164 | 165 | 169 | 170 | 171 | -------------------------------------------------------------------------------- /src/main/resources/esmapper/span_query.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 20 | 21 | 22 | 23 | 24 | 30 | 31 | 32 | 33 | 34 | 43 | 44 | 45 | 46 | 47 | 71 | 72 | 73 | 74 | 75 | 110 | 111 | -------------------------------------------------------------------------------- /src/main/resources/sqlFile.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | #[log_id]]]> 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /src/test/java/com/example/esbboss/service/RestClientTest.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.service; 2 | /** 3 | * Copyright 2008 biaoping.yin 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.example.esbboss.entity.Demo; 19 | import com.example.esbboss.entity.DemoSearchResult; 20 | import org.frameworkset.elasticsearch.ElasticSearchException; 21 | import org.frameworkset.elasticsearch.boot.BBossESStarter; 22 | import org.frameworkset.elasticsearch.client.ClientInterface; 23 | import org.frameworkset.elasticsearch.entity.ESDatas; 24 | import org.junit.Test; 25 | import org.junit.runner.RunWith; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | import org.springframework.beans.factory.annotation.Autowired; 29 | import org.springframework.boot.test.context.SpringBootTest; 30 | import org.springframework.test.context.junit4.SpringRunner; 31 | 32 | import java.text.DateFormat; 33 | import java.text.ParseException; 34 | import java.text.SimpleDateFormat; 35 | import java.util.Date; 36 | import java.util.HashMap; 37 | import java.util.List; 38 | import java.util.Map; 39 | 40 | /** 41 | *

Description: for Elasticsearch 1.x,2.x,5.x,6.x,+

42 | *

43 | *

Copyright (c) 2018

44 | * @Date 2019/9/18 10:27 45 | * @author biaoping.yin 46 | * @version 1.0 47 | */ 48 | @RunWith(SpringRunner.class) 49 | @SpringBootTest 50 | public class RestClientTest { 51 | private Logger logger = LoggerFactory.getLogger(RestClientTest.class); 52 | @Autowired 53 | private BBossESStarter bbossESStarter; 54 | //DSL config file path 55 | private String mappath = "esmapper/demo.xml"; 56 | 57 | @Test 58 | public void test(){ 59 | this.dropAndCreateAndGetIndice(); 60 | addAndUpdateDocument(); 61 | searchAllPararrel(); 62 | this.search(); 63 | this.deleteDocuments(); 64 | } 65 | public void dropAndCreateAndGetIndice(){ 66 | //Create a client tool to load configuration files, single instance multithreaded security 67 | ClientInterface clientUtil = bbossESStarter.getConfigRestClient(mappath); 68 | try { 69 | //To determine whether the indice demo exists, it returns true if it exists and false if it does not 70 | boolean exist = clientUtil.existIndice("demo"); 71 | 72 | //Delete mapping if the indice demo already exists 73 | if(exist) { 74 | String r = clientUtil.dropIndice("demo"); 75 | logger.debug("clientUtil.dropIndice(\"demo\") response:"+r); 76 | 77 | } 78 | //Create index demo 79 | clientUtil.createIndiceMapping("demo",//The indice name 80 | "createDemoIndice");//Index mapping DSL script name, defined createDemoIndice in esmapper/demo.xml 81 | 82 | String demoIndice = clientUtil.getIndice("demo");//Gets the newly created indice structure 83 | logger.info("after createIndiceMapping clientUtil.getIndice(\"demo\") response:"+demoIndice); 84 | } catch (ElasticSearchException e) { 85 | // TODO Auto-generated catch block 86 | e.printStackTrace(); 87 | } 88 | 89 | } 90 | 91 | 92 | 93 | public void addAndUpdateDocument() { 94 | //Build a create/modify/get/delete document client object, single instance multi-thread security 95 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 96 | //Build an object as index document 97 | Demo demo = new Demo(); 98 | demo.setDemoId(2l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 99 | demo.setAgentStarttime(new Date()); 100 | demo.setAgentStarttimezh(new Date()); 101 | demo.setApplicationName("blackcatdemo2"); 102 | demo.setContentbody("this is content body2"); 103 | demo.setName("liudehua"); 104 | demo.setOrderId("NFZF15045871807281445364228"); 105 | demo.setContrastStatus(2); 106 | 107 | 108 | //Add the document and force refresh 109 | String response = clientUtil.addDocument("demo",//indice name 110 | "demo",//idnex type 111 | demo,"refresh=true"); 112 | 113 | 114 | 115 | logger.debug("Print the result：addDocument-------------------------"); 116 | logger.debug(response); 117 | 118 | demo = new Demo(); 119 | demo.setDemoId(3l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 120 | demo.setAgentStarttime(new Date()); 121 | demo.setApplicationName("blackcatdemo3"); 122 | demo.setContentbody("this is content body3"); 123 | demo.setName("zhangxueyou"); 124 | demo.setOrderId("NFZF15045871807281445364228"); 125 | demo.setContrastStatus(3); 126 | demo.setAgentStarttime(new Date()); 127 | demo.setAgentStarttimezh(new Date()); 128 | 129 | //Add the document and force refresh 130 | response = clientUtil.addDocument("demo",//indice name 131 | "demo",//idnex type 132 | demo,"refresh=true"); 133 | 134 | //Get the document object according to the document id, and return the Demo object 135 | demo = clientUtil.getDocument("demo",//indice name 136 | "demo",//idnex type 137 | "2",//document id 138 | Demo.class); 139 | 140 | //update document 141 | demo = new Demo(); 142 | demo.setDemoId(2l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 143 | demo.setAgentStarttime(new Date()); 144 | demo.setApplicationName("blackcatdemo2"); 145 | demo.setContentbody("this is modify content body2"); 146 | demo.setName("刘德华modify\t"); 147 | demo.setOrderId("NFZF15045871807281445364228"); 148 | demo.setContrastStatus(2); 149 | demo.setAgentStarttimezh(new Date()); 150 | //Execute update and force refresh 151 | response = clientUtil.addDocument("demo",//index name 152 | "demo",//idnex type 153 | demo,"refresh=true"); 154 | 155 | 156 | //Get the modified document object according to the document id and return the json message string 157 | response = clientUtil.getDocument("demo",//indice name 158 | "demo",//idnex type 159 | "2");//document id 160 | logger.debug("Print the modified result:getDocument-------------------------"); 161 | logger.debug(response); 162 | 163 | 164 | 165 | 166 | logger.debug("Print the modified result：getDocument-------------------------"); 167 | logger.debug(response); 168 | 169 | 170 | } 171 | 172 | public void deleteDocuments(){ 173 | //Build a create/modify/get/delete document client object, single instance multi-thread security 174 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 175 | //Batch delete documents 176 | clientUtil.deleteDocuments("demo",//indice name 177 | "demo",//idnex type 178 | new String[]{"2","3"});//Batch delete document ids 179 | } 180 | 181 | /** 182 | * Use slice parallel scoll query all documents of indice demo by 2 thread tasks. DEFAULT_FETCHSIZE is 5000 183 | */ 184 | public void searchAllPararrel(){ 185 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 186 | ESDatas esDatas = clientUtil.searchAllParallel("demo", Demo.class,2); 187 | } 188 | 189 | 190 | 191 | /** 192 | * Search the documents 193 | */ 194 | public DemoSearchResult search() { 195 | //Create a load DSL file client instance to retrieve documents, single instance multithread security 196 | ClientInterface clientUtil = bbossESStarter.getConfigRestClient(mappath); 197 | //Set query conditions, pass variable parameter values via map,key for variable names in DSL 198 | //There are four variables in the DSL: 199 | // applicationName1 200 | // applicationName2 201 | // startTime 202 | // endTime 203 | Map params = new HashMap(); 204 | //Set the values of applicationName1 and applicationName2 variables 205 | params.put("applicationName1","blackcatdemo2"); 206 | params.put("applicationName2","blackcatdemo3"); 207 | DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 208 | //Set the time range, and accept the long value as the time parameter 209 | try { 210 | params.put("startTime",dateFormat.parse("2017-09-02 00:00:00").getTime()); 211 | } catch (ParseException e) { 212 | e.printStackTrace(); 213 | } 214 | params.put("endTime",new Date().getTime()); 215 | 216 | 217 | //Execute the query 218 | ESDatas esDatas = //ESDatas contains a collection of currently retrieved records, up to 1000 records, specified by the size attribute in the DSL 219 | clientUtil.searchList("demo/_search",//demo as the indice, _search as the search action 220 | "searchDatas",//DSL statement name defined in esmapper/demo.xml 221 | params,//Query parameters 222 | Demo.class);//Data object type Demo returned 223 | 224 | 225 | //Gets a list of result objects and returns max up to 1000 records (specified in DSL) 226 | List demos = esDatas.getDatas(); 227 | 228 | // String json = clientUtil.executeRequest("demo/_search",//demo as the index table, _search as the search action 229 | // "searchDatas",//DSL statement name defined in esmapper/demo.xml 230 | // params);//Query parameters 231 | 232 | // String json = com.frameworkset.util.SimpleStringUtil.object2json(demos); 233 | //Gets the total number of records 234 | long totalSize = esDatas.getTotalSize(); 235 | DemoSearchResult demoSearchResult = new DemoSearchResult(); 236 | demoSearchResult.setDemos(demos); 237 | demoSearchResult.setTotalSize(totalSize); 238 | return demoSearchResult; 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /src/test/java/com/example/esbboss/service/RestClientTest7.java: -------------------------------------------------------------------------------- 1 | package com.example.esbboss.service; 2 | /** 3 | * Copyright 2008 biaoping.yin 4 | *

5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | *

9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | *

11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.example.esbboss.entity.Demo; 19 | import com.example.esbboss.entity.DemoSearchResult; 20 | import org.frameworkset.elasticsearch.ElasticSearchException; 21 | import org.frameworkset.elasticsearch.boot.BBossESStarter; 22 | import org.frameworkset.elasticsearch.client.ClientInterface; 23 | import org.frameworkset.elasticsearch.entity.ESDatas; 24 | import org.junit.Test; 25 | import org.junit.runner.RunWith; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | import org.springframework.beans.factory.annotation.Autowired; 29 | import org.springframework.boot.test.context.SpringBootTest; 30 | import org.springframework.test.context.junit4.SpringRunner; 31 | 32 | import java.text.DateFormat; 33 | import java.text.ParseException; 34 | import java.text.SimpleDateFormat; 35 | import java.util.Date; 36 | import java.util.HashMap; 37 | import java.util.List; 38 | import java.util.Map; 39 | 40 | /** 41 | *

Description: for Elasticsearch 7.x,8.x,+

42 | *

43 | *

Copyright (c) 2018

44 | * @Date 2019/9/18 10:27 45 | * @author biaoping.yin 46 | * @version 1.0 47 | */ 48 | @RunWith(SpringRunner.class) 49 | @SpringBootTest 50 | public class RestClientTest7 { 51 | private Logger logger = LoggerFactory.getLogger(RestClientTest7.class); 52 | @Autowired 53 | private BBossESStarter bbossESStarter; 54 | //DSL config file path 55 | private String mappath = "esmapper/demo7.xml"; 56 | 57 | @Test 58 | public void test(){ 59 | this.dropAndCreateAndGetIndice(); 60 | addAndUpdateDocument(); 61 | searchAllPararrel(); 62 | this.search(); 63 | this.deleteDocuments(); 64 | } 65 | public void dropAndCreateAndGetIndice(){ 66 | //Create a client tool to load configuration files, single instance multithreaded security 67 | ClientInterface clientUtil = bbossESStarter.getConfigRestClient(mappath); 68 | try { 69 | //To determine whether the indice demo exists, it returns true if it exists and false if it does not 70 | boolean exist = clientUtil.existIndice("demo"); 71 | 72 | //Delete mapping if the indice demo already exists 73 | if(exist) { 74 | String r = clientUtil.dropIndice("demo"); 75 | logger.debug("clientUtil.dropIndice(\"demo\") response:"+r); 76 | 77 | } 78 | //Create index demo 79 | clientUtil.createIndiceMapping("demo",//The indice name 80 | "createDemoIndice");//Index mapping DSL script name, defined createDemoIndice in esmapper/demo.xml 81 | 82 | String demoIndice = clientUtil.getIndice("demo");//Gets the newly created indice structure 83 | logger.info("after createIndiceMapping clientUtil.getIndice(\"demo\") response:"+demoIndice); 84 | } catch (ElasticSearchException e) { 85 | // TODO Auto-generated catch block 86 | e.printStackTrace(); 87 | } 88 | 89 | } 90 | 91 | 92 | 93 | public void addAndUpdateDocument() { 94 | //Build a create/modify/get/delete document client object, single instance multi-thread security 95 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 96 | //Build an object as index document 97 | Demo demo = new Demo(); 98 | demo.setDemoId(2l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 99 | demo.setAgentStarttime(new Date()); 100 | demo.setAgentStarttimezh(new Date()); 101 | demo.setApplicationName("blackcatdemo2"); 102 | demo.setContentbody("this is content body2"); 103 | demo.setName("liudehua"); 104 | demo.setOrderId("NFZF15045871807281445364228"); 105 | demo.setContrastStatus(2); 106 | 107 | 108 | //Add the document and force refresh 109 | String response = clientUtil.addDocument("demo",//indice name 110 | demo,"refresh=true"); 111 | 112 | 113 | 114 | logger.debug("Print the result：addDocument-------------------------"); 115 | logger.debug(response); 116 | 117 | demo = new Demo(); 118 | demo.setDemoId(3l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 119 | demo.setAgentStarttime(new Date()); 120 | demo.setApplicationName("blackcatdemo3"); 121 | demo.setContentbody("this is content body3"); 122 | demo.setName("zhangxueyou"); 123 | demo.setOrderId("NFZF15045871807281445364228"); 124 | demo.setContrastStatus(3); 125 | demo.setAgentStarttime(new Date()); 126 | demo.setAgentStarttimezh(new Date()); 127 | 128 | //Add the document and force refresh 129 | response = clientUtil.addDocument("demo",//indice name 130 | demo,"refresh=true"); 131 | 132 | //Get the document object according to the document id, and return the Demo object 133 | demo = clientUtil.getDocument("demo",//indice name 134 | "2",//document id 135 | Demo.class); 136 | 137 | //update document 138 | demo = new Demo(); 139 | demo.setDemoId(2l);//Specify the document id, the unique identity, and mark with the @ESId annotation. If the demoId already exists, modify the document; otherwise, add the document 140 | demo.setAgentStarttime(new Date()); 141 | demo.setApplicationName("blackcatdemo2"); 142 | demo.setContentbody("this is modify content body2"); 143 | demo.setName("刘德华modify\t"); 144 | demo.setOrderId("NFZF15045871807281445364228"); 145 | demo.setContrastStatus(2); 146 | demo.setAgentStarttimezh(new Date()); 147 | //Execute update and force refresh 148 | response = clientUtil.addDocument("demo",//index name 149 | demo,"refresh=true"); 150 | 151 | 152 | //Get the modified document object according to the document id and return the json message string 153 | response = clientUtil.getDocument("demo",//indice name 154 | "2");//document id 155 | logger.debug("Print the modified result:getDocument-------------------------"); 156 | logger.debug(response); 157 | 158 | 159 | 160 | 161 | logger.debug("Print the modified result：getDocument-------------------------"); 162 | logger.debug(response); 163 | 164 | 165 | } 166 | 167 | public void deleteDocuments(){ 168 | //Build a create/modify/get/delete document client object, single instance multi-thread security 169 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 170 | //Batch delete documents 171 | clientUtil.deleteDocuments("demo",//indice name 172 | new String[]{"2","3"});//Batch delete document ids 173 | } 174 | 175 | /** 176 | * Use slice parallel scoll query all documents of indice demo by 2 thread tasks. DEFAULT_FETCHSIZE is 5000 177 | */ 178 | public void searchAllPararrel(){ 179 | ClientInterface clientUtil = bbossESStarter.getRestClient(); 180 | ESDatas esDatas = clientUtil.searchAllParallel("demo", Demo.class,2); 181 | } 182 | 183 | 184 | 185 | /** 186 | * Search the documents 187 | */ 188 | public DemoSearchResult search() { 189 | //Create a load DSL file client instance to retrieve documents, single instance multithread security 190 | ClientInterface clientUtil = bbossESStarter.getConfigRestClient(mappath); 191 | //Set query conditions, pass variable parameter values via map,key for variable names in DSL 192 | //There are four variables in the DSL: 193 | // applicationName1 194 | // applicationName2 195 | // startTime 196 | // endTime 197 | Map params = new HashMap(); 198 | //Set the values of applicationName1 and applicationName2 variables 199 | params.put("applicationName1","blackcatdemo2"); 200 | params.put("applicationName2","blackcatdemo3"); 201 | DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 202 | //Set the time range, and accept the long value as the time parameter 203 | try { 204 | params.put("startTime",dateFormat.parse("2017-09-02 00:00:00").getTime()); 205 | } catch (ParseException e) { 206 | e.printStackTrace(); 207 | } 208 | params.put("endTime",new Date().getTime()); 209 | 210 | 211 | //Execute the query 212 | ESDatas esDatas = //ESDatas contains a collection of currently retrieved records, up to 1000 records, specified by the size attribute in the DSL 213 | clientUtil.searchList("demo/_search",//demo as the indice, _search as the search action 214 | "searchDatas",//DSL statement name defined in esmapper/demo.xml 215 | params,//Query parameters 216 | Demo.class);//Data object type Demo returned 217 | 218 | 219 | //Gets a list of result objects and returns max up to 1000 records (specified in DSL) 220 | List demos = esDatas.getDatas(); 221 | 222 | // String json = clientUtil.executeRequest("demo/_search",//demo as the index table, _search as the search action 223 | // "searchDatas",//DSL statement name defined in esmapper/demo.xml 224 | // params);//Query parameters 225 | 226 | // String json = com.frameworkset.util.SimpleStringUtil.object2json(demos); 227 | //Gets the total number of records 228 | long totalSize = esDatas.getTotalSize(); 229 | DemoSearchResult demoSearchResult = new DemoSearchResult(); 230 | demoSearchResult.setDemos(demos); 231 | demoSearchResult.setTotalSize(totalSize); 232 | return demoSearchResult; 233 | } 234 | } 235 | --------------------------------------------------------------------------------