Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 5 | * except in compliance with the License. You may obtain a copy of the License at 6 | * 7 | *
http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | *
Unless required by applicable law or agreed to in writing, software distributed under the 10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | * express or implied. See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.webank.blockchain.data.reconcile.constants; 15 | 16 | /** 17 | * @author wesleywang 18 | * @Description: 19 | * @date 2020/7/3 20 | */ 21 | public class ResponseCode { 22 | 23 | public static final int HTTP_SUCCESS_CODE = 200; 24 | 25 | public static final int RECONCILE_FAILED_CODE = 3000; 26 | } 27 | -------------------------------------------------------------------------------- /src/test/java/com/webank/blockchain/data/reconcile/BaseTests.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Webank. 3 | * 4 | *
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 5 | * except in compliance with the License. You may obtain a copy of the License at 6 | * 7 | *
http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | *
Unless required by applicable law or agreed to in writing, software distributed under the 10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | * express or implied. See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package com.webank.blockchain.data.reconcile; 15 | 16 | 17 | import org.junit.runner.RunWith; 18 | import org.springframework.boot.test.context.SpringBootTest; 19 | import org.springframework.test.context.junit4.SpringRunner; 20 | 21 | @RunWith(SpringRunner.class) 22 | @SpringBootTest 23 | public abstract class BaseTests { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/webank/blockchain/data/reconcile/message/NoticeService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Webank. 3 | * 4 | *
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 5 | * except in compliance with the License. You may obtain a copy of the License at 6 | * 7 | *
http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | *
Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.message;
15 |
16 | /**
17 | * An interface for message queues, used for extensions
18 | *
19 | * @author wesleywang
20 | * @Description:
21 | * @date 2020/6/17
22 | */
23 | public interface NoticeService Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.reconcile;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileExecuteData;
17 |
18 | /**
19 | * @author wesleywang
20 | * @Description: ReconcileTransfer
21 | * @date 2020/6/24
22 | */
23 | public interface ReconcileTransfer {
24 |
25 | void transferToExecutor(ReconcileExecuteData executeData);
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/exporter/bc/BCDataExportService.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.exporter.bc;
15 |
16 | import java.io.File;
17 | import java.util.Date;
18 |
19 | /**
20 | * @author wesleywang
21 | * @Description:
22 | * @date 2020/6/24
23 | */
24 | public interface BCDataExportService{
25 |
26 | File exportData(String filePath, Date dataRangeBeginTime, Date dataRangeEndTime) throws Exception;
27 |
28 | }
29 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | ### common ###
2 | log/
3 | src/main/resources
4 | src/test/resources
5 |
6 |
7 | ### mac ###
8 | .DS_Store
9 |
10 | ### gradle ###
11 | .gradle/
12 | build
13 | dist/
14 |
15 | ## eclipse ##
16 | .classpath
17 | .project
18 | .settings
19 | bin/
20 | out/
21 | conf/
22 |
23 | ## integration test files
24 | nodes/
25 | src/integration-test/resources/
26 | build_chain.sh
27 |
28 | ### STS ###
29 | .apt_generated
30 | .factorypath
31 | .springBeans
32 | .sts4-cache
33 |
34 | ### IntelliJ IDEA ###
35 | .idea
36 | *.iws
37 | *.iml
38 | *.ipr
39 |
40 | ### NetBeans ###
41 | /nbproject/private/
42 | /build/
43 | /nbbuild/
44 | /dist/
45 | /nbdist/
46 | /.nb-gradle/
47 |
48 | ### Compiled class file ###
49 | *.class
50 |
51 | ### Log file ###
52 | *.log
53 |
54 | ### BlueJ files ###
55 | *.ctxt
56 |
57 | ### Mobile Tools for Java (J2ME) ###
58 | .mtj.tmp/
59 |
60 | ### Package Files ###
61 | *.jar
62 | *.war
63 | *.nar
64 | *.ear
65 | *.zip
66 | *.tar.gz
67 | *.rar
68 |
69 | ### virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml ###
70 | hs_err_pid*
71 |
72 | ### repo ###
73 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/handler/Handler.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
17 |
18 | /**
19 | * The interface of the chain of responsibility
20 | *
21 | * @author wesleywang
22 | * @Description: Handler
23 | * @date 2020/6/17
24 | */
25 | public interface Handler {
26 |
27 | void invoke(ReconcileContext context, InvocationHandler handler) throws Exception;
28 |
29 | }
30 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/handler/InvocationHandler.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
17 |
18 | /**
19 | * The call interface for the chain of responsibility
20 | *
21 | * @author wesleywang
22 | * @Description: InvocationHandler
23 | * @date 2020/6/19
24 | */
25 | public interface InvocationHandler {
26 |
27 | void handle(ReconcileContext context) throws Exception;
28 | }
29 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/reconcile/ReconcileExecutor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.reconcile;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileExecuteData;
17 | import com.webank.blockchain.data.reconcile.entity.ReconcileResult;
18 |
19 | /**
20 | * @author wesleywang
21 | * @Description: ReconcileExecutor
22 | * @date 2020/6/24
23 | */
24 | public interface ReconcileExecutor{
25 |
26 | ReconcileResult execute(ReconcileExecuteData executeData);
27 |
28 | }
29 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/enums/TriggerType.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.enums;
15 |
16 | import lombok.AllArgsConstructor;
17 | import lombok.Getter;
18 | import lombok.ToString;
19 |
20 | /**
21 | * @author wesleywang
22 | * @Description:
23 | * @date 2020/6/19
24 | */
25 | @Getter
26 | @ToString
27 | @AllArgsConstructor
28 | public enum TriggerType {
29 |
30 | SCHEDULED(1, "timing trigger"),
31 | MANUAL(2,"manual trigger");
32 |
33 |
34 | private int type;
35 | private String describe;
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/filetransfer/FileTransferService.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.filetransfer;
15 |
16 | import java.io.File;
17 | import java.io.IOException;
18 |
19 | /**
20 | * File network transfer interface
21 | *
22 | * @Description: FileSender
23 | * @author graysonzhang
24 | * @date 2020-06-10 16:48:26
25 | *
26 | */
27 | public interface FileTransferService {
28 |
29 | String sendFile(File file) throws Exception;
30 |
31 | File obtainFile(String fileName, String localFilePath) throws IOException;
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/entity/ReconcileInfo.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.entity;
15 |
16 | import lombok.Data;
17 |
18 | import java.util.Map;
19 |
20 | /**
21 | * @author wesleywang
22 | * @Description:
23 | * @date 2020-06-12
24 | */
25 | @Data
26 | public class ReconcileInfo {
27 |
28 | /**
29 | * Unique identification of each reconciliation data
30 | */
31 | private String id;
32 |
33 | /**
34 | * Field mapping for reconciliation data
35 | */
36 | private Map Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.config;
15 |
16 | import org.springframework.boot.context.properties.ConfigurationProperties;
17 | import org.springframework.context.annotation.Configuration;
18 | import lombok.Data;
19 |
20 |
21 | /**
22 | * SystemEnvironmentConfig
23 | *
24 | * @Description: SystemEnvironmentConfig
25 | * @author graysonzhang
26 | * @date 2020-06-10 16:27:14
27 | *
28 | */
29 | @Configuration
30 | @ConfigurationProperties("system")
31 | @Data
32 | public class SystemEnvironmentConfig {
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/exception/ReconcileException.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.exception;
15 |
16 | /**
17 | * @author wesleywang
18 | * @Description:
19 | * @date 2020/6/16
20 | */
21 | public class ReconcileException extends Exception{
22 |
23 | private static final long serialVersionUID = 1L;
24 |
25 | public ReconcileException(){}
26 |
27 | public ReconcileException(String message, Throwable cause){
28 | super(message,cause);
29 | }
30 |
31 | public ReconcileException(String message){
32 | super(message);
33 | }
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/parser/FileParser.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.parser;
15 |
16 | import com.webank.blockchain.data.reconcile.exception.ReconcileException;
17 | import com.webank.blockchain.data.reconcile.reconcile.ReconcileTransfer;
18 |
19 | import java.io.File;
20 | import java.io.IOException;
21 |
22 | /**
23 | * @author wesleywang
24 | * @Description: FileParser
25 | * @date 2020/6/17
26 | */
27 | public interface FileParser {
28 |
29 | void parseAndTransfer(File businessReconFile, File bcReconFile, ReconcileTransfer transfer) throws ReconcileException, IOException;
30 |
31 | }
32 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/exporter/result/ResultDataExportService.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.exporter.result;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileResult;
17 |
18 | import java.io.File;
19 | import java.util.List;
20 | import java.util.concurrent.Future;
21 |
22 | /**
23 | * Export interface of reconciliation result data
24 | *
25 | * @author wesleywang
26 | * @Description:
27 | * @date 2020/7/1
28 | */
29 | public interface ResultDataExportService {
30 |
31 | File exportResultData(String filePath, List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.entity;
15 |
16 | import lombok.Data;
17 |
18 | /**
19 | * @author wesleywang
20 | * @Description:
21 | * @date 2020/6/12
22 | */
23 | @Data
24 | public class ReconcileResult {
25 |
26 | /**
27 | * Unique identification of each reconciliation data
28 | */
29 | private String id;
30 |
31 | /**
32 | * Is the reconciliation data consistent
33 | */
34 | private boolean success;
35 |
36 | /**
37 | * note
38 | */
39 | private String note;
40 |
41 | /**
42 | * state
43 | */
44 | private String state;
45 |
46 | }
47 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/filetransfer/local/LocalFileTransferService.java:
--------------------------------------------------------------------------------
1 | package com.webank.blockchain.data.reconcile.filetransfer.local;
2 |
3 | import cn.hutool.core.io.FileUtil;
4 | import com.webank.blockchain.data.reconcile.filetransfer.FileTransferService;
5 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
6 | import lombok.extern.slf4j.Slf4j;
7 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
8 | import org.springframework.stereotype.Component;
9 |
10 | import java.io.File;
11 | import java.io.IOException;
12 | import java.nio.file.Paths;
13 | import java.nio.file.StandardCopyOption;
14 |
15 | /**
16 | * @author wesleywang
17 | * @Description:
18 | * @date 2020/12/15
19 | */
20 | @ConditionalOnProperty(value = "local.enabled", havingValue = "true")
21 | @Component
22 | @Slf4j
23 | public class LocalFileTransferService implements FileTransferService {
24 |
25 | @Override
26 | public String sendFile(File file) throws Exception {
27 | return null;
28 | }
29 |
30 | @Override
31 | public File obtainFile(String fileName, String localFilePath) throws IOException {
32 | File localFile = Paths.get("./", fileName).toFile();
33 | return FileUtil.copyFile(localFile, FileUtils.newFile(localFilePath), StandardCopyOption.REPLACE_EXISTING);
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/db/entity/IdEntity.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 |
15 | package com.webank.blockchain.data.reconcile.db.entity;
16 |
17 | import lombok.Data;
18 | import lombok.experimental.Accessors;
19 |
20 | import javax.persistence.*;
21 | import java.io.Serializable;
22 |
23 | /**
24 | * @author wesleywang
25 | * @Description:
26 | * @date 2020/6/17
27 | */
28 | @Data
29 | @MappedSuperclass
30 | @Accessors(chain = true)
31 | public abstract class IdEntity implements Serializable {
32 |
33 | private static final long serialVersionUID = 5903397383140175895L;
34 | /** @Fields pkId : primary key */
35 | @Id
36 | @GeneratedValue(strategy = GenerationType.IDENTITY)
37 | @Column(name = "pk_id")
38 | protected Long pkId;
39 | }
40 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/config/BeanConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.config;
15 |
16 | import lombok.extern.slf4j.Slf4j;
17 | import org.springframework.context.annotation.Bean;
18 | import org.springframework.context.annotation.Configuration;
19 |
20 | import java.net.InetAddress;
21 | import java.net.UnknownHostException;
22 |
23 | /**
24 | * @author wesleywang
25 | * @Description:
26 | * @date 2020/6/19
27 | */
28 | @Configuration
29 | @Slf4j
30 | public class BeanConfig {
31 |
32 | @Bean
33 | public InetAddress getIpAddress() throws UnknownHostException {
34 | log.info("System confPath is {}", System.getProperty("confPath"));
35 | return InetAddress.getLocalHost();
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/src/main/resources/datasource.properties:
--------------------------------------------------------------------------------
1 | ## data export DB config
2 | spring.datasource.url=jdbc:mysql://[ip]:[port]/[database]?autoReconnect=true&characterEncoding=utf8&useSSL=false&serverTimezone=GMT%2b8
3 | spring.datasource.username=
4 | spring.datasource.password=
5 |
6 | #Specifies the minimum value at which the connection must be maintained.
7 | spring.datasource.min-idle=50
8 | #Specifies the maximum number of free connections to the connection pool.
9 | spring.datasource.max-idle=100
10 | #Specifies the maximum wait time, in milliseconds, for a connection pool to wait for a connection to return.
11 | spring.datasource.max-wait=1000
12 | #Verify the validity of the connection.
13 | spring.datasource.valdation-query=SELECT 1
14 | spring.datasource.test-while-idle=true
15 | #The interval between idle connections being reclaimed.
16 | spring.datasource.time-between-eviction-runs-millis=300000
17 | spring.datasource.encrypt=false
18 |
19 | #Configure database type
20 | spring.jpa.database=MYSQL
21 | #Configure whether to print SQL
22 | spring.jpa.show-sql=true
23 | #Configure the cascade level, which is configured by default to disable DDL
24 | spring.jpa.hibernate.ddl-auto=update
25 | #Naming policy, no modified naming by default
26 | spring.jpa.hibernate.naming-strategy=org.hibernate.cfg.ImprovedNamingStrategy
27 | spring.jpa.hibernate.database-platform=org.hibernate.dialect.MySQL5Dialect
28 |
29 |
30 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/entity/ReconcileExecuteData.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.entity;
15 |
16 | import lombok.Data;
17 |
18 | /**
19 | * @author wesleywang
20 | * @Description:
21 | * @date 2020/6/18
22 | */
23 | @Data
24 | public class ReconcileExecuteData {
25 |
26 | /**
27 | * Unique identification of each reconciliation data
28 | */
29 | private String id;
30 |
31 | /**
32 | * Reconciliation data of business parties
33 | */
34 | private ReconcileInfo busOrg;
35 |
36 | /**
37 | * Reconciliation data of BC parties
38 | */
39 | private ReconcileInfo bcOrg;
40 |
41 | /**
42 | * Whether or not id matching data was found
43 | */
44 | private boolean matchById;
45 |
46 | }
47 |
--------------------------------------------------------------------------------
/src/test/java/com/webank/blockchain/data/reconcile/FileTransferTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile;
15 |
16 | import com.webank.blockchain.data.reconcile.filetransfer.FileTransferService;
17 | import org.junit.Test;
18 | import org.springframework.beans.factory.annotation.Autowired;
19 |
20 | /**
21 | * @author wesleywang
22 | * @Description:
23 | * @date 2020/7/8
24 | */
25 | public class FileTransferTest extends BaseTests{
26 |
27 | @Autowired
28 | private FileTransferService transferService;
29 |
30 | @Test
31 | public void transferServiceTest() throws Exception {
32 | // transferService.obtainFile("xxxxx_2020-07-06.txt","out/business/xxxx_2020-07-06.txt");
33 | // transferService.sendFile(new File("out/result/result_xxxx_2020-07-06.txt"));
34 | }
35 |
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/utils/StringUtils.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.utils;
15 |
16 | /**
17 | * @author wesleywang
18 | * @Description:
19 | * @date 2020/7/3
20 | */
21 | public class StringUtils {
22 |
23 | public static final String EMPTY = "";
24 |
25 | public static final int INDEX_NOT_FOUND = -1;
26 |
27 |
28 | public static String substringAfterOfIgnoreCase(String str, String separator) {
29 | if (isEmpty(str)) {
30 | return str;
31 | }
32 | if (isEmpty(separator)) {
33 | return EMPTY;
34 | }
35 | int pos = org.apache.commons.lang3.StringUtils.indexOfIgnoreCase(str,separator);
36 | if (pos == INDEX_NOT_FOUND || pos == str.length() - separator.length()) {
37 | return EMPTY;
38 | }
39 | return str.substring(pos + separator.length());
40 | }
41 |
42 | public static boolean isEmpty(CharSequence cs) {
43 | return cs == null || cs.length() == 0;
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/config/FTPConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.config;
15 |
16 | import lombok.Data;
17 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
18 | import org.springframework.boot.context.properties.ConfigurationProperties;
19 | import org.springframework.context.annotation.Configuration;
20 | import org.springframework.context.annotation.PropertySource;
21 |
22 | /**
23 | * @author wesleywang
24 | * @Description:
25 | * @date 2020-06-11
26 | */
27 | @Configuration
28 | @ConditionalOnProperty(value = "ftp.enabled", havingValue = "true")
29 | @ConfigurationProperties(prefix="ftp")
30 | @Data
31 | public class FTPConfig {
32 |
33 | private String host;
34 | private int port;
35 | private String userName;
36 | private String passWord;
37 | private String workDir;
38 | private String encoding;
39 | private String root;
40 | private int maxTotal;
41 | private int minIdel;
42 | private int maxIdle;
43 | private int maxWaitMillis;
44 |
45 | }
46 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/entity/Responses.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.entity;
15 |
16 | import com.webank.blockchain.data.reconcile.constants.ResponseCode;
17 | import lombok.Data;
18 | import org.apache.commons.lang3.StringUtils;
19 |
20 | import java.io.Serializable;
21 |
22 | /**
23 | * @author wesleywang
24 | * @Description:
25 | * @date 2020/6/18
26 | */
27 | @Data
28 | public class Responses Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.config;
15 |
16 | import com.zaxxer.hikari.HikariDataSource;
17 | import lombok.Data;
18 | import org.springframework.boot.context.properties.ConfigurationProperties;
19 | import org.springframework.context.annotation.Bean;
20 | import org.springframework.context.annotation.Configuration;
21 | import org.springframework.context.annotation.PropertySource;
22 | import org.springframework.core.env.Environment;
23 |
24 | import javax.sql.DataSource;
25 | import java.io.IOException;
26 |
27 | /**
28 | * @author wesleywang
29 | * @Description:
30 | * @date 2020/6/18
31 | */
32 | @Configuration
33 | @Data
34 | public class DBConfig {
35 |
36 | @Bean(name = "dataSource")
37 | public DataSource dataSource(Environment env) throws IOException {
38 | HikariDataSource ds = new HikariDataSource();
39 | ds.setJdbcUrl(env.getProperty("spring.datasource.url"));
40 | ds.setUsername(env.getProperty("spring.datasource.username"));
41 | ds.setPassword(env.getProperty("spring.datasource.password"));
42 | return ds;
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 组件介绍
2 | 传统企业间的对账依赖于对账双⽅的中心化账本。中心化账本在对账期间如果出现账不平的情况，排查会⾮常耗时耗力。区块链作为信任的机器，具有不可篡改、分布式账本等特性，基于区块链的对账能够在对账不一致的情况下，找到⼀个可信的客观依据，从而减少因对账不平造成的排查成本。
3 | WeBankBlockchain-Data-Reconcile是一款基于区块链的对账组件，提供基于区块链智能合约账本的通用化数据对账解决方案，并提供了一套可动态扩展的对账框架，支持定制化开发。
4 |
5 | ## 关键特性
6 | - 支持自定义对账数据
7 | - 支持自定义对账规则
8 | - 支持多个对账任务, 提供对账任务配置
9 | - 支持多种对账文件格式，可扩展
10 | - 支持自定义对账任务和方式，手动或自动、任务频次、时间可配置
11 | - 支持多种对账文件托管方式，如文件管理系统或服务器，可扩展
12 |
13 | ## 环境要求
14 |
15 | 在使用本组件前，请确认系统环境已安装相关依赖软件，清单如下：
16 |
17 | | 依赖软件 | 说明 | 备注 |
18 | | ---------- | ------------------------------------------------------------ | ---- |
19 | | FISCO-BCOS | \>= 2.0， 1.x版本请参考V0.5版本 dev分支 | |
20 | | Bash | 需支持Bash（理论上来说支持所有ksh、zsh等其他unix shell，但未测试） | |
21 | | Java | \>= JDK[1.8] | |
22 | | Git | 下载的安装包使用Git | |
23 | | MySQL | \>= mysql-community-server[5.7] | |
24 | | FTP | 需要时安装 | |
25 |
26 |
27 | ## 文档
28 | - [**中文**](https://data-doc.readthedocs.io/zh_CN/dev/docs/WeBankBlockchain-Data-Reconcile/index.html)
29 | - [**快速安装**](https://data-doc.readthedocs.io/zh_CN/dev/docs/WeBankBlockchain-Data-Reconcile/install.html)
30 |
31 |
32 | ## 贡献代码
33 | 欢迎参与本项目的社区建设：
34 | - 如项目对您有帮助，欢迎点亮我们的小星星(点击项目右上方Star按钮)。
35 | - 欢迎提交代码(Pull requests)。
36 | - [提问和提交BUG](https://github.com/WeBankBlockchain/Data-Reconcile/issues)。
37 | - 如果发现代码存在安全漏洞，请在[这里](https://security.webank.com)上报。
38 |
39 |
40 | ## License
41 | 
42 |
43 | 开源协议为[Apache License 2.0](http://www.apache.org/licenses/). 详情参考[LICENSE](../LICENSE)。
44 |
45 |
--------------------------------------------------------------------------------
/src/test/java/com/webank/blockchain/data/reconcile/FileUtilsTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile;
15 |
16 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
17 | import org.junit.Test;
18 |
19 | import java.util.ArrayList;
20 | import java.util.List;
21 |
22 | /**
23 | * @author wesleywang
24 | * @Description:
25 | * @date 2020/7/8
26 | */
27 | public class FileUtilsTest extends BaseTests {
28 |
29 | @Test
30 | public void exportDataUtilTest() throws Exception {
31 | List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile;
15 |
16 | import org.springframework.boot.SpringApplication;
17 | import org.springframework.boot.autoconfigure.SpringBootApplication;
18 | import org.springframework.boot.context.properties.EnableConfigurationProperties;
19 | import org.springframework.context.annotation.PropertySource;
20 | import org.springframework.transaction.annotation.EnableTransactionManagement;
21 |
22 | @PropertySource(value = {
23 | "file:./config/application.properties",
24 | "file:./config/datasource.properties",
25 | "file:./config/reconcile.properties",
26 | "file:./config/filetransfer.properties",
27 | "classpath:application.properties",
28 | "classpath:datasource.properties",
29 | "classpath:reconcile.properties",
30 | "classpath:filetransfer.properties",
31 | }, encoding = "utf-8",ignoreResourceNotFound = true)
32 | @SpringBootApplication
33 | @EnableTransactionManagement
34 | @EnableConfigurationProperties
35 | public class BcreconcileApplication {
36 |
37 | public static void main(String[] args) {
38 | SpringApplication.run(BcreconcileApplication.class, args);
39 | }
40 |
41 | }
42 |
--------------------------------------------------------------------------------
/src/test/java/com/webank/blockchain/data/reconcile/ReconcileTaskTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
17 | import com.webank.blockchain.data.reconcile.enums.TriggerType;
18 | import com.webank.blockchain.data.reconcile.handler.InvocationHandler;
19 | import com.webank.blockchain.data.reconcile.handler.ReconcileHandlerFactory;
20 | import org.junit.Test;
21 | import org.springframework.beans.factory.annotation.Autowired;
22 |
23 | /**
24 | * @author wesleywang
25 | * @Description:
26 | * @date 2020/6/28
27 | */
28 | public class ReconcileTaskTest extends BaseTests {
29 |
30 |
31 | @Autowired
32 | private ReconcileHandlerFactory handlerFactory;
33 |
34 | @Test
35 | public void test() throws Exception {
36 | InvocationHandler handler = handlerFactory.getInvocationHandler();
37 | ReconcileContext context = ReconcileContext.builder()
38 | .triggerType(TriggerType.SCHEDULED).build();
39 | try {
40 | handler.handle(context);
41 | } catch (Exception e) {
42 | // e.printStackTrace();
43 | }
44 |
45 | // Thread.sleep(1000000000);
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/utils/ThreadUtils.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.utils;
15 |
16 | import java.util.concurrent.Executors;
17 | import java.util.concurrent.LinkedBlockingQueue;
18 | import java.util.concurrent.ScheduledExecutorService;
19 | import java.util.concurrent.ThreadPoolExecutor;
20 | import java.util.concurrent.TimeUnit;
21 |
22 | /**
23 | * @author wesleywang
24 | * @Description:
25 | * @date 2020/6/17
26 | */
27 | public final class ThreadUtils {
28 |
29 | public static ThreadPoolExecutor executor;
30 |
31 | public static ScheduledExecutorService timer;
32 |
33 | public static ThreadPoolExecutor transfer;
34 |
35 | static {
36 | executor = new ThreadPoolExecutor(5,20,1000L,
37 | TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(2048));
38 |
39 | timer = Executors.newScheduledThreadPool(4);
40 |
41 | transfer = new ThreadPoolExecutor(2,5,100L,
42 | TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(256));
43 |
44 | Runtime.getRuntime().addShutdownHook(new Thread(){
45 | @Override
46 | public void run() {
47 | executor.shutdown();
48 | timer.shutdown();
49 | transfer.shutdown();
50 | }
51 | });
52 | }
53 |
54 | }
55 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/handler/ReconcileHandlerFactory.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler;
15 |
16 | import org.springframework.beans.factory.annotation.Autowired;
17 | import org.springframework.stereotype.Service;
18 |
19 | import javax.annotation.PostConstruct;
20 | import java.util.List;
21 |
22 | /**
23 | * The assembly and construction of the responsibility chain, and the management of the processor
24 | *
25 | * @author wesleywang
26 | * @Description: ReconcileHandlerFactory
27 | * @date 2020/6/19
28 | */
29 | @Service
30 | public class ReconcileHandlerFactory {
31 |
32 | @Autowired
33 | private List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler.executor;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
17 | import com.webank.blockchain.data.reconcile.entity.ReconcileResult;
18 | import com.webank.blockchain.data.reconcile.handler.Handler;
19 | import com.webank.blockchain.data.reconcile.handler.InvocationHandler;
20 |
21 | import java.io.File;
22 | import java.util.List;
23 | import java.util.concurrent.Future;
24 |
25 | /**
26 | * An abstract class that handles the reconciliation execution steps and is responsible for parsing and comparing
27 | * reconciliation files. It contains a core method for reconciliation execution and provides a default implementation
28 | *
29 | * @author wesleywang
30 | * @Description: ReconcileExecuteHandler
31 | * @date 2020/6/17
32 | */
33 | public abstract class ReconcileExecuteHandler implements Handler {
34 |
35 | @Override
36 | public void invoke(ReconcileContext context, InvocationHandler handler) throws Exception {
37 | List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.enums;
15 |
16 | import lombok.AllArgsConstructor;
17 | import lombok.Getter;
18 | import lombok.ToString;
19 | import lombok.extern.slf4j.Slf4j;
20 |
21 | /**
22 | * reconciliation task status flow：
23 | * 1.INIT -> EXECUTING -> SUCCESS
24 | * 2.INIT -> EXECUTING -> FAILURE -> SUCCESS
25 | * 3.INIT -> EXECUTING -> FAILURE -> TERMINATE
26 | *
27 | * @author wesleywang
28 | * @Description:TaskStatus
29 | * @date 2020/6/19
30 | */
31 | @Getter
32 | @ToString
33 | @AllArgsConstructor
34 | @Slf4j
35 | public enum TaskStatus {
36 |
37 | INIT(0, "waiting for executing"),
38 | EXECUTING(1, "executing"),
39 | SUCCESS(2, "execute successfully"),
40 | FAILURE(3,"execute failed"),
41 | TERMINATE(4,"execute terminated");
42 |
43 |
44 | private int status;
45 | private String describe;
46 |
47 | public static TaskStatus getByDescribe(String Describe){
48 | for(TaskStatus type : TaskStatus.values()){
49 | if(type.describe.equals(Describe)){
50 | return type;
51 | }
52 | }
53 | log.error("TaskStatus type {} can't be converted.", Describe);
54 | return null;
55 | }
56 |
57 | public static TaskStatus getByStatus(int status){
58 | for(TaskStatus type : TaskStatus.values()){
59 | if(type.status == status){
60 | return type;
61 | }
62 | }
63 | log.error("TaskStatus type {} error.", status);
64 | return null;
65 | }
66 |
67 |
68 | }
69 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/db/dao/TaskDao.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.db.dao;
15 |
16 | import com.webank.blockchain.data.reconcile.db.entity.TaskInfo;
17 | import com.webank.blockchain.data.reconcile.db.repository.TaskInfoRepository;
18 | import org.springframework.beans.factory.annotation.Autowired;
19 | import org.springframework.stereotype.Service;
20 |
21 | import java.util.Date;
22 | import java.util.List;
23 |
24 | /**
25 | * @author wesleywang
26 | * @Description:
27 | * @date 2020/6/19
28 | */
29 | @Service
30 | public class TaskDao {
31 |
32 | @Autowired
33 | private TaskInfoRepository repository;
34 |
35 | public void updateTaskStatus(long pkId, int from, int to){
36 | repository.updateStatus(pkId, from,to);
37 | }
38 |
39 | public void updateTaskStatus(long pkId, int from, int to, Date lastExecuteEndTime){
40 | repository.updateStatus(pkId, from,to,lastExecuteEndTime);
41 | }
42 |
43 | public TaskInfo save(TaskInfo taskInfo){
44 | return repository.save(taskInfo);
45 | }
46 |
47 | public List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.exporter.result;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileResult;
17 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
18 | import lombok.extern.slf4j.Slf4j;
19 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
20 | import org.springframework.stereotype.Service;
21 |
22 | import java.io.File;
23 | import java.time.LocalTime;
24 | import java.util.ArrayList;
25 | import java.util.List;
26 | import java.util.concurrent.Future;
27 |
28 | /**
29 | * @author wesleywang
30 | * @Description:
31 | * @date 2020/6/28
32 | */
33 | @ConditionalOnProperty(value = "reconcile.file.type", havingValue = "txt")
34 | @Service
35 | @Slf4j
36 | public class ResultDataExportServiceTxtImpl implements ResultDataExportService{
37 |
38 |
39 | @Override
40 | public File exportResultData(String filePath, List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileExecuteData;
17 | import com.webank.blockchain.data.reconcile.entity.ReconcileInfo;
18 | import com.webank.blockchain.data.reconcile.reconcile.ReconcileExecutor;
19 | import org.junit.Test;
20 | import org.springframework.beans.factory.annotation.Autowired;
21 |
22 | import java.util.HashMap;
23 | import java.util.Map;
24 |
25 | /**
26 | * @author wesleywang
27 | * @Description:
28 | * @date 2020/7/8
29 | */
30 | public class ReconcileExecutorTest extends BaseTests{
31 |
32 |
33 | @Autowired
34 | private ReconcileExecutor reconcileExecutor;
35 |
36 | @Test
37 | public void reconcileExecutorTest() throws Exception {
38 | ReconcileExecuteData reconcileExecuteData = new ReconcileExecuteData();
39 | ReconcileInfo busOrg = new ReconcileInfo();
40 | Map Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.db.repository;
15 |
16 | import com.webank.blockchain.data.reconcile.db.entity.TaskInfo;
17 | import org.springframework.data.jpa.repository.JpaRepository;
18 | import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
19 | import org.springframework.data.jpa.repository.Modifying;
20 | import org.springframework.data.jpa.repository.Query;
21 | import org.springframework.stereotype.Repository;
22 |
23 | import javax.transaction.Transactional;
24 | import java.util.Date;
25 | import java.util.List;
26 |
27 | /**
28 | * @author wesleywang
29 | * @Description:
30 | * @date 2020/6/19
31 | */
32 | @Repository
33 | public interface TaskInfoRepository extends JpaRepository Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.entity;
15 |
16 | import com.webank.blockchain.data.reconcile.db.entity.TaskInfo;
17 | import com.webank.blockchain.data.reconcile.enums.TriggerType;
18 | import lombok.Builder;
19 | import lombok.Data;
20 |
21 | import java.io.File;
22 | import java.util.Date;
23 | import java.util.List;
24 | import java.util.concurrent.Future;
25 |
26 | /**
27 | * @author wesleywang
28 | * @Description:
29 | * @date 2020/6/17
30 | */
31 | @Data
32 | @Builder
33 | public class ReconcileContext {
34 |
35 | /**
36 | * Reconciliation task trigger mode
37 | */
38 | private TriggerType triggerType;
39 |
40 | /**
41 | * Whether the reconciliation task is enabled asynchronously
42 | */
43 | private boolean async;
44 |
45 | /**
46 | * Name of reconciliation document of business party
47 | */
48 | private String businessFileName;
49 |
50 | /**
51 | * The start time of the reconciliation data range
52 | */
53 | private Date dataRangeBeginTime;
54 |
55 | /**
56 | * The end time of the reconciliation data range
57 | */
58 | private Date dataRangeEndTime;
59 |
60 | /**
61 | * Reconciliation task detail object
62 | */
63 | private TaskInfo taskInfo;
64 |
65 | /**
66 | * Reconciliation business party documents
67 | */
68 | private File businessReconFile;
69 |
70 | /**
71 | * Reconciliation BC party documents
72 | */
73 | private File bcReconFile;
74 |
75 | /**
76 | * Reconciliation task results
77 | */
78 | private List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.task;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
17 | import com.webank.blockchain.data.reconcile.enums.TriggerType;
18 | import com.webank.blockchain.data.reconcile.handler.InvocationHandler;
19 | import com.webank.blockchain.data.reconcile.handler.ReconcileHandlerFactory;
20 | import lombok.extern.slf4j.Slf4j;
21 | import org.springframework.beans.factory.annotation.Autowired;
22 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
23 | import org.springframework.scheduling.annotation.EnableScheduling;
24 | import org.springframework.scheduling.annotation.Scheduled;
25 | import org.springframework.stereotype.Component;
26 |
27 | import java.time.LocalDate;
28 |
29 | /**
30 | * Timed trigger processing of reconciliation
31 | *
32 | * @author wesleywang
33 | * @Description:
34 | * @date 2020/6/17
35 | */
36 | @Component
37 | @Slf4j
38 | @EnableScheduling
39 | @ConditionalOnProperty(value = "reconcile.task.timer.enable", havingValue = "true")
40 | public class ReconcileTaskTimer {
41 |
42 | @Autowired
43 | private ReconcileHandlerFactory handlerFactory;
44 |
45 | @Scheduled(cron = "${reconcile.task.time.rule}")
46 | public void execute() {
47 | InvocationHandler handler = handlerFactory.getInvocationHandler();
48 | ReconcileContext context = ReconcileContext.builder()
49 | .triggerType(TriggerType.SCHEDULED).build();
50 | log.info("reconcile schedule task start , time :" + LocalDate.now().toString());
51 | try {
52 | handler.handle(context);
53 | } catch (Exception e) {
54 | log.error("reconcile schedule task failed, reason :", e);
55 | }
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/handler/executor/DefaultReconcileExecuteHandler.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler.executor;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileResult;
17 | import com.webank.blockchain.data.reconcile.parser.FileParser;
18 | import com.webank.blockchain.data.reconcile.reconcile.ReconcileExecutor;
19 | import com.webank.blockchain.data.reconcile.reconcile.ReconcileTransfer;
20 | import com.webank.blockchain.data.reconcile.utils.ThreadUtils;
21 | import org.springframework.beans.factory.annotation.Autowired;
22 | import org.springframework.core.annotation.Order;
23 | import org.springframework.stereotype.Service;
24 |
25 | import java.io.File;
26 | import java.util.ArrayList;
27 | import java.util.List;
28 | import java.util.concurrent.Future;
29 |
30 | /**
31 | * The default implementation for the reconciliation processing step consists of two modules, parsing
32 | * {@link FileParser} and executing {@link ReconcileExecutor}, with a decoupling between the modules through
33 | * the {@link ReconcileTransfer} interface
34 | *
35 | * @author wesleywang
36 | * @Description: DefaultReconcileExecuteHandler
37 | * @date 2020/6/23
38 | */
39 | @Service
40 | @Order(3)
41 | public class DefaultReconcileExecuteHandler extends ReconcileExecuteHandler{
42 |
43 | @Autowired
44 | private FileParser fileParser;
45 | @Autowired
46 | private ReconcileExecutor reconcileExecutor;
47 |
48 | @Override
49 | List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.exporter.bc;
15 |
16 | import com.webank.blockchain.data.reconcile.db.dao.BCInfoDao;
17 | import com.webank.blockchain.data.reconcile.utils.BufferedRandomAccessFile;
18 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
19 | import org.apache.tomcat.util.http.fileupload.IOUtils;
20 | import org.springframework.beans.factory.annotation.Autowired;
21 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
22 | import org.springframework.data.domain.Page;
23 | import org.springframework.stereotype.Service;
24 |
25 | import java.io.File;
26 | import java.util.Date;
27 | import java.util.List;
28 | import java.util.Map;
29 |
30 | /**
31 | * @author wesleywang
32 | * @Description:
33 | * @date 2020/6/24
34 | */
35 | @Service
36 | @ConditionalOnProperty(value = "reconcile.file.type", havingValue = "txt")
37 | public class BCDataExportServiceTxtImpl implements BCDataExportService{
38 |
39 | @Autowired
40 | private BCInfoDao bcInfoDao;
41 |
42 | @Override
43 | public File exportData(String filePath, Date dataRangeBeginTime, Date dataRangeEndTime) throws Exception {
44 | File file = FileUtils.newFile(filePath);
45 | Page Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler.finish;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
17 | import com.webank.blockchain.data.reconcile.entity.ReconcileResult;
18 | import com.webank.blockchain.data.reconcile.exception.ReconcileException;
19 | import com.webank.blockchain.data.reconcile.handler.Handler;
20 | import com.webank.blockchain.data.reconcile.handler.InvocationHandler;
21 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
22 |
23 | import java.io.File;
24 | import java.util.List;
25 | import java.util.concurrent.Future;
26 |
27 | /**
28 | * The abstract class for reconciliation result processing, which is the end of the entire chain of responsibility,
29 | * is responsible for exporting and transmitting the reconciliation result and provides the default implementation
30 | *
31 | * @author wesleywang
32 | * @Description: ReconcileResultHandler
33 | * @date 2020/6/17
34 | */
35 | public abstract class ReconcileResultHandler implements Handler {
36 |
37 |
38 | private ReconcileContext reconcileContext;
39 |
40 | @Override
41 | public void invoke(ReconcileContext context, InvocationHandler handler) throws Exception {
42 | reconcileContext = context;
43 | File file = null;
44 | try {
45 | file = exportData(context.getReconcileResults());
46 | }catch (Exception e){
47 | FileUtils.clearReconcileFileCache(context);
48 | throw new ReconcileException("reconcile result data export failed", e);
49 | }
50 | context.getTaskInfo().setResultFilePath(file.getPath());
51 | context.setResultFile(file);
52 | transferResultFile(file);
53 | }
54 |
55 | abstract File exportData(List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.exporter.result;
15 |
16 | import com.fasterxml.jackson.core.JsonEncoding;
17 | import com.fasterxml.jackson.core.JsonFactory;
18 | import com.fasterxml.jackson.core.JsonGenerator;
19 | import com.fasterxml.jackson.databind.ObjectMapper;
20 | import com.webank.blockchain.data.reconcile.entity.ReconcileResult;
21 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
22 | import lombok.extern.slf4j.Slf4j;
23 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
24 | import org.springframework.stereotype.Service;
25 |
26 | import java.io.File;
27 | import java.time.LocalTime;
28 | import java.util.ArrayList;
29 | import java.util.List;
30 | import java.util.concurrent.Future;
31 |
32 | /**
33 | * @author wesleywang
34 | * @Description:
35 | * @date 2020/7/1
36 | */
37 | @ConditionalOnProperty(value = "reconcile.file.type", havingValue = "json")
38 | @Service
39 | @Slf4j
40 | public class ResultDataExportServiceJsonImpl implements ResultDataExportService{
41 |
42 | @Override
43 | public File exportResultData(String filePath, List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler.filesource;
15 |
16 | import com.webank.blockchain.data.reconcile.db.entity.TaskInfo;
17 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
18 | import com.webank.blockchain.data.reconcile.exception.ReconcileException;
19 | import com.webank.blockchain.data.reconcile.handler.Handler;
20 | import com.webank.blockchain.data.reconcile.handler.InvocationHandler;
21 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
22 |
23 | import java.io.File;
24 |
25 | /**
26 | * An abstract class that obtain reconciliation files, responsible for the steps of file acquisition,
27 | * including business side reconciliation files and block chain reconciliation file acquisition template method,
28 | * providing default implementation
29 | *
30 | * @author wesleywang
31 | * @Description: ReconcileFileObtainHandler
32 | * @date 2020/6/17
33 | */
34 | public abstract class ReconcileFileObtainHandler implements Handler {
35 |
36 | @Override
37 | public void invoke(ReconcileContext context, InvocationHandler handler) throws Exception {
38 | File businessReconcileFile = null;
39 | File bcReconcileFile = null;
40 | try {
41 | businessReconcileFile = getBusinessReconcileFile(context.getTaskInfo());
42 | bcReconcileFile = getBCReconcileFile(context.getTaskInfo());
43 | context.setBcReconFile(bcReconcileFile);
44 | context.setBusinessReconFile(businessReconcileFile);
45 | }catch (Exception e){
46 | FileUtils.clearReconcileFileCache(context);
47 | throw new ReconcileException("reconcile file obtain failed", e);
48 | }
49 | TaskInfo taskInfo = context.getTaskInfo();
50 | context.setBusinessFileName(taskInfo.getBusinessFileName());
51 | taskInfo.setBcFilePath(bcReconcileFile.getPath());
52 | taskInfo.setBusinessFilePath(businessReconcileFile.getPath());
53 |
54 | handler.handle(context);
55 | }
56 |
57 | abstract File getBCReconcileFile(TaskInfo taskInfo) throws Exception;
58 |
59 | abstract File getBusinessReconcileFile(TaskInfo taskInfo) throws Exception;
60 |
61 | }
62 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/handler/finish/DefaultReconcileResultHandler.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler.finish;
15 |
16 | import com.webank.blockchain.data.reconcile.entity.ReconcileResult;
17 | import com.webank.blockchain.data.reconcile.exporter.result.ResultDataExportService;
18 | import com.webank.blockchain.data.reconcile.filetransfer.FileTransferService;
19 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
20 | import com.webank.blockchain.data.reconcile.utils.ThreadUtils;
21 | import lombok.extern.slf4j.Slf4j;
22 | import org.springframework.beans.factory.annotation.Autowired;
23 | import org.springframework.core.annotation.Order;
24 | import org.springframework.stereotype.Service;
25 |
26 | import java.io.File;
27 | import java.util.List;
28 | import java.util.concurrent.Future;
29 |
30 | /**
31 | * The default implementation of the abstract class of account checking results contains two modules:
32 | * export {@link FileTransferService} of the result data and
33 | * transfer {@link ResultDataExportService} of the reconciliation result file.
34 | *
35 | * @author wesleywang
36 | * @Description: DefaultReconcileResultHandler
37 | * @date 2020/6/23
38 | */
39 | @Service
40 | @Slf4j
41 | @Order(4)
42 | public class DefaultReconcileResultHandler extends ReconcileResultHandler{
43 |
44 | @Autowired
45 | private FileTransferService fileTransferService;
46 | @Autowired
47 | private ResultDataExportService resultDataExportService;
48 | @Override
49 | File exportData(List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.filetransfer.ftp;
15 |
16 | import com.webank.blockchain.data.reconcile.config.FTPConfig;
17 | import lombok.extern.slf4j.Slf4j;
18 | import org.apache.commons.net.ftp.FTPClient;
19 | import org.apache.commons.pool2.impl.GenericObjectPool;
20 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
21 | import org.springframework.beans.factory.annotation.Autowired;
22 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
23 | import org.springframework.stereotype.Component;
24 |
25 | /**
26 | * @author wesleywang
27 | * @Description:
28 | * @date 2020/6/16
29 | */
30 | @ConditionalOnProperty(value = "ftp.enabled", havingValue = "true")
31 | @Component
32 | @Slf4j
33 | public class FtpPool {
34 |
35 | FtpClientFactory factory;
36 | private final GenericObjectPool Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.task;
15 |
16 | import cn.hutool.core.collection.CollectionUtil;
17 | import com.webank.blockchain.data.reconcile.db.dao.TaskDao;
18 | import com.webank.blockchain.data.reconcile.db.entity.TaskInfo;
19 | import com.webank.blockchain.data.reconcile.enums.TaskStatus;
20 | import com.webank.blockchain.data.reconcile.enums.TriggerType;
21 | import org.springframework.beans.factory.annotation.Autowired;
22 | import org.springframework.stereotype.Service;
23 |
24 | import java.net.InetAddress;
25 | import java.util.Date;
26 | import java.util.List;
27 | import java.util.UUID;
28 |
29 | /**
30 | * @Description: ReconclieTask
31 | * @author graysonzhang
32 | * @date 2020-06-10 16:30:06
33 | */
34 | @Service
35 | public class ReconcileTaskService {
36 |
37 | @Autowired
38 | private TaskDao taskDao;
39 | @Autowired
40 | private InetAddress inetAddress;
41 |
42 | public TaskInfo createReconcileTask(TriggerType triggered, String businessFileName,
43 | Date dataRangeBeginTime, Date dataRangeEndTime){
44 | TaskInfo task = new TaskInfo();
45 | task.setNodeId(inetAddress.getHostAddress())
46 | .setTaskId(UUID.randomUUID().toString())
47 | .setStatus(TaskStatus.INIT.getStatus())
48 | .setBusinessFileName(businessFileName)
49 | .setDataRangeBeginTime(dataRangeBeginTime)
50 | .setDataRangeEndTime(dataRangeEndTime)
51 | .setTriggered(triggered.getType());
52 | return save(task);
53 | }
54 |
55 | public TaskInfo save(TaskInfo taskInfo){
56 | return taskDao.save(taskInfo);
57 | }
58 |
59 | public void changeReconcileTaskStatus(long pkId, int from, int to){
60 | taskDao.updateTaskStatus(pkId, from,to);
61 | }
62 |
63 | public void changeReconcileTaskStatus(long pkId, int from, int to, Date lastExecuteEndTime){
64 | taskDao.updateTaskStatus(pkId, from,to,lastExecuteEndTime);
65 | }
66 |
67 | public List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.exporter.bc;
15 |
16 | import com.fasterxml.jackson.databind.ObjectMapper;
17 | import com.webank.blockchain.data.reconcile.db.dao.BCInfoDao;
18 | import com.webank.blockchain.data.reconcile.utils.BufferedRandomAccessFile;
19 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
20 | import org.apache.tomcat.util.http.fileupload.IOUtils;
21 | import org.springframework.beans.factory.annotation.Autowired;
22 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
23 | import org.springframework.data.domain.Page;
24 | import org.springframework.stereotype.Service;
25 |
26 | import java.io.File;
27 | import java.util.Date;
28 | import java.util.List;
29 | import java.util.Map;
30 |
31 | /**
32 | * @author wesleywang
33 | * @Description:
34 | * @date 2020/7/14
35 | */
36 | @Service
37 | @ConditionalOnProperty(value = "reconcile.file.type", havingValue = "json")
38 | public class BCDataExportServiceJsonImpl implements BCDataExportService{
39 |
40 |
41 | @Autowired
42 | private BCInfoDao bcInfoDao;
43 |
44 | @Override
45 | public File exportData(String filePath, Date dataRangeBeginTime, Date dataRangeEndTime) throws Exception {
46 | File file = FileUtils.newFile(filePath);
47 | Page Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.reconcile;
15 |
16 | import com.webank.blockchain.data.reconcile.config.ReconcileConfig;
17 | import com.webank.blockchain.data.reconcile.entity.ReconcileExecuteData;
18 | import com.webank.blockchain.data.reconcile.entity.ReconcileResult;
19 | import lombok.extern.slf4j.Slf4j;
20 | import org.springframework.beans.factory.annotation.Autowired;
21 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
22 | import org.springframework.stereotype.Component;
23 |
24 | import java.util.Map;
25 |
26 | /**
27 | * @author wesleywang
28 | * @Description:
29 | * @date 2020/6/24
30 | */
31 | @ConditionalOnProperty(value = "reconcile.general.enabled", havingValue = "true")
32 | @Component
33 | @Slf4j
34 | public class ReconcileExecutorImpl implements ReconcileExecutor {
35 |
36 | @Autowired
37 | private ReconcileConfig reconcileConfig;
38 |
39 | @Override
40 | public ReconcileResult execute(ReconcileExecuteData executeData) {
41 | Map Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.db.dao;
15 |
16 | import cn.hutool.core.date.DatePattern;
17 | import cn.hutool.core.date.DateUtil;
18 | import com.google.common.collect.Lists;
19 | import com.webank.blockchain.data.reconcile.config.ReconcileConfig;
20 | import com.webank.blockchain.data.reconcile.utils.StringUtils;
21 | import org.hibernate.query.internal.NativeQueryImpl;
22 | import org.hibernate.transform.Transformers;
23 | import org.springframework.beans.factory.annotation.Autowired;
24 | import org.springframework.data.domain.Page;
25 | import org.springframework.data.domain.PageImpl;
26 | import org.springframework.data.domain.PageRequest;
27 | import org.springframework.data.domain.Pageable;
28 | import org.springframework.stereotype.Service;
29 | import org.springframework.transaction.annotation.Transactional;
30 |
31 | import javax.annotation.PostConstruct;
32 | import javax.persistence.EntityManager;
33 | import javax.persistence.PersistenceContext;
34 | import javax.persistence.Query;
35 | import java.math.BigInteger;
36 | import java.util.Date;
37 | import java.util.List;
38 | import java.util.Map;
39 |
40 | /**
41 | * @author wesleywang
42 | * @Description:
43 | * @date 2020/6/22
44 | */
45 | @Service
46 | @Transactional(readOnly = true)
47 | public class BCInfoDao {
48 |
49 | @Autowired
50 | private ReconcileConfig reconcileConfig;
51 | @PersistenceContext
52 | private EntityManager entityManager;
53 |
54 | private String bcDataQuerySql;
55 |
56 | private String bcDataCountSql;
57 |
58 | @PostConstruct
59 | private void sqlInit(){
60 | bcDataQuerySql = reconcileConfig.getReconcileQuerySql();
61 | bcDataCountSql = reconcileConfig.getReconcileCountSql();
62 | }
63 |
64 | public Page Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler.filesource;
15 |
16 | import cn.hutool.core.date.DateUtil;
17 | import com.webank.blockchain.data.reconcile.config.ReconcileConfig;
18 | import com.webank.blockchain.data.reconcile.db.entity.TaskInfo;
19 | import com.webank.blockchain.data.reconcile.exporter.bc.BCDataExportService;
20 | import com.webank.blockchain.data.reconcile.filetransfer.FileTransferService;
21 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
22 | import org.springframework.beans.factory.annotation.Autowired;
23 | import org.springframework.core.annotation.Order;
24 | import org.springframework.stereotype.Service;
25 |
26 | import java.io.File;
27 | import java.time.LocalDate;
28 | import java.util.Date;
29 |
30 | /**
31 | * The default implementation of the abstract class that gets the reconciliation file includes the business data
32 | * and blockchain data,where the business data is obtained through the {@link FileTransferService} interface and
33 | * the blockchain data is obtained through {@link BCDataExportService}
34 | *
35 | * @author wesleywang
36 | * @Description: DefaultReconcileFileObtainHandler
37 | * @date 2020/6/19
38 | */
39 | @Service
40 | @Order(2)
41 | public class DefaultReconcileFileObtainHandler extends ReconcileFileObtainHandler {
42 |
43 | @Autowired
44 | private FileTransferService fileService;
45 | @Autowired
46 | private BCDataExportService bcDataExportService;
47 | @Autowired
48 | private ReconcileConfig reconcileConfig;
49 |
50 | @Override
51 | File getBCReconcileFile(TaskInfo taskInfo) throws Exception {
52 | // Block chain reconciliation file name generation
53 | String filePath = FileUtils.BC_RECONCILE_FILEPATH + "BC_" + taskInfo.getBusinessFileName().split("_")[1];
54 |
55 | taskInfo.setBcFilePath(filePath);
56 | return bcDataExportService.exportData(filePath, taskInfo.getDataRangeBeginTime(),
57 | taskInfo.getDataRangeEndTime());
58 | }
59 |
60 | @Override
61 | File getBusinessReconcileFile(TaskInfo taskInfo) throws Exception {
62 | String fileName = taskInfo.getBusinessFileName();
63 | if (fileName == null) {
64 | // Timing task filename generation rule: reconciliation is carried out across days, and the beginning time
65 | // of reconciliation data is taken as part of reconciliation name
66 | String time = LocalDate.now().minusDays(reconcileConfig.getDaysTimeRange()).toString();
67 | fileName = reconcileConfig.getBusinessName() + "_" + time + "." + reconcileConfig.getFileType();
68 | Date startDate = DateUtil.parse(time);
69 | Date endDate = DateUtil.offsetDay(startDate, reconcileConfig.getDaysTimeRange());
70 |
71 | taskInfo.setDataRangeBeginTime(startDate);
72 | taskInfo.setDataRangeEndTime(endDate);
73 | taskInfo.setBusinessFileName(fileName);
74 | }
75 | taskInfo.setBusinessFilePath(FileUtils.BUS_RECONCILE_FILEPATH + fileName);
76 | return fileService.obtainFile(fileName, taskInfo.getBusinessFilePath());
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/config/ReconcileConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.config;
15 |
16 | import cn.hutool.core.collection.CollectionUtil;
17 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
18 | import lombok.Data;
19 | import lombok.extern.slf4j.Slf4j;
20 | import org.springframework.beans.factory.annotation.Value;
21 | import org.springframework.boot.context.properties.ConfigurationProperties;
22 | import org.springframework.context.annotation.Configuration;
23 | import org.springframework.core.annotation.Order;
24 |
25 | import javax.annotation.PostConstruct;
26 | import java.util.HashMap;
27 | import java.util.Map;
28 |
29 | /**
30 | * @author wesleywang
31 | * @Description:
32 | * @date 2020-06-11
33 | */
34 | @Configuration
35 | @ConfigurationProperties("reconcile")
36 | @Data
37 | @Order(1)
38 | @Slf4j
39 | public class ReconcileConfig {
40 |
41 | @Value("${reconcile.task.timer.enable}")
42 | private boolean timerEnable;
43 |
44 | @Value("${reconcile.task.time.range.days}")
45 | private int daysTimeRange;
46 |
47 | @Value("${reconcile.task.time.rule}")
48 | private String taskTimeRule;
49 |
50 | @Value("${reconcile.task.timeout}")
51 | private long taskTimeout;
52 |
53 | @Value("${reconcile.task.retry.interval.time}")
54 | private long retryIntervalTime;
55 |
56 | @Value("${reconcile.task.retry.count}")
57 | private int retryCount;
58 |
59 | @Value("${reconcile.business.name}")
60 | private String businessName;
61 |
62 | @Value("${reconcile.general.enabled}")
63 | private Boolean generalEnabled;
64 |
65 | @Value("${reconcile.file.type}")
66 | private String fileType;
67 |
68 | @Value("${reconcile.field.business.uniqueColumn}")
69 | private String businessUniqueColumn;
70 |
71 | @Value("${reconcile.field.bc.uniqueColumn}")
72 | private String bcUniqueColumn;
73 |
74 | @Value("${reconcile.bc.reconcileQuerySql}")
75 | private String reconcileQuerySql;
76 |
77 | @Value("${reconcile.bc.reconcileCountSql}")
78 | private String reconcileCountSql;
79 |
80 | @Value("${reconcile.bc.QueryTimeField}")
81 | private String reconcileQueryTimeField;
82 |
83 | private Map Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.filetransfer.ftp;
15 |
16 | import com.webank.blockchain.data.reconcile.config.FTPConfig;
17 | import org.apache.commons.net.ftp.FTP;
18 | import org.apache.commons.net.ftp.FTPClient;
19 | import org.apache.commons.pool2.PooledObject;
20 | import org.apache.commons.pool2.PooledObjectFactory;
21 | import org.apache.commons.pool2.impl.DefaultPooledObject;
22 | import org.springframework.beans.factory.annotation.Autowired;
23 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
24 | import org.springframework.stereotype.Component;
25 |
26 | import java.io.IOException;
27 |
28 | /**
29 | * @author wesleywang
30 | * @Description: FtpClientFactory
31 | * @date 2020/6/16
32 | */
33 | @ConditionalOnProperty(value = "ftp.enabled", havingValue = "true")
34 | @Component
35 | public class FtpClientFactory implements PooledObjectFactory Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.db.entity;
15 |
16 | import lombok.Data;
17 | import lombok.EqualsAndHashCode;
18 | import lombok.experimental.Accessors;
19 | import org.hibernate.annotations.CreationTimestamp;
20 | import org.hibernate.annotations.UpdateTimestamp;
21 | import org.springframework.data.annotation.LastModifiedDate;
22 |
23 | import javax.persistence.Column;
24 | import javax.persistence.Entity;
25 | import javax.persistence.Index;
26 | import javax.persistence.Table;
27 | import javax.persistence.Temporal;
28 | import javax.persistence.TemporalType;
29 | import java.util.Date;
30 |
31 | /**
32 | * @author wesleywang
33 | * @Description:
34 | * @date 2020/6/17
35 | */
36 | @Data
37 | @Accessors(chain = true)
38 | @Entity(name = "task_info")
39 | @Table(name = "task_info",indexes = { @Index(name = "status", columnList = "status")})
40 | @EqualsAndHashCode(callSuper = true)
41 | public class TaskInfo extends IdEntity {
42 | private static final long serialVersionUID = 9096084645189100625L;
43 |
44 | /**
45 | * Task unique id
46 | */
47 | @Column(name = "task_id", unique = true)
48 | private String taskId;
49 |
50 | /**
51 | * Execution node ip
52 | */
53 | @Column(name = "node_id")
54 | private String nodeId;
55 |
56 | /**
57 | * task status
58 | */
59 | @Column(name = "status")
60 | private int status;
61 |
62 | /**
63 | * Mode of triggering
64 | */
65 | @Column(name = "triggered")
66 | private int triggered;
67 |
68 | /**
69 | * Name of reconciliation document of business party
70 | */
71 | @Column(name = "business_file_name")
72 | private String businessFileName;
73 |
74 | /**
75 | * Path of reconciliation document of business party
76 | */
77 | @Column(name = "business_file_path")
78 | private String businessFilePath;
79 |
80 | /**
81 | * Path of reconciliation document of block chain
82 | */
83 | @Column(name = "bc_file_path")
84 | private String bcFilePath;
85 |
86 | /**
87 | * Path of reconciliation document of result
88 | */
89 | @Column(name = "result_file_path")
90 | private String resultFilePath;
91 |
92 | /**
93 | * Last execution start time
94 | */
95 | @Column(name = "last_execute_starttime")
96 | private Date lastExecuteStartTime;
97 |
98 | /**
99 | * Last execution end time
100 | */
101 | @Column(name = "last_execute_endtime")
102 | private Date lastExecuteEndTime;
103 |
104 | /**
105 | * Number of failed retries
106 | */
107 | @Column(name = "retry_count")
108 | private int retryCount;
109 |
110 | /**
111 | * The start time of the reconciliation data range
112 | */
113 | @Column(name = "data_range_begintime")
114 | private Date dataRangeBeginTime;
115 |
116 | /**
117 | * The end time of the reconciliation data range
118 | */
119 | @Column(name = "data_range_endtime")
120 | private Date dataRangeEndTime;
121 |
122 | @CreationTimestamp
123 | @Column(name = "createtime", columnDefinition = "timestamp NOT NULL DEFAULT '2019-01-01 00:00:00'", insertable = true)
124 | @Temporal(TemporalType.TIMESTAMP)
125 | private Date createTime;
126 |
127 | @UpdateTimestamp
128 | @Column(name = "updatetime", columnDefinition = "timestamp not null DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP", updatable = true)
129 | @Temporal(TemporalType.TIMESTAMP)
130 | @LastModifiedDate
131 | protected Date updateTime;
132 |
133 | }
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/controller/ReconcileController.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.controller;
15 |
16 | import cn.hutool.core.io.FileUtil;
17 | import com.webank.blockchain.data.reconcile.constants.ResponseCode;
18 | import com.webank.blockchain.data.reconcile.db.entity.TaskInfo;
19 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
20 | import com.webank.blockchain.data.reconcile.entity.Responses;
21 | import com.webank.blockchain.data.reconcile.enums.TriggerType;
22 | import com.webank.blockchain.data.reconcile.handler.InvocationHandler;
23 | import com.webank.blockchain.data.reconcile.handler.ReconcileHandlerFactory;
24 | import com.webank.blockchain.data.reconcile.task.ReconcileTaskService;
25 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
26 | import lombok.extern.slf4j.Slf4j;
27 | import org.springframework.beans.factory.annotation.Autowired;
28 | import org.springframework.format.annotation.DateTimeFormat;
29 | import org.springframework.web.bind.annotation.GetMapping;
30 | import org.springframework.web.bind.annotation.PathVariable;
31 | import org.springframework.web.bind.annotation.RequestMapping;
32 | import org.springframework.web.bind.annotation.RequestParam;
33 | import org.springframework.web.bind.annotation.RestController;
34 |
35 | import java.time.LocalDate;
36 | import java.util.Date;
37 |
38 | /**
39 | * ReconcileController
40 | *
41 | * @Description: ReconcileController
42 | * @author graysonzhang
43 | * @date 2020-06-10 16:28:22
44 | *
45 | */
46 | @RestController
47 | @Slf4j
48 | @RequestMapping("reconcile")
49 | public class ReconcileController {
50 |
51 | @Autowired
52 | private ReconcileHandlerFactory handlerFactory;
53 | @Autowired
54 | private ReconcileTaskService taskService;
55 |
56 | @RequestMapping(value = "/requestReconcile")
57 | public Responses Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.filetransfer.ftp;
15 |
16 | import com.webank.blockchain.data.reconcile.config.FTPConfig;
17 | import com.webank.blockchain.data.reconcile.filetransfer.FileTransferService;
18 | import lombok.extern.slf4j.Slf4j;
19 | import org.apache.commons.net.ftp.FTPClient;
20 | import org.apache.commons.net.ftp.FTPFile;
21 | import org.springframework.beans.factory.annotation.Autowired;
22 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
23 | import org.springframework.stereotype.Component;
24 |
25 | import java.io.File;
26 | import java.io.FileInputStream;
27 | import java.io.FileOutputStream;
28 | import java.io.IOException;
29 | import java.io.OutputStream;
30 |
31 | /**
32 | * @author wesleywang
33 | * @Description:
34 | * @date 2020/6/16
35 | */
36 | @ConditionalOnProperty(value = "ftp.enabled", havingValue = "true")
37 | @Component
38 | @Slf4j
39 | public class FtpFileTransferService implements FileTransferService {
40 |
41 | @Autowired
42 | FTPConfig config;
43 | @Autowired
44 | FtpPool pool;
45 |
46 |
47 | /**
48 | * Upload files to the FTP server
49 | *
50 | * @param file Upload files to an FTP server
51 | * @return Returns file name successfully, otherwise returns NULL
52 | */
53 | public String upload(File file) throws Exception {
54 | FTPClient ftpClient = pool.getFTPClient();
55 | //开始进行文件上传
56 | String fileName = file.getName();
57 | try (FileInputStream input = new FileInputStream(file)) {
58 | boolean result = ftpClient.storeFile(fileName, input);//执行文件传输
59 | if (!result) {
60 | throw new RuntimeException("Upload failed");
61 | }
62 | } catch (Exception e) {
63 | log.error("file upload failed , reason : ", e);
64 | return null;
65 | } finally {
66 | log.info("Start returning the connection");
67 | pool.returnFTPClient(ftpClient);//Return the resources
68 | }
69 | return fileName;
70 | }
71 |
72 | /**
73 | * Download files from the FTP server
74 | *
75 | * @param fileName File name in the FTP server
76 | * @param localFilePath Local save path
77 | */
78 | public File downLoad(String fileName, String localFilePath) throws IOException {
79 | FTPClient ftpClient = pool.getFTPClient();
80 | File file = new File(localFilePath);
81 | OutputStream out = new FileOutputStream(file);
82 | if(ftpClient.retrieveFile(fileName, out)){
83 | out.flush();
84 | out.close();
85 | }else {
86 | log.error("origin file downLoad failed, fileName = " + fileName);
87 | }
88 | pool.returnFTPClient(ftpClient);
89 | return file;
90 | }
91 |
92 |
93 | /**
94 | * Query directory file
95 | * @param ftpDirectory
96 | */
97 | public FTPFile[] getFileList(String ftpDirectory) throws IOException {
98 | FTPFile[] ftpFiles = null;
99 | FTPClient ftpClient = pool.getFTPClient();
100 | try {
101 | ftpFiles = ftpClient.listFiles(ftpDirectory);
102 | }
103 | finally {
104 | log.info("Start returning the connection");
105 | pool.returnFTPClient(ftpClient);
106 | }
107 | return ftpFiles;
108 | }
109 |
110 | /**
111 | * Delete files on FTP
112 | * @param ftpDirAndFileName
113 | */
114 | public boolean deleteFile(String ftpDirAndFileName) throws IOException {
115 | FTPClient ftpClient = pool.getFTPClient();
116 | try {
117 | ftpClient.deleteFile(ftpDirAndFileName);
118 | } finally {
119 | log.info("Start returning the connection");
120 | pool.returnFTPClient(ftpClient);
121 | }
122 | return true;
123 | }
124 |
125 | /**
126 | * Delete FTP directory
127 | * @param ftpDirectory
128 | */
129 | public boolean deleteDirectory(String ftpDirectory) throws IOException {
130 | FTPClient ftpClient = pool.getFTPClient();
131 | try {
132 | ftpClient.removeDirectory(ftpDirectory);
133 | } finally {
134 | log.info("Start returning the connection");
135 | pool.returnFTPClient(ftpClient);
136 | }
137 | return true;
138 | }
139 |
140 | @Override
141 | public String sendFile(File file) throws Exception {
142 | return upload(file);
143 | }
144 |
145 | @Override
146 | public File obtainFile(String fileName, String localFilePath) throws IOException {
147 | return downLoad(fileName,localFilePath);
148 | }
149 | }
150 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/handler/task/ReconcileTaskHandler.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.handler.task;
15 |
16 | import com.webank.blockchain.data.reconcile.db.entity.TaskInfo;
17 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
18 | import com.webank.blockchain.data.reconcile.enums.TaskStatus;
19 | import com.webank.blockchain.data.reconcile.exception.ReconcileException;
20 | import com.webank.blockchain.data.reconcile.handler.Handler;
21 | import com.webank.blockchain.data.reconcile.handler.InvocationHandler;
22 | import com.webank.blockchain.data.reconcile.task.ReconcileTaskService;
23 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
24 | import com.webank.blockchain.data.reconcile.utils.ThreadUtils;
25 | import lombok.extern.slf4j.Slf4j;
26 | import org.springframework.beans.factory.annotation.Autowired;
27 | import org.springframework.core.annotation.Order;
28 | import org.springframework.stereotype.Service;
29 | import org.springframework.transaction.annotation.Transactional;
30 | import org.springframework.transaction.support.TransactionSynchronizationAdapter;
31 | import org.springframework.transaction.support.TransactionSynchronizationManager;
32 |
33 | import java.util.Date;
34 | import java.util.concurrent.TimeUnit;
35 |
36 | /**
37 | * Reconciliation task flow management, all exceptions will be thrown to this layer to resolve, is the beginning
38 | * of the entire chain of responsibility
39 | *
40 | * @author wesleywang
41 | * @Description: ReconcileTaskHandler
42 | * @date 2020/6/17
43 | */
44 | @Service
45 | @Order(1)
46 | @Slf4j
47 | public class ReconcileTaskHandler implements Handler {
48 |
49 | @Autowired
50 | private ReconcileTaskService taskService;
51 |
52 | @Transactional
53 | @Override
54 | public void invoke(ReconcileContext context, InvocationHandler handler) throws ReconcileException {
55 | TaskInfo taskInfo = context.getTaskInfo();
56 | if(taskInfo != null && taskInfo.getStatus() == TaskStatus.FAILURE.getStatus()){
57 | int retryCount = taskInfo.getRetryCount();
58 | taskInfo.setRetryCount(++retryCount);
59 | log.info("the failed reconcile task retry time ++, the task id=" + taskInfo.getTaskId());
60 | }else {
61 | if (taskService.existTaskInfo(context.getBusinessFileName())){
62 | throw new ReconcileException("the reconcile task already exists，filename = " + context.getBusinessFileName());
63 | }
64 | taskInfo = taskService.createReconcileTask(context.getTriggerType(), context.getBusinessFileName(),
65 | context.getDataRangeBeginTime(),context.getDataRangeEndTime());
66 | log.info("create a reconcile task, the task id=" + taskInfo.getTaskId());
67 | context.setTaskInfo(taskInfo);
68 | }
69 |
70 | if (context.isAsync()){
71 | ThreadUtils.timer.schedule(() ->
72 | taskExecute(context,handler),100L, TimeUnit.MILLISECONDS);
73 | return;
74 | }
75 | taskExecute(context,handler);
76 | }
77 |
78 | private void taskExecute(ReconcileContext context, InvocationHandler handler){
79 | TaskInfo taskInfo = context.getTaskInfo();
80 | //reconcile begins
81 | taskInfo.setStatus(TaskStatus.EXECUTING.getStatus())
82 | .setLastExecuteStartTime(new Date());
83 | taskService.save(taskInfo);
84 | log.info("the reconcile task begins, the task id=" + taskInfo.getTaskId());
85 | try{
86 | handler.handle(context);
87 | taskService.save(taskInfo);
88 | }catch (Exception e){
89 | //reconcile failed
90 | taskInfo.setStatus(TaskStatus.FAILURE.getStatus());
91 | taskInfo.setLastExecuteEndTime(new Date());
92 | taskService.changeReconcileTaskStatus(taskInfo.getPkId(),
93 | TaskStatus.EXECUTING.getStatus(),taskInfo.getStatus(),taskInfo.getLastExecuteEndTime());
94 | FileUtils.clearReconcileFileCache(context);
95 | log.error("the reconcile task execute failed,the task id=" + taskInfo.getTaskId() + " reason:",e);
96 | return;
97 | }
98 | //reconcile success
99 | taskInfo.setStatus(TaskStatus.SUCCESS.getStatus());
100 | taskInfo.setLastExecuteEndTime(new Date());
101 | taskService.changeReconcileTaskStatus(taskInfo.getPkId(),
102 | TaskStatus.EXECUTING.getStatus(),taskInfo.getStatus(),taskInfo.getLastExecuteEndTime());
103 | log.info("the reconcile task execute success,the task id=" + taskInfo.getTaskId());
104 | // file consistency processing
105 | TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() {
106 | @Override
107 | public void afterCommit() {
108 | TaskInfo validTask = taskService.queryTaskInfoByPkId(taskInfo.getPkId());
109 | if (validTask.getStatus() != TaskStatus.SUCCESS.getStatus()){
110 | FileUtils.clearReconcileFileCache(context);
111 | }
112 | }
113 | });
114 | }
115 | }
116 |
--------------------------------------------------------------------------------
/src/main/java/com/webank/blockchain/data/reconcile/task/TaskCompensateTimer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2020 Webank.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.task;
15 |
16 | import com.webank.blockchain.data.reconcile.config.ReconcileConfig;
17 | import com.webank.blockchain.data.reconcile.db.entity.TaskInfo;
18 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
19 | import com.webank.blockchain.data.reconcile.enums.TaskStatus;
20 | import com.webank.blockchain.data.reconcile.handler.InvocationHandler;
21 | import com.webank.blockchain.data.reconcile.handler.ReconcileHandlerFactory;
22 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
23 | import lombok.extern.slf4j.Slf4j;
24 | import org.springframework.beans.factory.annotation.Autowired;
25 | import org.springframework.scheduling.annotation.EnableScheduling;
26 | import org.springframework.scheduling.annotation.Scheduled;
27 | import org.springframework.stereotype.Component;
28 |
29 | import java.time.LocalDate;
30 | import java.util.Date;
31 | import java.util.List;
32 |
33 | /**
34 | * Reconciliation task compensation processing
35 | * reconciliation task status flow：
36 | * 1.INIT -> EXECUTING -> SUCCESS
37 | * 2.INIT -> EXECUTING -> FAILURE -> SUCCESS
38 | * 3.INIT -> EXECUTING -> FAILURE -> TERMINATE
39 | * Here we need to compensate the EXECUTING and FAILURE intermediate states
40 | *
41 | * @author wesleywang
42 | * @Description:
43 | * @date 2020/6/17
44 | */
45 | @Component
46 | @EnableScheduling
47 | @Slf4j
48 | public class TaskCompensateTimer {
49 |
50 | @Autowired
51 | private ReconcileTaskService taskService;
52 | @Autowired
53 | private ReconcileConfig reconcileConfig;
54 | @Autowired
55 | private ReconcileHandlerFactory handlerFactory;
56 |
57 |
58 | @Scheduled(cron = "${reconcile.executing.compensate.rule}")
59 | public void executingStateCompensate(){
60 | log.info("executingState compensate task start at " + LocalDate.now().toString());
61 | List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.parser;
15 |
16 | import com.fasterxml.jackson.core.JsonFactory;
17 | import com.fasterxml.jackson.core.JsonParser;
18 | import com.fasterxml.jackson.core.JsonToken;
19 | import com.fasterxml.jackson.databind.MappingJsonFactory;
20 | import com.fasterxml.jackson.databind.ObjectMapper;
21 | import com.fasterxml.jackson.databind.type.MapType;
22 | import com.webank.blockchain.data.reconcile.config.ReconcileConfig;
23 | import com.webank.blockchain.data.reconcile.entity.ReconcileExecuteData;
24 | import com.webank.blockchain.data.reconcile.entity.ReconcileInfo;
25 | import com.webank.blockchain.data.reconcile.exception.ReconcileException;
26 | import com.webank.blockchain.data.reconcile.reconcile.ReconcileTransfer;
27 | import com.webank.blockchain.data.reconcile.utils.BufferedRandomAccessFile;
28 | import lombok.extern.slf4j.Slf4j;
29 | import org.apache.tomcat.util.http.fileupload.IOUtils;
30 | import org.springframework.beans.factory.annotation.Autowired;
31 | import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
32 | import org.springframework.stereotype.Service;
33 |
34 | import java.io.File;
35 | import java.io.IOException;
36 | import java.util.HashMap;
37 | import java.util.Map;
38 |
39 | /**
40 | * Json file parsing, the same principle as TXT
41 | *
42 | * @author wesleywang
43 | * @Description:
44 | * @date 2020/6/12
45 | */
46 | @ConditionalOnExpression("'json'.equals('${reconcile.file.type}') && ${reconcile.general.enabled:true}")
47 | @Service
48 | @Slf4j
49 | public class JsonFileDataParser implements FileParser {
50 |
51 | @Autowired
52 | private ReconcileConfig reconcileConfig;
53 |
54 | @Override
55 | public void parseAndTransfer(File businessReconFile, File bcReconFile, ReconcileTransfer transfer)
56 | throws ReconcileException, IOException {
57 | if (businessReconFile.length() <= 0) {
58 | throw new ReconcileException("businessReconFile is empty");
59 | }
60 | if (bcReconFile.length() <= 0) {
61 | throw new ReconcileException("bcReconFile is empty");
62 | }
63 |
64 | BufferedRandomAccessFile bcReader = null;
65 | JsonFactory f = new MappingJsonFactory();
66 | JsonParser jp = null;
67 | try {
68 | bcReader = new BufferedRandomAccessFile(bcReconFile, "r");
69 | ObjectMapper objectMapper = new ObjectMapper();
70 | MapType mapType = objectMapper.getTypeFactory().constructMapType(HashMap.class,String.class,String.class);
71 | jp = f.createParser(businessReconFile);
72 |
73 | Map Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.parser;
15 |
16 |
17 | import com.webank.blockchain.data.reconcile.config.ReconcileConfig;
18 | import com.webank.blockchain.data.reconcile.entity.ReconcileExecuteData;
19 | import com.webank.blockchain.data.reconcile.entity.ReconcileInfo;
20 | import com.webank.blockchain.data.reconcile.exception.ReconcileException;
21 | import com.webank.blockchain.data.reconcile.reconcile.ReconcileTransfer;
22 | import com.webank.blockchain.data.reconcile.utils.BufferedRandomAccessFile;
23 | import com.webank.blockchain.data.reconcile.utils.FileUtils;
24 | import lombok.extern.slf4j.Slf4j;
25 | import org.apache.tomcat.util.http.fileupload.IOUtils;
26 | import org.springframework.beans.factory.annotation.Autowired;
27 | import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
28 | import org.springframework.stereotype.Component;
29 |
30 | import java.io.File;
31 | import java.io.IOException;
32 | import java.util.HashMap;
33 | import java.util.Map;
34 |
35 | /**
36 | * txt file analysis, implementation principle:
37 | * First of all, take one of the files in memory by reading generate index on the map < String, Long >, then, according
38 | * to the line read another file, parse the only key value in each row, according to the value of these index map to
39 | * find the corresponding position of the data line, reads the specified line, and then parse, analytical results
40 | * in {@link ReconcileExecuteData} object, give it to the executor for processing.
41 | *
42 | * @author wesleywang
43 | * @Description:
44 | * @date 2020/6/12
45 | */
46 | @ConditionalOnExpression("'txt'.equals('${reconcile.file.type}') && ${reconcile.general.enabled:true}")
47 | @Component
48 | @Slf4j
49 | public class TxtFileDataParser implements FileParser {
50 |
51 | @Autowired
52 | private ReconcileConfig reconcileConfig;
53 |
54 | @Override
55 | public void parseAndTransfer(File businessReconFile, File bcReconFile, ReconcileTransfer transfer)
56 | throws ReconcileException {
57 | if (businessReconFile.length() <= 0){
58 | throw new ReconcileException("businessReconFile is empty, fileName = " + businessReconFile.getName());
59 | }
60 | if (bcReconFile.length() <= 0){
61 | throw new ReconcileException("bcReconFile is empty, fileName = " + bcReconFile.getName());
62 | }
63 |
64 | BufferedRandomAccessFile bcReader = null;
65 | BufferedRandomAccessFile busReader = null;
66 | try {
67 | bcReader = new BufferedRandomAccessFile(bcReconFile, "r");
68 | busReader = new BufferedRandomAccessFile(businessReconFile, "r");
69 | String bcHead = bcReader.readLine();
70 | String busHead = busReader.readLine();
71 |
72 | Integer bcUniqueIndex = findUniqueColumnIndex(bcHead,reconcileConfig.getBcUniqueColumn());
73 | if (bcUniqueIndex == null){
74 | throw new ReconcileException("TxtFileDataParser parseAndMatching failed, can not find BcUniqueColumn");
75 | }
76 |
77 | Integer busUniqueIndex = findUniqueColumnIndex(busHead,reconcileConfig.getBusinessUniqueColumn());
78 | if (busUniqueIndex == null){
79 | throw new ReconcileException("TxtFileDataParser parseAndMatching failed, can not find BcUniqueColumn");
80 | }
81 | Map Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.utils;
15 |
16 | import cn.hutool.core.collection.CollectionUtil;
17 | import cn.hutool.core.io.FileUtil;
18 | import com.google.common.collect.Lists;
19 | import com.webank.blockchain.data.reconcile.entity.ReconcileContext;
20 | import com.webank.blockchain.data.reconcile.exception.ReconcileException;
21 | import lombok.extern.slf4j.Slf4j;
22 | import org.apache.commons.lang3.StringUtils;
23 | import org.apache.tomcat.util.http.fileupload.IOUtils;
24 |
25 | import java.io.File;
26 | import java.io.IOException;
27 | import java.lang.reflect.Field;
28 | import java.nio.file.Files;
29 | import java.nio.file.Path;
30 | import java.nio.file.Paths;
31 | import java.util.HashMap;
32 | import java.util.List;
33 | import java.util.Map;
34 |
35 | /**
36 | * @author wesleywang
37 | * @Description:
38 | * @date 2020/6/28
39 | */
40 | @Slf4j
41 | public final class FileUtils {
42 |
43 | public static String BC_RECONCILE_FILEPATH;
44 |
45 | public static String BUS_RECONCILE_FILEPATH;
46 |
47 | public static String RESULT_RECONCILE_FILEPATH;
48 |
49 | public static final String TXT_FILE_SUFFIX = ".txt";
50 |
51 | public static final String JSON_FILE_SUFFIX = ".json";
52 |
53 | public static final String TXT_FILE_SPLIT = "#_#";
54 |
55 | public static final String NEWLINE_SYMBOL = "\n";
56 |
57 | public static String fileDir;
58 |
59 | public static final List Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5 | * except in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * http://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the
10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11 | * express or implied. See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package com.webank.blockchain.data.reconcile.utils;
15 |
16 | import java.io.File;
17 | import java.io.FileNotFoundException;
18 | import java.io.IOException;
19 | import java.io.RandomAccessFile;
20 | import java.util.Arrays;
21 |
22 | /**
23 | *
24 | * @Description:
25 | * @date 2020/6/23
26 | */
27 |
28 | public class BufferedRandomAccessFile extends RandomAccessFile {
29 | static final int LogBuffSz_ = 16; // 64K buffer
30 | public static final int BuffSz_ = (1 << LogBuffSz_);
31 | static final long BuffMask_ = -((long) BuffSz_);
32 |
33 | private String path_;
34 |
35 | /*
36 | * This implementation is based on the buffer implementation in Modula-3's
37 | * "Rd", "Wr", "RdClass", and "WrClass" interfaces.
38 | */
39 | private boolean dirty_; // true iff unflushed bytes exist
40 | private boolean syncNeeded_; // dirty_ can be cleared by e.g. seek, so track sync separately
41 | private long curr_; // current position in file
42 | private long lo_, hi_; // bounds on characters in "buff"
43 | private byte[] buff_; // local buffer
44 | private long maxHi_; // this.lo + this.buff.length
45 | private boolean hitEOF_; // buffer contains last file block?
46 | private long diskPos_; // disk position
47 |
48 | /*
49 | * To describe the above fields, we introduce the following abstractions for
50 | * the file "f":
51 | *
52 | * len(f) the length of the file curr(f) the current position in the file
53 | * c(f) the abstract contents of the file disk(f) the contents of f's
54 | * backing disk file closed(f) true iff the file is closed
55 | *
56 | * "curr(f)" is an index in the closed interval [0, len(f)]. "c(f)" is a
57 | * character sequence of length "len(f)". "c(f)" and "disk(f)" may differ if
58 | * "c(f)" contains unflushed writes not reflected in "disk(f)". The flush
59 | * operation has the effect of making "disk(f)" identical to "c(f)".
60 | *
61 | * A file is said to be *valid* if the following conditions hold:
62 | *
63 | * V1. The "closed" and "curr" fields are correct:
64 | *
65 | * f.closed == closed(f) f.curr == curr(f)
66 | *
67 | * V2. The current position is either contained in the buffer, or just past
68 | * the buffer:
69 | *
70 | * f.lo <= f.curr <= f.hi
71 | *
72 | * V3. Any (possibly) unflushed characters are stored in "f.buff":
73 | *
74 | * (forall i in [f.lo, f.curr): c(f)[i] == f.buff[i - f.lo])
75 | *
76 | * V4. For all characters not covered by V3, c(f) and disk(f) agree:
77 | *
78 | * (forall i in [f.lo, len(f)): i not in [f.lo, f.curr) => c(f)[i] ==
79 | * disk(f)[i])
80 | *
81 | * V5. "f.dirty" is true iff the buffer contains bytes that should be
82 | * flushed to the file; by V3 and V4, only part of the buffer can be dirty.
83 | *
84 | * f.dirty == (exists i in [f.lo, f.curr): c(f)[i] != f.buff[i - f.lo])
85 | *
86 | * V6. this.maxHi == this.lo + this.buff.length
87 | *
88 | * Note that "f.buff" can be "null" in a valid file, since the range of
89 | * characters in V3 is empty when "f.lo == f.curr".
90 | *
91 | * A file is said to be *ready* if the buffer contains the current position,
92 | * i.e., when:
93 | *
94 | * R1. !f.closed && f.buff != null && f.lo <= f.curr && f.curr < f.hi
95 | *
96 | * When a file is ready, reading or writing a single byte can be performed
97 | * by reading or writing the in-memory buffer without performing a disk
98 | * operation.
99 | */
100 |
101 | /**
102 | * Open a new
205 | *
206 | * On exit from this routine BufferedRandomAccessFile on file
103 | * in mode mode, which should be "r" for reading only, or
104 | * "rw" for reading and writing.
105 | */
106 | public BufferedRandomAccessFile(File file, String mode) throws IOException {
107 | this(file, mode, 0);
108 | }
109 |
110 | public BufferedRandomAccessFile(File file, String mode, int size) throws IOException {
111 | super(file, mode);
112 | path_ = file.getAbsolutePath();
113 | this.init(size);
114 | }
115 |
116 | /**
117 | * Open a new BufferedRandomAccessFile on the file named
118 | * name in mode mode, which should be "r" for
119 | * reading only, or "rw" for reading and writing.
120 | */
121 | public BufferedRandomAccessFile(String name, String mode) throws IOException {
122 | this(name, mode, 0);
123 | }
124 |
125 | public BufferedRandomAccessFile(String name, String mode, int size) throws FileNotFoundException {
126 | super(name, mode);
127 | path_ = name;
128 | this.init(size);
129 | }
130 |
131 | private void init(int size) {
132 | this.dirty_ = false;
133 | this.lo_ = this.curr_ = this.hi_ = 0;
134 | this.buff_ = (size > BuffSz_) ? new byte[size] : new byte[BuffSz_];
135 | this.maxHi_ = (long) BuffSz_;
136 | this.hitEOF_ = false;
137 | this.diskPos_ = 0L;
138 | }
139 |
140 | public String getPath() {
141 | return path_;
142 | }
143 |
144 | public void sync() throws IOException {
145 | if (syncNeeded_) {
146 | flush();
147 | getChannel().force(true);
148 | syncNeeded_ = false;
149 | }
150 | }
151 |
152 | public void close() throws IOException {
153 | this.flush();
154 | this.buff_ = null;
155 | super.close();
156 | }
157 |
158 | /**
159 | * Flush any bytes in the file's buffer that have not yet been written to
160 | * disk. If the file was created read-only, this method is a no-op.
161 | */
162 | public void flush() throws IOException {
163 | this.flushBuffer();
164 | }
165 |
166 | /* Flush any dirty bytes in the buffer to disk. */
167 | private void flushBuffer() throws IOException {
168 | if (this.dirty_) {
169 | if (this.diskPos_ != this.lo_)
170 | super.seek(this.lo_);
171 | int len = (int) (this.curr_ - this.lo_);
172 | super.write(this.buff_, 0, len);
173 | this.diskPos_ = this.curr_;
174 | this.dirty_ = false;
175 | }
176 | }
177 |
178 | /*
179 | * Read at most "this.buff.length" bytes into "this.buff", returning the
180 | * number of bytes read. If the return result is less than
181 | * "this.buff.length", then EOF was read.
182 | */
183 | private int fillBuffer() throws IOException {
184 | int cnt = 0;
185 | int rem = this.buff_.length;
186 | while (rem > 0) {
187 | int n = super.read(this.buff_, cnt, rem);
188 | if (n < 0)
189 | break;
190 | cnt += n;
191 | rem -= n;
192 | }
193 | if ((cnt < 0) && (this.hitEOF_ = (cnt < this.buff_.length))) {
194 | // make sure buffer that wasn't read is initialized with -1
195 | Arrays.fill(this.buff_, cnt, this.buff_.length, (byte) 0xff);
196 | }
197 | this.diskPos_ += cnt;
198 | return cnt;
199 | }
200 |
201 | /*
202 | * This method positions this.curr at position pos.
203 | * If pos does not fall in the current buffer, it flushes the
204 | * current buffer and loads the correct one.this.curr == this.hi iff pos
207 | * is at or past the end-of-file, which can only happen if the file was
208 | * opened in read-only mode.
209 | */
210 | public void seek(long pos) throws IOException {
211 | if (pos >= this.hi_ || pos < this.lo_) {
212 | // seeking outside of current buffer -- flush and read
213 | this.flushBuffer();
214 | this.lo_ = pos & BuffMask_; // start at BuffSz boundary
215 | this.maxHi_ = this.lo_ + (long) this.buff_.length;
216 | if (this.diskPos_ != this.lo_) {
217 | super.seek(this.lo_);
218 | this.diskPos_ = this.lo_;
219 | }
220 | int n = this.fillBuffer();
221 | this.hi_ = this.lo_ + (long) n;
222 | } else {
223 | // seeking inside current buffer -- no read required
224 | if (pos < this.curr_) {
225 | // if seeking backwards, we must flush to maintain V4
226 | this.flushBuffer();
227 | }
228 | }
229 | this.curr_ = pos;
230 | }
231 |
232 | public long getFilePointer() {
233 | return this.curr_;
234 | }
235 |
236 | public long length() throws IOException {
237 | // max accounts for the case where we have written past the old file length, but not yet flushed our buffer
238 | return Math.max(this.curr_, super.length());
239 | }
240 |
241 | public int read() throws IOException {
242 | if (this.curr_ >= this.hi_) {
243 | // test for EOF
244 | // if (this.hi < this.maxHi) return -1;
245 | if (this.hitEOF_)
246 | return -1;
247 |
248 | // slow path -- read another buffer
249 | this.seek(this.curr_);
250 | if (this.curr_ == this.hi_)
251 | return -1;
252 | }
253 | byte res = this.buff_[(int) (this.curr_ - this.lo_)];
254 | this.curr_++;
255 | return ((int) res) & 0xFF; // convert byte -> int
256 | }
257 |
258 | public int read(byte[] b) throws IOException {
259 | return this.read(b, 0, b.length);
260 | }
261 |
262 | public int read(byte[] b, int off, int len) throws IOException {
263 | if (this.curr_ >= this.hi_) {
264 | // test for EOF
265 | // if (this.hi < this.maxHi) return -1;
266 | if (this.hitEOF_)
267 | return -1;
268 |
269 | // slow path -- read another buffer
270 | this.seek(this.curr_);
271 | if (this.curr_ == this.hi_)
272 | return -1;
273 | }
274 | len = Math.min(len, (int) (this.hi_ - this.curr_));
275 | int buffOff = (int) (this.curr_ - this.lo_);
276 | System.arraycopy(this.buff_, buffOff, b, off, len);
277 | this.curr_ += len;
278 | return len;
279 | }
280 |
281 | public void write(int b) throws IOException {
282 | if (this.curr_ >= this.hi_) {
283 | if (this.hitEOF_ && this.hi_ < this.maxHi_) {
284 | // at EOF -- bump "hi"
285 | this.hi_++;
286 | } else {
287 | // slow path -- write current buffer; read next one
288 | this.seek(this.curr_);
289 | if (this.curr_ == this.hi_) {
290 | // appending to EOF -- bump "hi"
291 | this.hi_++;
292 | }
293 | }
294 | }
295 | this.buff_[(int) (this.curr_ - this.lo_)] = (byte) b;
296 | this.curr_++;
297 | this.dirty_ = true;
298 | syncNeeded_ = true;
299 | }
300 |
301 | public void write(byte[] b) throws IOException {
302 | this.write(b, 0, b.length);
303 | }
304 |
305 | public void write(byte[] b, int off, int len) throws IOException {
306 | while (len > 0) {
307 | int n = this.writeAtMost(b, off, len);
308 | off += n;
309 | len -= n;
310 | this.dirty_ = true;
311 | syncNeeded_ = true;
312 | }
313 | }
314 |
315 | /*
316 | * Write at most "len" bytes to "b" starting at position "off", and return
317 | * the number of bytes written.
318 | */
319 | private int writeAtMost(byte[] b, int off, int len) throws IOException {
320 | if (this.curr_ >= this.hi_) {
321 | if (this.hitEOF_ && this.hi_ < this.maxHi_) {
322 | // at EOF -- bump "hi"
323 | this.hi_ = this.maxHi_;
324 | } else {
325 | // slow path -- write current buffer; read next one
326 | this.seek(this.curr_);
327 | if (this.curr_ == this.hi_) {
328 | // appending to EOF -- bump "hi"
329 | this.hi_ = this.maxHi_;
330 | }
331 | }
332 | }
333 | len = Math.min(len, (int) (this.hi_ - this.curr_));
334 | int buffOff = (int) (this.curr_ - this.lo_);
335 | System.arraycopy(b, off, this.buff_, buffOff, len);
336 | this.curr_ += len;
337 | return len;
338 | }
339 |
340 | }
341 |
--------------------------------------------------------------------------------