├── .gitignore ├── LICENSE ├── README.md ├── pom.xml └── src └── main └── java └── io └── loli └── util └── osm ├── OsmApplication.java ├── source ├── Storage.java ├── StorageFactory.java ├── StorageProperties.java └── impl │ ├── AliOssStorage.java │ ├── BaiduBosStorage.java │ ├── HuaweiObsStorage.java │ ├── JdOssStorage.java │ ├── QiniuPrivateStorage.java │ ├── QiniuPublicStorage.java │ ├── TencentCosStorage.java │ └── UpaiStorage.java └── ui ├── OsmUi.form └── OsmUi.java /.gitignore: -------------------------------------------------------------------------------- 1 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 2 | hs_err_pid* 3 | 4 | broker/.classpath 5 | broker/.project 6 | broker/.settings/org.eclipse.jdt.core.prefs 7 | broker/target/classes/.gitignore 8 | broker/target/test-classes/.gitignore 9 | 10 | common/.classpath 11 | common/.project 12 | common/.settings/org.eclipse.jdt.core.prefs 13 | common/target/classes/.gitignore 14 | common/target/test-classes/.gitignore 15 | 16 | core/.classpath 17 | core/.project 18 | core/.settings/org.eclipse.jdt.core.prefs 19 | core/target/classes/.gitignore 20 | core/target/test-classes/.gitignore 21 | 22 | dal/.classpath 23 | dal/.project 24 | dal/.settings/org.eclipse.jdt.core.prefs 25 | dal/target/classes/.gitignore 26 | dal/target/test-classes/.gitignore 27 | 28 | facade/.classpath 29 | facade/.project 30 | facade/.settings/org.eclipse.jdt.core.prefs 31 | facade/target/classes/.gitignore 32 | facade/target/test-classes/.gitignore 33 | 34 | service/.classpath 35 | service/.project 36 | service/.settings/org.eclipse.jdt.core.prefs 37 | service/target/classes/.gitignore 38 | service/target/test-classes/.gitignore 39 | 40 | web/.classpath 41 | web/.project 42 | web/.settings/org.eclipse.jdt.core.prefs 43 | web/target/classes/.gitignore 44 | web/target/test-classes/.gitignore 45 | 46 | target/mvn-eclipse-cache.properties 47 | 48 | *.class 49 | target/ 50 | *.iml 51 | .idea/ 52 | target/ 53 | logs/ 54 | .classpath 55 | .project 56 | .settings/ 57 | .amateras 58 | /target* 59 | *.log* 60 | *.jar 61 | *.war 62 | *.ear 63 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 chocotan (loli@linux.com) 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 云存储迁移工具 2 | --- 3 | 4 | ## 功能 5 | 1. 支持阿里OSS、腾讯COS、百度BOS、京东OSS、华为OBS、七牛、又拍的互相迁移 6 | 2. 批量删除bucket文件 7 | 8 | ## 打包 9 | ```bash 10 | mvn clean assembly:single 11 | ``` 12 | 13 | ## 截图 14 | ![image.png](https://b1.loli.io/images/e0gox.png) 15 | 16 | ## 17 | 1. 界面的`导出`按钮暂时没啥用 18 | 2. 暂不支持失败重推 19 | 3. 本程序是先将文件下载到本地再上传，会消耗流量 20 | 4. 请根据自己带宽合理设置线程数 21 | 5. 代码不漂亮，甚至还有中文变量名，请勿吐槽 22 | 6. 功能如有问题，请在issue中写明自己的操作流程，或者你直接改代码 23 | 24 | ## 其他 25 | 1. BAT、华为、京东的sdk很像，代码复制粘贴就完事了，我怀疑他们是不是互相抄袭的 26 | 2. 又拍和七牛的都特立独行，尤其是七牛，居然还区分公开和私有，更加离谱的是，不同的区域要调用不同的方法，都不能直接传个参数进去 27 | 28 | 29 | ## Region和endpoint 30 | 31 | > 注意所有的endpoint必须以`http://`开头，不能以`/`结尾 32 | 33 | 34 | ### 阿里OSS 35 | 阿里OSS不需要填写region，只需要填写endpoint，点击 https://help.aliyun.com/document_detail/31837.html 36 | 37 | ### 腾讯COS 38 | 腾讯直接填入region，不需要endpoint： https://cloud.tencent.com/document/product/436/6224 39 | 40 | ### 百度BOS 41 | 百度和阿里一样，不须要region，使用endpoint： https://cloud.baidu.com/doc/BOS/Java-SDK.html 42 | 43 | ### 华为OBS 44 | 华为也是不需要region，使用endpoint： https://support.huaweicloud.com/devg-obs/zh-cn_topic_0105713153.html 45 | 46 | ### 京东OSS 47 | 京东两者都需要，使用s3 endpoint：https://docs.jdcloud.com/cn/object-storage-service/regions-and-endpoints 48 | 49 | ### 七牛 50 | 七牛需要填写region和endpoint等，https://developer.qiniu.com/kodo/sdk/1239/java#4 51 | 52 | region可以不填写，将自动识别 53 | 54 | ### 又拍云 55 | 又拍云无需填写region和endpoint 56 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | io.loli.util 8 | osm 9 | 0.0.1-SNAPSHOT 10 | 11 | 12 | 1.8 13 | UTF-8 14 | 15 | 16 | 17 | com.aliyun.oss 18 | aliyun-sdk-oss 19 | 2.8.3 20 | 21 | 22 | org.apache.httpcomponents 23 | httpclient 24 | 25 | 26 | 27 | 28 | com.qcloud 29 | cos_api 30 | 5.4.6 31 | 32 | 33 | org.slf4j 34 | slf4j-log4j12 35 | 36 | 37 | 38 | 39 | io.javaslang 40 | javaslang 41 | 2.0.6 42 | 43 | 44 | org.apache.commons 45 | commons-lang3 46 | 3.8.1 47 | 48 | 49 | org.apache.commons 50 | commons-collections4 51 | 4.2 52 | 53 | 54 | commons-io 55 | commons-io 56 | 2.6 57 | 58 | 59 | com.alibaba 60 | fastjson 61 | 1.2.49 62 | 63 | 64 | com.google.guava 65 | guava 66 | 23.0 67 | 68 | 69 | com.baidubce 70 | bce-java-sdk 71 | 0.10.48 72 | 73 | 74 | ch.qos.logback 75 | logback-core 76 | 77 | 78 | ch.qos.logback 79 | logback-classic 80 | 81 | 82 | 83 | 84 | com.qiniu 85 | qiniu-java-sdk 86 | [7.2.0, 7.2.99] 87 | 88 | 89 | com.intellij 90 | forms_rt 91 | 7.0.3 92 | 93 | 94 | ch.qos.logback 95 | logback-classic 96 | 1.2.3 97 | 98 | 99 | com.huawei.storage 100 | esdk-obs-java 101 | 3.0.2 102 | 103 | 104 | org.apache.logging.log4j 105 | log4j-core 106 | 107 | 108 | org.apache.logging.log4j 109 | log4j-api 110 | 111 | 112 | 113 | 114 | com.amazonaws 115 | aws-java-sdk 116 | 1.11.136 117 | 118 | 119 | com.fasterxml.jackson.core 120 | jackson-annotations 121 | 2.9.8 122 | 123 | 124 | com.fasterxml.jackson.core 125 | jackson-core 126 | 2.9.8 127 | 128 | 129 | com.upyun 130 | java-sdk 131 | 4.0.1 132 | 133 | 134 | 135 | 136 | 137 | 138 | org.codehaus.mojo 139 | ideauidesigner-maven-plugin 140 | 1.0-beta-1 141 | 142 | 143 | 144 | javac2 145 | 146 | 147 | 148 | 149 | 150 | true 151 | true 152 | true 153 | 154 | 155 | 156 | org.apache.maven.plugins 157 | maven-compiler-plugin 158 | 3.8.0 159 | 160 | ${java.version} 161 | ${java.version} 162 | 163 | true 164 | true 165 | 166 | 167 | 168 | org.apache.maven.plugins 169 | maven-assembly-plugin 170 | 171 | 172 | 173 | io.loli.util.osm.OsmApplication 174 | 175 | 176 | 177 | jar-with-dependencies 178 | 179 | 180 | 181 | 182 | 183 | org.apache.maven.plugins 184 | maven-resources-plugin 185 | 2.7 186 | 187 | ${project.build.sourceEncoding} 188 | 189 | 190 | 191 | org.apache.maven.plugins 192 | maven-source-plugin 193 | 194 | 195 | attach-sources 196 | 197 | jar 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | huawei 208 | https://repo.huaweicloud.com/repository/maven/huaweicloudsdk/ 209 | 210 | 211 | 212 | 213 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/OsmApplication.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm; 2 | 3 | 4 | import ch.qos.logback.classic.Level; 5 | import ch.qos.logback.classic.Logger; 6 | import io.loli.util.osm.ui.OsmUi; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import javax.swing.*; 10 | 11 | public class OsmApplication { 12 | public static void main(String[] args) throws InterruptedException { 13 | try { 14 | // Set cross-platform Java L&F (also called "Metal") 15 | UIManager.setLookAndFeel( 16 | UIManager.getSystemLookAndFeelClassName()); 17 | } catch (Exception e) { 18 | } 19 | ch.qos.logback.classic.Logger logger = (Logger) LoggerFactory.getLogger("root"); 20 | logger.setLevel(Level.INFO); 21 | SwingUtilities.invokeLater(OsmUi::new); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/Storage.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source; 2 | 3 | import javaslang.Tuple2; 4 | import javaslang.Tuple3; 5 | 6 | import java.io.File; 7 | import java.util.List; 8 | 9 | public interface Storage { 10 | public String getName(); 11 | 12 | public Tuple3> list(String bucket, String prefix, String marker, Integer size); 13 | 14 | public void downloadToFile(String bucket, String key, File file); 15 | 16 | public void upload(String bucket, String key, File file); 17 | 18 | boolean exist(String bucket, String key); 19 | 20 | void delete(String bucket, String key); 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/StorageFactory.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source; 2 | 3 | import io.loli.util.osm.source.impl.*; 4 | 5 | public class StorageFactory { 6 | public static Storage getStorage(StorageProperties properties) { 7 | switch (properties.getType()) { 8 | case "阿里OSS": 9 | return new AliOssStorage(properties); 10 | case "腾讯COS": 11 | return new TencentCosStorage(properties); 12 | case "百度BOS": 13 | return new BaiduBosStorage(properties); 14 | case "华为OBS": 15 | return new HuaweiObsStorage(properties); 16 | case "京东OSS": 17 | return new JdOssStorage(properties); 18 | case "七牛公有": 19 | return new QiniuPublicStorage(properties); 20 | case "七牛私有": 21 | return new QiniuPrivateStorage(properties); 22 | case "又拍云": 23 | return new UpaiStorage(properties); 24 | } 25 | return null; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/StorageProperties.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source; 2 | 3 | public class StorageProperties { 4 | private String type; 5 | private String key; 6 | private String secret; 7 | private String bucket; 8 | private String region; 9 | private String endpoint; 10 | 11 | 12 | public String getEndpoint() { 13 | return endpoint; 14 | } 15 | 16 | public void setEndpoint(String endpoint) { 17 | this.endpoint = endpoint; 18 | } 19 | 20 | public String getType() { 21 | return type; 22 | } 23 | 24 | public void setType(String type) { 25 | this.type = type; 26 | } 27 | 28 | public String getKey() { 29 | return key; 30 | } 31 | 32 | public void setKey(String key) { 33 | this.key = key; 34 | } 35 | 36 | public String getSecret() { 37 | return secret; 38 | } 39 | 40 | public void setSecret(String secret) { 41 | this.secret = secret; 42 | } 43 | 44 | public String getBucket() { 45 | return bucket; 46 | } 47 | 48 | public void setBucket(String bucket) { 49 | this.bucket = bucket; 50 | } 51 | 52 | public String getRegion() { 53 | return region; 54 | } 55 | 56 | public void setRegion(String region) { 57 | this.region = region; 58 | } 59 | 60 | public static final class StoragePropertiesBuilder { 61 | private String type; 62 | private String key; 63 | private String secret; 64 | private String bucket; 65 | private String region; 66 | private String endpoint; 67 | 68 | private StoragePropertiesBuilder() { 69 | } 70 | 71 | public static StoragePropertiesBuilder a() { 72 | return new StoragePropertiesBuilder(); 73 | } 74 | 75 | public StoragePropertiesBuilder withType(String type) { 76 | this.type = type; 77 | return this; 78 | } 79 | 80 | public StoragePropertiesBuilder withKey(String key) { 81 | this.key = key; 82 | return this; 83 | } 84 | 85 | public StoragePropertiesBuilder withSecret(String secret) { 86 | this.secret = secret; 87 | return this; 88 | } 89 | 90 | public StoragePropertiesBuilder withBucket(String bucket) { 91 | this.bucket = bucket; 92 | return this; 93 | } 94 | 95 | public StoragePropertiesBuilder withRegion(String region) { 96 | this.region = region; 97 | return this; 98 | } 99 | 100 | public StoragePropertiesBuilder withEndpoint(String endpoint) { 101 | this.endpoint = endpoint; 102 | return this; 103 | } 104 | 105 | public StorageProperties build() { 106 | StorageProperties storageProperties = new StorageProperties(); 107 | storageProperties.setType(type); 108 | storageProperties.setKey(key); 109 | storageProperties.setSecret(secret); 110 | storageProperties.setBucket(bucket); 111 | storageProperties.setRegion(region); 112 | storageProperties.setEndpoint(endpoint); 113 | return storageProperties; 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/impl/AliOssStorage.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source.impl; 2 | 3 | import com.aliyun.oss.OSSClient; 4 | import com.aliyun.oss.model.GetObjectRequest; 5 | import com.aliyun.oss.model.ListObjectsRequest; 6 | import com.aliyun.oss.model.OSSObjectSummary; 7 | import com.aliyun.oss.model.ObjectListing; 8 | import io.loli.util.osm.source.Storage; 9 | import io.loli.util.osm.source.StorageProperties; 10 | import javaslang.Tuple; 11 | import javaslang.Tuple2; 12 | import javaslang.Tuple3; 13 | 14 | import java.io.File; 15 | import java.io.UnsupportedEncodingException; 16 | import java.net.URLEncoder; 17 | import java.util.List; 18 | import java.util.stream.Collectors; 19 | 20 | public class AliOssStorage implements Storage { 21 | private OSSClient client; 22 | 23 | public AliOssStorage(StorageProperties properties) { 24 | client = new OSSClient(properties.getEndpoint(), properties.getKey(), properties.getSecret()); 25 | } 26 | 27 | @Override 28 | public String getName() { 29 | return "阿里OSS"; 30 | } 31 | 32 | @Override 33 | public Tuple3> list(String bucket, String prefix, String marker, Integer size) { 34 | ListObjectsRequest request = new ListObjectsRequest(bucket).withMarker(marker) 35 | .withMaxKeys(size); 36 | ObjectListing objectListing = client.listObjects(request); 37 | return Tuple.of(objectListing.isTruncated(), objectListing.getNextMarker(), objectListing.getObjectSummaries().stream() 38 | .map(OSSObjectSummary::getKey).collect(Collectors.toList())); 39 | } 40 | 41 | @Override 42 | public void downloadToFile(String bucket, String key, File file) { 43 | client.getObject(new GetObjectRequest(bucket, key), file); 44 | } 45 | 46 | @Override 47 | public void upload(String bucket, String key, File file) { 48 | client.putObject(bucket, key, file); 49 | } 50 | 51 | @Override 52 | public boolean exist(String bucket, String key) { 53 | return client.doesObjectExist(bucket, key); 54 | } 55 | 56 | public void delete(String bucket, String key) { 57 | client.deleteObject(bucket, key); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/impl/BaiduBosStorage.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source.impl; 2 | 3 | import com.baidubce.auth.DefaultBceCredentials; 4 | import com.baidubce.services.bos.BosClient; 5 | import com.baidubce.services.bos.BosClientConfiguration; 6 | import com.baidubce.services.bos.model.BosObjectSummary; 7 | import com.baidubce.services.bos.model.GetObjectRequest; 8 | import com.baidubce.services.bos.model.ListObjectsRequest; 9 | import com.baidubce.services.bos.model.ListObjectsResponse; 10 | import io.loli.util.osm.source.Storage; 11 | import io.loli.util.osm.source.StorageProperties; 12 | import javaslang.Tuple; 13 | import javaslang.Tuple2; 14 | import javaslang.Tuple3; 15 | 16 | import java.io.File; 17 | import java.util.List; 18 | import java.util.stream.Collectors; 19 | 20 | public class BaiduBosStorage implements Storage { 21 | private BosClient client; 22 | 23 | public BaiduBosStorage(StorageProperties properties) { 24 | BosClientConfiguration config = new BosClientConfiguration(); 25 | config.setCredentials(new DefaultBceCredentials(properties.getKey(), properties.getSecret())); 26 | config.setEndpoint(properties.getEndpoint()); 27 | client = new BosClient(config); 28 | } 29 | 30 | @Override 31 | public String getName() { 32 | return "百度BOS"; 33 | } 34 | 35 | @Override 36 | public Tuple3> list(String bucket, String prefix, String marker, Integer size) { 37 | ListObjectsRequest request = new ListObjectsRequest(bucket).withMarker(marker) 38 | .withMaxKeys(size); 39 | ListObjectsResponse objectListing = client.listObjects(request); 40 | 41 | return Tuple.of(objectListing.isTruncated(), objectListing.getNextMarker(), objectListing.getContents().stream() 42 | .map(BosObjectSummary::getKey).collect(Collectors.toList())); 43 | } 44 | 45 | @Override 46 | public void downloadToFile(String bucket, String key, File file) { 47 | client.getObject(new GetObjectRequest(bucket, key), file); 48 | } 49 | 50 | @Override 51 | public void upload(String bucket, String key, File file) { 52 | client.putObject(bucket, key, file); 53 | } 54 | 55 | @Override 56 | public boolean exist(String bucket, String key) { 57 | return client.doesObjectExist(bucket,key); 58 | } 59 | 60 | public void delete(String bucket, String key){ 61 | client.deleteObject(bucket, key); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/impl/HuaweiObsStorage.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source.impl; 2 | 3 | import com.obs.services.ObsClient; 4 | import com.obs.services.model.*; 5 | import io.loli.util.osm.source.Storage; 6 | import io.loli.util.osm.source.StorageProperties; 7 | import javaslang.Tuple; 8 | import javaslang.Tuple2; 9 | import javaslang.Tuple3; 10 | import org.apache.commons.io.FileUtils; 11 | 12 | import java.io.*; 13 | import java.util.List; 14 | import java.util.stream.Collectors; 15 | 16 | public class HuaweiObsStorage implements Storage { 17 | private ObsClient client; 18 | 19 | public HuaweiObsStorage(StorageProperties properties) { 20 | client = new ObsClient(properties.getKey(), properties.getSecret(), properties.getEndpoint()); 21 | } 22 | 23 | @Override 24 | public String getName() { 25 | return "华为OBS"; 26 | } 27 | 28 | @Override 29 | public Tuple3> list(String bucket, String prefix, String marker, Integer size) { 30 | ListObjectsRequest request = new ListObjectsRequest(bucket); 31 | request.setMarker(marker); 32 | request.setMaxKeys(size); 33 | ObjectListing objectListing = client.listObjects(request); 34 | 35 | return Tuple.of(objectListing.isTruncated(), objectListing.getNextMarker(), objectListing.getObjects().stream() 36 | .map(ObsObject::getObjectKey).collect(Collectors.toList())); 37 | } 38 | 39 | @Override 40 | public void downloadToFile(String bucket, String key, File file) { 41 | 42 | try { 43 | client.downloadFile(new DownloadFileRequest(bucket, key, file.getPath())); 44 | } catch (Exception e) { 45 | throw new RuntimeException(e); 46 | } 47 | } 48 | 49 | @Override 50 | public void upload(String bucket, String key, File file) { 51 | client.putObject(bucket, key, file); 52 | } 53 | 54 | @Override 55 | public boolean exist(String bucket, String key) { 56 | try { 57 | ObsObject object = client.getObject(bucket, key); 58 | return object != null; 59 | } catch (Exception e) { 60 | return false; 61 | } 62 | } 63 | 64 | public void delete(String bucket, String key) { 65 | client.deleteObject(bucket, key); 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/impl/JdOssStorage.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source.impl; 2 | 3 | import com.amazonaws.ClientConfiguration; 4 | import com.amazonaws.SDKGlobalConfiguration; 5 | import com.amazonaws.auth.AWSCredentials; 6 | import com.amazonaws.auth.AWSCredentialsProvider; 7 | import com.amazonaws.auth.AWSStaticCredentialsProvider; 8 | import com.amazonaws.auth.BasicAWSCredentials; 9 | import com.amazonaws.client.builder.AwsClientBuilder; 10 | import com.amazonaws.services.s3.AmazonS3; 11 | import com.amazonaws.services.s3.AmazonS3Client; 12 | import com.amazonaws.services.s3.model.GetObjectRequest; 13 | import com.amazonaws.services.s3.model.ListObjectsRequest; 14 | import com.amazonaws.services.s3.model.ObjectListing; 15 | import com.amazonaws.services.s3.model.S3ObjectSummary; 16 | import io.loli.util.osm.source.Storage; 17 | import io.loli.util.osm.source.StorageProperties; 18 | import javaslang.Tuple; 19 | import javaslang.Tuple3; 20 | 21 | import java.io.File; 22 | import java.util.List; 23 | import java.util.stream.Collectors; 24 | 25 | public class JdOssStorage implements Storage { 26 | @Override 27 | public String getName() { 28 | return "京东OSS"; 29 | } 30 | 31 | private AmazonS3 client; 32 | 33 | public JdOssStorage(StorageProperties properties) { 34 | final String accessKey = properties.getKey(); 35 | final String secretKey = properties.getSecret(); 36 | final String endpoint = properties.getEndpoint(); 37 | System.setProperty(SDKGlobalConfiguration.ENABLE_S3_SIGV4_SYSTEM_PROPERTY, "true"); 38 | ClientConfiguration config = new ClientConfiguration(); 39 | 40 | AwsClientBuilder.EndpointConfiguration endpointConfig = 41 | new AwsClientBuilder.EndpointConfiguration(endpoint, properties.getRegion()); 42 | 43 | AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey); 44 | AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(awsCredentials); 45 | 46 | client = AmazonS3Client.builder() 47 | .withEndpointConfiguration(endpointConfig) 48 | .withClientConfiguration(config) 49 | .withCredentials(awsCredentialsProvider) 50 | .disableChunkedEncoding() 51 | .withPathStyleAccessEnabled(true) 52 | .build(); 53 | } 54 | 55 | 56 | @Override 57 | public Tuple3> list(String bucket, String prefix, String marker, Integer size) { 58 | ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket) 59 | .withMarker(marker) 60 | .withMaxKeys(size); 61 | ObjectListing objectListing = client.listObjects(request); 62 | return Tuple.of(objectListing.isTruncated(), objectListing.getNextMarker(), objectListing.getObjectSummaries().stream() 63 | .map(S3ObjectSummary::getKey).collect(Collectors.toList())); 64 | } 65 | 66 | @Override 67 | public void downloadToFile(String bucket, String key, File file) { 68 | client.getObject(new GetObjectRequest(bucket, key), file); 69 | } 70 | 71 | @Override 72 | public void upload(String bucket, String key, File file) { 73 | client.putObject(bucket, key, file); 74 | } 75 | 76 | @Override 77 | public boolean exist(String bucket, String key) { 78 | return client.doesObjectExist(bucket, key); 79 | } 80 | 81 | public void delete(String bucket, String key) { 82 | client.deleteObject(bucket, key); 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/impl/QiniuPrivateStorage.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source.impl; 2 | 3 | import com.qiniu.common.QiniuException; 4 | import com.qiniu.common.Zone; 5 | import com.qiniu.storage.BucketManager; 6 | import com.qiniu.storage.Configuration; 7 | import com.qiniu.storage.UploadManager; 8 | import com.qiniu.storage.model.FileListing; 9 | import com.qiniu.util.Auth; 10 | import io.loli.util.osm.source.Storage; 11 | import io.loli.util.osm.source.StorageProperties; 12 | import javaslang.Tuple; 13 | import javaslang.Tuple2; 14 | import org.apache.commons.io.FileUtils; 15 | 16 | import java.io.File; 17 | import java.io.IOException; 18 | import java.lang.reflect.InvocationTargetException; 19 | import java.net.URL; 20 | import java.net.URLEncoder; 21 | import java.util.Arrays; 22 | import java.util.List; 23 | import java.util.stream.Collectors; 24 | 25 | public class QiniuPrivateStorage extends QiniuPublicStorage { 26 | 27 | public QiniuPrivateStorage(StorageProperties properties) { 28 | super(properties); 29 | } 30 | 31 | @Override 32 | public String getName() { 33 | return "七牛私有"; 34 | } 35 | 36 | @Override 37 | public void downloadToFile(String bucket, String key, File file) { 38 | try { 39 | String domainOfBucket = properties.getEndpoint(); 40 | String encodedFileName = URLEncoder.encode(key, "utf-8"); 41 | String publicUrl = String.format("%s/%s", domainOfBucket, encodedFileName); 42 | String finalUrl = auth.privateDownloadUrl(publicUrl); 43 | FileUtils.copyURLToFile(new URL(finalUrl), file); 44 | } catch (IOException e) { 45 | throw new RuntimeException(e); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/impl/QiniuPublicStorage.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source.impl; 2 | 3 | import com.qiniu.common.QiniuException; 4 | import com.qiniu.common.Zone; 5 | import com.qiniu.storage.BucketManager; 6 | import com.qiniu.storage.Configuration; 7 | import com.qiniu.storage.UploadManager; 8 | import com.qiniu.storage.model.FileListing; 9 | import com.qiniu.util.Auth; 10 | import io.loli.util.osm.source.Storage; 11 | import io.loli.util.osm.source.StorageProperties; 12 | import javaslang.Tuple; 13 | import javaslang.Tuple2; 14 | import javaslang.Tuple3; 15 | import org.apache.commons.io.FileUtils; 16 | 17 | import java.io.File; 18 | import java.io.IOException; 19 | import java.lang.reflect.InvocationTargetException; 20 | import java.net.URL; 21 | import java.net.URLEncoder; 22 | import java.util.Arrays; 23 | import java.util.List; 24 | import java.util.stream.Collectors; 25 | 26 | public class QiniuPublicStorage implements Storage { 27 | protected BucketManager client; 28 | protected UploadManager uploadManager; 29 | protected StorageProperties properties; 30 | protected Auth auth; 31 | 32 | @Override 33 | public Tuple3> list(String bucket, String prefix, String marker, Integer size) { 34 | 35 | FileListing objectListing = null; 36 | try { 37 | objectListing = client.listFiles(bucket, "", marker, size, null); 38 | return Tuple.of(objectListing.isEOF(), objectListing.marker, Arrays.stream(objectListing.items) 39 | .map(i -> i.key).collect(Collectors.toList())); 40 | } catch (QiniuException e) { 41 | throw new RuntimeException(e); 42 | } 43 | } 44 | 45 | public QiniuPublicStorage(StorageProperties properties) { 46 | this.properties = properties; 47 | //构造一个带指定Zone对象的配置类 48 | Configuration cfg = new Configuration(getZone(properties.getRegion())); 49 | auth = Auth.create(properties.getKey(), properties.getSecret()); 50 | client = new BucketManager(auth, cfg); 51 | uploadManager = new UploadManager(cfg); 52 | } 53 | 54 | @Override 55 | public String getName() { 56 | return "七牛公有"; 57 | } 58 | 59 | @Override 60 | public void downloadToFile(String bucket, String key, File file) { 61 | try { 62 | String domainOfBucket = properties.getEndpoint(); 63 | String encodedFileName; 64 | encodedFileName = URLEncoder.encode(key, "utf-8"); 65 | String finalUrl = String.format("%s/%s", domainOfBucket, encodedFileName); 66 | FileUtils.copyURLToFile(new URL(finalUrl), file); 67 | } catch (IOException e) { 68 | throw new RuntimeException(e); 69 | } 70 | } 71 | 72 | @Override 73 | public void upload(String bucket, String key, File file) { 74 | String upToken = auth.uploadToken(bucket); 75 | try { 76 | uploadManager.put(file.getPath(), key, upToken); 77 | } catch (QiniuException ex) { 78 | throw new RuntimeException(ex); 79 | } 80 | 81 | } 82 | 83 | @Override 84 | public boolean exist(String bucket, String key) { 85 | FileListing objectListing = null; 86 | try { 87 | objectListing = client.listFiles(bucket, key, null, 1, null); 88 | return objectListing.items != null && objectListing.items.length > 0; 89 | } catch (QiniuException e) { 90 | throw new RuntimeException(e); 91 | } 92 | } 93 | 94 | public void delete(String bucket, String key) { 95 | try { 96 | client.delete(bucket, key); 97 | } catch (QiniuException e) { 98 | throw new RuntimeException(e); 99 | } 100 | } 101 | 102 | 103 | private Zone getZone(String region) { 104 | try { 105 | return (Zone) Zone.class.getMethod(region).invoke(null); 106 | } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { 107 | return Zone.autoZone(); 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/impl/TencentCosStorage.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source.impl; 2 | 3 | import com.qcloud.cos.COSClient; 4 | import com.qcloud.cos.ClientConfig; 5 | import com.qcloud.cos.auth.BasicCOSCredentials; 6 | import com.qcloud.cos.auth.COSCredentials; 7 | import com.qcloud.cos.model.COSObjectSummary; 8 | import com.qcloud.cos.model.GetObjectRequest; 9 | import com.qcloud.cos.model.ListObjectsRequest; 10 | import com.qcloud.cos.model.ObjectListing; 11 | import com.qcloud.cos.region.Region; 12 | import io.loli.util.osm.source.Storage; 13 | import io.loli.util.osm.source.StorageProperties; 14 | import javaslang.Tuple; 15 | import javaslang.Tuple2; 16 | import javaslang.Tuple3; 17 | 18 | import java.io.File; 19 | import java.util.List; 20 | import java.util.stream.Collectors; 21 | 22 | public class TencentCosStorage implements Storage { 23 | private COSClient client; 24 | 25 | public TencentCosStorage(StorageProperties properties) { 26 | COSCredentials cred = new BasicCOSCredentials(properties.getKey(), properties.getSecret()); 27 | ClientConfig clientConfig = new ClientConfig(new Region(properties.getRegion())); 28 | client = new COSClient(cred, clientConfig); 29 | } 30 | 31 | @Override 32 | public String getName() { 33 | return "腾讯COS"; 34 | } 35 | 36 | @Override 37 | public Tuple3> list(String bucket, String prefix, String marker, Integer size) { 38 | ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket) 39 | .withMarker(marker) 40 | .withMaxKeys(size); 41 | ObjectListing objectListing = client.listObjects(request); 42 | return Tuple.of(objectListing.isTruncated(), objectListing.getNextMarker(), objectListing.getObjectSummaries().stream() 43 | .map(COSObjectSummary::getKey).collect(Collectors.toList())); 44 | } 45 | 46 | @Override 47 | public void downloadToFile(String bucket, String key, File file) { 48 | client.getObject(new GetObjectRequest(bucket, key), file); 49 | } 50 | 51 | @Override 52 | public void upload(String bucket, String key, File file) { 53 | client.putObject(bucket, key, file); 54 | } 55 | 56 | @Override 57 | public boolean exist(String bucket, String key) { 58 | return client.doesObjectExist(bucket, key); 59 | } 60 | 61 | @Override 62 | public void delete(String bucket, String key) { 63 | client.deleteObject(bucket, key); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/source/impl/UpaiStorage.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.source.impl; 2 | 3 | import com.UpYun; 4 | import com.upyun.UpException; 5 | import io.loli.util.osm.source.Storage; 6 | import io.loli.util.osm.source.StorageProperties; 7 | import javaslang.Tuple; 8 | import javaslang.Tuple3; 9 | 10 | import java.io.File; 11 | import java.io.FileNotFoundException; 12 | import java.io.IOException; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.stream.Collectors; 16 | 17 | public class UpaiStorage implements Storage { 18 | private UpYun client; 19 | 20 | public UpaiStorage(StorageProperties properties) { 21 | client = new UpYun(properties.getBucket(), properties.getKey(), properties.getSecret()); 22 | } 23 | 24 | @Override 25 | public String getName() { 26 | return "又拍云"; 27 | } 28 | 29 | @Override 30 | public Tuple3> list(String bucket, String prefix, String marker, Integer size) { 31 | try { 32 | List folderItems = client.readDir(prefix); 33 | return Tuple.of(true, null, folderItems.stream().map(i -> i.name).collect(Collectors.toList())); 34 | } catch (IOException | UpException e) { 35 | throw new RuntimeException(e); 36 | } 37 | } 38 | 39 | @Override 40 | public void downloadToFile(String bucket, String key, File file) { 41 | try { 42 | client.writeFile(key, file); 43 | } catch (IOException | UpException e) { 44 | throw new RuntimeException(e); 45 | } 46 | } 47 | 48 | @Override 49 | public void upload(String bucket, String key, File file) { 50 | try { 51 | client.writeFile("/" + key, file, true); 52 | } catch (IOException | UpException e) { 53 | throw new RuntimeException(e); 54 | } 55 | } 56 | 57 | @Override 58 | public boolean exist(String bucket, String key) { 59 | Map fileInfo = null; 60 | try { 61 | fileInfo = client.getFileInfo("/" + key); 62 | } catch (IOException | UpException e) { 63 | if (e instanceof FileNotFoundException) { 64 | return false; 65 | } 66 | throw new RuntimeException(e); 67 | } 68 | return fileInfo != null; 69 | } 70 | 71 | @Override 72 | public void delete(String bucket, String key) { 73 | try { 74 | client.deleteFile("/" + key); 75 | } catch (IOException | UpException e) { 76 | throw new RuntimeException(e); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/ui/OsmUi.form: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 |
419 | -------------------------------------------------------------------------------- /src/main/java/io/loli/util/osm/ui/OsmUi.java: -------------------------------------------------------------------------------- 1 | package io.loli.util.osm.ui; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.alibaba.fastjson.JSONObject; 5 | import io.loli.util.osm.source.Storage; 6 | import io.loli.util.osm.source.StorageFactory; 7 | import io.loli.util.osm.source.StorageProperties; 8 | import javaslang.Tuple2; 9 | import javaslang.Tuple3; 10 | import org.apache.commons.collections4.QueueUtils; 11 | import org.apache.commons.collections4.queue.CircularFifoQueue; 12 | import org.apache.commons.io.FileUtils; 13 | import org.apache.commons.lang3.StringUtils; 14 | 15 | import javax.swing.*; 16 | import java.awt.*; 17 | import java.io.File; 18 | import java.io.IOException; 19 | import java.util.List; 20 | import java.util.Queue; 21 | import java.util.*; 22 | import java.util.concurrent.ExecutorService; 23 | import java.util.concurrent.Executors; 24 | import java.util.concurrent.ScheduledExecutorService; 25 | import java.util.concurrent.TimeUnit; 26 | import java.util.concurrent.atomic.AtomicBoolean; 27 | import java.util.concurrent.atomic.AtomicInteger; 28 | import java.util.function.Consumer; 29 | import java.util.stream.Collectors; 30 | 31 | public class OsmUi extends JFrame { 32 | private JTabbedPane tabbedPane1; 33 | private JPanel panel1; 34 | private JComboBox type1; 35 | private JTextField region1; 36 | private JCheckBox 是否覆盖CheckBox; 37 | private JButton 开始Button; 38 | private JButton 暂停Button; 39 | private JButton 停止Button; 40 | private JButton 导出错误Button; 41 | private JTextField key1; 42 | private JTextField secret1; 43 | private JTextField bucket1; 44 | private JTextField region2; 45 | private JTextField key2; 46 | private JTextField secret2; 47 | private JTextField bucket2; 48 | private JComboBox type2; 49 | private JTextArea textArea1; 50 | private JLabel statusBarLabel; 51 | private JButton 批量删除Button; 52 | private JButton 批量删除Button1; 53 | private JTextField threadNumberfield; 54 | private JTextField endpoint1; 55 | private JTextField endpoint2; 56 | private JButton 互换Button; 57 | private Task task; 58 | 59 | public OsmUi() { 60 | setDefaultCloseOperation(EXIT_ON_CLOSE); 61 | setSize(new Dimension(485, 600)); 62 | setPreferredSize(new Dimension(485, 600)); 63 | setResizable(false); 64 | add(panel1); 65 | setVisible(true); 66 | this.repaint(); 67 | 暂停Button.setEnabled(false); 68 | 停止Button.setEnabled(false); 69 | 开始Button.addActionListener(e -> { 70 | if (startPrepare()) return; 71 | task.setFromProperties(fromProperties()); 72 | task.setToProperties(toProperties()); 73 | task.start(); 74 | }); 75 | 76 | 暂停Button.addActionListener(e -> { 77 | if ("暂停".equals(暂停Button.getText())) { 78 | 暂停Button.setText("继续"); 79 | } else if ("继续".equals(暂停Button.getText())) { 80 | 暂停Button.setText("暂停"); 81 | } 82 | task.pauseOrResume(); 83 | }); 84 | 停止Button.addActionListener(e -> { 85 | 开始Button.setEnabled(true); 86 | 批量删除Button.setEnabled(true); 87 | 批量删除Button1.setEnabled(true); 88 | 暂停Button.setEnabled(false); 89 | 停止Button.setEnabled(false); 90 | 导出错误Button.setEnabled(true); 91 | task.stop(); 92 | }); 93 | 94 | 批量删除Button.addActionListener(e -> { 95 | if (startPrepare()) return; 96 | task.setFromProperties(fromProperties()); 97 | task.setToProperties(fromProperties()); 98 | task.delete1(); 99 | }); 100 | 批量删除Button1.addActionListener(e -> { 101 | if (startPrepare()) return; 102 | task.setFromProperties(toProperties()); 103 | task.setToProperties(toProperties()); 104 | task.delete2(); 105 | }); 106 | 互换Button.addActionListener(e->{ 107 | String type1Str = type1.getSelectedItem().toString(); 108 | String type2Str = type2.getSelectedItem().toString(); 109 | String key1Str = key1.getText(); 110 | String key2Str = key2.getText(); 111 | String bucket1Str = bucket1.getText(); 112 | String bucket2Str = bucket2.getText(); 113 | String secret1Str = secret1.getText(); 114 | String secret2Str = secret2.getText(); 115 | String region1Str = region1.getText(); 116 | String region2Str = region2.getText(); 117 | String endpoint1Str = endpoint1.getText(); 118 | String endpoint2Str = endpoint2.getText(); 119 | type1.setSelectedItem(type2Str); 120 | type2.setSelectedItem(type1Str); 121 | key1.setText(key2Str); 122 | key2.setText(key1Str); 123 | region1.setText(region2Str); 124 | region2.setText(region1Str); 125 | secret1.setText(secret2Str); 126 | secret2.setText(secret1Str); 127 | bucket1.setText(bucket2Str); 128 | bucket2.setText(bucket1Str); 129 | endpoint1.setText(endpoint2Str); 130 | endpoint2.setText(endpoint1Str); 131 | repaint(); 132 | revalidate(); 133 | }); 134 | loadConfig(); 135 | } 136 | 137 | private boolean startPrepare() { 138 | if (!checkValue()) { 139 | return true; 140 | } 141 | saveConfig(); 142 | System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism", threadNumberfield.getText()); 143 | 开始Button.setEnabled(false); 144 | 暂停Button.setEnabled(true); 145 | 停止Button.setEnabled(true); 146 | 导出错误Button.setEnabled(true); 147 | 批量删除Button.setEnabled(false); 148 | 批量删除Button1.setEnabled(false); 149 | textArea1.setText(""); 150 | task = new Task(); 151 | return false; 152 | } 153 | 154 | private StorageProperties fromProperties() { 155 | return StorageProperties.StoragePropertiesBuilder.a() 156 | .withType(type1.getSelectedItem().toString()) 157 | .withBucket(bucket1.getText()) 158 | .withKey(key1.getText()) 159 | .withSecret(secret1.getText()) 160 | .withRegion(region1.getText()) 161 | .withEndpoint(endpoint1.getText()) 162 | .build(); 163 | } 164 | 165 | private StorageProperties toProperties() { 166 | return StorageProperties.StoragePropertiesBuilder.a() 167 | .withType(type2.getSelectedItem().toString()) 168 | .withBucket(bucket2.getText()) 169 | .withKey(key2.getText()) 170 | .withSecret(secret2.getText()) 171 | .withRegion(region2.getText()) 172 | .withEndpoint(endpoint2.getText()) 173 | .build(); 174 | } 175 | 176 | private void saveConfig() { 177 | try { 178 | String type1Str = type1.getSelectedItem().toString(); 179 | String type2Str = type2.getSelectedItem().toString(); 180 | String key1Str = key1.getText(); 181 | String key2Str = key2.getText(); 182 | String bucket1Str = bucket1.getText(); 183 | String bucket2Str = bucket2.getText(); 184 | String secret1Str = secret1.getText(); 185 | String secret2Str = secret2.getText(); 186 | String region1Str = region1.getText(); 187 | String region2Str = region2.getText(); 188 | String endpoint1Str = endpoint1.getText(); 189 | String endpoint2Str = endpoint2.getText(); 190 | JSONObject cfg = new JSONObject(); 191 | cfg.put("type1", type1Str); 192 | cfg.put("type2", type2Str); 193 | cfg.put("key1", key1Str); 194 | cfg.put("key2", key2Str); 195 | cfg.put("secret1", secret1Str); 196 | cfg.put("secret2", secret2Str); 197 | cfg.put("bucket1", bucket1Str); 198 | cfg.put("bucket2", bucket2Str); 199 | cfg.put("region1", region1Str); 200 | cfg.put("region2", region2Str); 201 | cfg.put("endpoint1", endpoint1Str); 202 | cfg.put("endpoint2", endpoint2Str); 203 | String json = cfg.toString(); 204 | String cfgFilePath = System.getProperty("java.io.tmpdir") + File.separator + "osmcfg.json"; 205 | FileUtils.writeStringToFile(new File(cfgFilePath), json, "UTF-8"); 206 | } catch (Exception ignored) { 207 | } 208 | } 209 | 210 | private void loadConfig() { 211 | try { 212 | String cfgFilePath = System.getProperty("java.io.tmpdir") + File.separator + "osmcfg.json"; 213 | String s = FileUtils.readFileToString(new File(cfgFilePath), "UTF-8"); 214 | JSONObject cfg = JSON.parseObject(s); 215 | String type1Str = cfg.getString("type1"); 216 | String type2Str = cfg.getString("type2"); 217 | String key1Str = cfg.getString("key1"); 218 | String key2Str = cfg.getString("key2"); 219 | String bucket1Str = cfg.getString("bucket1"); 220 | String bucket2Str = cfg.getString("bucket2"); 221 | String secret1Str = cfg.getString("secret1"); 222 | String secret2Str = cfg.getString("secret2"); 223 | String region1Str = cfg.getString("region1"); 224 | String region2Str = cfg.getString("region2"); 225 | String endpoint1Str = cfg.getString("endpoint1"); 226 | String endpoint2Str = cfg.getString("endpoint2"); 227 | type1.setSelectedItem(type1Str); 228 | type2.setSelectedItem(type2Str); 229 | key1.setText(key1Str); 230 | key2.setText(key2Str); 231 | region1.setText(region1Str); 232 | region2.setText(region2Str); 233 | secret1.setText(secret1Str); 234 | secret2.setText(secret2Str); 235 | bucket1.setText(bucket1Str); 236 | bucket2.setText(bucket2Str); 237 | endpoint1.setText(endpoint1Str); 238 | endpoint2.setText(endpoint2Str); 239 | repaint(); 240 | revalidate(); 241 | } catch (Exception ignored) { 242 | } 243 | } 244 | 245 | private boolean checkValue() { 246 | if ( 247 | StringUtils.isBlank(key1.getText()) || 248 | StringUtils.isBlank(key2.getText()) || 249 | StringUtils.isBlank(secret1.getText()) || 250 | StringUtils.isBlank(secret2.getText()) || 251 | StringUtils.isBlank(bucket1.getText()) || 252 | StringUtils.isBlank(bucket2.getText())) { 253 | JOptionPane.showMessageDialog(this, "参数不能为空", 254 | "检查参数", 255 | JOptionPane.ERROR_MESSAGE); 256 | return false; 257 | } 258 | return true; 259 | } 260 | 261 | private class Task { 262 | 263 | private AtomicInteger successCount = new AtomicInteger(); 264 | private AtomicInteger failedCount = new AtomicInteger(); 265 | private final Queue log = QueueUtils.synchronizedQueue(new CircularFifoQueue<>(1000)); 266 | private Map failedReason = new HashMap<>(); 267 | private StorageProperties fromProperties; 268 | private StorageProperties toProperties; 269 | private Storage from; 270 | private Storage to; 271 | private ExecutorService exe = Executors.newFixedThreadPool(2); 272 | private ExecutorService lockPool = Executors.newSingleThreadExecutor(); 273 | private ScheduledExecutorService schedule = Executors.newScheduledThreadPool(2); 274 | private final Object pauseLock = new Object(); 275 | private AtomicBoolean pauseStatus = new AtomicBoolean(false); 276 | private AtomicBoolean stopStatus = new AtomicBoolean(false); 277 | 278 | private void start() { 279 | listAndAction(key -> { 280 | try { 281 | downloadAndUpload(key, 是否覆盖CheckBox.isSelected()); 282 | } catch (IOException ignored) { 283 | } 284 | }); 285 | } 286 | 287 | private void listAndAction(Consumer action) { 288 | from = StorageFactory.getStorage(fromProperties); 289 | to = StorageFactory.getStorage(toProperties); 290 | schedule.scheduleAtFixedRate(() -> { 291 | statusBarLabel.setText(successCount.get() + "/" + failedCount.get()); 292 | synchronized (log) { 293 | textArea1.setText(log.stream().sorted(Collections.reverseOrder()).collect(Collectors.joining("\r\n"))); 294 | } 295 | textArea1.repaint(); 296 | textArea1.revalidate(); 297 | }, 1, 1, TimeUnit.SECONDS); 298 | exe.execute(new Runnable() { 299 | @Override 300 | public void run() { 301 | String lastMarker = null; 302 | for (; !stopStatus.get(); ) { 303 | try { 304 | // 检查暂停状态，如果已暂停，则等待唤醒 305 | if (pauseStatus.get()) { 306 | waitForNotify(); 307 | } 308 | Tuple3> list = from.list(fromProperties.getBucket(), "", lastMarker, 1000); 309 | 310 | // 如果都上传完了，用返回的marker 311 | List fileKeys = list._3; 312 | fileKeys.parallelStream().forEach(key -> { 313 | // 如果已停止，就不执行了 314 | 315 | // 检查暂停状态，如果已暂停，则等待唤醒 316 | if (pauseStatus.get()) { 317 | waitForNotify(); 318 | } 319 | // 检查停止状态，如果已停止，则不处理 320 | if (stopStatus.get()) { 321 | return; 322 | } 323 | try { 324 | action.accept(key); 325 | } catch (Exception e) { 326 | failedCount.incrementAndGet(); 327 | String message = e.getClass().getName() + "^^" + e.getMessage(); 328 | addLog(key, message); 329 | failedReason.put(key, message); 330 | } 331 | }); 332 | lastMarker = list._2; 333 | if (list._2 == null) { 334 | // 处理结束 335 | log.add("处理结束，共有" + 336 | successCount + 337 | "个成功，共有" + 338 | failedCount + 339 | "个失败"); 340 | break; 341 | } 342 | } catch (Exception e) { 343 | addLog("发生错误", e.getClass().getName() + "^^" + e.getMessage()); 344 | if (successCount.get() == 0) { 345 | JOptionPane.showMessageDialog(textArea1, "第一次查询就报错了，请检查参数", 346 | "检查参数", 347 | JOptionPane.ERROR_MESSAGE); 348 | break; 349 | } 350 | } 351 | } 352 | } 353 | }); 354 | } 355 | 356 | private void addLog(String key, String message) { 357 | String e = key + "^^" + message; 358 | synchronized (log) { 359 | log.add(e); 360 | } 361 | } 362 | 363 | private void downloadAndUpload(String key, boolean override) throws IOException { 364 | File file = File.createTempFile("mig", "f"); 365 | try { 366 | if (override) { 367 | from.downloadToFile(fromProperties.getBucket(), key, file); 368 | to.upload(toProperties.getBucket(), key, file); 369 | addLog(key, "处理成功"); 370 | } else { 371 | if (!to.exist(toProperties.getBucket(), key)) { 372 | from.downloadToFile(fromProperties.getBucket(), key, file); 373 | to.upload(toProperties.getBucket(), key, file); 374 | addLog(key, "处理成功"); 375 | } else { 376 | addLog(key, "已存在"); 377 | } 378 | } 379 | successCount.incrementAndGet(); 380 | 381 | } finally { 382 | file.delete(); 383 | } 384 | } 385 | 386 | private void stop() { 387 | stopStatus.set(true); 388 | exe.shutdownNow(); 389 | schedule.shutdownNow(); 390 | } 391 | 392 | public void repushFailed() { 393 | // 处理失败的 394 | Map failed = new HashMap<>(); 395 | exe.execute(() -> failedReason.forEach((key, v) -> { 396 | // 检查暂停状态，如果已暂停，则等待唤醒 397 | if (pauseStatus.get()) { 398 | waitForNotify(); 399 | } 400 | try { 401 | downloadAndUpload(key, 是否覆盖CheckBox.isSelected()); 402 | failedCount.decrementAndGet(); 403 | } catch (Exception e) { 404 | failed.put("key", e.getClass().getName() + "^^" + e.getMessage()); 405 | } 406 | })); 407 | failedReason = failed; 408 | } 409 | 410 | private void waitForNotify() { 411 | synchronized (pauseLock) { 412 | try { 413 | pauseLock.wait(); 414 | } catch (InterruptedException ignored) { 415 | } 416 | } 417 | } 418 | 419 | private void pauseOrResume() { 420 | if (pauseStatus.get()) { 421 | // 如果已暂停，则唤醒 422 | pauseStatus.set(false); 423 | synchronized (pauseLock) { 424 | pauseLock.notifyAll(); 425 | } 426 | } else { 427 | // 如果未暂停，则暂停 428 | pauseStatus.set(true); 429 | } 430 | } 431 | 432 | private void setFromProperties(StorageProperties fromProperties) { 433 | this.fromProperties = fromProperties; 434 | } 435 | 436 | private void setToProperties(StorageProperties toProperties) { 437 | this.toProperties = toProperties; 438 | } 439 | 440 | private void delete1() { 441 | listAndAction(key -> { 442 | from.delete(fromProperties.getBucket(), key); 443 | successCount.incrementAndGet(); 444 | addLog(key, "删除成功"); 445 | }); 446 | } 447 | 448 | private void delete2() { 449 | listAndAction(key -> { 450 | to.delete(toProperties.getBucket(), key); 451 | successCount.incrementAndGet(); 452 | addLog(key, "删除成功"); 453 | }); 454 | } 455 | } 456 | } 457 | --------------------------------------------------------------------------------