├── .gitignore ├── README.md ├── pom.xml └── src ├── main ├── assemblies │ └── plugin.xml ├── java │ └── org │ │ └── elasticsearch │ │ ├── aliyun │ │ └── oss │ │ │ ├── blobstore │ │ │ ├── OssBlobContainer.java │ │ │ └── OssBlobStore.java │ │ │ └── service │ │ │ ├── OssClientSettings.java │ │ │ ├── OssService.java │ │ │ ├── OssServiceImpl.java │ │ │ ├── OssStorageClient.java │ │ │ └── exception │ │ │ └── CreateStsOssClientException.java │ │ ├── plugin │ │ └── repository │ │ │ └── oss │ │ │ └── OssRepositoryPlugin.java │ │ ├── repository │ │ └── oss │ │ │ └── OssRepository.java │ │ └── utils │ │ ├── DateHelper.java │ │ ├── HttpClientHelper.java │ │ └── PermissionHelper.java └── resources │ ├── plugin-descriptor.properties │ └── plugin-security.policy └── test └── java └── org └── elasticsearch ├── aliyun └── oss │ └── blobstore │ ├── MockOssService.java │ ├── OssBlobContainerTest.java │ └── OssBlobStoreTest.java ├── bootstrap └── JarHell.java └── repository └── oss └── OssRepositoryTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | # Files for the Dalvik VM 2 | *.dex 3 | 4 | # Java class files 5 | *.class 6 | 7 | # Generated files 8 | */bin/ 9 | */gen/ 10 | */out/ 11 | 12 | # Gradle files 13 | .gradle/ 14 | build/ 15 | */build/ 16 | gradlew 17 | gradlew.bat 18 | 19 | # Local configuration file (sdk path, etc) 20 | local.properties 21 | 22 | # Proguard folder generated by Eclipse 23 | proguard/ 24 | 25 | # Log Files 26 | *.log 27 | 28 | # Android Studio Navigation editor temp files 29 | .navigation/ 30 | 31 | # Android Studio captures folder 32 | captures/ 33 | 34 | # Intellij 35 | *.iml 36 | */*.iml 37 | 38 | # Keystore files 39 | #*.jks 40 | #gradle wrapper 41 | gradle/ 42 | 43 | #some local files 44 | */.settings/ 45 | */.DS_Store 46 | .DS_Store 47 | */.idea/ 48 | .idea/ 49 | target/ 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elasticsearch-repository-oss 2 | 3 | 4 | 要备份你的集群，你可以使用 `snapshot` API。这个会拿到你集群里当前的状态和数据然后保存到一个共享仓库里。这个备份过程是"智能"的。你的第一个快照会是一个数据的完整拷贝，但是所有后续的快照会保留的是已存快照和新数据之间的差异。随着你不时的对数据进行快照，备份也在增量的添加和删除。这意味着后续备份会相当快速，因为它们只传输很小的数据量。 5 | 6 | * [OSS快照迁移文档](https://github.com/zhichen/elasticsearch-repository-oss/wiki/OSS快照迁移) 7 | 8 | 9 | ## 创建仓库 10 | ``` 11 | PUT _snapshot/my_backup 12 | { 13 | "type": "oss", 14 | "settings": { 15 | "endpoint": "http://oss-cn-hangzhou-internal.aliyuncs.com", <1> 16 | "access_key_id": "xxxx", 17 | "secret_access_key": "xxxxxx", 18 | "bucket": "xxxxxx", <2> 19 | "compress": true 20 | } 21 | } 22 | ``` 23 | * <1> 本处的OSS, 要求和你的elasticsearch集群在同一个region中, 这里的endpoint填的是这个region对应的内网地址 ,具体参考 https://help.aliyun.com/document_detail/31837.html?spm=5176.doc31922.6.577.YxqZYt 中`ECS访问的内网Endpoint`一栏 24 | * <2> 需要一个已经存在的OSS bucket 25 | 26 | 27 | 28 | 29 | 假设我们上传的数据非常大, 我们可以限制snapshot过程中分块的大小,超过这个大小，数据将会被分块上传到OSS中 30 | 31 | ``` 32 | POST _snapshot/my_backup/ <1> 33 | { 34 | "type": "oss", 35 | "settings": { 36 | "endpoint": "http://oss-cn-hangzhou-internal.aliyuncs.com", 37 | "access_key_id": "xxxx", 38 | "secret_access_key": "xxxxxx", 39 | "bucket": "xxxxxx", 40 | "chunk_size": "500mb", 41 | "base_path": "snapshot/" <2> 42 | } 43 | } 44 | ``` 45 | * <1> 注意我们用的是 `POST` 而不是 `PUT` 。这会更新已有仓库的设置。 46 | * <2> base_path 设置仓库的起始位置默认为根目录 47 | 48 | ## 列出仓库信息 49 | ``` 50 | GET _snapshot 51 | ``` 52 | * 也可以使用 `GET _snapshot/my_backup` 获取指定仓库的信息 53 | 54 | ### 备份快照迁移 55 | 如果需要将快照迁移到另一个集群.只需要备份到OSS, 然后再在新的集群上注册一个快照仓库(相同的OSS),设置`base_path`的位置为备份文件所在的地方，然后执行恢复备份的命令即可。 56 | 57 | ### 快照所有打开的索引 （以下内容和官方一致） 58 | 59 | 一个仓库可以包含多个快照。每个快照跟一系列索引相关（比如所有索引，一部分索引，或者单个索引）。当创建快照的时候，你指定你感兴趣的索引然后给快照取一个唯一的名字。 60 | 61 | 让我们从最基础的快照命令开始： 62 | 63 | ``` 64 | PUT _snapshot/my_backup/snapshot_1 65 | ``` 66 | 67 | 这个会备份所有打开的索引到 `my_backup` 仓库下一个命名为 `snapshot_1` 的快照里。这个调用会立刻返回，然后快照会在后台运行。 68 | 69 | 70 | 71 | 通常你会希望你的快照作为后台进程运行，不过有时候你会希望在你的脚本中一直等待到完成。这可以通过添加一个 `wait_for_completion` 标记实现： 72 | 73 | ``` 74 | PUT _snapshot/my_backup/snapshot_1?wait_for_completion=true 75 | ``` 76 | 77 | 这个会阻塞调用直到快照完成。注意大型快照会花很长时间才返回。 78 | 79 | 80 | 81 | 82 | ### 快照指定索引 83 | 84 | 默认行为是备份所有打开的索引。不过如果你在用 Kibana，你不是真的想要把所有诊断相关的 `.kibana` 索引也备份起来。可能你就压根没那么大空间备份所有数据。 85 | 86 | 这种情况下，你可以在快照你的集群的时候指定备份哪些索引： 87 | 88 | ``` 89 | PUT _snapshot/my_backup/snapshot_2 90 | { 91 | "indices": "index_1,index_2" 92 | } 93 | ``` 94 | 95 | 这个快照命令现在只会备份 `index1` 和 `index2` 了。 96 | 97 | ### 列出快照相关的信息 98 | 99 | 一旦你开始在你的仓库里积攒起快照了，你可能就慢慢忘记里面各自的细节了——特别是快照按照时间划分命名的时候（比如， `backup_2014_10_28` ）。 100 | 101 | 要获得单个快照的信息，直接对仓库和快照名发起一个 `GET` 请求： 102 | 103 | ``` 104 | GET _snapshot/my_backup/snapshot_2 105 | ``` 106 | 107 | 这个会返回一个小响应，包括快照相关的各种信息： 108 | 109 | ``` 110 | { 111 | "snapshots": [ 112 | { 113 | "snapshot": "snapshot_1", 114 | "indices": [ 115 | ".marvel_2014_28_10", 116 | "index1", 117 | "index2" 118 | ], 119 | "state": "SUCCESS", 120 | "start_time": "2014-09-02T13:01:43.115Z", 121 | "start_time_in_millis": 1409662903115, 122 | "end_time": "2014-09-02T13:01:43.439Z", 123 | "end_time_in_millis": 1409662903439, 124 | "duration_in_millis": 324, 125 | "failures": [], 126 | "shards": { 127 | "total": 10, 128 | "failed": 0, 129 | "successful": 10 130 | } 131 | } 132 | ] 133 | } 134 | ``` 135 | 136 | 要获取一个仓库中所有快照的完整列表，使用 `_all` 占位符替换掉具体的快照名称： 137 | 138 | ``` 139 | GET _snapshot/my_backup/_all 140 | ``` 141 | 142 | ### 删除快照 143 | 144 | 最后，我们需要一个命令来删除所有不再有用的旧快照。这只要对仓库/快照名称发一个简单的 `DELETE` HTTP 调用： 145 | 146 | ``` 147 | DELETE _snapshot/my_backup/snapshot_2 148 | ``` 149 | 150 | 用 API 删除快照很重要，而不能用其他机制（比如手动删除）。因为快照是增量的，有可能很多快照依赖于过去的段。`delete` API 知道哪些数据还在被更多近期快照使用，然后会只删除不再被使用的段。 151 | 152 | 但是，如果你做了一次人工文件删除，你将会面临备份严重损坏的风险，因为你在删除的是可能还在使用中的数据。 153 | 154 | 155 | ### 监控快照进度 156 | 157 | `wait_for_completion` 标记提供了一个监控的基础形式，但哪怕只是对一个中等规模的集群做快照恢复的时候，它都真的不够用。 158 | 159 | 另外两个 API 会给你有关快照状态更详细的信息。首先你可以给快照 ID 执行一个 `GET`，就像我们之前获取一个特定快照的信息时做的那样： 160 | 161 | ``` 162 | GET _snapshot/my_backup/snapshot_3 163 | ``` 164 | 165 | 如果你调用这个命令的时候快照还在进行中，你会看到它什么时候开始，运行了多久等等信息。不过要注意，这个 API 用的是快照机制相同的线程池。如果你在快照非常大的分片，状态更新的间隔会很大，因为 API 在竞争相同的线程池资源。 166 | 167 | 更好的方案是拽取 `_status` API 数据： 168 | 169 | ``` 170 | GET _snapshot/my_backup/snapshot_3/_status 171 | ``` 172 | 173 | `_status` API 立刻返回，然后给出详细的多的统计值输出： 174 | 175 | ``` 176 | { 177 | "snapshots": [ 178 | { 179 | "snapshot": "snapshot_3", 180 | "repository": "my_backup", 181 | "state": "IN_PROGRESS", <1> 182 | "shards_stats": { 183 | "initializing": 0, 184 | "started": 1, <2> 185 | "finalizing": 0, 186 | "done": 4, 187 | "failed": 0, 188 | "total": 5 189 | }, 190 | "stats": { 191 | "number_of_files": 5, 192 | "processed_files": 5, 193 | "total_size_in_bytes": 1792, 194 | "processed_size_in_bytes": 1792, 195 | "start_time_in_millis": 1409663054859, 196 | "time_in_millis": 64 197 | }, 198 | "indices": { 199 | "index_3": { 200 | "shards_stats": { 201 | "initializing": 0, 202 | "started": 0, 203 | "finalizing": 0, 204 | "done": 5, 205 | "failed": 0, 206 | "total": 5 207 | }, 208 | "stats": { 209 | "number_of_files": 5, 210 | "processed_files": 5, 211 | "total_size_in_bytes": 1792, 212 | "processed_size_in_bytes": 1792, 213 | "start_time_in_millis": 1409663054859, 214 | "time_in_millis": 64 215 | }, 216 | "shards": { 217 | "0": { 218 | "stage": "DONE", 219 | "stats": { 220 | "number_of_files": 1, 221 | "processed_files": 1, 222 | "total_size_in_bytes": 514, 223 | "processed_size_in_bytes": 514, 224 | "start_time_in_millis": 1409663054862, 225 | "time_in_millis": 22 226 | } 227 | }, 228 | ... 229 | ``` 230 | * <1> 一个正在运行的快照会显示 `IN_PROGRESS` 作为状态。 231 | * <2> 这个特定快照有一个分片还在传输（另外四个已经完成）。 232 | 233 | 响应包括快照的总体状况，但也包括下钻到每个索引和每个分片的统计值。这个给你展示了有关快照进展的非常详细的视图。分片可以在不同的完成状态： 234 | 235 | `INITIALIZING`:: 236 | 分片在检查集群状态看看自己是否可以被快照。这个一般是非常快的。 237 | 238 | `STARTED`:: 239 | 数据正在被传输到仓库。 240 | 241 | `FINALIZING`:: 242 | 数据传输完成；分片现在在发送快照元数据。 243 | 244 | `DONE`:: 245 | 快照完成！ 246 | 247 | `FAILED`:: 248 | 快照处理的时候碰到了错误，这个分片/索引/快照不可能完成了。检查你的日志获取更多信息。 249 | 250 | 251 | ### 取消一个快照 252 | 253 | 最后，你可能想取消一个快照或恢复。因为它们是长期运行的进程，执行操作的时候一个笔误或者过错就会花很长时间来解决——而且同时还会耗尽有价值的资源。 254 | 255 | 要取消一个快照，在他进行中的时候简单的删除快照就可以： 256 | 257 | ``` 258 | DELETE _snapshot/my_backup/snapshot_3 259 | ``` 260 | 261 | 这个会中断快照进程。然后删除仓库里进行到一半的快照。 262 | 263 | 264 | ### 从快照恢复 265 | 266 | 一旦你备份过了数据，恢复它就简单了：只要在你希望恢复回集群的快照 ID 后面加上 `_restore` 即可： 267 | 268 | ``` 269 | POST _snapshot/my_backup/snapshot_1/_restore 270 | ``` 271 | 272 | 默认行为是把这个快照里存有的所有索引都恢复。如果 `snapshot_1` 包括五个索引，这五个都会被恢复到我们集群里。和 `snapshot` API 一样，我们也可以选择希望恢复具体哪个索引。 273 | 274 | 还有附加的选项用来重命名索引。这个选项允许你通过模式匹配索引名称，然后通过恢复进程提供一个新名称。如果你想在不替换现有数据的前提下，恢复老数据来验证内容，或者做其他处理，这个选项很有用。让我们从快照里恢复单个索引并提供一个替换的名称： 275 | 276 | ``` 277 | POST /_snapshot/my_backup/snapshot_1/_restore 278 | { 279 | "indices": "index_1", <1> 280 | "rename_pattern": "index_(.+)", <2> 281 | "rename_replacement": "restored_index_$1" <3> 282 | } 283 | ``` 284 | * <1> 只恢复 `index_1` 索引，忽略快照中存在的其余索引。 285 | * <2> 查找所提供的模式能匹配上的正在恢复的索引。 286 | * <3> 然后把它们重命名成替代的模式。 287 | 288 | 这个会恢复 `index_1` 到你及群里，但是重命名成了 `restored_index_1` 。 289 | 290 | 291 | 292 | 和快照类似， `restore` 命令也会立刻返回，恢复进程会在后台进行。如果你更希望你的 HTTP 调用阻塞直到恢复完成，添加 `wait_for_completion` 标记： 293 | 294 | ``` 295 | POST _snapshot/my_backup/snapshot_1/_restore?wait_for_completion=true 296 | ``` 297 | 298 | 299 | 300 | 301 | ### 监控恢复操作 302 | 303 | 从仓库恢复数据借鉴了 Elasticsearch 里已有的现行恢复机制。在内部实现上，从仓库恢复分片和从另一个节点恢复是等价的。 304 | 305 | 如果你想监控恢复的进度，你可以使用 `recovery` API。这是一个通用目的的 API，用来展示你集群中移动着的分片状态。 306 | 307 | 这个 API 可以为你在恢复的指定索引单独调用： 308 | 309 | ``` 310 | GET restored_index_3/_recovery 311 | ``` 312 | 313 | 或者查看你集群里所有索引，可能包括跟你的恢复进程无关的其他分片移动： 314 | 315 | ``` 316 | GET /_recovery/ 317 | ``` 318 | 319 | 输出会跟这个类似（注意，根据你集群的活跃度，输出可能会变得非常啰嗦！）： 320 | 321 | ``` 322 | { 323 | "restored_index_3" : { 324 | "shards" : [ { 325 | "id" : 0, 326 | "type" : "snapshot", <1> 327 | "stage" : "index", 328 | "primary" : true, 329 | "start_time" : "2014-02-24T12:15:59.716", 330 | "stop_time" : 0, 331 | "total_time_in_millis" : 175576, 332 | "source" : { <2> 333 | "repository" : "my_backup", 334 | "snapshot" : "snapshot_3", 335 | "index" : "restored_index_3" 336 | }, 337 | "target" : { 338 | "id" : "ryqJ5lO5S4-lSFbGntkEkg", 339 | "hostname" : "my.fqdn", 340 | "ip" : "10.0.1.7", 341 | "name" : "my_es_node" 342 | }, 343 | "index" : { 344 | "files" : { 345 | "total" : 73, 346 | "reused" : 0, 347 | "recovered" : 69, 348 | "percent" : "94.5%" <3> 349 | }, 350 | "bytes" : { 351 | "total" : 79063092, 352 | "reused" : 0, 353 | "recovered" : 68891939, 354 | "percent" : "87.1%" 355 | }, 356 | "total_time_in_millis" : 0 357 | }, 358 | "translog" : { 359 | "recovered" : 0, 360 | "total_time_in_millis" : 0 361 | }, 362 | "start" : { 363 | "check_index_time" : 0, 364 | "total_time_in_millis" : 0 365 | } 366 | } ] 367 | } 368 | } 369 | ``` 370 | * <1> `type` 字段告诉你恢复的本质；这个分片是在从一个快照恢复。 371 | * <2> `source` 哈希描述了作为恢复来源的特定快照和仓库。 372 | * <3> `percent` 字段让你对恢复的状态有个概念。这个特定分片目前已经恢复了 94% 的文件；它就快完成了。 373 | 374 | 输出会列出所有目前正在经历恢复的索引，然后列出这些索引里的所有分片。每个分片里会有启动/停止时间、持续时间、恢复百分比、传输字节数等统计值。 375 | 376 | ### 取消一个恢复 377 | 378 | 要取消一个恢复，你需要删除正在恢复的索引。因为恢复进程其实就是分片恢复，发送一个 `删除索引` API 修改集群状态，就可以停止恢复进程。比如： 379 | 380 | ``` 381 | DELETE /restored_index_3 382 | ``` 383 | 384 | 如果 `restored_index_3` 正在恢复中，这个删除命令会停止恢复，同时删除所有已经恢复到集群里的数据。 385 | 386 | 387 | 388 | 389 | 390 | 391 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.opensearch.es 8 | elasticsearch-repository-oss 9 | 6.7.0 10 | 11 | 12 | 13 | 6.7.0 14 | 7.7.0 15 | 1.8 16 | elasticsearch-repository-oss 17 | ${project.basedir}/src/main/assemblies/plugin.xml 18 | 19 | elasticsearch-repository-oss 20 | 21 | org.elasticsearch.plugin.repository.oss.OssRepositoryPlugin 22 | 23 | true 24 | true 25 | false 26 | true 27 | 4E899B30 28 | true 29 | UTF-8 30 | 31 | 32 | 33 | org.elasticsearch 34 | elasticsearch 35 | ${elasticsearch.version} 36 | compile 37 | 38 | 39 | securemock 40 | org.elasticsearch 41 | 42 | 43 | 44 | 45 | org.elasticsearch.test 46 | framework 47 | ${elasticsearch.version} 48 | test 49 | 50 | 51 | org.apache.lucene 52 | lucene-test-framework 53 | ${lucene.version} 54 | test 55 | 56 | 57 | com.google.guava 58 | guava 59 | 18.0 60 | compile 61 | 62 | 63 | org.slf4j 64 | slf4j-log4j12 65 | 1.7.12 66 | provided 67 | 68 | 69 | log4j 70 | log4j 71 | 1.2.17 72 | provided 73 | 74 | 75 | 76 | 77 | org.apache.logging.log4j 78 | log4j-core 79 | 2.9.1 80 | provided 81 | 82 | 83 | 84 | com.aliyun.oss 85 | aliyun-sdk-oss 86 | 2.8.1 87 | 88 | 89 | commons-beanutils 90 | commons-beanutils 91 | 92 | 93 | commons-collections 94 | commons-collections 95 | 96 | 97 | 98 | 99 | com.squareup.okhttp3 100 | okhttp 101 | 3.10.0 102 | 103 | 104 | com.alibaba 105 | fastjson 106 | 1.2.47 107 | 108 | 109 | junit 110 | junit 111 | 4.12 112 | test 113 | 114 | 115 | org.mockito 116 | mockito-core 117 | 1.10.19 118 | test 119 | 120 | 121 | com.squareup.okhttp3 122 | mockwebserver 123 | 3.10.0 124 | test 125 | 126 | 127 | org.mockito 128 | mockito-core 129 | 2.18.0 130 | test 131 | 132 | 133 | org.powermock 134 | powermock-api-mockito2 135 | 2.0.0-beta.5 136 | test 137 | 138 | 139 | org.powermock 140 | powermock-module-junit4 141 | 2.0.0-beta.5 142 | test 143 | 144 | 145 | org.elasticsearch 146 | securemock 147 | 1.2 148 | test 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | maven-assembly-plugin 157 | 2.3 158 | 159 | false 160 | 161 | 162 | ${basedir}/src/main/assemblies/plugin.xml 163 | 164 | 165 | 166 | 167 | package 168 | 169 | single 170 | 171 | 172 | 173 | 174 | 175 | 176 | org.apache.maven.plugins 177 | maven-compiler-plugin 178 | 179 | ${maven.compiler.target} 180 | ${maven.compiler.target} 181 | 182 | 183 | 184 | 185 | 186 | 187 | -------------------------------------------------------------------------------- /src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | elasticsearch-repository-oss 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | elasticsearch 11 | true 12 | true 13 | 14 | org.elasticsearch:elasticsearch 15 | 16 | 17 | 18 | 19 | 20 | src/main/resources 21 | /elasticsearch 22 | 23 | plugin-security.policy 24 | plugin-descriptor.properties 25 | log4j2.properties 26 | 27 | 28 | 29 | 30 | lib 31 | 32 | 33 | *.jar 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/aliyun/oss/blobstore/OssBlobContainer.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.blobstore; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.nio.file.FileAlreadyExistsException; 6 | import java.nio.file.NoSuchFileException; 7 | import java.util.Map; 8 | 9 | import com.aliyun.oss.ClientException; 10 | import com.aliyun.oss.OSSException; 11 | import org.apache.commons.lang.StringUtils; 12 | import org.apache.logging.log4j.LogManager; 13 | import org.apache.logging.log4j.Logger; 14 | import org.elasticsearch.common.blobstore.BlobMetaData; 15 | import org.elasticsearch.common.blobstore.BlobPath; 16 | import org.elasticsearch.common.blobstore.BlobStoreException; 17 | import org.elasticsearch.common.blobstore.support.AbstractBlobContainer; 18 | /** 19 | * A class for managing a oss repository of blob entries, where each blob entry is just a named group of bytes 20 | * Created by yangkongshi on 2017/11/24. 21 | */ 22 | public class OssBlobContainer extends AbstractBlobContainer { 23 | private static final Logger logger = LogManager.getLogger(OssBlobContainer.class); 24 | protected final OssBlobStore blobStore; 25 | protected final String keyPath; 26 | 27 | public OssBlobContainer(BlobPath path, OssBlobStore blobStore) { 28 | super(path); 29 | this.keyPath = path.buildAsString(); 30 | this.blobStore = blobStore; 31 | } 32 | 33 | /** 34 | * Tests whether a blob with the given blob name exists in the container. 35 | * 36 | * @param blobName The name of the blob whose existence is to be determined. 37 | * @return {@code true} if a blob exists in the BlobContainer with the given name, and {@code false} otherwise. 38 | */ 39 | @Override 40 | public boolean blobExists(String blobName) { 41 | logger.trace("blobExists({})", blobName); 42 | try { 43 | return blobStore.blobExists(buildKey(blobName)); 44 | } catch (OSSException | ClientException | IOException e) { 45 | logger.warn("can not access [{}] : {}", blobName, e.getMessage()); 46 | throw new BlobStoreException("Failed to check if blob [" + blobName + "] exists", e); 47 | } 48 | } 49 | 50 | /** 51 | * Creates a new {@link InputStream} for the given blob name. 52 | * 53 | * @param blobName The name of the blob to get an {@link InputStream} for. 54 | * @return The {@code InputStream} to read the blob. 55 | * @throws NoSuchFileException if the blob does not exist 56 | * @throws IOException if the blob can not be read. 57 | */ 58 | @Override 59 | public InputStream readBlob(String blobName) throws IOException { 60 | logger.trace("readBlob({})", blobName); 61 | if (!blobExists(blobName)) { 62 | throw new NoSuchFileException("[" + blobName + "] blob not found"); 63 | } 64 | return blobStore.readBlob(buildKey(blobName)); 65 | } 66 | 67 | /** 68 | * Reads blob content from the input stream and writes it to the container in a new blob with the given name. 69 | * This method assumes the container does not already contain a blob of the same blobName. If a blob by the 70 | * same name already exists, the operation will fail and an {@link IOException} will be thrown. 71 | * 72 | * @param blobName The name of the blob to write the contents of the input stream to. 73 | * @param inputStream The input stream from which to retrieve the bytes to write to the blob. 74 | * @param blobSize The size of the blob to be written, in bytes. It is implementation dependent whether 75 | * this value is used in writing the blob to the repository. 76 | * @param failIfAlreadyExists whether to throw a FileAlreadyExistsException if the given blob already exists 77 | * @throws FileAlreadyExistsException if failIfAlreadyExists is true and a blob by the same name already exists 78 | * @throws IOException if the input stream could not be read, or the target blob could not be written to. 79 | */ 80 | @Override 81 | public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) 82 | throws IOException { 83 | if (blobExists(blobName)) { 84 | if (failIfAlreadyExists) { 85 | throw new FileAlreadyExistsException( 86 | "blob [" + blobName + "] already exists, cannot overwrite"); 87 | } else { 88 | deleteBlobIgnoringIfNotExists(blobName); 89 | } 90 | } 91 | logger.trace("writeBlob({}, stream, {})", blobName, blobSize); 92 | blobStore.writeBlob(buildKey(blobName), inputStream, blobSize); 93 | } 94 | 95 | /** 96 | * Deletes a blob with giving name, if the blob exists. If the blob does not exist, this method throws an 97 | * IOException. 98 | * 99 | * @param blobName The name of the blob to delete. 100 | * @throws NoSuchFileException if the blob does not exist 101 | * @throws IOException if the blob exists but could not be deleted. 102 | */ 103 | @Override 104 | public void deleteBlob(String blobName) throws IOException { 105 | logger.trace("deleteBlob({})", blobName); 106 | if (!blobExists(blobName)) { 107 | throw new NoSuchFileException("Blob [" + blobName + "] does not exist"); 108 | } 109 | try { 110 | blobStore.deleteBlob(buildKey(blobName)); 111 | } catch (OSSException | ClientException e) { 112 | logger.warn("can not access [{}] : {}", blobName, 113 | e.getMessage()); 114 | throw new IOException(e); 115 | } 116 | 117 | } 118 | 119 | /** 120 | * Lists all blobs in the container. 121 | * 122 | * @return A map of all the blobs in the container. The keys in the map are the names of the blobs and 123 | * the values are {@link BlobMetaData}, containing basic information about each blob. 124 | * @throws IOException if there were any failures in reading from the blob container. 125 | */ 126 | @Override 127 | public Map listBlobs() throws IOException { 128 | return listBlobsByPrefix(null); 129 | } 130 | 131 | /** 132 | * Lists all blobs in the container. 133 | * 134 | * @return A map of all the blobs in the container. The keys in the map are the names of the blobs and 135 | * the values are {@link BlobMetaData}, containing basic information about each blob. 136 | * @throws IOException if there were any failures in reading from the blob container. 137 | */ 138 | @Override 139 | public Map listBlobsByPrefix(String blobNamePrefix) 140 | throws IOException { 141 | logger.trace("listBlobsByPrefix({})", blobNamePrefix); 142 | try { 143 | return blobStore.listBlobsByPrefix(keyPath, blobNamePrefix); 144 | } catch (IOException e) { 145 | logger.warn("can not access [{}] : {}", blobNamePrefix, e.getMessage()); 146 | throw new IOException(e); 147 | } 148 | } 149 | 150 | protected String buildKey(String blobName) { 151 | return keyPath + (blobName == null ? StringUtils.EMPTY : blobName); 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/aliyun/oss/blobstore/OssBlobStore.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.blobstore; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.security.PrivilegedExceptionAction; 6 | import java.util.ArrayList; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | import com.aliyun.oss.ClientException; 12 | import com.aliyun.oss.OSSException; 13 | import com.aliyun.oss.model.DeleteObjectsRequest; 14 | import com.aliyun.oss.model.ListObjectsRequest; 15 | import com.aliyun.oss.model.OSSObjectSummary; 16 | import com.aliyun.oss.model.ObjectListing; 17 | import com.aliyun.oss.model.ObjectMetadata; 18 | import org.apache.commons.lang.StringUtils; 19 | import org.elasticsearch.aliyun.oss.service.OssService; 20 | import org.elasticsearch.common.blobstore.BlobContainer; 21 | import org.elasticsearch.common.blobstore.BlobMetaData; 22 | import org.elasticsearch.common.blobstore.BlobPath; 23 | import org.elasticsearch.common.blobstore.BlobStore; 24 | import org.elasticsearch.common.blobstore.BlobStoreException; 25 | import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; 26 | import org.elasticsearch.common.collect.MapBuilder; 27 | import org.elasticsearch.utils.PermissionHelper; 28 | 29 | /** 30 | * An oss blob store for managing oss client write and read blob directly 31 | * Created by yangkongshi on 2017/11/24. 32 | */ 33 | public class OssBlobStore implements BlobStore { 34 | 35 | private final OssService client; 36 | private final String bucket; 37 | 38 | public OssBlobStore(String bucket, OssService client) { 39 | this.client = client; 40 | this.bucket = bucket; 41 | if (!doesBucketExist(bucket)) { 42 | throw new BlobStoreException("bucket does not exist"); 43 | } 44 | } 45 | 46 | public String getBucket() { 47 | return this.bucket; 48 | } 49 | 50 | @Override 51 | public BlobContainer blobContainer(BlobPath blobPath) { 52 | return new OssBlobContainer(blobPath, this); 53 | } 54 | 55 | @Override 56 | public void delete(BlobPath blobPath) throws IOException { 57 | DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket); 58 | Map blobs = listBlobsByPrefix(blobPath.buildAsString(), null); 59 | List toBeDeletedBlobs = new ArrayList<>(); 60 | Iterator blobNameIterator = blobs.keySet().iterator(); 61 | while (blobNameIterator.hasNext()) { 62 | String blobName = blobNameIterator.next(); 63 | toBeDeletedBlobs.add(blobPath.buildAsString() + blobName); 64 | if (toBeDeletedBlobs.size() > DeleteObjectsRequest.DELETE_OBJECTS_ONETIME_LIMIT / 2 65 | || !blobNameIterator.hasNext()) { 66 | deleteRequest.setKeys(toBeDeletedBlobs); 67 | deleteObjects(deleteRequest); 68 | toBeDeletedBlobs.clear(); 69 | } 70 | } 71 | } 72 | 73 | @Override 74 | public void close() { 75 | client.shutdown(); 76 | } 77 | 78 | /** 79 | * Return true if the given bucket exists 80 | * 81 | * @param bucketName name of the bucket 82 | * @return true if the bucket exists, false otherwise 83 | */ 84 | boolean doesBucketExist(String bucketName) { 85 | try { 86 | return doPrivilegedAndRefreshClient(() -> this.client.doesBucketExist(bucketName)); 87 | } catch (IOException e) { 88 | throw new BlobStoreException("do privileged has failed", e); 89 | } 90 | } 91 | 92 | /** 93 | * List all blobs in the bucket which have a prefix 94 | * 95 | * @param prefix prefix of the blobs to list 96 | * @return a map of blob names and their metadata 97 | */ 98 | Map listBlobsByPrefix(String keyPath, String prefix) throws IOException { 99 | MapBuilder blobsBuilder = MapBuilder.newMapBuilder(); 100 | String actualPrefix = keyPath + (prefix == null ? StringUtils.EMPTY : prefix); 101 | String nextMarker = null; 102 | ObjectListing blobs; 103 | do { 104 | blobs = listBlobs(actualPrefix, nextMarker); 105 | for (OSSObjectSummary summary : blobs.getObjectSummaries()) { 106 | String blobName = summary.getKey().substring(keyPath.length()); 107 | blobsBuilder.put(blobName, new PlainBlobMetaData(blobName, summary.getSize())); 108 | } 109 | nextMarker = blobs.getNextMarker(); 110 | } while (blobs.isTruncated()); 111 | return blobsBuilder.immutableMap(); 112 | } 113 | 114 | /** 115 | * list blob with privilege check 116 | * 117 | * @param actualPrefix actual prefix of the blobs to list 118 | * @param nextMarker blobs next marker 119 | * @return {@link ObjectListing} 120 | */ 121 | ObjectListing listBlobs(String actualPrefix, String nextMarker) throws IOException { 122 | return doPrivilegedAndRefreshClient(() -> this.client.listObjects( 123 | new ListObjectsRequest(bucket).withPrefix(actualPrefix).withMarker(nextMarker) 124 | )); 125 | } 126 | 127 | /** 128 | * Delete Objects 129 | * 130 | * @param deleteRequest {@link DeleteObjectsRequest} 131 | */ 132 | void deleteObjects(DeleteObjectsRequest deleteRequest) throws IOException { 133 | doPrivilegedAndRefreshClient(() -> this.client.deleteObjects(deleteRequest)); 134 | } 135 | 136 | /** 137 | * Returns true if the blob exists in the bucket 138 | * 139 | * @param blobName name of the blob 140 | * @return true if the blob exists, false otherwise 141 | */ 142 | boolean blobExists(String blobName) throws OSSException, ClientException, IOException { 143 | return doPrivilegedAndRefreshClient(() -> this.client.doesObjectExist(bucket, blobName)); 144 | } 145 | 146 | /** 147 | * Returns an {@link java.io.InputStream} for a given blob 148 | * 149 | * @param blobName name of the blob 150 | * @return an InputStream 151 | */ 152 | InputStream readBlob(String blobName) throws OSSException, ClientException, IOException { 153 | return doPrivilegedAndRefreshClient(() -> this.client.getObject(bucket, blobName).getObjectContent()); 154 | } 155 | 156 | /** 157 | * Writes a blob in the bucket. 158 | * 159 | * @param inputStream content of the blob to be written 160 | * @param blobSize expected size of the blob to be written 161 | */ 162 | void writeBlob(String blobName, InputStream inputStream, long blobSize) 163 | throws OSSException, ClientException, IOException { 164 | ObjectMetadata meta = new ObjectMetadata(); 165 | meta.setContentLength(blobSize); 166 | doPrivilegedAndRefreshClient(() -> this.client.putObject(bucket, blobName, inputStream, meta)); 167 | } 168 | 169 | /** 170 | * Deletes a blob in the bucket 171 | * 172 | * @param blobName name of the blob 173 | */ 174 | void deleteBlob(String blobName) throws OSSException, ClientException, IOException { 175 | doPrivilegedAndRefreshClient(() -> { 176 | this.client.deleteObject(bucket, blobName); 177 | return null; 178 | }); 179 | } 180 | 181 | public void move(String sourceBlobName, String targetBlobName) 182 | throws OSSException, ClientException, IOException { 183 | doPrivilegedAndRefreshClient(() -> { 184 | this.client.copyObject(bucket, sourceBlobName, bucket, targetBlobName); 185 | return null; 186 | }); 187 | doPrivilegedAndRefreshClient(() -> { 188 | this.client.deleteObject(bucket, sourceBlobName); 189 | return null; 190 | }); 191 | } 192 | 193 | /** 194 | * Executes a {@link PrivilegedExceptionAction} with privileges enabled. 195 | */ 196 | T doPrivilegedAndRefreshClient(PrivilegedExceptionAction operation) throws IOException { 197 | refreshStsOssClient(); 198 | return PermissionHelper.doPrivileged(operation); 199 | } 200 | 201 | private void refreshStsOssClient() throws IOException { 202 | if (this.client.isUseStsOssClient()) { 203 | this.client.refreshStsOssClient();//refresh token to avoid expired 204 | } 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/aliyun/oss/service/OssClientSettings.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.service; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.elasticsearch.common.settings.SecureString; 5 | import org.elasticsearch.common.settings.Setting; 6 | import org.elasticsearch.common.unit.ByteSizeUnit; 7 | import org.elasticsearch.common.unit.ByteSizeValue; 8 | 9 | import static org.elasticsearch.common.settings.Setting.Property; 10 | import static org.elasticsearch.common.settings.Setting.boolSetting; 11 | import static org.elasticsearch.common.settings.Setting.byteSizeSetting; 12 | import static org.elasticsearch.common.settings.Setting.simpleString; 13 | 14 | /** 15 | * OSS client configuration 16 | * Created by yangkongshi on 2017/11/27. 17 | */ 18 | public class OssClientSettings { 19 | private static final ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.MB); 20 | private static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.GB); 21 | 22 | public static final Setting ACCESS_KEY_ID = 23 | new Setting<>("access_key_id", StringUtils.EMPTY, SecureString::new, Property.Filtered, 24 | Property.Dynamic, Property.NodeScope); 25 | public static final Setting SECRET_ACCESS_KEY = 26 | new Setting<>("secret_access_key", StringUtils.EMPTY, SecureString::new, Property.Filtered, 27 | Property.Dynamic, Property.NodeScope); 28 | public static final Setting ENDPOINT = 29 | Setting.simpleString("endpoint", Setting.Property.NodeScope, Setting.Property.Dynamic); 30 | public static final Setting SECURITY_TOKEN = 31 | new Setting<>("security_token", StringUtils.EMPTY, SecureString::new, Property.Filtered, 32 | Property.Dynamic, Property.NodeScope); 33 | public static final Setting BUCKET = 34 | simpleString("bucket", Setting.Property.NodeScope, Setting.Property.Dynamic); 35 | public static final Setting BASE_PATH = simpleString("base_path", Setting.Property.NodeScope, 36 | Setting.Property.Dynamic); 37 | public static final Setting COMPRESS = 38 | boolSetting("compress", false, Setting.Property.NodeScope, Setting.Property.Dynamic); 39 | public static final Setting CHUNK_SIZE = 40 | byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, 41 | Setting.Property.NodeScope, Setting.Property.Dynamic); 42 | public static final Setting ECS_RAM_ROLE = 43 | new Setting<>("ecs_ram_role", StringUtils.EMPTY, SecureString::new, Property.Filtered, 44 | Property.Dynamic, Property.NodeScope); 45 | public static final Setting AUTO_SNAPSHOT_BUCKET = 46 | new Setting<>("auto_snapshot_bucket", StringUtils.EMPTY, SecureString::new, Property.Filtered, 47 | Property.Dynamic, Property.NodeScope); 48 | public static final Setting SUPPORT_CNAME = 49 | boolSetting("support_cname", true, Setting.Property.NodeScope, Setting.Property.Dynamic); 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/aliyun/oss/service/OssService.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.service; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | 6 | import com.aliyun.oss.ClientException; 7 | import com.aliyun.oss.OSSException; 8 | import com.aliyun.oss.model.Bucket; 9 | import com.aliyun.oss.model.CopyObjectResult; 10 | import com.aliyun.oss.model.DeleteObjectsRequest; 11 | import com.aliyun.oss.model.DeleteObjectsResult; 12 | import com.aliyun.oss.model.ListObjectsRequest; 13 | import com.aliyun.oss.model.OSSObject; 14 | import com.aliyun.oss.model.ObjectListing; 15 | import com.aliyun.oss.model.ObjectMetadata; 16 | import com.aliyun.oss.model.PutObjectResult; 17 | import org.elasticsearch.aliyun.oss.service.exception.CreateStsOssClientException; 18 | 19 | /** 20 | * OSS Service interface for creating oss client. 21 | * Created by yangkongshi on 2017/11/24. 22 | */ 23 | public interface OssService { 24 | 25 | /** 26 | * Bulk delete the specified bucket {@link OSSObject}. 27 | * 28 | * @param deleteObjectsRequest request parameters {@link DeleteObjectsRequest} 29 | * @return delete results in bulk. 30 | */ 31 | DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) 32 | throws OSSException, ClientException; 33 | 34 | /** 35 | * Determines if the specified {@link OSSObject} exists under the specified {@link Bucket}. 36 | * 37 | * @param bucketName bucket name 38 | * @param key Object Key 39 | * @return {@code true} if a blob exists with the given name, and {@code false} otherwise. 40 | */ 41 | boolean doesObjectExist(String bucketName, String key) throws OSSException, ClientException; 42 | 43 | /** 44 | * Determines if a given {@link Bucket} exists. 45 | * 46 | * @param bucketName bucket name 47 | */ 48 | boolean doesBucketExist(String bucketName) throws OSSException, ClientException; 49 | 50 | /** 51 | * List {@link OSSObject} under the specified {@link Bucket}. 52 | * 53 | * @param listObjectsRequest request information 54 | * @return object list {@link ObjectListing} 55 | * @throws OSSException 56 | * @throws ClientException 57 | */ 58 | ObjectListing listObjects(ListObjectsRequest listObjectsRequest) 59 | throws OSSException, ClientException; 60 | 61 | /** 62 | * Export {@link OSSObject} from the OSS specified {@link Bucket}. 63 | * 64 | * @param bucketName Bucket name. 65 | * @param key Object Key. 66 | * @return Request result {@link OSSObject} instance. After use, you need to manually 67 | * close the ObjectContent release request connection. 68 | */ 69 | OSSObject getObject(String bucketName, String key) 70 | throws OSSException, ClientException, IOException; 71 | 72 | /** 73 | * Upload the specified {@link OSSObject} to the {@link Bucket} specified in OSS. 74 | * 75 | * @param bucketName Bucket name 76 | * @param key object key 77 | * @param input inputStream 78 | * @param metadata the meta-information of the object {@link ObjectMetadata}, 79 | * if the meta-information does not contain Content-Length, 80 | * transmits the request data in chunked encoding. 81 | */ 82 | PutObjectResult putObject(String bucketName, String key, InputStream input, 83 | ObjectMetadata metadata) throws OSSException, ClientException, IOException; 84 | 85 | /** 86 | * Delete the specified {@link OSSObject}. 87 | * 88 | * @param bucketName Bucket name. 89 | * @param key Object key. 90 | * @throws OSSException 91 | * @throws ClientException 92 | */ 93 | void deleteObject(String bucketName, String key) throws OSSException, ClientException; 94 | 95 | /** 96 | * Copy an Object that already exists on the OSS into another Object. 97 | * 98 | * @param sourceBucketName the name of the bucket where the source object resides. 99 | * @param sourceKey key of source Object. 100 | * @param destinationBucketName the name of the bucket where the target object is. 101 | * @param destinationKey key of the target Object. 102 | * @return request result {@link CopyObjectResult} instance. 103 | * @throws OSSException 104 | * @throws ClientException 105 | */ 106 | CopyObjectResult copyObject(String sourceBucketName, String sourceKey, 107 | String destinationBucketName, String destinationKey) throws OSSException, ClientException; 108 | 109 | /** 110 | * Closes the Client instance and frees all the resources that are in use. 111 | * Once closed, no requests to the OSS will be processed anymore. 112 | */ 113 | void shutdown(); 114 | 115 | /** 116 | * refresh StsOssClient instance which construct via sts-accessKeyId, sts-accessKeySecret, sts-securityToken 117 | * Once refreshed, sts-securityToken is valid. 118 | */ 119 | void refreshStsOssClient() throws CreateStsOssClientException; 120 | 121 | /** 122 | * judge if use Sts OssClient 123 | */ 124 | boolean isUseStsOssClient(); 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/aliyun/oss/service/OssServiceImpl.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.service; 2 | 3 | import java.io.InputStream; 4 | 5 | import com.aliyun.oss.ClientException; 6 | import com.aliyun.oss.OSSException; 7 | import com.aliyun.oss.model.CopyObjectResult; 8 | import com.aliyun.oss.model.DeleteObjectsRequest; 9 | import com.aliyun.oss.model.DeleteObjectsResult; 10 | import com.aliyun.oss.model.ListObjectsRequest; 11 | import com.aliyun.oss.model.OSSObject; 12 | import com.aliyun.oss.model.ObjectListing; 13 | import com.aliyun.oss.model.ObjectMetadata; 14 | import com.aliyun.oss.model.PutObjectResult; 15 | import org.elasticsearch.aliyun.oss.service.exception.CreateStsOssClientException; 16 | import org.elasticsearch.cluster.metadata.RepositoryMetaData; 17 | 18 | /** 19 | * OSS Service implementation for creating oss client 20 | * Created by yangkongshi on 2017/11/24. 21 | */ 22 | public class OssServiceImpl implements OssService { 23 | 24 | private OssStorageClient ossStorageClient; 25 | 26 | public OssServiceImpl(RepositoryMetaData metadata) throws CreateStsOssClientException { 27 | this.ossStorageClient = new OssStorageClient(metadata); 28 | } 29 | 30 | @Override 31 | public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) 32 | throws OSSException, ClientException { 33 | return this.ossStorageClient.deleteObjects(deleteObjectsRequest); 34 | } 35 | 36 | @Override 37 | public boolean doesObjectExist(String bucketName, String key) 38 | throws OSSException, ClientException { 39 | return this.ossStorageClient.doesObjectExist(bucketName, key); 40 | } 41 | 42 | @Override 43 | public boolean doesBucketExist(String bucketName) 44 | throws OSSException, ClientException { 45 | return this.ossStorageClient.doesBucketExist(bucketName); 46 | } 47 | 48 | @Override 49 | public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) 50 | throws OSSException, ClientException { 51 | return this.ossStorageClient.listObjects(listObjectsRequest); 52 | } 53 | 54 | @Override 55 | public OSSObject getObject(String bucketName, String key) 56 | throws OSSException, ClientException { 57 | return this.ossStorageClient.getObject(bucketName, key); 58 | } 59 | 60 | @Override 61 | public PutObjectResult putObject(String bucketName, String key, InputStream input, 62 | ObjectMetadata metadata) throws OSSException, ClientException { 63 | return this.ossStorageClient.putObject(bucketName, key, input, metadata); 64 | } 65 | 66 | @Override 67 | public void deleteObject(String bucketName, String key) 68 | throws OSSException, ClientException { 69 | this.ossStorageClient.deleteObject(bucketName, key); 70 | } 71 | 72 | @Override 73 | public CopyObjectResult copyObject(String sourceBucketName, String sourceKey, 74 | String destinationBucketName, String destinationKey) throws OSSException, ClientException { 75 | return this.ossStorageClient 76 | .copyObject(sourceBucketName, sourceKey, destinationBucketName, destinationKey); 77 | } 78 | 79 | @Override 80 | public void shutdown() { 81 | ossStorageClient.shutdown(); 82 | } 83 | 84 | @Override 85 | public void refreshStsOssClient() throws CreateStsOssClientException { 86 | ossStorageClient.refreshStsOssClient(); 87 | } 88 | 89 | @Override 90 | public boolean isUseStsOssClient() { 91 | return ossStorageClient.isStsOssClient(); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/aliyun/oss/service/OssStorageClient.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.service; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.Date; 6 | import java.util.concurrent.locks.ReadWriteLock; 7 | import java.util.concurrent.locks.ReentrantReadWriteLock; 8 | 9 | import com.alibaba.fastjson.JSON; 10 | import com.alibaba.fastjson.JSONObject; 11 | 12 | import com.aliyun.oss.ClientConfiguration; 13 | import com.aliyun.oss.ClientException; 14 | import com.aliyun.oss.OSSClient; 15 | import com.aliyun.oss.OSSException; 16 | import com.aliyun.oss.model.CopyObjectResult; 17 | import com.aliyun.oss.model.DeleteObjectsRequest; 18 | import com.aliyun.oss.model.DeleteObjectsResult; 19 | import com.aliyun.oss.model.ListObjectsRequest; 20 | import com.aliyun.oss.model.OSSObject; 21 | import com.aliyun.oss.model.ObjectListing; 22 | import com.aliyun.oss.model.ObjectMetadata; 23 | import com.aliyun.oss.model.PutObjectResult; 24 | import okhttp3.Response; 25 | import org.apache.commons.lang.StringUtils; 26 | import org.apache.logging.log4j.LogManager; 27 | import org.apache.logging.log4j.Logger; 28 | import org.elasticsearch.aliyun.oss.blobstore.OssBlobContainer; 29 | import org.elasticsearch.aliyun.oss.service.exception.CreateStsOssClientException; 30 | import org.elasticsearch.cluster.metadata.RepositoryMetaData; 31 | import org.elasticsearch.common.settings.SecureString; 32 | import org.elasticsearch.repository.oss.OssRepository; 33 | import org.elasticsearch.utils.DateHelper; 34 | import org.elasticsearch.utils.HttpClientHelper; 35 | 36 | import static java.lang.Thread.sleep; 37 | 38 | /** 39 | * @author hanqing.zhq@alibaba-inc.com 40 | * @date 2018/6/26 41 | */ 42 | public class OssStorageClient { 43 | private static final Logger logger = LogManager.getLogger(OssBlobContainer.class); 44 | 45 | private RepositoryMetaData metadata; 46 | private OSSClient client; 47 | private Date stsTokenExpiration; 48 | private String ECS_METADATA_SERVICE = "http://100.100.100.200/latest/meta-data/ram/security-credentials/"; 49 | private final int IN_TOKEN_EXPIRED_MS = 5000; 50 | private final String ACCESS_KEY_ID = "AccessKeyId"; 51 | private final String ACCESS_KEY_SECRET = "AccessKeySecret"; 52 | private final String SECURITY_TOKEN = "SecurityToken"; 53 | private final int REFRESH_RETRY_COUNT = 3; 54 | private boolean isStsOssClient; 55 | private ReadWriteLock readWriteLock; 56 | 57 | private final String EXPIRATION = "Expiration"; 58 | 59 | public OssStorageClient(RepositoryMetaData metadata) throws CreateStsOssClientException { 60 | this.metadata = metadata; 61 | if (StringUtils.isNotEmpty(OssClientSettings.ECS_RAM_ROLE.get(metadata.settings()).toString())) { 62 | isStsOssClient = true; 63 | } else { 64 | isStsOssClient = false; 65 | } 66 | readWriteLock = new ReentrantReadWriteLock(); 67 | client = createClient(metadata); 68 | 69 | } 70 | 71 | public boolean isStsOssClient() { 72 | return isStsOssClient; 73 | } 74 | 75 | public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) 76 | throws OSSException, ClientException { 77 | if (isStsOssClient) { 78 | readWriteLock.readLock().lock(); 79 | try { 80 | return this.client.deleteObjects(deleteObjectsRequest); 81 | } finally { 82 | readWriteLock.readLock().unlock(); 83 | } 84 | } else { 85 | return this.client.deleteObjects(deleteObjectsRequest); 86 | } 87 | } 88 | 89 | public boolean doesObjectExist(String bucketName, String key) 90 | throws OSSException, ClientException { 91 | if (isStsOssClient) { 92 | readWriteLock.readLock().lock(); 93 | try { 94 | return this.client.doesObjectExist(bucketName, key); 95 | } finally { 96 | readWriteLock.readLock().unlock(); 97 | } 98 | } else { 99 | return this.client.doesObjectExist(bucketName, key); 100 | } 101 | } 102 | 103 | public boolean doesBucketExist(String bucketName) 104 | throws OSSException, ClientException { 105 | if (isStsOssClient) { 106 | readWriteLock.readLock().lock(); 107 | try { 108 | return this.client.doesBucketExist(bucketName); 109 | } finally { 110 | readWriteLock.readLock().unlock(); 111 | } 112 | } else { 113 | return this.client.doesBucketExist(bucketName); 114 | } 115 | } 116 | 117 | public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) 118 | throws OSSException, ClientException { 119 | if (isStsOssClient) { 120 | readWriteLock.readLock().lock(); 121 | try { 122 | return this.client.listObjects(listObjectsRequest); 123 | } finally { 124 | readWriteLock.readLock().unlock(); 125 | } 126 | } else { 127 | return this.client.listObjects(listObjectsRequest); 128 | } 129 | } 130 | 131 | public OSSObject getObject(String bucketName, String key) 132 | throws OSSException, ClientException { 133 | if (isStsOssClient) { 134 | readWriteLock.readLock().lock(); 135 | try { 136 | return this.client.getObject(bucketName, key); 137 | } finally { 138 | readWriteLock.readLock().unlock(); 139 | } 140 | } else { 141 | return this.client.getObject(bucketName, key); 142 | } 143 | } 144 | 145 | public PutObjectResult putObject(String bucketName, String key, InputStream input, 146 | ObjectMetadata metadata) throws OSSException, ClientException { 147 | if (isStsOssClient) { 148 | readWriteLock.readLock().lock(); 149 | try { 150 | return this.client.putObject(bucketName, key, input, metadata); 151 | } finally { 152 | readWriteLock.readLock().unlock(); 153 | } 154 | } else { 155 | return this.client.putObject(bucketName, key, input, metadata); 156 | } 157 | } 158 | 159 | public void deleteObject(String bucketName, String key) 160 | throws OSSException, ClientException { 161 | if (isStsOssClient) { 162 | readWriteLock.readLock().lock(); 163 | try { 164 | this.client.deleteObject(bucketName, key); 165 | } finally { 166 | readWriteLock.readLock().unlock(); 167 | } 168 | } else { 169 | this.client.deleteObject(bucketName, key); 170 | } 171 | 172 | } 173 | 174 | public CopyObjectResult copyObject(String sourceBucketName, String sourceKey, 175 | String destinationBucketName, String destinationKey) throws OSSException, ClientException { 176 | 177 | if (isStsOssClient) { 178 | readWriteLock.readLock().lock(); 179 | try { 180 | return this.client 181 | .copyObject(sourceBucketName, sourceKey, destinationBucketName, destinationKey); 182 | } finally { 183 | readWriteLock.readLock().unlock(); 184 | } 185 | } else { 186 | return this.client 187 | .copyObject(sourceBucketName, sourceKey, destinationBucketName, destinationKey); 188 | } 189 | } 190 | 191 | public void refreshStsOssClient() throws CreateStsOssClientException { 192 | int retryCount = 0; 193 | while (isStsTokenExpired() || isTokenWillExpired()) { 194 | retryCount++; 195 | if (retryCount > REFRESH_RETRY_COUNT) { 196 | logger.error("Can't get valid token after retry {} times", REFRESH_RETRY_COUNT); 197 | throw new CreateStsOssClientException( 198 | "Can't get valid token after retry " + REFRESH_RETRY_COUNT + " times"); 199 | } 200 | this.client = createStsOssClient(this.metadata); 201 | try { 202 | if (isStsTokenExpired() || isTokenWillExpired()) { 203 | sleep(IN_TOKEN_EXPIRED_MS * 2); 204 | } 205 | } catch (InterruptedException e) { 206 | logger.error("refresh sleep exception", e); 207 | throw new CreateStsOssClientException(e); 208 | } 209 | } 210 | } 211 | 212 | public void shutdown() { 213 | if (isStsOssClient) { 214 | readWriteLock.writeLock().lock(); 215 | try { 216 | if (null != this.client) { 217 | this.client.shutdown(); 218 | } 219 | } finally { 220 | readWriteLock.writeLock().unlock(); 221 | } 222 | } else { 223 | if (null != this.client) { 224 | this.client.shutdown(); 225 | } 226 | } 227 | } 228 | 229 | private boolean isStsTokenExpired() { 230 | boolean expired = true; 231 | Date now = new Date(); 232 | if (null != stsTokenExpiration) { 233 | if (stsTokenExpiration.after(now)) { 234 | expired = false; 235 | } 236 | } 237 | return expired; 238 | } 239 | 240 | private boolean isTokenWillExpired() { 241 | boolean in = true; 242 | Date now = new Date(); 243 | long millisecond = stsTokenExpiration.getTime() - now.getTime(); 244 | if (millisecond >= IN_TOKEN_EXPIRED_MS) { 245 | in = false; 246 | } 247 | return in; 248 | } 249 | 250 | private OSSClient createClient(RepositoryMetaData repositoryMetaData) throws CreateStsOssClientException { 251 | OSSClient client; 252 | 253 | String ecsRamRole = OssClientSettings.ECS_RAM_ROLE.get(repositoryMetaData.settings()).toString(); 254 | String stsToken = OssClientSettings.SECURITY_TOKEN.get(repositoryMetaData.settings()).toString(); 255 | /* 256 | * If ecsRamRole exist 257 | * means use ECS metadata service to get ststoken for auto snapshot. 258 | * */ 259 | if (StringUtils.isNotEmpty(ecsRamRole.toString())) { 260 | client = createStsOssClient(repositoryMetaData); 261 | } else if (StringUtils.isNotEmpty(stsToken)) { 262 | //no used still now. 263 | client = createAKStsTokenClient(repositoryMetaData); 264 | } else { 265 | client = createAKOssClient(repositoryMetaData); 266 | } 267 | return client; 268 | } 269 | 270 | private ClientConfiguration extractClientConfiguration(RepositoryMetaData repositoryMetaData) { 271 | ClientConfiguration configuration = new ClientConfiguration(); 272 | configuration.setSupportCname(OssRepository.getSetting(OssClientSettings.SUPPORT_CNAME, repositoryMetaData)); 273 | return configuration; 274 | } 275 | 276 | private OSSClient createAKOssClient(RepositoryMetaData repositoryMetaData) { 277 | SecureString accessKeyId = 278 | OssRepository.getSetting(OssClientSettings.ACCESS_KEY_ID, repositoryMetaData); 279 | SecureString secretAccessKey = 280 | OssRepository.getSetting(OssClientSettings.SECRET_ACCESS_KEY, repositoryMetaData); 281 | String endpoint = OssRepository.getSetting(OssClientSettings.ENDPOINT, repositoryMetaData); 282 | return new OSSClient(endpoint, accessKeyId.toString(), secretAccessKey.toString(), 283 | extractClientConfiguration(repositoryMetaData)); 284 | } 285 | 286 | private OSSClient createAKStsTokenClient(RepositoryMetaData repositoryMetaData) { 287 | SecureString securityToken = OssClientSettings.SECURITY_TOKEN.get(repositoryMetaData.settings()); 288 | SecureString accessKeyId = 289 | OssRepository.getSetting(OssClientSettings.ACCESS_KEY_ID, repositoryMetaData); 290 | SecureString secretAccessKey = 291 | OssRepository.getSetting(OssClientSettings.SECRET_ACCESS_KEY, repositoryMetaData); 292 | String endpoint = OssRepository.getSetting(OssClientSettings.ENDPOINT, repositoryMetaData); 293 | return new OSSClient(endpoint, accessKeyId.toString(), secretAccessKey.toString(), 294 | securityToken.toString(), extractClientConfiguration(repositoryMetaData)); 295 | } 296 | 297 | private synchronized OSSClient createStsOssClient(RepositoryMetaData repositoryMetaData) 298 | throws CreateStsOssClientException { 299 | if (isStsTokenExpired() || isTokenWillExpired()) { 300 | try { 301 | if (null == repositoryMetaData) { 302 | throw new IOException("repositoryMetaData is null"); 303 | } 304 | String ecsRamRole = OssClientSettings.ECS_RAM_ROLE.get(repositoryMetaData.settings()).toString(); 305 | String endpoint = OssRepository.getSetting(OssClientSettings.ENDPOINT, repositoryMetaData); 306 | 307 | String fullECSMetaDataServiceUrl = ECS_METADATA_SERVICE + ecsRamRole; 308 | Response response = HttpClientHelper.httpRequest(fullECSMetaDataServiceUrl); 309 | if (!response.isSuccessful()) { 310 | throw new IOException("ECS meta service server error"); 311 | } 312 | String jsonStringResponse = response.body().string(); 313 | JSONObject jsonObjectResponse = JSON.parseObject(jsonStringResponse); 314 | String accessKeyId = jsonObjectResponse.getString(ACCESS_KEY_ID); 315 | String accessKeySecret = jsonObjectResponse.getString(ACCESS_KEY_SECRET); 316 | String securityToken = jsonObjectResponse.getString(SECURITY_TOKEN); 317 | stsTokenExpiration = DateHelper.convertStringToDate(jsonObjectResponse.getString(EXPIRATION)); 318 | try { 319 | readWriteLock.writeLock().lock(); 320 | if (null != this.client) { 321 | this.client.shutdown(); 322 | } 323 | this.client = new OSSClient(endpoint, accessKeyId, accessKeySecret, securityToken, 324 | extractClientConfiguration(repositoryMetaData)); 325 | } finally { 326 | readWriteLock.writeLock().unlock(); 327 | } 328 | response.close(); 329 | } catch (IOException e) { 330 | logger.error("create stsOssClient exception", e); 331 | throw new CreateStsOssClientException(e); 332 | } 333 | return this.client; 334 | } else { 335 | return this.client; 336 | } 337 | } 338 | } 339 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/aliyun/oss/service/exception/CreateStsOssClientException.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.service.exception; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | * @author hanqing.zhq@alibaba-inc.com 7 | * @date 2018/4/17 8 | */ 9 | public class CreateStsOssClientException extends IOException { 10 | public CreateStsOssClientException(Throwable e) { 11 | super(e); 12 | } 13 | 14 | public CreateStsOssClientException(String s) { 15 | super(s); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/plugin/repository/oss/OssRepositoryPlugin.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.plugin.repository.oss; 2 | 3 | import java.security.AccessController; 4 | import java.security.PrivilegedAction; 5 | import java.util.Arrays; 6 | import java.util.Collections; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import org.apache.logging.log4j.LogManager; 11 | import org.apache.logging.log4j.Logger; 12 | import org.elasticsearch.SpecialPermission; 13 | import org.elasticsearch.aliyun.oss.blobstore.OssBlobContainer; 14 | import org.elasticsearch.aliyun.oss.service.OssClientSettings; 15 | import org.elasticsearch.aliyun.oss.service.OssService; 16 | import org.elasticsearch.aliyun.oss.service.OssServiceImpl; 17 | import org.elasticsearch.aliyun.oss.service.exception.CreateStsOssClientException; 18 | import org.elasticsearch.cluster.metadata.RepositoryMetaData; 19 | import org.elasticsearch.common.settings.Setting; 20 | import org.elasticsearch.common.xcontent.NamedXContentRegistry; 21 | import org.elasticsearch.env.Environment; 22 | import org.elasticsearch.plugins.Plugin; 23 | import org.elasticsearch.plugins.RepositoryPlugin; 24 | import org.elasticsearch.repositories.Repository; 25 | import org.elasticsearch.repository.oss.OssRepository; 26 | 27 | /** 28 | * A plugin to add a repository type that writes to and from OSS. 29 | * Created by yangkongshi on 2017/11/24. 30 | */ 31 | public class OssRepositoryPlugin extends Plugin implements RepositoryPlugin { 32 | 33 | private static final Logger logger = LogManager.getLogger(OssBlobContainer.class); 34 | 35 | static { 36 | SecurityManager sm = System.getSecurityManager(); 37 | if (sm != null) { 38 | sm.checkPermission(new SpecialPermission()); 39 | } 40 | AccessController.doPrivileged((PrivilegedAction)() -> { 41 | return null; 42 | }); 43 | } 44 | 45 | protected OssService createStorageService(RepositoryMetaData metadata) 46 | throws CreateStsOssClientException { 47 | return new OssServiceImpl(metadata); 48 | } 49 | 50 | @Override 51 | public Map getRepositories(Environment env, 52 | NamedXContentRegistry namedXContentRegistry) { 53 | return Collections.singletonMap(OssRepository.TYPE, 54 | (metadata) -> new OssRepository(metadata, env, namedXContentRegistry, 55 | createStorageService(metadata))); 56 | } 57 | 58 | @Override 59 | public List> getSettings() { 60 | return Arrays.asList(OssClientSettings.ACCESS_KEY_ID, OssClientSettings.SECRET_ACCESS_KEY, 61 | OssClientSettings.ENDPOINT, OssClientSettings.BUCKET, OssClientSettings.SECURITY_TOKEN, 62 | OssClientSettings.BASE_PATH, OssClientSettings.COMPRESS, OssClientSettings.CHUNK_SIZE, 63 | OssClientSettings.AUTO_SNAPSHOT_BUCKET, OssClientSettings.ECS_RAM_ROLE, OssClientSettings.SUPPORT_CNAME); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/repository/oss/OssRepository.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.repository.oss; 2 | 3 | import java.io.File; 4 | 5 | import org.apache.commons.lang.StringUtils; 6 | import org.apache.logging.log4j.LogManager; 7 | import org.apache.logging.log4j.Logger; 8 | import org.elasticsearch.aliyun.oss.blobstore.OssBlobContainer; 9 | import org.elasticsearch.aliyun.oss.blobstore.OssBlobStore; 10 | import org.elasticsearch.aliyun.oss.service.OssClientSettings; 11 | import org.elasticsearch.aliyun.oss.service.OssService; 12 | import org.elasticsearch.cluster.metadata.RepositoryMetaData; 13 | import org.elasticsearch.common.Strings; 14 | import org.elasticsearch.common.blobstore.BlobPath; 15 | import org.elasticsearch.common.blobstore.BlobStore; 16 | import org.elasticsearch.common.settings.Setting; 17 | import org.elasticsearch.common.unit.ByteSizeValue; 18 | import org.elasticsearch.common.xcontent.NamedXContentRegistry; 19 | import org.elasticsearch.env.Environment; 20 | import org.elasticsearch.repositories.RepositoryException; 21 | import org.elasticsearch.repositories.blobstore.BlobStoreRepository; 22 | 23 | /** 24 | * An oss repository working for snapshot and restore. 25 | * Implementations are responsible for reading and writing both metadata and shard data to and from 26 | * a repository backend. 27 | * Created by yangkongshi on 2017/11/24. 28 | */ 29 | public class OssRepository extends BlobStoreRepository { 30 | 31 | private static final Logger logger = LogManager.getLogger(OssBlobContainer.class); 32 | 33 | public static final String TYPE = "oss"; 34 | private final BlobPath basePath; 35 | private final boolean compress; 36 | private final ByteSizeValue chunkSize; 37 | private final String bucket; 38 | private final OssService ossService; 39 | 40 | public OssRepository(RepositoryMetaData metadata, Environment env, 41 | NamedXContentRegistry namedXContentRegistry, OssService ossService) { 42 | super(metadata, env.settings(), namedXContentRegistry); 43 | this.ossService = ossService; 44 | String ecsRamRole = OssClientSettings.ECS_RAM_ROLE.get(metadata.settings()).toString(); 45 | if (StringUtils.isNotEmpty(ecsRamRole)) { 46 | this.bucket = getSetting(OssClientSettings.AUTO_SNAPSHOT_BUCKET, metadata).toString(); 47 | } else { 48 | this.bucket = getSetting(OssClientSettings.BUCKET, metadata); 49 | } 50 | String basePath = OssClientSettings.BASE_PATH.get(metadata.settings()); 51 | if (Strings.hasLength(basePath)) { 52 | BlobPath path = new BlobPath(); 53 | for (String elem : basePath.split(File.separator)) { 54 | path = path.add(elem); 55 | } 56 | this.basePath = path; 57 | } else { 58 | this.basePath = BlobPath.cleanPath(); 59 | } 60 | this.compress = getSetting(OssClientSettings.COMPRESS, metadata); 61 | this.chunkSize = getSetting(OssClientSettings.CHUNK_SIZE, metadata); 62 | logger.info("Using base_path [{}], chunk_size [{}], compress [{}]", 63 | basePath, chunkSize, compress); 64 | } 65 | 66 | @Override 67 | protected BlobStore createBlobStore() throws Exception { 68 | return new OssBlobStore(bucket, ossService); 69 | } 70 | 71 | @Override 72 | protected BlobPath basePath() { 73 | return basePath; 74 | } 75 | 76 | @Override 77 | protected boolean isCompress() { 78 | return compress; 79 | } 80 | 81 | @Override 82 | protected ByteSizeValue chunkSize() { 83 | return chunkSize; 84 | } 85 | 86 | public static T getSetting(Setting setting, RepositoryMetaData metadata) { 87 | T value = setting.get(metadata.settings()); 88 | if (value == null) { 89 | throw new RepositoryException(metadata.name(), 90 | "Setting [" + setting.getKey() + "] is not defined for repository"); 91 | } 92 | if ((value instanceof String) && (Strings.hasText((String)value)) == false) { 93 | throw new RepositoryException(metadata.name(), 94 | "Setting [" + setting.getKey() + "] is empty for repository"); 95 | } 96 | return value; 97 | } 98 | 99 | /** mainly for test **/ 100 | public OssService getOssService() { 101 | return ossService; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/utils/DateHelper.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.utils; 2 | 3 | import java.text.ParseException; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | import java.util.TimeZone; 7 | 8 | import org.apache.logging.log4j.LogManager; 9 | import org.apache.logging.log4j.Logger; 10 | 11 | /** 12 | * @author hanqing.zhq@alibaba-inc.com 13 | * @date 2018/4/16 14 | */ 15 | public class DateHelper { 16 | private static final Logger logger = LogManager.getLogger(DateHelper.class); 17 | 18 | public static Date convertStringToDate(String dateString) { 19 | String dateRegex = "yyyy-MM-dd'T'HH:mm:ss'Z'"; 20 | SimpleDateFormat sdf = new SimpleDateFormat(dateRegex); 21 | sdf.setTimeZone(TimeZone.getTimeZone("GMT")); 22 | Date returnDate = null; 23 | try { 24 | returnDate = sdf.parse(dateString); 25 | } catch (ParseException e) { 26 | logger.error("convert String to Date type error", e); 27 | } 28 | return returnDate; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/utils/HttpClientHelper.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.utils; 2 | 3 | import java.io.IOException; 4 | 5 | import okhttp3.OkHttpClient; 6 | import okhttp3.Request; 7 | import okhttp3.Response; 8 | 9 | /** 10 | * @author hanqing.zhq@alibaba-inc.com 11 | * @date 2018/4/22 12 | */ 13 | public class HttpClientHelper { 14 | 15 | private static OkHttpClient httpClient; 16 | 17 | private HttpClientHelper() { } 18 | 19 | public static synchronized OkHttpClient getHTTPClient() { 20 | if (null == httpClient) { 21 | try { 22 | PermissionHelper.doPrivileged(() -> { 23 | httpClient = new OkHttpClient(); 24 | return httpClient; 25 | }); 26 | } catch (IOException e) { 27 | return httpClient; 28 | } 29 | } 30 | return httpClient; 31 | } 32 | 33 | public static Response httpRequest(String url) throws IOException { 34 | return PermissionHelper.doPrivileged(() -> { 35 | Request request = new Request.Builder().get().url(url).build(); 36 | return getHTTPClient().newCall(request).execute(); 37 | }); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/utils/PermissionHelper.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.utils; 2 | 3 | import java.io.IOException; 4 | import java.security.AccessController; 5 | import java.security.PrivilegedActionException; 6 | import java.security.PrivilegedExceptionAction; 7 | 8 | import org.elasticsearch.SpecialPermission; 9 | 10 | /** 11 | * @author hanqing.zhq@alibaba-inc.com 12 | * @date 2018/4/18 13 | */ 14 | public class PermissionHelper { 15 | 16 | /** 17 | * Executes a {@link PrivilegedExceptionAction} with privileges enabled. 18 | */ 19 | 20 | public static T doPrivileged(PrivilegedExceptionAction operation) throws IOException { 21 | SecurityManager sm = System.getSecurityManager(); 22 | if (sm != null) { 23 | sm.checkPermission(new SpecialPermission()); 24 | } 25 | try { 26 | return AccessController.doPrivileged((PrivilegedExceptionAction)operation::run); 27 | } catch (PrivilegedActionException e) { 28 | throw (IOException)e.getException(); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/resources/plugin-descriptor.properties: -------------------------------------------------------------------------------- 1 | # Elasticsearch plugin descriptor file 2 | # This file must exist as 'plugin-descriptor.properties' at 3 | # the root directory of all plugins. 4 | # 5 | # A plugin can be 'site', 'jvm', or both. 6 | # 7 | ### example site plugin for "foo": 8 | # 9 | # foo.zip <-- zip file for the plugin, with this structure: 10 | # _site/ <-- the contents that will be served 11 | # plugin-descriptor.properties <-- example contents below: 12 | # 13 | # site=true 14 | # description=My cool plugin 15 | # version=1.0 16 | # 17 | ### example jvm plugin for "foo" 18 | # 19 | # foo.zip <-- zip file for the plugin, with this structure: 20 | # .jar <-- classes, resources, dependencies 21 | # .jar <-- any number of jars 22 | # plugin-descriptor.properties <-- example contents below: 23 | # 24 | # jvm=true 25 | # classname=foo.bar.BazPlugin 26 | # description=My cool plugin 27 | # version=2.0.0-rc1 28 | # elasticsearch.version=2.0 29 | # java.version=1.7 30 | # 31 | ### mandatory elements for all plugins: 32 | # 33 | # 'description': simple summary of the plugin 34 | description=The OSS repository plugin adds support for using Alibaba Cloud OSS as a repository for Snapshot/Restore. 35 | # 36 | # 'version': plugin's version 37 | version=6.7.0 38 | # 39 | # 'name': the plugin name 40 | name=elasticsearch-repository-oss 41 | 42 | ### mandatory elements for site plugins: 43 | # 44 | # 'site': set to true to indicate contents of the _site/ 45 | # directory in the root of the plugin should be served. 46 | #site=${elasticsearch.plugin.site} 47 | # 48 | ### mandatory elements for jvm plugins : 49 | # 50 | # 'jvm': true if the 'classname' class should be loaded 51 | # from jar files in the root directory of the plugin. 52 | # Note that only jar files in the root directory are 53 | # added to the classpath for the plugin! If you need 54 | # other resources, package them into a resources jar. 55 | # jvm=true 56 | # 57 | # 'classname': the name of the class to load, fully-qualified. 58 | classname=org.elasticsearch.plugin.repository.oss.OssRepositoryPlugin 59 | # 60 | # 'java.version' version of java the code is built against 61 | # use the system property java.specification.version 62 | # version string must be a sequence of nonnegative decimal integers 63 | # separated by "."'s and may have leading zeros 64 | java.version=1.8 65 | # 66 | # 'elasticsearch.version' version of elasticsearch compiled against 67 | # You will have to release a new version of the plugin for each new 68 | # elasticsearch release. This version is checked when the plugin 69 | # is loaded so Elasticsearch will refuse to start in the presence of 70 | # plugins with the incorrect elasticsearch.version. 71 | elasticsearch.version=6.7.0 72 | # 73 | ### deprecated elements for jvm plugins : 74 | # 75 | # 'isolated': true if the plugin should have its own classloader. 76 | # passing false is deprecated, and only intended to support plugins 77 | # that have hard dependencies against each other. If this is 78 | # not specified, then the plugin is isolated by default. 79 | # isolated=true 80 | # date:2017-06-27 81 | 82 | -------------------------------------------------------------------------------- /src/main/resources/plugin-security.policy: -------------------------------------------------------------------------------- 1 | grant { 2 | permission java.security.AllPermission; 3 | permission java.net.SocketPermission "*", "connect,resolve"; 4 | permission java.lang.RuntimePermission "accessDeclaredMembers"; 5 | permission java.lang.RuntimePermission "getClassLoader"; 6 | permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; 7 | permission java.net.NetPermission "getProxySelector"; 8 | }; 9 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/aliyun/oss/blobstore/MockOssService.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.blobstore; 2 | 3 | import java.io.ByteArrayInputStream; 4 | import java.io.ByteArrayOutputStream; 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | import java.util.Locale; 8 | import java.util.Map; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | 11 | import com.aliyun.oss.ClientException; 12 | import com.aliyun.oss.OSSException; 13 | import com.aliyun.oss.model.CopyObjectResult; 14 | import com.aliyun.oss.model.DeleteObjectsRequest; 15 | import com.aliyun.oss.model.DeleteObjectsResult; 16 | import com.aliyun.oss.model.ListObjectsRequest; 17 | import com.aliyun.oss.model.OSSObject; 18 | import com.aliyun.oss.model.OSSObjectSummary; 19 | import com.aliyun.oss.model.ObjectListing; 20 | import com.aliyun.oss.model.ObjectMetadata; 21 | import com.aliyun.oss.model.PutObjectResult; 22 | import org.elasticsearch.aliyun.oss.service.OssService; 23 | import org.elasticsearch.aliyun.oss.service.exception.CreateStsOssClientException; 24 | import org.elasticsearch.common.Strings; 25 | import org.elasticsearch.common.component.AbstractComponent; 26 | import org.elasticsearch.common.io.Streams; 27 | 28 | /** 29 | * Created by yangkongshi on 2017/11/28. 30 | */ 31 | public class MockOssService extends AbstractComponent implements OssService { 32 | 33 | protected final Map blobs = new ConcurrentHashMap<>(); 34 | 35 | public MockOssService() { 36 | super(); 37 | } 38 | 39 | @Override 40 | public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) 41 | throws OSSException, ClientException { 42 | for (String key : deleteObjectsRequest.getKeys()) { 43 | blobs.remove(key); 44 | } 45 | return null; 46 | } 47 | 48 | @Override 49 | public boolean doesObjectExist(String bucketName, String key) 50 | throws OSSException, ClientException { 51 | return blobs.containsKey(key); 52 | } 53 | 54 | @Override 55 | public boolean doesBucketExist(String bucketName) 56 | throws OSSException, ClientException { 57 | return true; 58 | } 59 | 60 | @Override 61 | public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) 62 | throws OSSException, ClientException { 63 | ObjectListing objectListing = new ObjectListing(); 64 | objectListing.setTruncated(false); 65 | String prefix = listObjectsRequest.getPrefix(); 66 | blobs.forEach((String blobName, OSSObject ossObject) -> { 67 | if (!Strings.hasLength(prefix) || startsWithIgnoreCase(blobName, prefix)) { 68 | OSSObjectSummary summary = new OSSObjectSummary(); 69 | summary.setKey(blobName); 70 | summary.setSize(ossObject.getObjectMetadata().getContentLength()); 71 | objectListing.addObjectSummary(summary); 72 | } 73 | }); 74 | return objectListing; 75 | } 76 | 77 | @Override 78 | public OSSObject getObject(String bucketName, String key) 79 | throws OSSException, ClientException, IOException { 80 | OSSObject ossObject = new OSSObject(); 81 | synchronized (this) { 82 | OSSObject oldObject = blobs.get(key); 83 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 84 | Streams.copy(oldObject.getObjectContent(), outputStream); 85 | oldObject.setObjectContent(new ByteArrayInputStream(outputStream.toByteArray())); 86 | ossObject.setObjectContent(new ByteArrayInputStream(outputStream.toByteArray())); 87 | } 88 | return ossObject; 89 | } 90 | 91 | @Override 92 | public PutObjectResult putObject(String bucketName, String key, InputStream input, 93 | ObjectMetadata metadata) throws OSSException, ClientException, IOException { 94 | synchronized (this) { 95 | OSSObject ossObject = new OSSObject(); 96 | ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 97 | Streams.copy(input, outputStream); 98 | ossObject.setObjectContent(new ByteArrayInputStream(outputStream.toByteArray())); 99 | ossObject.setObjectMetadata(metadata); 100 | blobs.put(key, ossObject); 101 | } 102 | return null; 103 | } 104 | 105 | @Override 106 | public void deleteObject(String bucketName, String key) 107 | throws OSSException, ClientException { 108 | blobs.remove(key); 109 | } 110 | 111 | @Override 112 | public CopyObjectResult copyObject(String sourceBucketName, String sourceKey, 113 | String destinationBucketName, String destinationKey) throws OSSException, ClientException { 114 | OSSObject sourceOssObject = blobs.get(sourceKey); 115 | blobs.put(destinationKey, sourceOssObject); 116 | return null; 117 | } 118 | 119 | @Override 120 | public void shutdown() { 121 | blobs.clear(); 122 | } 123 | 124 | @Override 125 | public void refreshStsOssClient() throws CreateStsOssClientException { 126 | 127 | } 128 | 129 | @Override 130 | public boolean isUseStsOssClient() { 131 | return false; 132 | } 133 | 134 | /** 135 | * Test if the given String starts with the specified prefix, 136 | * ignoring upper/lower case. 137 | * 138 | * @param str the String to check 139 | * @param prefix the prefix to look for 140 | * @see java.lang.String#startsWith 141 | */ 142 | public static boolean startsWithIgnoreCase(String str, String prefix) { 143 | if (str == null || prefix == null) { 144 | return false; 145 | } 146 | if (str.startsWith(prefix)) { 147 | return true; 148 | } 149 | if (str.length() < prefix.length()) { 150 | return false; 151 | } 152 | String lcStr = str.substring(0, prefix.length()).toLowerCase(Locale.ROOT); 153 | String lcPrefix = prefix.toLowerCase(Locale.ROOT); 154 | return lcStr.equals(lcPrefix); 155 | } 156 | 157 | } 158 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/aliyun/oss/blobstore/OssBlobContainerTest.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.blobstore; 2 | 3 | import java.io.IOException; 4 | import java.util.Locale; 5 | 6 | import org.elasticsearch.common.blobstore.BlobStore; 7 | import org.elasticsearch.repositories.ESBlobStoreContainerTestCase; 8 | 9 | /** 10 | * Created by yangkongshi on 2017/11/28. 11 | */ 12 | public class OssBlobContainerTest extends ESBlobStoreContainerTestCase { 13 | @Override 14 | protected BlobStore newBlobStore() throws IOException { 15 | String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); 16 | MockOssService client = new MockOssService(); 17 | return new OssBlobStore(bucket, client); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/aliyun/oss/blobstore/OssBlobStoreTest.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.aliyun.oss.blobstore; 2 | 3 | import java.io.IOException; 4 | import java.util.Locale; 5 | 6 | import org.elasticsearch.common.blobstore.BlobStore; 7 | import org.elasticsearch.repositories.ESBlobStoreTestCase; 8 | 9 | /** 10 | * Created by yangkongshi on 2017/11/28. 11 | */ 12 | public class OssBlobStoreTest extends ESBlobStoreTestCase { 13 | 14 | @Override 15 | protected BlobStore newBlobStore() throws IOException { 16 | String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); 17 | MockOssService client = new MockOssService(); 18 | return new OssBlobStore(bucket, client); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/bootstrap/JarHell.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.bootstrap; 2 | 3 | import java.net.URL; 4 | import java.util.HashSet; 5 | import java.util.Set; 6 | import java.util.function.Consumer; 7 | 8 | /** 9 | * Created by yangkongshi on 2017/12/4. 10 | */ 11 | public class JarHell { 12 | private JarHell() { 13 | } 14 | 15 | public static void checkJarHell() throws Exception { 16 | } 17 | 18 | public static void checkJarHell(URL urls[]) throws Exception { 19 | } 20 | 21 | public static void checkJarHell(Consumer output) throws Exception { 22 | } 23 | 24 | public static void checkVersionFormat(String targetVersion) { 25 | } 26 | 27 | public static void checkJavaVersion(String resource, String targetVersion) { 28 | } 29 | 30 | public static Set parseClassPath() {return new HashSet<>();} 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/repository/oss/OssRepositoryTest.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.repository.oss; 2 | 3 | import java.util.Arrays; 4 | import java.util.Collection; 5 | 6 | import org.apache.commons.lang.StringUtils; 7 | import org.elasticsearch.aliyun.oss.blobstore.MockOssService; 8 | import org.elasticsearch.aliyun.oss.service.OssClientSettings; 9 | import org.elasticsearch.aliyun.oss.service.OssService; 10 | import org.elasticsearch.cluster.metadata.RepositoryMetaData; 11 | import org.elasticsearch.common.settings.Settings; 12 | import org.elasticsearch.common.unit.ByteSizeUnit; 13 | import org.elasticsearch.plugin.repository.oss.OssRepositoryPlugin; 14 | import org.elasticsearch.plugins.Plugin; 15 | import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase; 16 | 17 | import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; 18 | 19 | /** 20 | * Created by yangkongshi on 2017/11/28. 21 | */ 22 | public class OssRepositoryTest extends ESBlobStoreRepositoryIntegTestCase { 23 | private static final String BUCKET = "oss-repository-test"; 24 | private static final OssService client = new MockOssService(); 25 | 26 | @Override 27 | protected Collection> nodePlugins() { 28 | return Arrays.asList(MockOssRepositoryPlugin.class); 29 | } 30 | 31 | @Override 32 | protected void createTestRepository(String name, boolean verify) { 33 | assertAcked( 34 | client().admin().cluster().preparePutRepository(name).setType(OssRepository.TYPE).setVerify(verify) 35 | .setSettings(Settings.builder().put(OssClientSettings.BUCKET.getKey(), BUCKET) 36 | .put(OssClientSettings.BASE_PATH.getKey(), StringUtils.EMPTY) 37 | .put(OssClientSettings.ACCESS_KEY_ID.getKey(), "test_access_key_id") 38 | .put(OssClientSettings.SECRET_ACCESS_KEY.getKey(), "test_secret_access_key") 39 | .put(OssClientSettings.ENDPOINT.getKey(), "test_endpoint") 40 | .put(OssClientSettings.COMPRESS.getKey(), randomBoolean()) 41 | .put(OssClientSettings.SUPPORT_CNAME.getKey(), randomBoolean()) 42 | .put(OssClientSettings.CHUNK_SIZE.getKey(), randomIntBetween(100, 1000), 43 | ByteSizeUnit.MB))); 44 | } 45 | 46 | public static class MockOssRepositoryPlugin extends OssRepositoryPlugin { 47 | @Override 48 | protected OssService createStorageService(RepositoryMetaData metadata) { 49 | return client; 50 | } 51 | } 52 | 53 | } 54 | --------------------------------------------------------------------------------