├── LICENSE ├── README.md ├── benchmark.go ├── client.go ├── doc.go ├── example_test.go ├── option ├── create_directory.go ├── create_file.go ├── delete.go ├── doc.go ├── exists.go ├── free.go ├── get_status.go ├── list_status.go ├── mount.go ├── open_file.go ├── rename.go ├── set_attribute.go └── unmount.go ├── optiontest ├── create_directory.go ├── create_file.go ├── delete.go ├── doc.go ├── exists.go ├── free.go ├── get_status.go ├── list_status.go ├── mount.go ├── open_file.go ├── rename.go ├── set_attribute.go └── unmount.go ├── paths.go ├── paths_test.go ├── streams.go ├── streams_test.go ├── wire ├── bits.go ├── block_info.go ├── block_location.go ├── doc.go ├── file_block_info.go ├── file_info.go ├── load_metadata_type.go ├── mode.go ├── read_type.go ├── ttl_action.go ├── worker_net_address.go └── write_type.go └── wiretest ├── bits.go ├── block_info.go ├── block_location.go ├── doc.go ├── file_block_info.go ├── file_info.go ├── load_metadata_type.go ├── mode.go ├── read_type.go ├── ttl_action.go ├── util.go ├── worker_net_address.go └── write_type.go /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2016 Alluxio Open Foundation 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | 204 | ======================================================================= 205 | 206 | Alluxio Subcomponents: 207 | 208 | The Alluxio project contains subcomponents with separate copyright 209 | notices and license terms. Your use of the source code for the these 210 | subcomponents is subject to the terms and conditions of the following 211 | licenses. 212 | 213 | ----------------------------------------------------------------------- 214 | The Apache License 215 | ----------------------------------------------------------------------- 216 | 217 | Alluxio bundles portions of the following under the Apache License 2.0 as detailed 218 | above: 219 | 220 | - Bootstrap 2 (http://getbootstrap.com) - Copyright 2011-2015 Twitter Inc 221 | 222 | ----------------------------------------------------------------------- 223 | The MIT License 224 | ----------------------------------------------------------------------- 225 | 226 | Alluxio bundles portions of the following under the MIT License: 227 | 228 | - jQuery (http://jquery.com) - Copyright 2014 jQuery Foundation and other contributors 229 | - Modernizr (http://modernizr.com) - Copyright 2009-2015 Modernizr 230 | 231 | Permission is hereby granted, free of charge, to any person obtaining a copy 232 | of this software and associated documentation files (the "Software"), to deal 233 | in the Software without restriction, including without limitation the rights 234 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 235 | copies of the Software, and to permit persons to whom the Software is 236 | furnished to do so, subject to the following conditions: 237 | 238 | The above copyright notice and this permission notice shall be included in 239 | all copies or substantial portions of the Software. 240 | 241 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 242 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 243 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 244 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 245 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 246 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 247 | THE SOFTWARE. 248 | 249 | ----------------------------------------------------------------------- 250 | The BSD 2-Clause License 251 | ----------------------------------------------------------------------- 252 | 253 | Alluxio bundles portions of the following under the BSD 2-Clause License: 254 | 255 | - Pygments (http://pygments.org) - Copyright 2006-2015 various contributors 256 | 257 | All rights reserved. 258 | 259 | Redistribution and use in source and binary forms, with or without 260 | modification, are permitted provided that the following conditions are 261 | met: 262 | 263 | * Redistributions of source code must retain the above copyright 264 | notice, this list of conditions and the following disclaimer. 265 | 266 | * Redistributions in binary form must reproduce the above copyright 267 | notice, this list of conditions and the following disclaimer in the 268 | documentation and/or other materials provided with the distribution. 269 | 270 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 271 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 272 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 273 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 274 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 275 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 276 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 277 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 278 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 279 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 280 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 281 | 282 | ----------------------------------------------------------------------- 283 | Eclipse Public License 284 | ----------------------------------------------------------------------- 285 | 286 | Alluxio depends on Jetty the following portions of which are under the Eclipse Public License 287 | 288 | - org.eclipse.jetty.orbit:org.eclipse.jdt.core (https://www.eclipse.org/jetty/) 289 | 290 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). 291 | ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS 292 | AGREEMENT. 293 | 294 | 1. DEFINITIONS 295 | 296 | "Contribution" means: 297 | 298 | a) in the case of the initial Contributor, the initial code and documentation distributed under this 299 | Agreement, and 300 | 301 | b) in the case of each subsequent Contributor: 302 | 303 | i) changes to the Program, and 304 | 305 | ii) additions to the Program; 306 | 307 | where such changes and/or additions to the Program originate from and are distributed by that 308 | particular Contributor. A Contribution 'originates' from a Contributor if it was added to the 309 | Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do 310 | not include additions to the Program which: (i) are separate modules of software distributed in 311 | conjunction with the Program under their own license agreement, and (ii) are not derivative works of 312 | the Program. 313 | 314 | "Contributor" means any person or entity that distributes the Program. 315 | 316 | "Licensed Patents mean patent claims licensable by a Contributor which are necessarily infringed by 317 | "the use or sale of its Contribution alone or when combined with the Program. 318 | 319 | "Program" means the Contributions distributed in accordance with this Agreement. 320 | 321 | "Recipient" means anyone who receives the Program under this Agreement, including all Contributors. 322 | 323 | 2. GRANT OF RIGHTS 324 | 325 | a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, 326 | worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly 327 | display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, 328 | and such derivative works, in source code and object code form. 329 | 330 | b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, 331 | worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, 332 | import and otherwise transfer the Contribution of such Contributor, if any, in source code and 333 | object code form. This patent license shall apply to the combination of the Contribution and the 334 | Program if, at the time the Contribution is added by the Contributor, such addition of the 335 | Contribution causes such combination to be covered by the Licensed Patents. The patent license shall 336 | not apply to any other combinations which include the Contribution. No hardware per se is licensed 337 | hereunder. 338 | 339 | c) Recipient understands that although each Contributor grants the licenses to its Contributions set 340 | forth herein, no assurances are provided by any Contributor that the Program does not infringe the 341 | patent or other intellectual property rights of any other entity. Each Contributor disclaims any 342 | liability to Recipient for claims brought by any other entity based on infringement of intellectual 343 | property rights or otherwise. As a condition to exercising the rights and licenses granted 344 | hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual 345 | property rights needed, if any. For example, if a third party patent license is required to allow 346 | Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before 347 | distributing the Program. 348 | 349 | d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its 350 | Contribution, if any, to grant the copyright license set forth in this Agreement. 351 | 352 | 3. REQUIREMENTS 353 | 354 | A Contributor may choose to distribute the Program in object code form under its own license 355 | agreement, provided that: 356 | 357 | a) it complies with the terms and conditions of this Agreement; and 358 | 359 | b) its license agreement: 360 | 361 | i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and 362 | implied, including warranties or conditions of title and non-infringement, and implied warranties or 363 | conditions of merchantability and fitness for a particular purpose; 364 | 365 | ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, 366 | indirect, special, incidental and consequential damages, such as lost profits; 367 | 368 | iii) states that any provisions which differ from this Agreement are offered by that Contributor 369 | alone and not by any other party; and 370 | 371 | iv) states that source code for the Program is available from such Contributor, and informs 372 | licensees how to obtain it in a reasonable manner on or through a medium customarily used for 373 | software exchange. 374 | 375 | When the Program is made available in source code form: 376 | 377 | a) it must be made available under this Agreement; and 378 | 379 | b) a copy of this Agreement must be included with each copy of the Program. 380 | 381 | Contributors may not remove or alter any copyright notices contained within the Program. 382 | 383 | Each Contributor must identify itself as the originator of its Contribution, if any, in a manner 384 | that reasonably allows subsequent Recipients to identify the originator of the Contribution. 385 | 386 | 4. COMMERCIAL DISTRIBUTION 387 | 388 | Commercial distributors of software may accept certain responsibilities with respect to end users, 389 | business partners and the like. While this license is intended to facilitate the commercial use of 390 | the Program, the Contributor who includes the Program in a commercial product offering should do so 391 | in a manner which does not create potential liability for other Contributors. Therefore, if a 392 | Contributor includes the Program in a commercial product offering, such Contributor ("Commercial 393 | Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified 394 | Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, 395 | lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the 396 | extent caused by the acts or omissions of such Commercial Contributor in connection with its 397 | distribution of the Program in a commercial product offering. The obligations in this section do not 398 | apply to any claims or Losses relating to any actual or alleged intellectual property infringement. 399 | In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor 400 | in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the 401 | Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified 402 | Contributor may participate in any such claim at its own expense. 403 | 404 | For example, a Contributor might include the Program in a commercial product offering, Product X. 405 | That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes 406 | performance claims, or offers warranties related to Product X, those performance claims and 407 | warranties are such Commercial Contributor's responsibility alone. Under this section, the 408 | Commercial Contributor would have to defend claims against the other Contributors related to those 409 | performance claims and warranties, and if a court requires any other Contributor to pay any damages 410 | as a result, the Commercial Contributor must pay those damages. 411 | 412 | 5. NO WARRANTY 413 | 414 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, 415 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT 416 | LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR 417 | A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of 418 | using and distributing the Program and assumes all risks associated with its exercise of rights 419 | under this Agreement , including but not limited to the risks and costs of program errors, 420 | compliance with applicable laws, damage to or loss of data, programs or equipment, and 421 | unavailability or interruption of operations. 422 | 423 | 6. DISCLAIMER OF LIABILITY 424 | 425 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE 426 | ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 427 | (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 428 | IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 429 | THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF 430 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 431 | 432 | 7. GENERAL 433 | 434 | If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not 435 | affect the validity or enforceability of the remainder of the terms of this Agreement, and without 436 | further action by the parties hereto, such provision shall be reformed to the minimum extent 437 | necessary to make such provision valid and enforceable. 438 | 439 | If Recipient institutes patent litigation against any entity (including a cross-claim or 440 | counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program 441 | with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights 442 | granted under Section 2(b) shall terminate as of the date such litigation is filed. 443 | 444 | All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the 445 | material terms or conditions of this Agreement and does not cure such failure in a reasonable period 446 | of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement 447 | terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably 448 | practicable. However, Recipient's obligations under this Agreement and any licenses granted by 449 | Recipient relating to the Program shall continue and survive. 450 | 451 | Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid 452 | inconsistency the Agreement is copyrighted and may only be modified in the following manner. The 453 | Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement 454 | from time to time. No one other than the Agreement Steward has the right to modify this Agreement. 455 | The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the 456 | responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of 457 | the Agreement will be given a distinguishing version number. The Program (including Contributions) 458 | may always be distributed subject to the version of the Agreement under which it was received. In 459 | addition, after a new version of the Agreement is published, Contributor may elect to distribute the 460 | Program (including its Contributions) under the new version. Except as expressly stated in Sections 461 | 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any 462 | Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All 463 | rights in the Program not expressly granted under this Agreement are reserved. 464 | 465 | This Agreement is governed by the laws of the State of New York and the intellectual property laws 466 | of the United States of America. No party to this Agreement will bring a legal action under this 467 | Agreement more than one year after the cause of action arose. Each party waives its rights to a jury 468 | trial in any resulting litigation. 469 | 470 | ----------------------------------------------------------------------- 471 | Public Domain 472 | ----------------------------------------------------------------------- 473 | 474 | Alluxio bundles portions of the following which are in the public domain: 475 | 476 | - Cookies.js (https://github.com/ScottHamper/Cookies/) 477 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Go Library for the Alluxio File System API 2 | 3 | To get started, fetch the repository into your Go workspace using `go get -d github.com/Alluxio/alluxio-go`. 4 | 5 | Your Go applications can then interact with Alluxio as follows: 6 | 7 | ``` 8 | package main 9 | 10 | import ( 11 | "fmt" 12 | "log" 13 | 14 | alluxio "github.com/Alluxio/alluxio-go" 15 | "github.com/Alluxio/alluxio-go/option" 16 | ) 17 | 18 | 19 | func main() { 20 | fs := alluxio.NewClient(, , ) 21 | ok, err := fs.Exists("/test_path", &option.Exists{}) 22 | if err != nil { 23 | log.Fatal(err) 24 | } 25 | fmt.Println(ok) 26 | } 27 | ``` 28 | 29 | For a list of all supported API calls, see https://github.com/Alluxio/alluxio-go/blob/master/paths.go. 30 | -------------------------------------------------------------------------------- /benchmark.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "bytes" 7 | "crypto/rand" 8 | "flag" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "log" 13 | "sync" 14 | "time" 15 | 16 | alluxio "github.com/Alluxio/alluxio-go" 17 | "github.com/Alluxio/alluxio-go/option" 18 | ) 19 | 20 | func metadataWorker(fs *alluxio.Client, path string) { 21 | for i := 0; i < numOperationsFlag; i++ { 22 | if _, err := fs.Exists(path, &option.Exists{}); err != nil { 23 | log.Fatal(err) 24 | } 25 | } 26 | } 27 | 28 | func metadataBenchmark() { 29 | clients, path := []*alluxio.Client{}, "/test_file" 30 | for i := 0; i < numThreadsFlag; i++ { 31 | clients = append(clients, alluxio.NewClient("localhost", 39999-i, time.Minute)) 32 | } 33 | totalRPCs := numOperationsFlag * numThreadsFlag 34 | var totalTime int64 35 | data := make([]byte, bufferSizeFlag) 36 | if _, err := rand.Read(data); err != nil { 37 | log.Fatal(err) 38 | } 39 | if ok, err := clients[0].Exists(path, &option.Exists{}); err != nil { 40 | log.Fatal(err) 41 | } else if ok { 42 | if err := clients[0].Delete(path, &option.Delete{}); err != nil { 43 | log.Fatal(err) 44 | } 45 | } 46 | upload(clients[0], data, path) 47 | for j := 0; j < numIterationsFlag; j++ { 48 | start := time.Now() 49 | wg := sync.WaitGroup{} 50 | wg.Add(numThreadsFlag) 51 | for i := 0; i < numThreadsFlag; i++ { 52 | i := i 53 | go func() { 54 | defer wg.Done() 55 | metadataWorker(clients[i], path) 56 | }() 57 | } 58 | wg.Wait() 59 | end := time.Now() 60 | fmt.Printf("Iteration #%d: %d RPCs in %v.\n", j+1, totalRPCs, end.Sub(start)) 61 | totalTime += end.Sub(start).Nanoseconds() 62 | } 63 | average := time.Duration(totalTime / int64(numIterationsFlag)) 64 | fmt.Printf("Average time %v, average throughput %.2f RPC/s\n", average, float64(totalRPCs)/average.Seconds()) 65 | } 66 | 67 | func readBenchmark() { 68 | clients, path := []*alluxio.Client{}, "/test_file" 69 | for i := 0; i < numThreadsFlag; i++ { 70 | clients = append(clients, alluxio.NewClient("localhost", 39999-i, time.Minute)) 71 | } 72 | totalBytes := float64(numOperationsFlag*numThreadsFlag*fileSizeFlag) / float64(GB) 73 | var totalTime int64 74 | data := make([]byte, bufferSizeFlag) 75 | if _, err := rand.Read(data); err != nil { 76 | log.Fatal(err) 77 | } 78 | if ok, err := clients[0].Exists(path, &option.Exists{}); err != nil { 79 | log.Fatal(err) 80 | } else if ok { 81 | if err := clients[0].Delete(path, &option.Delete{}); err != nil { 82 | log.Fatal(err) 83 | } 84 | } 85 | upload(clients[0], data, path) 86 | for j := 0; j < numIterationsFlag; j++ { 87 | start := time.Now() 88 | wg := sync.WaitGroup{} 89 | wg.Add(numThreadsFlag) 90 | for i := 0; i < numThreadsFlag; i++ { 91 | i := i 92 | go func() { 93 | defer wg.Done() 94 | readWorker(clients[i], path) 95 | }() 96 | } 97 | wg.Wait() 98 | end := time.Now() 99 | fmt.Printf("Iteration #%d: read %.2fGBs in %v.\n", j+1, totalBytes, end.Sub(start)) 100 | totalTime += end.Sub(start).Nanoseconds() 101 | } 102 | average := time.Duration(totalTime / int64(numIterationsFlag)) 103 | fmt.Printf("Average time %v, average throughput %.2f GB/s\n", average, float64(totalBytes)/average.Seconds()) 104 | } 105 | 106 | func readWorker(fs *alluxio.Client, path string) { 107 | for i := 0; i < numOperationsFlag; i++ { 108 | id, err := fs.OpenFile(path, &option.OpenFile{}) 109 | if err != nil { 110 | log.Fatal(err) 111 | } 112 | read(fs, id) 113 | } 114 | } 115 | 116 | func read(fs *alluxio.Client, id int) { 117 | r, err := fs.Read(id) 118 | if err != nil { 119 | log.Fatal(err) 120 | } 121 | defer r.Close() 122 | defer fs.Close(id) 123 | if _, err := io.Copy(ioutil.Discard, r); err != nil { 124 | log.Fatal(err) 125 | } 126 | } 127 | 128 | func writeBenchmark() { 129 | fs, path := alluxio.NewClient("localhost", 39999, time.Minute), "/test-file" 130 | totalBytes := float64(numOperationsFlag*numThreadsFlag*fileSizeFlag) / float64(GB) 131 | var totalTime int64 132 | for j := 0; j < numIterationsFlag; j++ { 133 | for i := 0; i < numOperationsFlag; i++ { 134 | for k := 0; k < numThreadsFlag; k++ { 135 | if ok, err := fs.Exists(fmt.Sprintf("%s-%d-%d", path, i, k), &option.Exists{}); err != nil { 136 | log.Fatal(err) 137 | } else if ok { 138 | if err := fs.Delete(fmt.Sprintf("%s-%d-%d", path, i, k), &option.Delete{}); err != nil { 139 | log.Fatal(err) 140 | } 141 | } 142 | } 143 | } 144 | data := make([]byte, bufferSizeFlag) 145 | if _, err := rand.Read(data); err != nil { 146 | log.Fatal(err) 147 | } 148 | start := time.Now() 149 | wg := sync.WaitGroup{} 150 | wg.Add(numThreadsFlag) 151 | for i := 0; i < numThreadsFlag; i++ { 152 | i := i 153 | go func() { 154 | defer wg.Done() 155 | writeWorker(fs, data, path, i) 156 | }() 157 | } 158 | wg.Wait() 159 | end := time.Now() 160 | fmt.Printf("Iteration #%d: wrote %.2fGBs in %v.\n", j+1, totalBytes, end.Sub(start)) 161 | totalTime += end.Sub(start).Nanoseconds() 162 | } 163 | average := time.Duration(totalTime / int64(numIterationsFlag)) 164 | fmt.Printf("Average time %v, average throughput %.2f GB/s\n", average, float64(totalBytes)/average.Seconds()) 165 | } 166 | 167 | func writeWorker(fs *alluxio.Client, data []byte, path string, id int) { 168 | for i := 0; i < numOperationsFlag; i++ { 169 | upload(fs, data, fmt.Sprintf("%s-%d-%d", path, i, id)) 170 | } 171 | } 172 | 173 | func upload(fs *alluxio.Client, data []byte, path string) { 174 | bytesToWrite := fileSizeFlag 175 | id, err := fs.CreateFile(path, &option.CreateFile{}) 176 | if err != nil { 177 | log.Fatal(err) 178 | } 179 | for bytesToWrite > 0 { 180 | n, err := fs.Write(id, bytes.NewBuffer(data)) 181 | if err != nil { 182 | log.Fatal(err) 183 | } 184 | bytesToWrite -= n 185 | } 186 | fs.Close(id) 187 | } 188 | 189 | const ( 190 | KB = 1024 191 | MB = KB * KB 192 | GB = KB * KB * KB 193 | ) 194 | 195 | func init() { 196 | log.SetFlags(log.LstdFlags | log.Lshortfile) 197 | flag.IntVar(&benchmarkFlag, "benchmark", 0, "the benchmark to run") 198 | flag.IntVar(&fileSizeFlag, "file-size", MB<<8, "the size of the file to use") 199 | flag.IntVar(&bufferSizeFlag, "buffer-size", MB, "the size of the buffer to use") 200 | flag.IntVar(&numOperationsFlag, "num-operations", 1, "the numbers of operations to run") 201 | flag.IntVar(&numIterationsFlag, "num-iterations", 10, "the number of iterations to run") 202 | flag.IntVar(&numThreadsFlag, "num-threads", 1, "the number of threads to run") 203 | flag.Parse() 204 | } 205 | 206 | var ( 207 | benchmarkFlag int 208 | bufferSizeFlag int 209 | fileSizeFlag int 210 | numOperationsFlag int 211 | numIterationsFlag int 212 | numThreadsFlag int 213 | ) 214 | 215 | func main() { 216 | switch benchmarkFlag { 217 | case 0: 218 | readBenchmark() 219 | case 1: 220 | writeBenchmark() 221 | case 2: 222 | metadataBenchmark() 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package alluxio 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "net/url" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | const ( 15 | apiPrefix = "api/v1" 16 | pathsPrefix = "paths" 17 | streamsPrefix = "streams" 18 | 19 | // Path endpoints 20 | createDirectory = "create-directory" 21 | createFile = "create-file" 22 | delete = "delete" 23 | exists = "exists" 24 | free = "free" 25 | getStatus = "get-status" 26 | listStatus = "list-status" 27 | mount = "mount" 28 | openFile = "open-file" 29 | rename = "rename" 30 | setAttribute = "set-attribute" 31 | unmount = "unmount" 32 | 33 | // Stream endpoints 34 | close = "close" 35 | read = "read" 36 | write = "write" 37 | ) 38 | 39 | type Client struct { 40 | host string 41 | port int 42 | prefix string 43 | http http.Client 44 | } 45 | 46 | // NewClient is the Alluxio file system client factory. 47 | func NewClient(host string, port int, timeout time.Duration) *Client { 48 | return &Client{ 49 | host: host, 50 | port: port, 51 | prefix: apiPrefix, 52 | http: http.Client{ 53 | Timeout: timeout, 54 | }, 55 | } 56 | } 57 | 58 | func join(components ...string) string { 59 | return strings.Join(components, "/") 60 | } 61 | 62 | func (client *Client) endpointURL(resource string, params map[string]string) string { 63 | paramPairs := []string{} 64 | for key, value := range params { 65 | paramPairs = append(paramPairs, key+"="+url.QueryEscape(value)) 66 | } 67 | return fmt.Sprintf("http://%v:%v/%v/%v?%v", client.host, client.port, client.prefix, resource, strings.Join(paramPairs, "&")) 68 | } 69 | 70 | func (client *Client) post(resource string, params map[string]string, input interface{}, output interface{}) error { 71 | var b bytes.Buffer 72 | if err := json.NewEncoder(&b).Encode(input); err != nil { 73 | return fmt.Errorf("Encode() failed: %v", err) 74 | } 75 | resp, err := client.http.Post(client.endpointURL(resource, params), "application/json", &b) 76 | if err != nil { 77 | return err 78 | } 79 | if err := check(resp); err != nil { 80 | return err 81 | } 82 | if err := process(resp, output); err != nil { 83 | return err 84 | } 85 | return nil 86 | } 87 | 88 | func check(resp *http.Response) error { 89 | if resp.StatusCode != http.StatusOK { 90 | bytes, err := ioutil.ReadAll(resp.Body) 91 | if err != nil { 92 | return fmt.Errorf("Response status: %v (%v):\nFailed to ready response body: %v", resp.Status, resp.StatusCode, err) 93 | } 94 | return fmt.Errorf("Response status: %v (%v):\nResponse body:\n%s", resp.Status, resp.StatusCode, bytes) 95 | } 96 | return nil 97 | } 98 | 99 | func process(resp *http.Response, output interface{}) error { 100 | defer resp.Body.Close() 101 | if output != nil { 102 | contentType := resp.Header.Get("Content-Type") 103 | switch contentType { 104 | case "application/json": 105 | if err := json.NewDecoder(resp.Body).Decode(output); err != nil { 106 | return fmt.Errorf("Decode() failed: %v", err) 107 | } 108 | default: 109 | return fmt.Errorf("Unsupported response type: %v", contentType) 110 | } 111 | } 112 | return nil 113 | } 114 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package alluxio provides the Alluxio file system client. 2 | package alluxio 3 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package alluxio 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "time" 10 | 11 | "github.com/Alluxio/alluxio-go/option" 12 | ) 13 | 14 | func ExampleClient_download() { 15 | fs := NewClient("localhost", 39999, time.Minute) 16 | id, err := fs.OpenFile("/test_file", &option.OpenFile{}) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | r, err := fs.Read(id) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | defer r.Close() 25 | defer fs.Close(id) 26 | if _, err := io.Copy(ioutil.Discard, r); err != nil { 27 | log.Fatal(err) 28 | } 29 | } 30 | 31 | func ExampleClient_exists() { 32 | fs := NewClient("localhost", 39999, time.Minute) 33 | ok, err := fs.Exists("/test_path", &option.Exists{}) 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | fmt.Println(ok) 38 | } 39 | 40 | func ExampleClient_listStatus() { 41 | fs := NewClient("localhost", 39999, time.Minute) 42 | fileInfos, err := fs.ListStatus("/test_path", &option.ListStatus{}) 43 | if err != nil { 44 | log.Fatal(err) 45 | } 46 | for _, fileInfo := range fileInfos { 47 | fmt.Println(fileInfo.Name) 48 | } 49 | } 50 | 51 | func ExampleClient_upload() { 52 | fs := NewClient("localhost", 39999, time.Minute) 53 | id, err := fs.CreateFile("/test_file", &option.CreateFile{}) 54 | if err != nil { 55 | log.Fatal(err) 56 | } 57 | n, err := fs.Write(id, bytes.NewBuffer(make([]byte, 1024))) 58 | if err != nil { 59 | log.Fatal(err) 60 | } 61 | defer fs.Close(id) 62 | fmt.Println(n) 63 | } 64 | -------------------------------------------------------------------------------- /option/create_directory.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/Alluxio/alluxio-go/wire" 5 | ) 6 | 7 | // CreateDirectory holds the options for creating a directory. 8 | type CreateDirectory struct { 9 | AllowExists *bool `json:"allowExists,omitempty"` 10 | Mode *wire.Mode `json:"mode,omitempty"` 11 | Recursive *bool `json:"recursive,omitempty"` 12 | WriteType *wire.WriteType `json:"writeType,omitempty"` 13 | } 14 | 15 | // SetAllowExists sets the bit that determines whether the directory to be 16 | // created is allowed to exist. 17 | func (option *CreateDirectory) SetAllowExists(value bool) { 18 | option.AllowExists = &value 19 | } 20 | 21 | // SetMode sets the access mode. 22 | func (option *CreateDirectory) SetMode(value wire.Mode) { 23 | option.Mode = &value 24 | } 25 | 26 | // SetRecursive sets the bit that determines whether non-existent ancestors 27 | // of the directory to be created should be created as well. 28 | func (option *CreateDirectory) SetRecursive(value bool) { 29 | option.Recursive = &value 30 | } 31 | 32 | // SetWriteType sets the write type. 33 | func (option *CreateDirectory) SetWriteType(value wire.WriteType) { 34 | option.WriteType = &value 35 | } 36 | -------------------------------------------------------------------------------- /option/create_file.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import "github.com/Alluxio/alluxio-go/wire" 4 | 5 | // CreateFile holds the options for creating a file. 6 | type CreateFile struct { 7 | BlockSizeBytes *int64 `json:"blockSizeBytes,omitempty"` 8 | LocationPolicyClass *string `json:"locationPolicyClass,omitempty"` 9 | Mode *wire.Mode `json:"mode,omitempty"` 10 | Recursive *bool `json:"recursive,omitempty"` 11 | TTL *int64 `json:"ttl,omitempty"` 12 | TTLAction *wire.TTLAction `json:"ttlAction,omitempty"` 13 | WriteType *wire.WriteType `json:"writeType,omitempty"` 14 | } 15 | 16 | // SetBlockSizeBytes sets the block size (in bytes). 17 | func (option *CreateFile) SetBlockSizeBytes(value int64) { 18 | option.BlockSizeBytes = &value 19 | } 20 | 21 | // SetLocationPolicyClass sets the location policy class. 22 | func (option *CreateFile) SetLocationPolicyClass(value string) { 23 | option.LocationPolicyClass = &value 24 | } 25 | 26 | // SetMode sets the mode. 27 | func (option *CreateFile) SetMode(value wire.Mode) { 28 | option.Mode = &value 29 | } 30 | 31 | // SetRecursive sets the bit that determines whether non-existent ancestors 32 | // of the file to be created should be created as well. 33 | func (option *CreateFile) SetRecursive(value bool) { 34 | option.Recursive = &value 35 | } 36 | 37 | // SetTTL sets the time-to-live window. Once this window expires a time-to-live 38 | // action is performed. 39 | func (option *CreateFile) SetTTL(value int64) { 40 | option.TTL = &value 41 | } 42 | 43 | // SetTTLAction sets the time-to-live action, which is execute once the 44 | // time-to-live window expires. 45 | func (option *CreateFile) SetTTLAction(value wire.TTLAction) { 46 | option.TTLAction = &value 47 | } 48 | 49 | // SetWriteType sets the write type. 50 | func (option *CreateFile) SetWriteType(value wire.WriteType) { 51 | option.WriteType = &value 52 | } 53 | -------------------------------------------------------------------------------- /option/delete.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | // Delete holds the options for deleting a path. 4 | type Delete struct { 5 | Recursive *bool `json:"recursive,omitempty"` 6 | } 7 | 8 | // SetRecursive sets the bit which determines whether the path should be 9 | // deleted recursively. 10 | func (option *Delete) SetRecursive(value bool) { 11 | option.Recursive = &value 12 | } 13 | -------------------------------------------------------------------------------- /option/doc.go: -------------------------------------------------------------------------------- 1 | // Package option contains types for method options. 2 | package option 3 | -------------------------------------------------------------------------------- /option/exists.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | // Exists holds the options for checking whether a path exists. 4 | type Exists struct{} 5 | -------------------------------------------------------------------------------- /option/free.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | // Free holds the options for freeing a path. 4 | type Free struct { 5 | Recursive *bool `json:"recursive,omitempty"` 6 | } 7 | 8 | // SetRecursive sets the bit which determines whether the path should be 9 | // freed recursively. 10 | func (option *Free) SetRecursive(value bool) { 11 | option.Recursive = &value 12 | } 13 | -------------------------------------------------------------------------------- /option/get_status.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | // GetStatus holds the options for fetching a path status. 4 | type GetStatus struct{} 5 | -------------------------------------------------------------------------------- /option/list_status.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/Alluxio/alluxio-go/wire" 5 | ) 6 | 7 | // ListStatus holds the options for listing a path. 8 | type ListStatus struct { 9 | LoadMetadataType *wire.LoadMetadataType `json:"loadMetadataType,omitempty"` 10 | } 11 | 12 | // SetLoadMetadataType sets the type, which determines whether to load metadata 13 | // for children of a directory. 14 | func (option *ListStatus) SetLoadMetadataType(value wire.LoadMetadataType) { 15 | option.LoadMetadataType = &value 16 | } 17 | -------------------------------------------------------------------------------- /option/mount.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | // Mount holds the options for mounting a path. 4 | type Mount struct { 5 | Properties map[string]string `json:"properties,omitempty"` 6 | ReadOnly *bool `json:"readOnly,omitempty"` 7 | Shared *bool `json:"shared,omitempty"` 8 | } 9 | 10 | // SetProperties sets the mount point properties. 11 | func (option *Mount) SetProperties(value map[string]string) { 12 | option.Properties = value 13 | } 14 | 15 | // SetReadOnly sets the bit which determines whether the mount point is 16 | // read-only. 17 | func (option *Mount) SetReadOnly(value bool) { 18 | option.ReadOnly = &value 19 | } 20 | 21 | // SetShared which determines whether this mount point is shared among multiple 22 | // users. 23 | func (option *Mount) SetShared(value bool) { 24 | option.Shared = &value 25 | } 26 | -------------------------------------------------------------------------------- /option/open_file.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/Alluxio/alluxio-go/wire" 5 | ) 6 | 7 | // OpenFile holds the options for opening a file. 8 | type OpenFile struct { 9 | LocationPolicyClass *string `json:"locationPolicyClass,omitempty"` 10 | ReadType *wire.ReadType `json:"readType,omitempty"` 11 | } 12 | 13 | // SetLocationPolicyClass sets the location policy class. 14 | func (option *OpenFile) SetLocationPolicyClass(value string) { 15 | option.LocationPolicyClass = &value 16 | } 17 | 18 | // SetReadType sets the read type. 19 | func (option *OpenFile) SetReadType(value wire.ReadType) { 20 | option.ReadType = &value 21 | } 22 | -------------------------------------------------------------------------------- /option/rename.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | // Rename holds the options for renaming a path. 4 | type Rename struct{} 5 | -------------------------------------------------------------------------------- /option/set_attribute.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import "github.com/Alluxio/alluxio-go/wire" 4 | 5 | // SetAttribute holds the options for setting path attributes. 6 | type SetAttribute struct { 7 | Group *string `json:"group,omitempty"` 8 | Mode *wire.Mode `json:"mode,omitempty"` 9 | Owner *string `json:"owner,omitempty"` 10 | Persisted *bool `json:"persisted,omitempty"` 11 | Pinned *bool `json:"pinned,omitempty"` 12 | Recursive *bool `json:"recursive,omitempty"` 13 | TTL *int64 `json:"ttl,omitempty"` 14 | TTLAction *wire.TTLAction `json:"ttlAction,omitempty"` 15 | } 16 | 17 | // SetGroup sets the group. 18 | func (option *SetAttribute) SetGroup(value string) { 19 | option.Group = &value 20 | } 21 | 22 | // SetMode sets the access mode. 23 | func (option *SetAttribute) SetMode(value wire.Mode) { 24 | option.Mode = &value 25 | } 26 | 27 | // SetOwner sets the owner. 28 | func (option *SetAttribute) SetOwner(value string) { 29 | option.Owner = &value 30 | } 31 | 32 | // SetPersisted sets the bit which determines whether the path is persisted. 33 | func (option *SetAttribute) SetPersisted(value bool) { 34 | option.Persisted = &value 35 | } 36 | 37 | // SetPinned sets the bit which determines whether a path is pinned. 38 | func (option *SetAttribute) SetPinned(value bool) { 39 | option.Pinned = &value 40 | } 41 | 42 | // SetRecursive sets the bit which determines whether the attributes should 43 | // be set recursively. 44 | func (option *SetAttribute) SetRecursive(value bool) { 45 | option.Recursive = &value 46 | } 47 | 48 | // SetTTL sets the time-to-live window. Once this window expires a time-to-live 49 | // action is performed. 50 | func (option *SetAttribute) SetTTL(value int64) { 51 | option.TTL = &value 52 | } 53 | 54 | // SetTTLAction sets the time-to-live action, which is execute once the 55 | // time-to-live window expires. 56 | func (option *SetAttribute) SetTTLAction(value wire.TTLAction) { 57 | option.TTLAction = &value 58 | } 59 | -------------------------------------------------------------------------------- /option/unmount.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | // Unmount holds the options for unmounting a path. 4 | type Unmount struct{} 5 | -------------------------------------------------------------------------------- /optiontest/create_directory.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import ( 4 | "github.com/Alluxio/alluxio-go/option" 5 | "github.com/Alluxio/alluxio-go/wiretest" 6 | ) 7 | 8 | // RandomCreateDirectory creates a random instance of option.CreateDirectory. 9 | func RandomCreateDirectory() option.CreateDirectory { 10 | var option option.CreateDirectory 11 | option.SetAllowExists(wiretest.RandomBool()) 12 | option.SetMode(wiretest.RandomMode()) 13 | option.SetRecursive(wiretest.RandomBool()) 14 | option.SetWriteType(wiretest.RandomWriteType()) 15 | return option 16 | } 17 | -------------------------------------------------------------------------------- /optiontest/create_file.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/option" 7 | "github.com/Alluxio/alluxio-go/wiretest" 8 | ) 9 | 10 | // RandomCreateFile creates a random instance of option.CreateFile. 11 | func RandomCreateFile() option.CreateFile { 12 | var option option.CreateFile 13 | option.SetBlockSizeBytes(rand.Int63()) 14 | option.SetLocationPolicyClass(wiretest.RandomString()) 15 | option.SetMode(wiretest.RandomMode()) 16 | option.SetRecursive(wiretest.RandomBool()) 17 | option.SetTTL(rand.Int63()) 18 | option.SetTTLAction(wiretest.RandomTTLAction()) 19 | option.SetWriteType(wiretest.RandomWriteType()) 20 | return option 21 | } 22 | -------------------------------------------------------------------------------- /optiontest/delete.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import ( 4 | "github.com/Alluxio/alluxio-go/option" 5 | "github.com/Alluxio/alluxio-go/wiretest" 6 | ) 7 | 8 | // RandomDelete creates a random instance of option.Delete. 9 | func RandomDelete() option.Delete { 10 | var option option.Delete 11 | option.SetRecursive(wiretest.RandomBool()) 12 | return option 13 | } 14 | -------------------------------------------------------------------------------- /optiontest/doc.go: -------------------------------------------------------------------------------- 1 | // Package optiontest contains utilities for testing method options. 2 | package optiontest 3 | -------------------------------------------------------------------------------- /optiontest/exists.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import "github.com/Alluxio/alluxio-go/option" 4 | 5 | // RandomExists creates a random instance of option.Exists. 6 | func RandomExists() option.Exists { 7 | var option option.Exists 8 | return option 9 | } 10 | -------------------------------------------------------------------------------- /optiontest/free.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import ( 4 | "github.com/Alluxio/alluxio-go/option" 5 | "github.com/Alluxio/alluxio-go/wiretest" 6 | ) 7 | 8 | // RandomFree creates a random instance of option.Free. 9 | func RandomFree() option.Free { 10 | var option option.Free 11 | option.SetRecursive(wiretest.RandomBool()) 12 | return option 13 | } 14 | -------------------------------------------------------------------------------- /optiontest/get_status.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import "github.com/Alluxio/alluxio-go/option" 4 | 5 | // RandomGetStatus creates a random instance of option.GetStatus. 6 | func RandomGetStatus() option.GetStatus { 7 | var option option.GetStatus 8 | return option 9 | } 10 | -------------------------------------------------------------------------------- /optiontest/list_status.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import ( 4 | "github.com/Alluxio/alluxio-go/option" 5 | "github.com/Alluxio/alluxio-go/wiretest" 6 | ) 7 | 8 | // RandomListStatus creates a random instance of option.ListStatus. 9 | func RandomListStatus() option.ListStatus { 10 | var option option.ListStatus 11 | option.SetLoadMetadataType(wiretest.RandomLoadMetadataType()) 12 | return option 13 | } 14 | -------------------------------------------------------------------------------- /optiontest/mount.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/option" 7 | "github.com/Alluxio/alluxio-go/wiretest" 8 | ) 9 | 10 | // RandomMount creates a random instance of option.Mount. 11 | func RandomMount() option.Mount { 12 | var option option.Mount 13 | properties, n := map[string]string{}, rand.Intn(10)+1 14 | for i := 0; i < n; i++ { 15 | properties[wiretest.RandomString()] = wiretest.RandomString() 16 | } 17 | option.SetProperties(properties) 18 | option.SetReadOnly(wiretest.RandomBool()) 19 | option.SetShared(wiretest.RandomBool()) 20 | return option 21 | } 22 | -------------------------------------------------------------------------------- /optiontest/open_file.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import ( 4 | "github.com/Alluxio/alluxio-go/option" 5 | "github.com/Alluxio/alluxio-go/wiretest" 6 | ) 7 | 8 | // RandomOpenFile creates a random instance of option.OpenFile. 9 | func RandomOpenFile() option.OpenFile { 10 | var option option.OpenFile 11 | option.SetLocationPolicyClass(wiretest.RandomString()) 12 | option.SetReadType(wiretest.RandomReadType()) 13 | return option 14 | } 15 | -------------------------------------------------------------------------------- /optiontest/rename.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import "github.com/Alluxio/alluxio-go/option" 4 | 5 | // RandomRename creates a random instance of option.Rename. 6 | func RandomRename() option.Rename { 7 | var option option.Rename 8 | return option 9 | } 10 | -------------------------------------------------------------------------------- /optiontest/set_attribute.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/option" 7 | "github.com/Alluxio/alluxio-go/wiretest" 8 | ) 9 | 10 | // RandomSetAttribute creates a random instance of option.SetAttribute. 11 | func RandomSetAttribute() option.SetAttribute { 12 | var option option.SetAttribute 13 | option.SetGroup(wiretest.RandomString()) 14 | option.SetMode(wiretest.RandomMode()) 15 | option.SetOwner(wiretest.RandomString()) 16 | option.SetPersisted(wiretest.RandomBool()) 17 | option.SetPinned(wiretest.RandomBool()) 18 | option.SetRecursive(wiretest.RandomBool()) 19 | option.SetTTL(rand.Int63()) 20 | option.SetTTLAction(wiretest.RandomTTLAction()) 21 | return option 22 | } 23 | -------------------------------------------------------------------------------- /optiontest/unmount.go: -------------------------------------------------------------------------------- 1 | package optiontest 2 | 3 | import "github.com/Alluxio/alluxio-go/option" 4 | 5 | // RandomUnmount creates a random instance of option.Unmount. 6 | func RandomUnmount() option.Unmount { 7 | var option option.Unmount 8 | return option 9 | } 10 | -------------------------------------------------------------------------------- /paths.go: -------------------------------------------------------------------------------- 1 | package alluxio 2 | 3 | import ( 4 | "github.com/Alluxio/alluxio-go/option" 5 | "github.com/Alluxio/alluxio-go/wire" 6 | ) 7 | 8 | var ( 9 | noParams = map[string]string{} 10 | ) 11 | 12 | // CreateDirectory creates a directory. 13 | func (client *Client) CreateDirectory(path string, options *option.CreateDirectory) error { 14 | return client.post(join(pathsPrefix, path, createDirectory), noParams, options, nil) 15 | } 16 | 17 | // CreateFile creates a file, returning a stream id. 18 | func (client *Client) CreateFile(path string, options *option.CreateFile) (int, error) { 19 | var result int 20 | if err := client.post(join(pathsPrefix, path, createFile), noParams, options, &result); err != nil { 21 | return -1, err 22 | } 23 | return result, nil 24 | } 25 | 26 | // Delete deletes a path. 27 | func (client *Client) Delete(path string, options *option.Delete) error { 28 | return client.post(join(pathsPrefix, path, delete), noParams, options, nil) 29 | } 30 | 31 | // Exists checks whether a path exists. 32 | func (client *Client) Exists(path string, options *option.Exists) (bool, error) { 33 | var result bool 34 | if err := client.post(join(pathsPrefix, path, exists), noParams, options, &result); err != nil { 35 | return false, err 36 | } 37 | return result, nil 38 | } 39 | 40 | // Free frees a path. 41 | func (client *Client) Free(path string, options *option.Free) error { 42 | return client.post(join(pathsPrefix, path, free), noParams, options, nil) 43 | } 44 | 45 | // GetStatus gets a path status. 46 | func (client *Client) GetStatus(path string, options *option.GetStatus) (*wire.FileInfo, error) { 47 | var result wire.FileInfo 48 | if err := client.post(join(pathsPrefix, path, getStatus), noParams, options, &result); err != nil { 49 | return nil, err 50 | } 51 | return &result, nil 52 | } 53 | 54 | // ListStatus lists path statuses. 55 | func (client *Client) ListStatus(path string, options *option.ListStatus) (wire.FileInfos, error) { 56 | var result wire.FileInfos 57 | if err := client.post(join(pathsPrefix, path, listStatus), noParams, options, &result); err != nil { 58 | return nil, err 59 | } 60 | return result, nil 61 | } 62 | 63 | // Mount mounts a UFS source to a path. 64 | func (client *Client) Mount(path, src string, options *option.Mount) error { 65 | params := map[string]string{ 66 | "src": src, 67 | } 68 | return client.post(join(pathsPrefix, path, mount), params, options, nil) 69 | } 70 | 71 | // OpenFile opens a file, returning a stream id. 72 | func (client *Client) OpenFile(path string, options *option.OpenFile) (int, error) { 73 | var result int 74 | if err := client.post(join(pathsPrefix, path, openFile), noParams, options, &result); err != nil { 75 | return -1, err 76 | } 77 | return result, nil 78 | } 79 | 80 | // Rename renames a path. 81 | func (client *Client) Rename(path, dst string, options *option.Rename) error { 82 | params := map[string]string{ 83 | "dst": dst, 84 | } 85 | return client.post(join(pathsPrefix, path, rename), params, options, nil) 86 | } 87 | 88 | // SetAttribute sets path attributes. 89 | func (client *Client) SetAttribute(path string, options *option.SetAttribute) error { 90 | return client.post(join(pathsPrefix, path, setAttribute), noParams, options, nil) 91 | } 92 | 93 | // Unmount unmounts a path. 94 | func (client *Client) Unmount(path string, options *option.Unmount) error { 95 | return client.post(join(pathsPrefix, path, unmount), noParams, options, nil) 96 | } 97 | -------------------------------------------------------------------------------- /paths_test.go: -------------------------------------------------------------------------------- 1 | package alluxio 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "log" 8 | "math/rand" 9 | "net" 10 | "net/http" 11 | "net/http/httptest" 12 | "net/url" 13 | "reflect" 14 | "strconv" 15 | "testing" 16 | "time" 17 | 18 | "github.com/Alluxio/alluxio-go/option" 19 | "github.com/Alluxio/alluxio-go/optiontest" 20 | "github.com/Alluxio/alluxio-go/wire" 21 | "github.com/Alluxio/alluxio-go/wiretest" 22 | ) 23 | 24 | func init() { 25 | seed := time.Now().UnixNano() 26 | log.Printf("Using seed: %v", seed) 27 | rand.Seed(seed) 28 | } 29 | 30 | // setupClient sets up a instance of the Alluxio file system client connected 31 | // to a local test server that uses the given handler. 32 | func setupClient(t *testing.T, handler http.Handler) (*Client, func()) { 33 | testServer := httptest.NewServer(handler) 34 | url, err := url.Parse(testServer.URL) 35 | if err != nil { 36 | t.Fatalf("Parse(%v) failed: %v", testServer.URL, err) 37 | } 38 | host, portStr, err := net.SplitHostPort(url.Host) 39 | if err != nil { 40 | t.Fatalf("SplitHostPort(%v) failed: %v", url.Host, err) 41 | } 42 | port, err := strconv.Atoi(portStr) 43 | if err != nil { 44 | t.Fatalf("Atoi(%v) failed: %v", portStr, err) 45 | } 46 | return NewClient(host, port, time.Minute), func() { 47 | testServer.Close() 48 | } 49 | } 50 | 51 | // jsonHandler creates a handler for the given resource that: 52 | // 1. JSON-decodes the input from the request body 53 | // 2. checks that the received input matches the expected input 54 | // 3. JSON-encodes the given output to the response body 55 | func jsonHandler(t *testing.T, resource string, input, output, expected interface{}) http.HandlerFunc { 56 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 57 | defer r.Body.Close() 58 | if got, want := r.URL.Path, join("", apiPrefix, pathsPrefix, resource); got != want { 59 | t.Fatalf("Unexpected path: got %v, want %v", got, want) 60 | } 61 | body, err := ioutil.ReadAll(r.Body) 62 | if err != nil { 63 | t.Fatalf("ReadAll() failed: %v", err) 64 | } 65 | decoder := json.NewDecoder(bytes.NewBuffer(body)) 66 | if err := decoder.Decode(&input); err != nil { 67 | t.Fatalf("Decode() failed: %v\nBody:\n%s", err, body) 68 | } 69 | if !reflect.DeepEqual(input, expected) { 70 | t.Fatalf("Unexpected input: got %v, want %v", input, expected) 71 | } 72 | w.Header().Set("Content-Type", "application/json") 73 | if err := json.NewEncoder(w).Encode(output); err != nil { 74 | t.Fatalf("Encode() failed: %v", err) 75 | } 76 | }) 77 | } 78 | 79 | func TestFileSystemCreateDirectory(t *testing.T) { 80 | input, expected := option.CreateDirectory{}, optiontest.RandomCreateDirectory() 81 | path := "/foo" 82 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, createDirectory), &input, nil, &expected)) 83 | defer cleanup() 84 | if err := fs.CreateDirectory(path, &expected); err != nil { 85 | t.Fatalf("CreateDirectory() failed: %v", err) 86 | } 87 | } 88 | 89 | func TestFileSystemCreateFile(t *testing.T) { 90 | input, expected := option.CreateFile{}, optiontest.RandomCreateFile() 91 | output := rand.Int() 92 | path := "/foo" 93 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, createFile), &input, output, &expected)) 94 | defer cleanup() 95 | if id, err := fs.CreateFile(path, &expected); err != nil { 96 | t.Fatalf("CreateFile() failed: %v", err) 97 | } else if id != output { 98 | t.Fatalf("Unexpected output: got %v, want %v", id, output) 99 | } 100 | } 101 | 102 | func TestFileSystemDelete(t *testing.T) { 103 | input, expected := option.Delete{}, optiontest.RandomDelete() 104 | path := "/foo" 105 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, delete), &input, nil, &expected)) 106 | defer cleanup() 107 | if err := fs.Delete(path, &expected); err != nil { 108 | t.Fatalf("Delete() failed: %v", err) 109 | } 110 | } 111 | 112 | func TestFileSystemExists(t *testing.T) { 113 | input, expected := option.Exists{}, optiontest.RandomExists() 114 | path := "/foo" 115 | output := wiretest.RandomBool() 116 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, exists), &input, output, &expected)) 117 | defer cleanup() 118 | if ok, err := fs.Exists(path, &expected); err != nil { 119 | t.Fatalf("Exists() failed: %v", err) 120 | } else if ok != output { 121 | t.Fatalf("Unexpected output: got %v, want %v", ok, output) 122 | } 123 | } 124 | 125 | func TestFileSystemFree(t *testing.T) { 126 | input, expected := option.Free{}, optiontest.RandomFree() 127 | path := "/foo" 128 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, free), &input, nil, &expected)) 129 | defer cleanup() 130 | if err := fs.Free(path, &expected); err != nil { 131 | t.Fatalf("Free() failed: %v", err) 132 | } 133 | } 134 | 135 | func TestFileSystemGetStatus(t *testing.T) { 136 | input, expected := option.GetStatus{}, optiontest.RandomGetStatus() 137 | output := wiretest.RandomFileInfo() 138 | path := "/foo" 139 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, getStatus), &input, output, &expected)) 140 | defer cleanup() 141 | if fileInfo, err := fs.GetStatus(path, &expected); err != nil { 142 | t.Fatalf("GetStatus() failed: %v", err) 143 | } else if !reflect.DeepEqual(&output, fileInfo) { 144 | t.Fatalf("Unexpected output: got %#v, want %#v", fileInfo, output) 145 | } 146 | } 147 | 148 | func TestFileSystemListStatus(t *testing.T) { 149 | input, expected := option.ListStatus{}, optiontest.RandomListStatus() 150 | output := wire.FileInfos(make([]wire.FileInfo, rand.Intn(10))) 151 | path := "/foo" 152 | for i := 0; i < len(output); i++ { 153 | output[i] = wiretest.RandomFileInfo() 154 | } 155 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, listStatus), &input, output, &expected)) 156 | defer cleanup() 157 | if fileInfos, err := fs.ListStatus(path, &expected); err != nil { 158 | t.Fatalf("ListStatus() failed: %v", err) 159 | } else if !reflect.DeepEqual(output, fileInfos) { 160 | t.Fatalf("Unexpected output: got %#v, want %#v", fileInfos, output) 161 | } 162 | } 163 | 164 | func TestFileSystemMount(t *testing.T) { 165 | input, expected := option.Mount{}, optiontest.RandomMount() 166 | path := "/foo" 167 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, mount), &input, nil, &expected)) 168 | defer cleanup() 169 | if err := fs.Mount(path, wiretest.RandomString(), &expected); err != nil { 170 | t.Fatalf("Mount() failed: %v", err) 171 | } 172 | } 173 | 174 | func TestFileSystemOpenFile(t *testing.T) { 175 | input, expected := option.OpenFile{}, optiontest.RandomOpenFile() 176 | output := rand.Int() 177 | path := "/foo" 178 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, openFile), &input, output, &expected)) 179 | defer cleanup() 180 | if id, err := fs.OpenFile(path, &expected); err != nil { 181 | t.Fatalf("OpenFile() failed: %v", err) 182 | } else if id != output { 183 | t.Fatalf("Unexpected output: got %v, want %v", id, output) 184 | } 185 | } 186 | 187 | func TestFileSystemRename(t *testing.T) { 188 | input, expected := option.Rename{}, optiontest.RandomRename() 189 | path := "/foo" 190 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, rename), &input, nil, &expected)) 191 | defer cleanup() 192 | if err := fs.Rename(path, wiretest.RandomString(), &expected); err != nil { 193 | t.Fatalf("Rename() failed: %v", err) 194 | } 195 | } 196 | 197 | func TestFileSystemSetAttribute(t *testing.T) { 198 | input, expected := option.SetAttribute{}, optiontest.RandomSetAttribute() 199 | path := "/foo" 200 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, setAttribute), &input, nil, &expected)) 201 | defer cleanup() 202 | if err := fs.SetAttribute(path, &expected); err != nil { 203 | t.Fatalf("SetAttribute() failed: %v", err) 204 | } 205 | } 206 | 207 | func TestFileSystemUnmount(t *testing.T) { 208 | input, expected := option.Unmount{}, optiontest.RandomUnmount() 209 | path := "/foo" 210 | fs, cleanup := setupClient(t, jsonHandler(t, join(path, unmount), &input, nil, &expected)) 211 | defer cleanup() 212 | if err := fs.Unmount(path, &expected); err != nil { 213 | t.Fatalf("Unmount() failed: %v", err) 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /streams.go: -------------------------------------------------------------------------------- 1 | package alluxio 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | ) 7 | 8 | // Close closes a stream. 9 | func (client *Client) Close(id int) error { 10 | return client.post(join(streamsPrefix, fmt.Sprintf("%d", id), close), nil, nil, nil) 11 | } 12 | 13 | // Read reads from a stream. 14 | func (client *Client) Read(id int) (io.ReadCloser, error) { 15 | suffix := join(streamsPrefix, fmt.Sprintf("%d", id), read) 16 | resp, err := client.http.Post(client.endpointURL(suffix, nil), "application/json", nil) 17 | if err != nil { 18 | return nil, err 19 | } 20 | if err := check(resp); err != nil { 21 | return nil, err 22 | } 23 | return resp.Body, nil 24 | } 25 | 26 | // Write writes to a stream. 27 | func (client *Client) Write(id int, input io.Reader) (int, error) { 28 | suffix := join(streamsPrefix, fmt.Sprintf("%d", id), write) 29 | resp, err := client.http.Post(client.endpointURL(suffix, nil), "application/octet-stream", input) 30 | if err != nil { 31 | return -1, err 32 | } 33 | if err := check(resp); err != nil { 34 | return -1, err 35 | } 36 | var result int 37 | if err := process(resp, &result); err != nil { 38 | return -1, err 39 | } 40 | return result, nil 41 | } 42 | -------------------------------------------------------------------------------- /streams_test.go: -------------------------------------------------------------------------------- 1 | package alluxio 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "math/rand" 10 | "net/http" 11 | "reflect" 12 | "testing" 13 | ) 14 | 15 | var message = []byte("Greetings traveller!") 16 | 17 | func streamHandler(t *testing.T, id int, method string, output io.Reader, expected []byte) http.HandlerFunc { 18 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 19 | if got, want := r.URL.Path, join("", apiPrefix, streamsPrefix, fmt.Sprintf("%d", id), method); got != want { 20 | t.Fatalf("Unexpected path: got %v, want %v", got, want) 21 | } 22 | if expected != nil { 23 | bytes, err := ioutil.ReadAll(r.Body) 24 | if err != nil { 25 | t.Fatalf("ReadAll() failed: %v", err) 26 | } 27 | defer r.Body.Close() 28 | if !reflect.DeepEqual(bytes, expected) { 29 | t.Fatalf("Unexpected input: got %s, want %s", bytes, expected) 30 | } 31 | } 32 | if output != nil { 33 | w.Header().Set("Content-Type", "application/octet-stream") 34 | io.Copy(w, output) 35 | } else { 36 | w.Header().Set("Content-Type", "application/json") 37 | json.NewEncoder(w).Encode(len(message)) 38 | } 39 | }) 40 | } 41 | 42 | func TestFileSystemClose(t *testing.T) { 43 | id := rand.Int() 44 | fs, cleanup := setupClient(t, streamHandler(t, id, close, nil, nil)) 45 | defer cleanup() 46 | if err := fs.Close(id); err != nil { 47 | t.Fatalf("Close() failed: %v", err) 48 | } 49 | } 50 | 51 | func TestFileSystemRead(t *testing.T) { 52 | id := rand.Int() 53 | fs, cleanup := setupClient(t, streamHandler(t, id, read, bytes.NewBuffer(message), nil)) 54 | defer cleanup() 55 | reader, err := fs.Read(id) 56 | if err != nil { 57 | t.Fatalf("Read() failed: %v", err) 58 | } 59 | bytes, err := ioutil.ReadAll(reader) 60 | if err != nil { 61 | t.Fatalf("ReadAll() failed: %v", err) 62 | } 63 | if !reflect.DeepEqual(bytes, message) { 64 | t.Fatalf("Unexpected output: got %s, want %s", bytes, message) 65 | } 66 | } 67 | 68 | func TestFileSystemWrite(t *testing.T) { 69 | id := rand.Int() 70 | fs, cleanup := setupClient(t, streamHandler(t, id, write, nil, message)) 71 | defer cleanup() 72 | if n, err := fs.Write(id, bytes.NewBuffer(message)); err != nil { 73 | t.Fatalf("Write() failed: %v", err) 74 | } else if got, want := n, len(message); got != want { 75 | t.Fatalf("Unexpected number of bytes written: got %v, want %v", got, want) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /wire/bits.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // Bits represents access mode's bits. 4 | type Bits string 5 | 6 | const ( 7 | // BitsNone represents no access. 8 | BitsNone Bits = "NONE" 9 | // BitsExecute represent execute access. 10 | BitsExecute = "EXECUTE" 11 | // BitsWrite represents write access. 12 | BitsWrite = "WRITE" 13 | // BitsWriteExecute represents write and execute access. 14 | BitsWriteExecute = "WRITE_EXECUTE" 15 | // BitsRead represents read access. 16 | BitsRead = "READ" 17 | // BitsReadExecute represents read and execute access. 18 | BitsReadExecute = "READ_EXECUTE" 19 | // BitsReadWrite represents read and write access. 20 | BitsReadWrite = "READ_WRITE" 21 | // BitsAll represents read, write, and execute access 22 | BitsAll = "ALL" 23 | ) 24 | -------------------------------------------------------------------------------- /wire/block_info.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // BlockInfo represents a block's information. 4 | type BlockInfo struct { 5 | // BlockID is the block id. 6 | BlockID int64 7 | // Length is the block length. 8 | Length int64 9 | // Locations holds the block locations. 10 | Locations []BlockLocation 11 | } 12 | -------------------------------------------------------------------------------- /wire/block_location.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // BlockLocation represents a block's location. 4 | type BlockLocation struct { 5 | // WorkerID is the worker id. 6 | WorkerID int64 7 | // WorkerAddress is the worker network address. 8 | WorkerAddress WorkerNetAddress 9 | // TierAlias is the tier alias. 10 | TierAlias string 11 | } 12 | -------------------------------------------------------------------------------- /wire/doc.go: -------------------------------------------------------------------------------- 1 | // Package wire contains Alluxio file system wire types. 2 | package wire 3 | -------------------------------------------------------------------------------- /wire/file_block_info.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // FileBlockInfo represents a file block's information. 4 | type FileBlockInfo struct { 5 | // BlockInfo is the block information 6 | BlockInfo BlockInfo 7 | // Offset is the file offset. 8 | Offset int64 9 | // UfsLocations holds the UFS locations. 10 | UfsLocations []string 11 | } 12 | -------------------------------------------------------------------------------- /wire/file_info.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | import "sort" 4 | 5 | // FileInfo represents a file's information. 6 | type FileInfo struct { 7 | // BlockIds holds the block ids. 8 | BlockIds []int64 9 | // BlockSizeBytes is the block size (in bytes). 10 | BlockSizeBytes int64 11 | // Cacheable determines whether the file is cacheable. 12 | Cacheable bool 13 | // Completed determines whether the file is completed. 14 | Completed bool 15 | // CreationTimesMs is the creation time (in milliseconds). 16 | CreationTimeMs int64 17 | // FileBlockInfos holds the file block information. 18 | FileBlockInfos []FileBlockInfo 19 | // FileID is the file id. 20 | FileID int64 21 | // Folder determines whether the file is a folder. 22 | Folder bool 23 | // Group is the group. 24 | Group string 25 | // InMemoryPercentage represents the in-memory percentage. 26 | InMemoryPercentage int32 27 | // LastModificationTimeMs is the last modification time (in milliseconds). 28 | LastModificationTimeMs int64 29 | // Length is the file length. 30 | Length int64 31 | // Name is the file name. 32 | Name string 33 | // Path is the file path. 34 | Path string 35 | // Persisted determines whether file is persisted. 36 | Persisted bool 37 | // PersistenceState represents the persistence state. 38 | PersistenceState string 39 | // Pinned determines whether the file is pinned. 40 | Pinned bool 41 | // Mode is the access mode. 42 | Mode int32 43 | // MountPoint determines whether the file is a mount point. 44 | MountPoint bool 45 | // Owner is the owner. 46 | Owner string 47 | // TTL is the time-to-live window. 48 | TTL int64 49 | // TTLAction si the time-to-live action. 50 | TTLAction string 51 | // UfsPath is the UFS path. 52 | UfsPath string 53 | } 54 | 55 | // FileInfos represents a list of file information. 56 | type FileInfos []FileInfo 57 | 58 | var _ sort.Interface = (*FileInfos)(nil) 59 | 60 | func (fis FileInfos) Len() int { 61 | return len(fis) 62 | } 63 | 64 | func (fis FileInfos) Swap(i, j int) { 65 | fis[i], fis[j] = fis[j], fis[i] 66 | } 67 | 68 | func (fis FileInfos) Less(i, j int) bool { 69 | return fis[i].Name < fis[j].Name 70 | } 71 | -------------------------------------------------------------------------------- /wire/load_metadata_type.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // LoadMetadataType represents the load metadata type. 4 | type LoadMetadataType string 5 | 6 | const ( 7 | // LoadMetadataTypeNever means metadata should never be loaded. 8 | LoadMetadataTypeNever LoadMetadataType = "Never" 9 | // LoadMetadataTypeOnce means metadata should be loaded once. 10 | LoadMetadataTypeOnce = "Once" 11 | // LoadMetadataTypeAlways means metadata should always be loaded. 12 | LoadMetadataTypeAlways = "Always" 13 | ) 14 | -------------------------------------------------------------------------------- /wire/mode.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // Mode represents the file's access mode. 4 | type Mode struct { 5 | // OwnerBits represents the owner access mode 6 | OwnerBits Bits `json:"ownerBits"` 7 | // GroupBits represents the group access mode 8 | GroupBits Bits `json:"groupBits"` 9 | // OtherBits represents the other access mode 10 | OtherBits Bits `json:"otherBits"` 11 | } 12 | -------------------------------------------------------------------------------- /wire/read_type.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // ReadType represents a read type. 4 | type ReadType string 5 | 6 | const ( 7 | // ReadTypeNoCache means data will be not cached. 8 | ReadTypeNoCache ReadType = "NO_CACHE" 9 | // ReadTypeCache means data will be cached. 10 | ReadTypeCache = "CACHE" 11 | // ReadTypeCachePromote mans data will be cached in the top tier. 12 | ReadTypeCachePromote = "CACHE_PROMOTE" 13 | ) 14 | -------------------------------------------------------------------------------- /wire/ttl_action.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // TTLAction represents a TTL action. 4 | type TTLAction string 5 | 6 | const ( 7 | // TTLActionDelete represents the action of deleting a path. 8 | TTLActionDelete TTLAction = "DELETE" 9 | // TTLActionFree represents the action of freeing a path. 10 | TTLActionFree = "FREE" 11 | ) 12 | -------------------------------------------------------------------------------- /wire/worker_net_address.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // WorkerNetAddress represents a worker's net address. 4 | type WorkerNetAddress struct { 5 | // Host is the hostname. 6 | Host string 7 | // RPCPort is the RPC port. 8 | RPCPort int 9 | // DataPort is the data port. 10 | DataPort int 11 | // WebPort is the web port. 12 | WebPort int 13 | } 14 | -------------------------------------------------------------------------------- /wire/write_type.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | // WriteType represents a write type. 4 | type WriteType string 5 | 6 | const ( 7 | // WriteTypeMustCache means the data will be stored in Alluxio. 8 | WriteTypeMustCache WriteType = "MUST_CACHE" 9 | // WriteTypeCacheThrough means the data will be stored in Alluxio and 10 | // synchronously written to UFS. 11 | WriteTypeCacheThrough = "CACHE_THROUGH" 12 | // WriteTypeThrough means the data will be sychrounously written to UFS. 13 | WriteTypeThrough = "THROUGH" 14 | // WriteTypeAsyncThrough means the data will be stored in Alluxio and 15 | // asynchrounously written to UFS. 16 | WriteTypeAsyncThrough = "ASYNC_THROUGH" 17 | // WriteTypeNone means the data will no be stored. 18 | WriteTypeNone = "NONE" 19 | ) 20 | -------------------------------------------------------------------------------- /wiretest/bits.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomBits generates a random instace of wire.Bits. 10 | func RandomBits() wire.Bits { 11 | var result wire.Bits 12 | switch rand.Intn(8) { 13 | case 0: 14 | result = wire.BitsNone 15 | case 1: 16 | result = wire.BitsExecute 17 | case 2: 18 | result = wire.BitsWrite 19 | case 3: 20 | result = wire.BitsWriteExecute 21 | case 4: 22 | result = wire.BitsRead 23 | case 5: 24 | result = wire.BitsReadExecute 25 | case 6: 26 | result = wire.BitsReadWrite 27 | case 7: 28 | result = wire.BitsAll 29 | } 30 | return result 31 | } 32 | -------------------------------------------------------------------------------- /wiretest/block_info.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomBlockInfo generates a random instance of wire.BlockInfo. 10 | func RandomBlockInfo() wire.BlockInfo { 11 | locations := make([]wire.BlockLocation, rand.Intn(10)) 12 | for i := 0; i < len(locations); i++ { 13 | locations[i] = RandomBlockLocation() 14 | } 15 | return wire.BlockInfo{ 16 | BlockID: rand.Int63(), 17 | Length: rand.Int63(), 18 | Locations: locations, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /wiretest/block_location.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomBlockLocation generates a random instance of wire.BlockLocation. 10 | func RandomBlockLocation() wire.BlockLocation { 11 | return wire.BlockLocation{ 12 | WorkerID: rand.Int63(), 13 | WorkerAddress: RandomWorkerNetAddress(), 14 | TierAlias: RandomString(), 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /wiretest/doc.go: -------------------------------------------------------------------------------- 1 | // Package wiretest contains utilites for testing Alluxio file system wire types. 2 | package wiretest 3 | -------------------------------------------------------------------------------- /wiretest/file_block_info.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomFileBlockInfo generates a random instance of wire.FileBlockInfo. 10 | func RandomFileBlockInfo() wire.FileBlockInfo { 11 | ufsLocations := make([]string, rand.Intn(10)) 12 | for i := 0; i < len(ufsLocations); i++ { 13 | ufsLocations[i] = RandomString() 14 | } 15 | return wire.FileBlockInfo{ 16 | BlockInfo: RandomBlockInfo(), 17 | Offset: rand.Int63(), 18 | UfsLocations: ufsLocations, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /wiretest/file_info.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomFileInfo generates a random instance of wire.FileInfo. 10 | func RandomFileInfo() wire.FileInfo { 11 | blockIDs := make([]int64, rand.Intn(10)) 12 | for i := 0; i < len(blockIDs); i++ { 13 | blockIDs[i] = rand.Int63() 14 | } 15 | fileBlockInfos := make([]wire.FileBlockInfo, rand.Intn(10)) 16 | for i := 0; i < len(fileBlockInfos); i++ { 17 | fileBlockInfos[i] = RandomFileBlockInfo() 18 | } 19 | return wire.FileInfo{ 20 | BlockIds: blockIDs, 21 | BlockSizeBytes: rand.Int63(), 22 | Cacheable: RandomBool(), 23 | Completed: RandomBool(), 24 | CreationTimeMs: rand.Int63(), 25 | FileBlockInfos: fileBlockInfos, 26 | FileID: rand.Int63(), 27 | Folder: RandomBool(), 28 | Group: RandomString(), 29 | InMemoryPercentage: rand.Int31(), 30 | LastModificationTimeMs: rand.Int63(), 31 | Length: rand.Int63(), 32 | Mode: rand.Int31(), 33 | MountPoint: RandomBool(), 34 | Name: RandomString(), 35 | Owner: RandomString(), 36 | Path: RandomString(), 37 | Persisted: RandomBool(), 38 | PersistenceState: RandomString(), 39 | Pinned: RandomBool(), 40 | TTL: rand.Int63(), 41 | TTLAction: RandomString(), 42 | UfsPath: RandomString(), 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /wiretest/load_metadata_type.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomLoadMetadataType generates a random instance of wire.LoadMetadataType. 10 | func RandomLoadMetadataType() wire.LoadMetadataType { 11 | var result wire.LoadMetadataType 12 | switch rand.Intn(3) { 13 | case 0: 14 | result = wire.LoadMetadataTypeNever 15 | case 1: 16 | result = wire.LoadMetadataTypeOnce 17 | case 2: 18 | result = wire.LoadMetadataTypeAlways 19 | } 20 | return result 21 | } 22 | -------------------------------------------------------------------------------- /wiretest/mode.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import "github.com/Alluxio/alluxio-go/wire" 4 | 5 | // RandomMode generates a random instance of wire.Mode. 6 | func RandomMode() wire.Mode { 7 | return wire.Mode{ 8 | OwnerBits: RandomBits(), 9 | GroupBits: RandomBits(), 10 | OtherBits: RandomBits(), 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /wiretest/read_type.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomReadType generates a random instance of wire.ReadType. 10 | func RandomReadType() wire.ReadType { 11 | var result wire.ReadType 12 | switch rand.Intn(3) { 13 | case 0: 14 | result = wire.ReadTypeNoCache 15 | case 1: 16 | result = wire.ReadTypeCache 17 | case 2: 18 | result = wire.ReadTypeCachePromote 19 | } 20 | return result 21 | } 22 | -------------------------------------------------------------------------------- /wiretest/ttl_action.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomTTLAction generates a random instance of wire.TTLAction. 10 | func RandomTTLAction() wire.TTLAction { 11 | var result wire.TTLAction 12 | switch rand.Intn(2) { 13 | case 0: 14 | result = wire.TTLActionDelete 15 | case 1: 16 | result = wire.TTLActionFree 17 | } 18 | return result 19 | } 20 | -------------------------------------------------------------------------------- /wiretest/util.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | ) 6 | 7 | // RandomBool generates a random bool. 8 | func RandomBool() bool { 9 | return rand.Int()%2 == 0 10 | } 11 | 12 | // RandomBytes generates a random byte slice. 13 | func RandomBytes() []byte { 14 | result := make([]byte, rand.Intn(128)) 15 | for i := 0; i < len(result); i++ { 16 | result[i] = byte(rand.Int() & 0xFF) 17 | } 18 | return result 19 | } 20 | 21 | // RandomString generates a random string. 22 | func RandomString() string { 23 | result, length := "", rand.Intn(128) 24 | for i := 0; i < length; i++ { 25 | result += string(rand.Intn(96) + 33) 26 | } 27 | return result 28 | } 29 | -------------------------------------------------------------------------------- /wiretest/worker_net_address.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomWorkerNetAddress generates a random instance of wire.WorkerNetAddress. 10 | func RandomWorkerNetAddress() wire.WorkerNetAddress { 11 | return wire.WorkerNetAddress{ 12 | Host: RandomString(), 13 | RPCPort: rand.Int(), 14 | DataPort: rand.Int(), 15 | WebPort: rand.Int(), 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /wiretest/write_type.go: -------------------------------------------------------------------------------- 1 | package wiretest 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/Alluxio/alluxio-go/wire" 7 | ) 8 | 9 | // RandomWriteType generates a random instance of wire.WriteType. 10 | func RandomWriteType() wire.WriteType { 11 | var result wire.WriteType 12 | switch rand.Intn(5) { 13 | case 0: 14 | result = wire.WriteTypeMustCache 15 | case 1: 16 | result = wire.WriteTypeCacheThrough 17 | case 2: 18 | result = wire.WriteTypeThrough 19 | case 3: 20 | result = wire.WriteTypeAsyncThrough 21 | case 4: 22 | result = wire.WriteTypeNone 23 | } 24 | return result 25 | } 26 | --------------------------------------------------------------------------------