├── .gitignore ├── .gitmodules ├── .travis.yml ├── CHANGES.md ├── LICENSE └── Mozilla_Public_License_2.0.txt ├── MANIFEST.in ├── README.md ├── build_simulator.sh ├── compile_proto.sh ├── deploy.sh ├── kinetic ├── __init__.py ├── admin.py ├── baseasync.py ├── baseclient.py ├── batch.py ├── benchmarks │ └── KeyGen.py ├── cmd │ └── __init__.py ├── common.py ├── deprecated │ ├── __init__.py │ ├── adminclient.py │ └── blockingclient.py ├── greenclient.py ├── kinetic_pb2.py ├── operations.py ├── secureclient.py ├── threadedclient.py ├── utils.py └── zero_copy.py ├── package.sh ├── requirements.txt ├── setup.cfg ├── setup.py └── test ├── __init__.py ├── __main__.py ├── base.py ├── test_adminclient.py ├── test_batch.py ├── test_client.py ├── test_cmd.py └── test_p2p.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_STORE 2 | 3 | # Python 4 | build/ 5 | dist/ 6 | *egg-info/ 7 | *.pyc 8 | 9 | # IDEs 10 | .idea/ 11 | 12 | # scripting 13 | tmp/ 14 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "kinetic-protocol"] 2 | path = kinetic-protocol 3 | url = https://github.com/Seagate/kinetic-protocol.git 4 | branch = 3.0.6 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "2.7" 5 | 6 | install: 7 | - "pip install ." 8 | - "pip install -r requirements.txt" 9 | 10 | env: 11 | KINETIC_JAR=$TRAVIS_BUILD_DIR/simulator.jar 12 | KINETIC_CONNECT_TIMEOUT=1.0 13 | 14 | before_script: ./build_simulator.sh 15 | 16 | script: nosetests 17 | sudo: false -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | Changes since 0.8.2 2 | =========================== 3 | >**NOTE:** this section will document changes to the library since the last release 4 | 5 | ## Important 6 | - Kinetic Protocol version updated to [3.0.6](https://github.com/Seagate/kinetic-protocol/tree/3.0.6) 7 | - Deprecrated classes and methods will be removed on the next major release. 8 | 9 | ## New features 10 | - Added support for Batch operations 11 | - Added `reverse` parameter on GetKeyRange operation 12 | 13 | ## Major changes 14 | - `AsyncClient` has been renamed to `Client` 15 | - A new `SecureClient` has been added to `kinetic`. 16 | 17 | ## Minor changes 18 | - Added env variable _KINETIC_CONNECT_TIMEOUT_ to control default connection timeout. 19 | 20 | ## Deprecated features 21 | - Old blocking `Client` has been moved to `kinetic.depracated.BlockingClient` 22 | - Old `AdminClient` has been moved to `kinetic.depracated.AdminClient` 23 | 24 | ## Misc 25 | - Added alias for `AsyncClient = Client` to smooth transition. 26 | - Added alias on `kinetic` for `AdminClient` to smooth transition. 27 | 28 | Changes from 0.8.1 to 0.8.2 29 | =========================== 30 | 31 | ## Bug fixes 32 | - Keys with empty values show correctly as '' instead of None 33 | - **WRITEBACK** set as default mode for `put` and `delete` operations 34 | 35 | ## Misc 36 | - OSX requirements for SSL connections updated. 37 | 38 | Changes from 0.8.0 to 0.8.1 39 | =========================== 40 | This section will document changes to the library since the last release 41 | 42 | ## Important 43 | - Kinetic Protocol version updated to [3.0.5](https://github.com/Seagate/kinetic-protocol/tree/3.0.5) 44 | 45 | ## New features 46 | - Added timeout, time_quanta, priority and early_exit support 47 | - Pin size added to limits 48 | - SHA1 calculation used as default on puts when algorithm and tag not specified. 49 | 50 | ## Minor changes 51 | - The client will use WRITEBACK as default when synchronization mode not specified on PUT and DELETE. 52 | 53 | Changes from 0.7.3 to 0.8.0 54 | =========================== 55 | 56 | ## Important 57 | - Kinetic Protocol version updated to [3.0.0](https://github.com/Seagate/kinetic-protocol/tree/3.0.0) 58 | - Everything requires proto 3.0.0 or higher on the device 59 | - Devices pre 3.0.0 do not have handshakes and will raise Handshake timeout 60 | 61 | ## New features 62 | - Added `--version` to cmd line tool 63 | - Added background operations Scan and Optimize 64 | - Added pin operations (Lock, unlock, erase, secureErase) 65 | - Client auto configures cluster_version based on initial handshake 66 | - ErasePin and LockPin can be set during the security operation 67 | - Client fields config and limits show device information 68 | - Added unsolicited status support 69 | - Pin based operations MUST have a pin set and SSL enabled 70 | - setSecurity MUST have SSL enabled 71 | - getLog / getLogAsync added to Client/AsyncClient 72 | 73 | ## Behavior changes 74 | - Removed GreenClient (Feature overlap with AsyncClient) 75 | - Removed PipelinedClient (Only used internally by the kineticc) 76 | - setPin removed from AdminClient 77 | 78 | Changes from 0.7.2 to 0.7.3 79 | =========================== 80 | 81 | ## Important 82 | - Kinetic Protocol version updated to [2.0.6](https://github.com/Seagate/kinetic-protocol/tree/2.0.6) 83 | 84 | ## New features 85 | - Added handshake to negotiate connection id during connect 86 | - Added TCP_NODELAY socket option to improve performance 87 | - GetKeyRange operations now accept None arguments on keys (Issue #10) 88 | 89 | ## Behavior changes 90 | - Removed DEVICE log from LogTypes.add() (Issue #15) 91 | 92 | ## Bug Fixes 93 | - Fixed problem with status validation on the AdminClient 94 | - Fixed error message when Magic number is invalid 95 | 96 | Changes from 0.7.1 to 0.7.2 97 | =========================== 98 | 99 | ## Important 100 | - Kinetic Protocol version updated to [2.0.5](https://github.com/Seagate/kinetic-protocol/tree/2.0.5) 101 | - The compiled python proto kinetic/kinetic_pb2.py is now included on the repo 102 | 103 | ## New features 104 | - Added zero copy support on puts and gets (Requires splice system call) 105 | - Added IPv6 address support on all clients (Issue #8) 106 | - Added new exception type ClusterVersionFailureException (Requires drive version 2.0.4) 107 | - Added new Device specific GetLog (Requires protocol 2.0.5) 108 | - Added SSL/TLS support on all clients 109 | 110 | ## Bug Fixes 111 | - Fixed bug on invalid magix number (PR #11 contributed by @rpcope1, ASOKVAD-313) 112 | - Fixex bug that caused close() and connect() to fail on a connection that faulted (Issue #7) 113 | - Fixed a bug that caused the AsyncClient to crash when calling close() 114 | 115 | Changes from 0.7.0 to 0.7.1 116 | =========================== 117 | 118 | ## New features 119 | - Added setSecurity on AdminClient 120 | - Added getVersion and getVersionAsync to the library. 121 | 122 | ## Bug Fixes 123 | - Fixed tests not running and testcases with hardcoded 'localhost' 124 | - Fixed flush operation build parameters (Merge #5, contributed by @rpcope1). 125 | - AsyncClient returns NotConnected exception when an operation is attempted on a client before calling connect(). 126 | - Lowered default number of keys asked on ranges to 200 (ASKOVAD-287). 127 | - Fixed typo on baset test case (Merge #2, contributed by @zaitcev). 128 | 129 | Changes from 0.6.0.2 to 0.7.0 130 | ============================= 131 | 132 | ## Important 133 | Kinetic Protocol version updated to 2.0.3 134 | 135 | ## New features 136 | - Added Flush command (Requires protocol 2.0.3). 137 | - Added Limits section on GetLog (Requires protocol 2.0.3). 138 | - Added protocol_version field on kinetic module. 139 | - Added version field on kinetic module. 140 | 141 | ## Breaking changes 142 | - Renamed Synchronization.ASYNC to Synchronization.WRITEBACK 143 | - Renamed Synchronization.SYNC to Synchronization.WRITETHROUGH 144 | 145 | ## Bug Fixes 146 | - Fixed issue with asynchronous clients leaving the socket open after _close_ was called (ASOKVAD-263). 147 | -------------------------------------------------------------------------------- /LICENSE/Mozilla_Public_License_2.0.txt: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # **Kinetic-py** 2 | [![Travis](https://img.shields.io/travis/Kinetic/kinetic-py.svg)](https://travis-ci.org/Kinetic/kinetic-py) 3 | [![PyPI](https://img.shields.io/pypi/v/kinetic.svg)](https://pypi.python.org/pypi/kinetic/) 4 | [![PyPI](https://img.shields.io/pypi/l/kinetic.svg)](https://github.com/Seagate/kinetic-py/blob/master/LICENSE/LGPL2.1.md) 5 | 6 | Introduction 7 | ============ 8 | The [kinetic-protocol](https://github.com/Seagate/kinetic-protocol) python client. 9 | 10 | ## Requirements 11 | - Requires Python 2.7.3 or higher. 12 | - Requires Python 2.7.9 on OSX to use SSL 13 | 14 | > **NOTE:** Python 3.x is not supported. 15 | 16 | Installing latest stable release 17 | ================================ 18 | pip install kinetic 19 | 20 | 21 | Installing from Source 22 | ====================== 23 | 24 | git clone https://github.com/Seagate/kinetic-py.git 25 | cd kinetic-py 26 | python setup.py develop 27 | 28 | > **NOTE:** for devices with old firmware code get version 0.7.3 of the libray 29 | git checkout 0.7.3 30 | 31 | Running Tests 32 | ============= 33 | The tests need a Kinetic device to run. You can use the simulator available at https://github.com/Seagate/kinetic-java. 34 | To configure the test environment: 35 | 36 | export KINETIC_HOST=192.168.0.20 37 | export KINETIC_PORT=8123 38 | 39 | Optionally you can point the tests to the simulator jar: 40 | 41 | export KINETIC_JAR=kinetic-simulator--jar-with-dependencies.jar 42 | 43 | Then to run the tests: 44 | 45 | python test/ 46 | 47 | Getting Started with the basic client 48 | ===================================== 49 | 50 | ```python 51 | from kinetic import Client 52 | c = Client('localhost', 8123) 53 | c.connect() 54 | c.put('message','hello world') 55 | print c.get('message').value 56 | ``` 57 | Should print out _hello world_ 58 | 59 | Troubleshooting during the installation 60 | ======================================= 61 | On a brand new system, you might be missing a few things. 62 | If you get an error saying setup tools not installed or missing. 63 | Check the python [setuptools intallation guide](https://pypi.python.org/pypi/setuptools#installation-instructions). 64 | If you needed to installed that, chances are you are missing some requirements to install and compile eventlet on your system. 65 | On debian systems the quickest way is `sudo apt-get install python-eventlet`. 66 | 67 | 68 | License 69 | ------- 70 | 71 | This project is licensed under Mozilla Public License, v. 2.0 72 | * [Original](LICENSE/Mozilla_Public_License_2.0.txt) version 73 | -------------------------------------------------------------------------------- /build_simulator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | git clone https://github.com/Seagate/kinetic-java.git 3 | cd kinetic-java 4 | mvn clean package -DskipTests -Dmaven.javadoc.skip=true 5 | jar=$(echo $(pwd)/kinetic-simulator/target/*-jar-with-dependencies.jar) 6 | mv $jar $(pwd)/../simulator.jar -------------------------------------------------------------------------------- /compile_proto.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | BASE_DIR=$(cd "$(dirname "$0")"; pwd) 4 | PROTO_DIR=$BASE_DIR/kinetic-protocol 5 | 6 | protoc -I $PROTO_DIR --python_out=$BASE_DIR/kinetic $PROTO_DIR/kinetic.proto 7 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | python setup.py sdist upload 3 | -------------------------------------------------------------------------------- /kinetic/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | # Logging 20 | import logging 21 | logging.basicConfig() 22 | 23 | # Protocol version 24 | from common import local 25 | 26 | protocol_version = local.protocolVersion 27 | 28 | from pkg_resources import get_distribution, DistributionNotFound 29 | import os.path 30 | 31 | try: 32 | _dist = get_distribution('kinetic') 33 | if not __file__.startswith(os.path.join(_dist.location, 'kinetic')): 34 | # not installed, but there is another version that *is* 35 | raise DistributionNotFound 36 | except DistributionNotFound: 37 | __version__ = 'Please install this project with setup.py' 38 | else: 39 | __version__ = _dist.version 40 | 41 | #utils 42 | from utils import buildRange 43 | 44 | # clients 45 | from greenclient import Client 46 | from secureclient import SecureClient 47 | from threadedclient import ThreadedClient 48 | 49 | # common 50 | from common import KeyRange 51 | from common import Entry 52 | from common import Peer 53 | 54 | # exceptions 55 | from common import KineticMessageException 56 | 57 | # backward compatibility alliases 58 | AsyncClient = Client 59 | from kinetic.deprecated.adminclient import AdminClient 60 | from kinetic import greenclient as client 61 | # Fake old asyncclient module 62 | class AsyncClientCompat(object): 63 | AsyncClient = Client 64 | asyncclient = AsyncClientCompat() -------------------------------------------------------------------------------- /kinetic/admin.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | ### This module is deprecated 20 | 21 | from kinetic.deprecated import AdminClient 22 | -------------------------------------------------------------------------------- /kinetic/baseasync.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import deprecated 20 | from common import Entry 21 | import common 22 | 23 | import logging 24 | import kinetic_pb2 as messages 25 | import operations 26 | import threading 27 | 28 | LOG = logging.getLogger(__name__) 29 | 30 | class BaseAsync(deprecated.BlockingClient): 31 | 32 | def __init__(self, *args, **kwargs): 33 | super(BaseAsync, self).__init__(*args, socket_timeout=None, **kwargs) 34 | self.unhandledException = lambda e: LOG.warn("Unhandled client exception. " + str(e)) 35 | self.faulted = False 36 | self.error = None 37 | # private attributes 38 | self._pending = dict() 39 | # start background workers 40 | self._initialize() 41 | 42 | 43 | def _initialize(self): pass 44 | 45 | 46 | def _raise(self, e, onError=None): 47 | if onError: 48 | try: 49 | onError(e) 50 | except Exception as ue: 51 | e = ue 52 | onError = None 53 | 54 | if onError == None: 55 | try: 56 | self.unhandledException(e) 57 | except Exception as e: 58 | LOG.warn("Unhandled exception when handling unhandled exception. " + str(e)) 59 | # just swallow it (the other option is faulting) 60 | 61 | 62 | def dispatch(self, fn, *args, **kwargs): 63 | fn(*args,**kwargs) 64 | 65 | 66 | def _fault_client(self, e): 67 | self.error = e 68 | self.faulted = True 69 | LOG.error("Connection {0} faulted. {1}".format(self,e)) 70 | for _,onError in self._pending.itervalues(): 71 | try: 72 | onError(e) 73 | except Exception as e2: 74 | LOG.error("Unhandled exception on callers code when reporting internal error. {0}".format(e2)) 75 | self._pending = {} 76 | 77 | 78 | def _async_recv(self): 79 | if self.faulted: 80 | raise common.ConnectionFaulted("Connection {0} is faulted. Can't receive message when connection is on a faulted state.".format(self)) 81 | 82 | try: 83 | m, resp, value = self.network_recv() 84 | if m.authType == messages.Message.UNSOLICITEDSTATUS: 85 | if self.on_unsolicited: 86 | try: 87 | self.dispatch(self.on_unsolicited,resp.status) 88 | except Exception as e: 89 | self._raise(e) 90 | else: 91 | LOG.warn('Unsolicited status %s received but nobody listening. %s' % (resp.status.code, resp.status.statusMessage)) 92 | else: 93 | seq = resp.header.ackSequence 94 | if LOG.isEnabledFor(logging.DEBUG): 95 | LOG.debug("Received message with ackSequence={0} on connection {1}.".format(seq,self)) 96 | onSuccess,_ = self._pending[seq] 97 | del self._pending[seq] 98 | try: 99 | self.dispatch(onSuccess, m, resp, value) 100 | except Exception as e: 101 | self._raise(e) 102 | except Exception as e: 103 | if not self.isConnected: 104 | raise common.ConnectionClosed("Connection closed by client.") 105 | else: 106 | self._fault_client(e) 107 | 108 | 109 | ### Override BaseClient methods 110 | 111 | def send(self, command, value): 112 | done = threading.Event() 113 | class Dummy : pass 114 | d = Dummy() 115 | d.error = None 116 | d.result = None 117 | 118 | def innerSuccess(m, resp, value): 119 | d.result = (m, resp, value) 120 | done.set() 121 | 122 | def innerError(e): 123 | d.error = e 124 | done.set() 125 | 126 | self.sendAsync(command, value, innerSuccess, innerError) 127 | 128 | done.wait() # TODO(Nacho): should be add a default timeout? 129 | if d.error: raise d.error 130 | return d.result 131 | 132 | ### 133 | 134 | 135 | def sendAsync(self, command, value, onSuccess, onError, no_ack=False): 136 | if self.faulted: # TODO(Nacho): should we fault through onError on fault or bow up on the callers face? 137 | self._raise(common.ConnectionFaulted("Can't send message when connection is on a faulted state."), onError) 138 | return #skip the rest 139 | 140 | # fail fast on NotConnected 141 | if not self.isConnected: # TODO(Nacho): should we fault through onError on fault or bow up on the callers face? 142 | self._raise(common.NotConnected("Not connected."), onError) 143 | return #skip the rest 144 | 145 | def innerSuccess(m, response, value): 146 | try: 147 | operations._check_status(response) 148 | onSuccess(m, response, value) 149 | except Exception as ex: 150 | onError(ex) 151 | 152 | # get sequence 153 | self.update_header(command) 154 | 155 | if not no_ack: 156 | # add callback to pending dictionary 157 | self._pending[command.header.sequence] = (innerSuccess, onError) 158 | 159 | # transmit 160 | self.network_send(command, value) 161 | 162 | 163 | def _process(self, op, *args, **kwargs): 164 | if not self.isConnected: raise common.NotConnected("Must call connect() before sending operations.") 165 | return super(BaseAsync, self)._process(op, *args, **kwargs) 166 | 167 | 168 | def _processAsync(self, op, onSuccess, onError, *args, **kwargs): 169 | if not self.isConnected: raise common.NotConnected("Must call connect() before sending operations.") 170 | 171 | def innerSuccess(m, header, value): 172 | onSuccess(op.parse(header, value)) 173 | 174 | def innerError(e): 175 | try: 176 | v = op.onError(e) 177 | onSuccess(v) 178 | except Exception as e2: 179 | onError(e2) 180 | 181 | if 'no_ack' in kwargs: 182 | send_no_ack = True 183 | del kwargs['no_ack'] 184 | else: 185 | send_no_ack = False 186 | 187 | header, value = op.build(*args, **kwargs) 188 | self.sendAsync(header, value, innerSuccess, innerError, send_no_ack) 189 | 190 | 191 | def putAsync(self, onSuccess, onError, *args, **kwargs): 192 | self._processAsync(operations.Put(), onSuccess, onError, *args, **kwargs) 193 | 194 | def getAsync(self, onSuccess, onError, *args, **kwargs): 195 | self._processAsync(operations.Get(), onSuccess, onError, *args, **kwargs) 196 | 197 | def getMetadataAsync(self, onSuccess, onError, *args, **kwargs): 198 | self._processAsync(operations.GetMetadata(), onSuccess, onError, *args, **kwargs) 199 | 200 | def deleteAsync(self, onSuccess, onError, *args, **kwargs): 201 | self._processAsync(operations.Delete(), onSuccess, onError, *args, **kwargs) 202 | 203 | def getNextAsync(self, onSuccess, onError, *args, **kwargs): 204 | self._processAsync(operations.GetNext(), onSuccess, onError, *args, **kwargs) 205 | 206 | def getPreviousAsync(self, onSuccess, onError, *args, **kwargs): 207 | self._processAsync(operations.GetPrevious(), onSuccess, onError, *args, **kwargs) 208 | 209 | def getKeyRangeAsync(self, onSuccess, onError, *args, **kwargs): 210 | self._processAsync(operations.GetKeyRange(), onSuccess, onError, *args, **kwargs) 211 | 212 | def getVersionAsync(self, onSuccess, onError, *args, **kwargs): 213 | self._processAsync(operations.GetVersion(), onSuccess, onError, *args, **kwargs) 214 | 215 | def flushAsync(self, onSuccess, onError, *args, **kwargs): 216 | self._processAsync(operations.Flush(), onSuccess, onError, *args, **kwargs) 217 | 218 | def noopAsync(self, onSuccess, onError, *args, **kwargs): 219 | self._processAsync(operations.Noop(), onSuccess, onError, *args, **kwargs) 220 | 221 | def mediaScanAsync(self, onSuccess, onError, *args, **kwargs): 222 | self._processAsync(operations.MediaScan(), onSuccess, onError, *args, **kwargs) 223 | 224 | def mediaOptimizeAsync(self, onSuccess, onError, *args, **kwargs): 225 | self._processAsync(operations.MediaOptimize(), onSuccess, onError, *args, **kwargs) 226 | 227 | def getLogAsync(self, onSuccess, onError, *args, **kwargs): 228 | self._processAsync(operations.GetLog(), onSuccess, onError, *args, **kwargs) 229 | 230 | -------------------------------------------------------------------------------- /kinetic/baseclient.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import itertools 20 | import logging 21 | import socket 22 | import time 23 | from binascii import hexlify 24 | from hashlib import sha1 25 | import hmac 26 | import struct 27 | import common 28 | import kinetic_pb2 as messages 29 | import ssl 30 | 31 | ss = socket 32 | 33 | LOG = logging.getLogger(__name__) 34 | 35 | def calculate_hmac(secret, command): 36 | mac = hmac.new(secret, digestmod=sha1) 37 | 38 | def update(entity): 39 | if not entity: 40 | return 41 | if hasattr(entity, 'SerializeToString'): 42 | entity = entity.SerializeToString() 43 | #converting to big endian to be compatible with java implementation. 44 | mac.update(struct.pack(">I", len(entity))) 45 | mac.update(entity) 46 | 47 | update(command) 48 | 49 | d = mac.digest() 50 | if LOG.isEnabledFor(logging.DEBUG): 51 | LOG.debug('command hmac: %s' % hexlify(d)) 52 | return d 53 | 54 | class BaseClient(object): 55 | 56 | # defaults 57 | HOSTNAME = 'localhost' 58 | PORT = 8123 59 | # drive default 60 | USER_ID = 1 61 | CLIENT_SECRET = 'asdfasdf' 62 | 63 | def __init__(self, hostname=HOSTNAME, port=PORT, identity=USER_ID, 64 | cluster_version=None, secret=CLIENT_SECRET, 65 | chunk_size=common.DEFAULT_CHUNK_SIZE, 66 | connect_timeout=common.KINETIC_CONNECT_TIMEOUT, 67 | socket_timeout=common.DEFAULT_SOCKET_TIMEOUT, 68 | socket_address=None, socket_port=0, 69 | defer_read=False, 70 | use_ssl=False, pin=None): 71 | self.hostname = hostname 72 | self.port = port 73 | self.identity = identity 74 | self.cluster_version = cluster_version 75 | self.secret = secret 76 | self.chunk_size = chunk_size 77 | self._socket = None 78 | self._buff = '' 79 | self.debug = False 80 | self._closed = True 81 | self.connect_timeout = connect_timeout 82 | self.socket_timeout = socket_timeout 83 | self.socket_address = socket_address 84 | self.socket_port = socket_port 85 | self.defer_read = defer_read 86 | self.wait_on_read = None 87 | self.use_ssl = use_ssl 88 | self.pin = pin 89 | self.on_unsolicited = None 90 | 91 | @property 92 | def socket(self): 93 | if not self._socket: 94 | raise NotConnected() 95 | return self._socket 96 | 97 | @property 98 | def isConnected(self): 99 | return not self._closed 100 | 101 | def build_socket(self, family=ss.AF_INET): 102 | return socket.socket(family) 103 | 104 | def wrap_secure_socket(self, s, ssl_version): 105 | return ssl.wrap_socket(s) #, ssl_version=ssl_version) 106 | 107 | def connect(self): 108 | if self._socket: 109 | raise common.AlreadyConnected("Client is already connected.") 110 | 111 | infos = socket.getaddrinfo(self.hostname, self.port, 0, 0, socket.SOL_TCP) 112 | (family,_,_,_, sockaddr) = infos[0] 113 | # Stage socket on a local variable first 114 | s = self.build_socket(family) 115 | if self.use_ssl: 116 | s = self.wrap_secure_socket(s, ssl.PROTOCOL_TLSv1_2) 117 | 118 | s.settimeout(self.connect_timeout) 119 | if self.socket_address: 120 | LOG.debug("Client local port address bound to " + self.socket_address) 121 | s.bind((self.socket_address, self.socket_port)) 122 | # if connect fails, there is nothing to clean up 123 | s.connect(sockaddr) # use first 124 | s.setsockopt(ss.IPPROTO_TCP, ss.TCP_NODELAY, 1) 125 | 126 | # We are connected now, update attributes 127 | self._socket = s 128 | try: 129 | self._handshake() 130 | self._socket.settimeout(self.socket_timeout) 131 | 132 | self._sequence = itertools.count() 133 | self._batch_id = itertools.count() 134 | self._closed = False 135 | except: 136 | self._socket = None 137 | raise 138 | 139 | def _handshake(self): 140 | # Connection id handshake 141 | try: 142 | _,cmd,v = self.network_recv() # unsolicited status 143 | except socket.timeout: 144 | raise common.KineticClientException("Handshake timeout") 145 | 146 | # device locked only allowed to continue over SSL 147 | if (cmd.status.code == messages.Command.Status.DEVICE_LOCKED): 148 | if not self.use_ssl: 149 | raise KineticMessageException(cmd.status) 150 | elif (cmd.status.code != messages.Command.Status.SUCCESS): 151 | raise KineticMessageException(cmd.status) 152 | 153 | self.connection_id = cmd.header.connectionID 154 | 155 | self.config = cmd.body.getLog.configuration 156 | self.limits = cmd.body.getLog.limits 157 | 158 | if self.cluster_version: 159 | if self.cluster_version != cmd.header.clusterVersion: 160 | cmd.status.code = messages.Command.Status.VERSION_FAILURE 161 | cmd.status.statusMessage = \ 162 | 'Cluster version missmatch detected during handshake' 163 | raise common.ClusterVersionFailureException( 164 | cmd.status, cmd.header.clusterVersion) 165 | else: 166 | self.cluster_version = cmd.header.clusterVersion 167 | 168 | def has_data_available(self): 169 | tmp = self._socket.recv(1, socket.MSG_PEEK) 170 | return len(tmp) > 0 171 | 172 | def close(self): 173 | self._closed = True 174 | if self._socket: 175 | try: 176 | self._socket.shutdown(socket.SHUT_RDWR) 177 | self._socket.close() 178 | except Exception as e: 179 | LOG.warning('Socket faulted on shutdown/close for {}.'.format(e)) 180 | self._buff = '' 181 | self._socket = None 182 | self.connection_id = None 183 | self._sequence = itertools.count() 184 | self._batch_id = itertools.count() 185 | 186 | 187 | def update_header(self, command): 188 | """ 189 | Updates the message header with connection specific information. 190 | The unique sequence is assigned by this method. 191 | 192 | :param message: message to be modified (message is modified in place) 193 | """ 194 | header = command.header 195 | header.clusterVersion = self.cluster_version 196 | header.connectionID = self.connection_id 197 | header.sequence = self._sequence.next() 198 | if LOG.isEnabledFor(logging.DEBUG): 199 | LOG.debug("Header updated. Connection=%s, Sequence=%s" % (header.connectionID, header.sequence)) 200 | 201 | 202 | def _send_delimited_v2(self, header, value): 203 | # build message (without value) to write 204 | out = header.SerializeToString() 205 | 206 | value_ln = 0 207 | if value: 208 | value_ln = len(value) 209 | 210 | # 1. write magic number 211 | # 2. write protobuf message message size, 4 bytes 212 | # 3. write attached value size, 4 bytes 213 | # 4. write protobuf message byte[] 214 | 215 | buff = struct.pack(">Bii",ord('F'), len(out), value_ln) 216 | 217 | # Send it all in one packet 218 | aux = bytearray(buff) 219 | aux.extend(out) 220 | self.socket.send(aux) 221 | 222 | # 5. (optional) write attached value if any 223 | send_op = getattr(value, "send", None) 224 | if callable(send_op): # if value has custom logic for sending over network, delegate 225 | send_op(self.socket) 226 | else: 227 | # 5 (optional) write attached value if any 228 | if value_ln > 0: 229 | # write value 230 | to_send = len(value) 231 | i = 0 232 | while i < to_send: 233 | nbytes = self.socket.send(value[i:i + self.chunk_size]) 234 | if not nbytes: 235 | raise common.ServerDisconnect('Server send disconnect') 236 | i += nbytes 237 | 238 | def authenticate(self, command): 239 | m = messages.Message() 240 | m.commandBytes = command.SerializeToString() 241 | 242 | if self.pin != None: 243 | m.authType = messages.Message.PINAUTH 244 | m.pinAuth.pin = self.pin 245 | else: # Hmac 246 | m.authType = messages.Message.HMACAUTH 247 | m.hmacAuth.identity = self.identity 248 | m.hmacAuth.hmac = calculate_hmac(self.secret, command) 249 | 250 | return m 251 | 252 | def network_send(self, command, value): 253 | """ 254 | Sends a raw message. 255 | The HMAC is calculated and added to the message. 256 | Important: update_header must be called before sending the message. 257 | 258 | :param message: the message to be sent 259 | """ 260 | # fail fast on NotConnected 261 | self.socket 262 | 263 | m = self.authenticate(command) 264 | 265 | if self.debug: 266 | print m 267 | print command 268 | 269 | self._send_delimited_v2(m, value) 270 | 271 | return m 272 | 273 | def toHexString(self, array): 274 | return ''.join('%02x ' % ord(byte) for byte in array) 275 | 276 | def bytearray_to_hex(self, array): 277 | return ''.join('%02x ' % byte for byte in array) 278 | 279 | def fast_read(self, toread): 280 | buf = bytearray(toread) 281 | view = memoryview(buf) 282 | while toread: 283 | nbytes = self.socket.recv_into(view, toread) 284 | if nbytes == 0: 285 | raise common.ServerDisconnect("Connection closed by peer") 286 | view = view[nbytes:] 287 | toread -= nbytes 288 | return buf 289 | 290 | def _recv_delimited_v2(self): 291 | # receive the leading 9 bytes 292 | 293 | if self.wait_on_read: 294 | self.wait_on_read.wait() 295 | self.wait_on_read = None 296 | 297 | msg = self.fast_read(9) 298 | 299 | magic, proto_ln, value_ln = struct.unpack_from(">bii", buffer(msg)) 300 | 301 | if magic!= 70: 302 | LOG.warn("Magic number = {0}".format(magic)) 303 | raise common.KineticClientException("Invalid Magic Value!") # 70 = 'F' 304 | 305 | # read proto message 306 | raw_proto = self.fast_read(proto_ln) 307 | 308 | value = '' 309 | if value_ln > 0: 310 | if self.defer_read: 311 | # let user handle the read from socket 312 | value = common.DeferedValue(self.socket, value_ln) 313 | self.wait_on_read = value 314 | else: 315 | # normal code path, read value 316 | value = self.fast_read(value_ln) 317 | 318 | proto = messages.Message() 319 | proto.ParseFromString(str(raw_proto)) 320 | 321 | return (proto, value) 322 | 323 | def network_recv(self): 324 | """ 325 | Receives a raw Kinetic message from the network. 326 | 327 | return: the message received 328 | """ 329 | 330 | (m, value) = self._recv_delimited_v2() 331 | 332 | if self.debug: 333 | print m 334 | 335 | if m.authType == messages.Message.HMACAUTH: 336 | if m.hmacAuth.identity == self.identity: 337 | hmac = calculate_hmac(self.secret, m.commandBytes) 338 | if not hmac == m.hmacAuth.hmac: 339 | raise Exception('Hmac does not match') 340 | else: 341 | raise Exception('Wrong identity received!') 342 | 343 | resp = messages.Command() 344 | resp.ParseFromString(m.commandBytes) 345 | 346 | if self.debug: 347 | print resp 348 | 349 | # update connectionId to whatever the drive said. 350 | if resp.header.connectionID: 351 | self.connection_id = resp.header.connectionID 352 | 353 | return (m, resp, value) 354 | 355 | def send(self, header, value): 356 | self.network_send(header, value) 357 | done = False 358 | while not done: 359 | m,cmd,value = self.network_recv() 360 | if m.authType == messages.Message.UNSOLICITEDSTATUS: 361 | if self.on_unsolicited: 362 | self.on_unsolicited(cmd.status) # uncatched exceptions by the handler will be raised to the caller 363 | else: 364 | LOG.warn('Unsolicited status %s received but nobody listening.' % cmd.status.code) 365 | else: done = True 366 | return m,cmd,value 367 | 368 | def send_no_ack(self, header, value): 369 | self.network_send(header, value) 370 | 371 | def next_batch_id(self): 372 | return self._batch_id.next() 373 | 374 | ### with statement support ### 375 | 376 | def __enter__(self): 377 | if not self.isConnected: 378 | self._temporaryConnection = True 379 | self.connect() 380 | else: 381 | self._temporaryConnection = False 382 | return self 383 | 384 | def __exit__(self, t, v, tb): 385 | if self._temporaryConnection: 386 | self.close() 387 | self._temporaryConnection = None 388 | 389 | ### Object overrides ### 390 | 391 | def __str__(self): 392 | return "{hostname}:{port}".format(hostname=self.hostname, port=self.port) 393 | 394 | -------------------------------------------------------------------------------- /kinetic/batch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Paul Dardeau 18 | 19 | import common 20 | import logging 21 | import operations 22 | 23 | LOG = logging.getLogger(__name__) 24 | 25 | class Batch(object): 26 | """ 27 | The Batch class is used for grouping a set of put and/or delete operations 28 | so that all are committed as one unit or all of them are canceled 29 | (aborted). 30 | 31 | A Batch object is obtained by calling :func:`~baseclient.BaseClient.begin_batch`. 32 | Once all relevant put and delete calls are made, 'commit' should be called 33 | to apply all of the operations, or 'abort' to cancel (abort) them. 34 | 35 | A Batch object cannot be reused for subsequent batches. After the 'commit' 36 | or 'abort' has completed successfully, a new Batch object should be 37 | requested for the next batch operation. 38 | """ 39 | 40 | 41 | def __init__(self, client, batch_id): 42 | """ 43 | Initialize instance with Kinetic client and batch identifier. 44 | 45 | Args: 46 | client: the Kinetic client to use for batch operations. 47 | batch_id: the batch identifier to be used for client connection. 48 | """ 49 | self._client = client 50 | self._batch_id = batch_id 51 | self._op_count = 0 52 | self._batch_completed = False # to detect attempted reuse 53 | 54 | def put(self, *args, **kwargs): 55 | """ 56 | Put an entry within the batch operation. 57 | 58 | The command is not committed until :func:`~batch.Batch.commit` is 59 | called and returns successfully. If a version is specified, it must 60 | match the one stored in the persistent storage. Otherwise, a 61 | KineticException is raised. 62 | 63 | Args: 64 | 65 | Kwargs: 66 | 67 | Raises: 68 | KineticException: if any internal error occurs. 69 | """ 70 | if self._batch_completed: 71 | raise common.BatchCompletedException() 72 | 73 | self._op_count += 1 74 | kwargs['batch_id'] = self._batch_id 75 | kwargs['no_ack'] = True 76 | 77 | self._client.put(*args, **kwargs) 78 | 79 | def delete(self, *args, **kwargs): 80 | """ 81 | Delete the entry associated with the specified key. 82 | 83 | The command is not committed until :func:`~batch.Batch.commit` is 84 | called and returns successfully. If a version is specified, it must 85 | match the one stored in persistent storage. Otherwise, a 86 | KineticException is raised. 87 | 88 | Args: 89 | 90 | Kwargs: 91 | 92 | Raises: 93 | KineticException: if any internal error occurs. 94 | """ 95 | if self._batch_completed: 96 | raise common.BatchCompletedException() 97 | 98 | self._op_count += 1 99 | kwargs['batch_id'] = self._batch_id 100 | kwargs['no_ack'] = True 101 | 102 | self._client.delete(*args, **kwargs) 103 | 104 | def commit(self, *args, **kwargs): 105 | """ 106 | Commit the current batch operation. 107 | 108 | When this call returned successfully, all the commands performed in the 109 | current batch are executed and committed to store successfully. 110 | 111 | Raises: 112 | KineticException: if any internal error occurred. The batch may 113 | or may not be committed. If committed, all commands are 114 | committed. Otherwise, no messages are committed. 115 | BatchAbortedException: if the commit failed. No messages within 116 | the batch were committed to the store. 117 | """ 118 | if self._batch_completed: 119 | raise common.BatchCompletedException() 120 | 121 | kwargs['batch_id'] = self._batch_id 122 | kwargs['batch_op_count'] = self._op_count 123 | try: 124 | self._client._process(operations.EndBatch(), *args, **kwargs) 125 | self._batch_completed = True 126 | except BatchAbortedException: 127 | self._batch_completed = True 128 | raise 129 | 130 | def abort(self, *args, **kwargs): 131 | """ 132 | Abort the current batch operation. 133 | 134 | When this call returned successfully, all the commands queued in the 135 | current batch are aborted. Resources related to the current batch are 136 | cleaned up and released. 137 | 138 | Raises: 139 | KineticException: if any internal error occurred. 140 | """ 141 | if self._batch_completed: 142 | raise common.BatchCompletedException() 143 | 144 | kwargs['batch_id'] = self._batch_id 145 | self._client._process(operations.AbortBatch(), *args, **kwargs) 146 | self._batch_completed = True 147 | 148 | def is_completed(self): 149 | """ 150 | Return boolean indicating whether the batch is completed (either 151 | committed or aborted) 152 | """ 153 | return self._batch_completed 154 | 155 | def __len__(self): 156 | """ 157 | Return the number of operations that have been included in the batch. 158 | """ 159 | return self._op_count 160 | 161 | -------------------------------------------------------------------------------- /kinetic/benchmarks/KeyGen.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Tim Feldman 18 | 19 | import random 20 | import string 21 | import sys 22 | 23 | # Generates a random key as a string of size 1 to maxKeySize 24 | # bytes with values of minByte to maxByte, inclusive, and 25 | # with equal probability for all possible keys. 26 | # Note that with a large number of possible keys, approximately 27 | # pow(1 + maxByte - minByte, maxKeySize), the limits of float precision 28 | # results in all keys being maxKeySize in length. This is the case, for 29 | # instance, with byte values of 0 to 255 and max key sizes of 128 or 30 | # larger. 31 | # Defaults are for common Kinetic keys. 32 | # Notes on the distribution of keys by size: 33 | # For a number of values per byte, v, v = 1 + maxByte - minByte, 34 | # and a number of keys, x, x = maxKeySize, 35 | # the number of keys by size, k, is 36 | # v^k / totalKeys 37 | # where totalKeys is v + v^2 + v^3 + ... + v^x. 38 | # A list of the probabilities of keys by key size is 39 | # size probability 40 | # 1 v / (v + v^2 + v^3 + ... + v^x) 41 | # 2 v^2 / (v + v^2 + v^3 + ... + v^x) 42 | # 3 v^3 / (v + v^2 + v^3 + ... + v^x) 43 | # ... ... 44 | # n v^n / (v + v^2 + v^3 + ... + v^x) 45 | # ... ... 46 | # x v^x / (v + v^2 + v^3 + ... + v^x) 47 | # These are very large, difficult to calculate numbers when v^x is large 48 | # compared to 1/epsilon. 49 | # An approximation for the likelihood of a key of size k is 50 | # (v / (v + 1))^(1 + x - k) 51 | # A list of these approximate probabilities of keys by key size is 52 | # size probability 53 | # 1 (v / (v + 1))^x 54 | # 2 (v / (v + 1))^(x - 1) 55 | # 3 (v / (v + 1))^(x - 3) 56 | # ... ... 57 | # n (v / (v + 1))^(1 + x - n) 58 | # ... 59 | # x v / (v + 1) 60 | # Note that in this approximation, many of the smaller probabilities will be 61 | # zero in this program and other limited-precision calculations. 62 | def GenerateRandomKey(minByte=0, maxByte=255, maxKeySize=32): 63 | assert(maxKeySize > 0) 64 | assert(0 <= minByte <= 255) 65 | assert(minByte <= maxByte <= 255) 66 | 67 | values = 1 + maxByte - minByte # local shorthand 68 | 69 | # First determine the size of the key to generate. 70 | # There is an accurate method that works only with smaller key ranges 71 | # and an approximate method that works for larger key ranges. 72 | try: 73 | totalKeys = sum(pow(values, k) for k in range(1, maxKeySize + 1)) 74 | # Compare a random pick to thresholds for the cumulative 75 | # likelihood of successively smaller key sizes. 76 | pick = random.random() 77 | numKeysOfKeySizeAndLarger = 0 78 | for keySize in range(maxKeySize, 1, -1): 79 | numKeysOfKeySizeAndLarger = numKeysOfKeySizeAndLarger + \ 80 | pow(values, keySize) 81 | portion = float(numKeysOfKeySizeAndLarger) / numKeys 82 | if pick <= portion: 83 | break 84 | except: 85 | while True: 86 | keySize = maxKeySize 87 | while keySize > 0: 88 | if random.randint(0, maxByte+1) == 0: # a 1 out of maxByte+1 chance 89 | keySize = keySize - 1 90 | else: 91 | break 92 | if keySize != 0: 93 | break 94 | # Else there were maxKeySize zeros in a row 95 | # and a valid key size was not found 96 | # so re-start the attempt to get a key size. 97 | 98 | # keySize is now the size of the key to generate 99 | # with a distribution of size proportional to the 100 | # representation in the set of all keys. 101 | 102 | key = "".join(chr(random.randint(minByte, maxByte)) \ 103 | for i in range(keySize)) 104 | 105 | return key 106 | 107 | 108 | # Generates the next key, a string with bytes of minByte to maxByte, 109 | # inclusive, in lexographical order after the specified key. 110 | # The key after a key of size maxKeySize with bytes of all maxByte is 111 | # a key with one byte whose value is minByte. 112 | # Defaults are for common Kinetic keys. 113 | def GenerateSequentialKey(key, minByte=0, maxByte=255, maxKeySize=32): 114 | assert(isinstance(key, str)) 115 | assert(len(key) <= maxKeySize) 116 | assert(0 <= minByte <= 255) 117 | assert(minByte <= maxByte <= 255) 118 | 119 | if len(key) == maxKeySize: 120 | while (len(key) > 0) and (ord(key[-1]) == maxByte): 121 | key = key[:-1] 122 | if len(key) == 0: 123 | key = chr(minByte) 124 | else: 125 | newChar = chr(ord(key[-1]) + 1) 126 | key = key[:-1] + newChar 127 | else: 128 | key = key + chr(minByte) 129 | 130 | return key 131 | 132 | 133 | # Generate a display string for a key. 134 | # Only keys with maximum values of 255 are supported; that is, a set 135 | # of bytes. The display string shows each byte as a pair of hexadecimal 136 | # numerals. Groups of four bytes or 8 hex numerals are delimited by a 137 | # space. There is a new line every 32 bytes. Each line is additionally 138 | # completed with the ASCII equivalent for non-whitespace printable 139 | # values and whitespace and non-printable values represented by a dot. 140 | def KeyToDisplayString (instr): 141 | bytesPerGroup = 4 142 | groupDelimiter = ' ' 143 | bytesPerLine = 32 144 | nonprintableGlyph = '.' 145 | outstr = "" 146 | asciistr = "" 147 | for i in range(len(instr)): 148 | c = instr[i] 149 | outstr = outstr + "".join("{0:02x}".format(ord(c))) 150 | if ((c in string.digits) or (c in string.letters) or 151 | (c in string.punctuation)): 152 | asciistr = asciistr + c 153 | else: 154 | asciistr = asciistr + nonprintableGlyph 155 | if (i % bytesPerLine == bytesPerLine - 1): 156 | outstr = outstr + " " + asciistr 157 | asciistr = "" 158 | if i != len(instr) - 1: 159 | outstr = outstr + '\n' 160 | elif (i % bytesPerGroup == bytesPerGroup - 1) and \ 161 | (i != len(instr) - 1): 162 | outstr = outstr + groupDelimiter 163 | asciistr = asciistr + groupDelimiter 164 | 165 | # Generate blank fill after hex and then append last asciistr. 166 | for j in range(bytesPerLine - (len(instr) % bytesPerLine)): 167 | outstr = outstr + " " 168 | if j % bytesPerGroup == bytesPerGroup - 1: 169 | outstr = outstr + " " 170 | outstr = outstr + " " + asciistr 171 | 172 | return outstr 173 | 174 | 175 | def main (): 176 | # The generate functions support strings of values from minByte to 177 | # maxByte, inclusive. 178 | # The intended use of low (maxByte-minByte) values is to get from 179 | # the random key generator a higher frequency of keys of less than 180 | # maxKeySize and keys for which the sequential key is shorter. 181 | # The intended use of the minByte to maxByte range as non-whitespace 182 | # printable values is to have complete ASCII strings. 183 | minByte = ord('a') # as low as 0 for Kinetic 184 | maxByte = ord('z') # up to 255 for Kinetic 185 | 186 | # Generate and print pairs of keys, the first of each pair is a 187 | # random key and the second is sequential to it. 188 | print ("Random keys and their sequential key") 189 | for i in range(50): 190 | randKey = GenerateRandomKey(minByte, maxByte) 191 | print "rand:\n", KeyToDisplayString(randKey) 192 | print "seql:\n", KeyToDisplayString(GenerateSequentialKey( 193 | randKey, minByte, maxByte)) 194 | print 195 | 196 | # Run the corner cases of the first key in the lexographical order 197 | # and the last key in the lexographical order. 198 | print "Corner cases, default size, min and max byte values" 199 | firstKey = chr(0) 200 | print "first key:\n", KeyToDisplayString(firstKey) 201 | seqlKey = GenerateSequentialKey(firstKey) 202 | print "seql:\n", KeyToDisplayString(seqlKey) 203 | lastKey = "".join([chr(255) for x in range(32)]) 204 | print "\nlast key:\n", KeyToDisplayString(lastKey) 205 | seqlKey = GenerateSequentialKey(lastKey) 206 | print "seql:\n", KeyToDisplayString(seqlKey) 207 | print 208 | 209 | # Generate random keys and bin the sizes to check on the 210 | # distribution of sizes. 211 | print "Histogram of key sizes computation in progress" 212 | minByte = 0 213 | maxByte = 255 214 | maxKeySize = 32 215 | numKeys = 1000000 216 | sizeBin = [0] * (maxKeySize + 1) # the index of sizeBin is the key size 217 | for i in range(numKeys): 218 | # Print a progress indicator. 219 | if i % 100 == 0: 220 | print "\r %f%% done " % (100.0 * i / numKeys), 221 | # Get a random key and bin it by size. 222 | length = len(GenerateRandomKey(minByte, maxByte, maxKeySize)) 223 | sizeBin[length] = sizeBin[length] + 1 224 | # Print the results. 225 | print "Histogram of key sizes" 226 | print "min byte value =", minByte, "; max byte value =", maxByte, \ 227 | "; max key size =", maxKeySize, \ 228 | "; number of keys to generate=", numKeys 229 | values = 1 + maxByte - minByte # local shorthand 230 | numPossibleKeys = sum(pow(values, k) for k in range(1, maxKeySize + 1)) 231 | print "key size num of keys portion of total num expected" 232 | for i in range(1, maxKeySize+1): 233 | try: 234 | numExpected = float(numKeys * pow(values, i)) / numPossibleKeys 235 | print "%8d %8d %8.4f%% %12.2f" % ( 236 | i, sizeBin[i], 100.0 * sizeBin[i] / numKeys, numExpected) 237 | except: 238 | numExpected = numKeys * pow(values, i) / numPossibleKeys 239 | print "%8d %8d %8.4f%% %10d" % ( 240 | i, sizeBin[i], 100.0 * sizeBin[i] / numKeys, numExpected) 241 | 242 | 243 | if __name__ == "__main__": 244 | main() 245 | 246 | -------------------------------------------------------------------------------- /kinetic/cmd/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2013-2015 Seagate Technology LLC. 4 | # 5 | # This Source Code Form is subject to the terms of the Mozilla 6 | # Public License, v. 2.0. If a copy of the MPL was not 7 | # distributed with this file, You can obtain one at 8 | # https://mozilla.org/MP:/2.0/. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 12 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 13 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 14 | # License for more details. 15 | # 16 | # See www.openkinetic.org for more project information 17 | # 18 | 19 | #@authors: [Ignacio Corderi, Clayg] 20 | 21 | import argparse 22 | import cmd 23 | import functools 24 | import logging 25 | import shlex 26 | import socket 27 | import sys 28 | 29 | import kinetic 30 | from kinetic import AsyncClient 31 | 32 | preparser = argparse.ArgumentParser(add_help=False) 33 | preparser.add_argument('-H', '--hostname', default='localhost') 34 | preparser.add_argument('-P', '--port', type=int, default=8123) 35 | preparser.add_argument('-v', '--version', action='store_true', 36 | help='Show kinetic version') 37 | preparser.add_argument('-vb', '--verbose', action='count', 38 | help='output more info (can stack)') 39 | parser = argparse.ArgumentParser(parents=[preparser]) 40 | command_parsers = parser.add_subparsers() 41 | 42 | # put 43 | put_parser = command_parsers.add_parser('put', help='store value at key') 44 | put_parser.add_argument('key', help='the key to access') 45 | put_parser.add_argument('value', help='the value to store') 46 | 47 | # get 48 | get_parser = command_parsers.add_parser('get', help='read value at key') 49 | get_parser.add_argument('key', help='the key to read') 50 | 51 | # delete 52 | delete_parser = command_parsers.add_parser('delete', help='remove a key') 53 | delete_parser.add_argument('key', help='the key to remove') 54 | 55 | # list 56 | list_parser = command_parsers.add_parser( 57 | 'list', help='list keys from start to end') 58 | list_parser.add_argument('start', help='the start of key range') 59 | list_parser.add_argument('end', help='the end of key range', nargs='?') 60 | 61 | # next 62 | next_parser = command_parsers.add_parser('next', help='read value at next key') 63 | next_parser.add_argument('key', help='the min key to read') 64 | 65 | # prev 66 | prev_parser = command_parsers.add_parser('prev', help='read value at prev key') 67 | prev_parser.add_argument('key', help='the max key to read') 68 | 69 | # getr 70 | getr_parser = command_parsers.add_parser( 71 | 'getr', help='get keys from start to end') 72 | getr_parser.add_argument('start', help='the start of key range') 73 | getr_parser.add_argument('end', help='the end of key range', nargs='?') 74 | 75 | # deleter 76 | deleter_parser = command_parsers.add_parser( 77 | 'deleter', help='get keys from start to end') 78 | deleter_parser.add_argument('start', help='the start of key range') 79 | deleter_parser.add_argument('end', help='the end of key range', nargs='?') 80 | 81 | 82 | def add_parser(parser): 83 | """ 84 | Create a decorator that uses the provided parser to pre-parse args based 85 | on line argument to the original do_* method. 86 | 87 | Also updates the parser's default func attribute to f for handle_command. 88 | """ 89 | def decorator(f): 90 | @functools.wraps(f) 91 | def wrapper(self, line): 92 | try: 93 | args = parser.parse_args(shlex.split(line)) 94 | except SystemExit: 95 | # let's not go quite that far... 96 | return 97 | args.verbose = self.verbose 98 | return f(self, args) 99 | parser.set_defaults(func=f) 100 | wrapper.parser = parser 101 | return wrapper 102 | return decorator 103 | 104 | 105 | def add_help(name, bases, attrs): 106 | """ 107 | Sorta magic, if we find a do_ method that has a parser hanging on 108 | it, use that to create a help_, there's some weird "scopy" stuff. 109 | """ 110 | attrs['sub_commands'] = [] 111 | for f_name, f in attrs.items(): 112 | if f_name.startswith('do_') and hasattr(f, 'parser'): 113 | command = f_name.split('_', 1)[-1] 114 | attrs['sub_commands'].append(command) 115 | f.parser.prog = command 116 | def make_help(print_help): 117 | def help_f(self): 118 | print_help() 119 | return help_f 120 | help_f = make_help(f.parser.print_help) 121 | help_f.__name__ == 'help_%s' % command 122 | attrs['help_%s' % command] = help_f 123 | 124 | return type(name, bases, attrs) 125 | 126 | 127 | def configure_logging(level): 128 | logging.basicConfig(level=logging.DEBUG) 129 | logger = logging.getLogger('kinetic') 130 | logger.propagate = True 131 | if not any(h.__class__ == logging.NullHandler for h in logger.handlers): 132 | logger.addHandler(logging.NullHandler()) 133 | if level >= 3: 134 | logger.setLevel(logging.DEBUG) 135 | elif level >= 2: 136 | logger.setLevel(logging.INFO) 137 | else: 138 | logger.propagate = False 139 | 140 | 141 | class Cmd(cmd.Cmd, object): 142 | 143 | __metaclass__ = add_help 144 | 145 | prompt = 'kinetic> ' 146 | intro = """ 147 | The Kinetic Protocol Command Line Interface Tools 148 | 149 | type help for commands 150 | """ 151 | 152 | def __init__(self, **options): 153 | cmd.Cmd.__init__(self) 154 | 155 | hostname = options.get('hostname', 'localhost') 156 | port = options.get('port', 8123) 157 | self.verbose = options.get('verbose') or 0 158 | self.client = AsyncClient(hostname, port) 159 | self.client.connect() 160 | 161 | def do_verbose(self, line): 162 | """Set active verbosity level [0-3]""" 163 | try: 164 | # see if the last word in the line is an integer 165 | level = int(line.strip().rsplit(None, 1)[-1]) 166 | except (ValueError, IndexError): 167 | if line: 168 | print 'Unknown level: %r' % line 169 | print 'Current level: %s' % self.verbose 170 | return 171 | configure_logging(level) 172 | self.verbose = level 173 | 174 | def do_quit(self, line): 175 | """Quit""" 176 | return StopIteration 177 | 178 | do_EOF = do_quit 179 | 180 | def postcmd(self, stop, line): 181 | if stop is StopIteration: 182 | print '\nuser quit' 183 | return True 184 | 185 | @add_parser(put_parser) 186 | def do_put(self, args): 187 | self.client.put(args.key, args.value) 188 | 189 | @add_parser(get_parser) 190 | def do_get(self, args): 191 | entry = self.client.get(args.key) 192 | if not entry: 193 | return 1 194 | print entry.value 195 | 196 | @add_parser(delete_parser) 197 | def do_delete(self, args): 198 | if not self.client.delete(args.key): 199 | return 1 200 | 201 | def _list(self, args): 202 | if not args.end: 203 | args.end = args.start + '\xff' 204 | keys = self.client.getKeyRange(args.start, args.end) 205 | # getKeyRange will return a maximum of 200 keys. if we have that many 206 | # we need to continue retrieving keys until we get none or less than 200. 207 | if len(keys) == 200: 208 | last_key_retrieved = keys[len(keys)-1] 209 | retrieving_keys = True 210 | start_key_inclusive = False 211 | while retrieving_keys: 212 | partial_keys = self.client.getKeyRange(last_key_retrieved, args.end, start_key_inclusive) 213 | if len(partial_keys) > 0: 214 | keys.extend(partial_keys) 215 | if len(partial_keys) == 200: 216 | last_key_retrieved = partial_keys[len(partial_keys)-1] 217 | else: 218 | retrieving_keys = False 219 | else: 220 | retrieving_keys = False 221 | return keys 222 | 223 | @add_parser(list_parser) 224 | def do_list(self, args): 225 | keys = self._list(args) 226 | for key in keys: 227 | print key 228 | 229 | @add_parser(next_parser) 230 | def do_next(self, args): 231 | entry = self.client.getNext(args.key) 232 | if not entry: 233 | return 1 234 | if args.verbose: 235 | print >> sys.stderr, 'key:', entry.key 236 | print entry.value 237 | 238 | @add_parser(prev_parser) 239 | def do_prev(self, args): 240 | entry = self.client.getPrevious(args.key) 241 | if not entry: 242 | return 1 243 | if args.verbose: 244 | print >> sys.stderr, 'key:', entry.key 245 | print entry.value 246 | 247 | @add_parser(getr_parser) 248 | def do_getr(self, args): 249 | # behave like a prefix 250 | if not args.end: 251 | args.end = args.start + '\xff' 252 | for entry in self.client.getRange(args.start, args.end): 253 | if args.verbose: 254 | print >> sys.stderr, 'key:', entry.key 255 | sys.stdout.write(entry.value) 256 | print '' 257 | 258 | @add_parser(deleter_parser) 259 | def do_deleter(self, args): 260 | def on_success(m): pass 261 | def on_error(ex): pass 262 | keys = self._list(args) 263 | for k in keys: 264 | self.client.deleteAsync(on_success, on_error, k, force=True) 265 | self.client.wait() 266 | 267 | 268 | def handle_loop(**options): 269 | while True: 270 | try: 271 | Cmd(**options).cmdloop() 272 | except socket.error as e: 273 | return 'ERROR: %s' % e 274 | except KeyboardInterrupt: 275 | print '' 276 | continue 277 | break 278 | 279 | 280 | def handle_command(args): 281 | args = parser.parse_args(args) 282 | c = Cmd(**vars(args)) 283 | try: 284 | return args.func(c, args) 285 | except socket.error as e: 286 | return 'ERROR: %s' % e 287 | except KeyboardInterrupt: 288 | print '' 289 | return 'ERROR: user quit' 290 | 291 | 292 | def main(args=None): 293 | """ 294 | :param args: only for testing, should be a string 295 | """ 296 | if args: 297 | args = shlex.split(args) 298 | else: 299 | args = sys.argv[1:] 300 | preparse_args, extra_args = preparser.parse_known_args(args) 301 | configure_logging(preparse_args.verbose) 302 | 303 | if preparse_args.version: 304 | # print version and leave 305 | print 'kinetic library version "%s"' % kinetic.__version__ 306 | print 'protocol version "%s"' % kinetic.protocol_version 307 | errorcode = 0 308 | elif extra_args: 309 | errorcode = handle_command(args) 310 | else: 311 | errorcode = handle_loop(**vars(preparse_args)) 312 | return errorcode 313 | 314 | 315 | if __name__ == "__main__": 316 | sys.exit(main()) 317 | -------------------------------------------------------------------------------- /kinetic/common.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import kinetic_pb2 as messages 20 | import eventlet 21 | import os 22 | 23 | MAX_KEY_SIZE = 4*1024 24 | MAX_VALUE_SIZE = 1024*1024 25 | 26 | DEFAULT_CONNECT_TIMEOUT = 0.1 27 | DEFAULT_SOCKET_TIMEOUT = 5 28 | DEFAULT_CHUNK_SIZE = 64*1024 29 | 30 | # Env variables 31 | try: 32 | KINETIC_CONNECT_TIMEOUT = float(os.environ.get('KINETIC_CONNECT_TIMEOUT', DEFAULT_CONNECT_TIMEOUT)) 33 | except: 34 | KINETIC_CONNECT_TIMEOUT = DEFAULT_CONNECT_TIMEOUT 35 | 36 | local = messages.Local() 37 | 38 | class DeferedValue(): 39 | 40 | def __init__(self, socket, value_ln): 41 | self.socket = socket 42 | self.length = value_ln 43 | self._evt = eventlet.event.Event() 44 | 45 | def set(self): 46 | self._evt.send() 47 | 48 | def wait(self): 49 | self._evt.wait() 50 | 51 | 52 | class Entry(object): 53 | 54 | #RPC: Note, you could build this as a class method, if you wanted the fromMessage to build 55 | #the subclass on a fromMessage. I suspect you always want to generate Entry objects, 56 | #in which case, a staticmethod is appropriate as a factory. 57 | @staticmethod 58 | def fromMessage(command, value): 59 | if not command: return None 60 | return Entry(command.body.keyValue.key, value, EntryMetadata.fromMessage(command)) 61 | 62 | @staticmethod 63 | def fromResponse(response, value): 64 | if (response.status.code == messages.Command.Status.SUCCESS): 65 | return Entry.fromMessage(response, value) 66 | elif (response.status.code == messages.Command.Status.NOT_FOUND): 67 | return None 68 | else: 69 | raise KineticClientException("Invalid response status, can' build entry from response.") 70 | 71 | def __init__(self, key, value, metadata=None): 72 | self.key = key 73 | self.value = value 74 | self.metadata = metadata or EntryMetadata() 75 | 76 | def __str__(self): 77 | if self.value: 78 | return "{key}={value}".format(key=self.key, value=self.value) 79 | else: 80 | return self.key 81 | 82 | 83 | class EntryMetadata(object): 84 | 85 | @staticmethod 86 | def fromMessage(command): 87 | if not command: return None 88 | return EntryMetadata(command.body.keyValue.dbVersion, command.body.keyValue.tag, 89 | command.body.keyValue.algorithm) 90 | 91 | def __init__(self, version=None, tag=None, algorithm=None): 92 | self.version = version 93 | self.tag = tag 94 | self.algorithm = algorithm 95 | 96 | def __str__(self): 97 | return self.version or "None" 98 | 99 | 100 | class KeyRange(object): 101 | 102 | def __init__(self, startKey, endKey, startKeyInclusive=True, 103 | endKeyInclusive=True): 104 | self.startKey = startKey 105 | self.endKey = endKey 106 | self.startKeyInclusive = startKeyInclusive 107 | self.endKeyInclusive = endKeyInclusive 108 | 109 | def getFrom(self, client, max=1024): 110 | return client.getKeyRange(self.startKey, self.endKey, self.startKeyInclusive, self.endKeyInclusive, max) 111 | 112 | 113 | class P2pOp(object): 114 | 115 | def __init__(self, key, version=None, newKey=None, force=None): 116 | self.key = key 117 | self.version = version 118 | self.newKey = newKey 119 | self.force = force 120 | 121 | 122 | class Peer(object): 123 | 124 | def __init__(self, hostname='localhost', port=8123, tls=None): 125 | self.hostname = hostname 126 | self.port = port 127 | self.tls = tls 128 | 129 | # Exceptions 130 | 131 | class KineticException(Exception): 132 | 133 | def __init__(self, value): 134 | self.value = value 135 | 136 | def __str__(self): 137 | return repr(self.value) 138 | 139 | class KineticClientException(KineticException): 140 | pass 141 | 142 | class NotConnected(KineticClientException): 143 | pass 144 | 145 | class AlreadyConnected(KineticClientException): 146 | pass 147 | 148 | class ServerDisconnect(KineticClientException): 149 | pass 150 | 151 | class ConnectionFaulted(KineticClientException): 152 | pass 153 | 154 | class ConnectionClosed(KineticClientException): 155 | pass 156 | 157 | class KineticMessageException(KineticException): 158 | 159 | def __init__(self, status): 160 | self.value = status.statusMessage 161 | self.status = status 162 | self.code = self.status.DESCRIPTOR.enum_types[0]\ 163 | .values_by_number[self.status.code].name 164 | 165 | def __str__(self): 166 | return self.code + (': %s' % self.value if self.value else '') 167 | 168 | class ClusterVersionFailureException(KineticMessageException): 169 | 170 | def __init__(self, status, cluster_version): 171 | super(ClusterVersionFailureException, self).__init__(status) 172 | self.cluster_version = cluster_version 173 | 174 | class BatchAbortedException(KineticException): 175 | def __init__(self, value): 176 | super(BatchAbortedException, self).__init__(value) 177 | self.failed_operation_index = -1 178 | 179 | class BatchCompletedException(KineticClientException): 180 | def __init__(self): 181 | super(BatchCompletedException, self).__init__('batch completed. no more operations are permitted within this batch.') 182 | 183 | class HmacAlgorithms: 184 | INVALID_HMAC_ALGORITHM = -1 # Must come first, so default is invalid 185 | HmacSHA1 = 1 # this is the default 186 | 187 | 188 | class Priority: 189 | NORMAL = 5 190 | LOWEST = 1 191 | LOWER = 3 192 | HIGHER = 7 193 | HIGHEST = 9 194 | 195 | 196 | class ACL(object): 197 | 198 | DEFAULT_IDENTITY=1 199 | DEFAULT_KEY = "asdfasdf" 200 | 201 | def __init__(self, identity=DEFAULT_IDENTITY, key=DEFAULT_KEY, algorithm=HmacAlgorithms.HmacSHA1, max_priority=Priority.NORMAL): 202 | self.identity = identity 203 | self.key = key 204 | self.hmacAlgorithm = algorithm 205 | self.domains = set() 206 | self.max_priority = max_priority 207 | 208 | 209 | class Domain(object): 210 | """ 211 | Domain object, which corresponds to the domain object in the Java client, 212 | and is the Scope object in the protobuf. 213 | """ 214 | def __init__(self, roles=None, tlsRequried=False, offset=None, value=None): 215 | if roles: 216 | self.roles = set(roles) 217 | else: 218 | self.roles = set() 219 | self.tlsRequired = tlsRequried 220 | self.offset = offset 221 | self.value = value 222 | 223 | 224 | class Roles(object): 225 | """ 226 | Role enumeration, which is the same thing as the permission field for each 227 | scope in the protobuf ACL list. 228 | """ 229 | READ = 0 230 | WRITE = 1 231 | DELETE = 2 232 | RANGE = 3 233 | SETUP = 4 234 | P2POP = 5 235 | GETLOG = 7 236 | SECURITY = 8 237 | 238 | @classmethod 239 | def all(cls): 240 | """ 241 | Return the set of all possible roles. 242 | """ 243 | return [cls.READ, cls.WRITE, cls.DELETE, cls.RANGE, cls.SETUP, cls.P2POP, cls.GETLOG, cls.SECURITY] 244 | 245 | 246 | class Synchronization: 247 | INVALID_SYNCHRONIZATION = -1 248 | WRITETHROUGH = 1 # Sync 249 | WRITEBACK = 2 # Async 250 | FLUSH = 3 251 | 252 | 253 | class IntegrityAlgorithms: 254 | SHA1 = 1 255 | SHA2 = 2 256 | SHA3 = 3 257 | CRC32 = 4 258 | CRC64 = 5 259 | # 6-99 are reserverd. 260 | # 100-inf are private algorithms 261 | 262 | 263 | class LogTypes: 264 | INVALID_TYPE = -1 265 | UTILIZATIONS = 0 266 | TEMPERATURES = 1 267 | CAPACITIES = 2 268 | CONFIGURATION = 3 269 | STATISTICS = 4 270 | MESSAGES = 5 271 | LIMITS = 6 272 | DEVICE = 7 273 | 274 | @classmethod 275 | def all(cls): 276 | """ 277 | LogTypes.all takes no arguments and returns a list of all valid log magic numbers (from the protobuf definition) 278 | that can be retrieved using the AdminClient .getLog method. Log types avaiable are: (0-> Utilizations, 1-> Temperatures, 279 | 2->Drive Capacity, 3-> Drive Configuration, 4->Drive usage statistics, and 5-> Drive messages). This can be passed as 280 | the sole argument to the AdminClient.getLog function. 281 | """ 282 | return [cls.UTILIZATIONS, cls.TEMPERATURES, cls.CAPACITIES, cls.CONFIGURATION, cls.STATISTICS, cls.MESSAGES, 283 | cls.LIMITS] 284 | -------------------------------------------------------------------------------- /kinetic/deprecated/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | from adminclient import AdminClient 18 | from blockingclient import BlockingClient -------------------------------------------------------------------------------- /kinetic/deprecated/adminclient.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import logging 20 | from kinetic import baseclient 21 | from kinetic import operations 22 | from kinetic.common import KineticException 23 | from functools import wraps 24 | import warnings 25 | 26 | def withPin(f): 27 | @wraps(f) 28 | def wrapper(self, *args, **kwargs): 29 | old = self.pin 30 | if 'pin' in kwargs: 31 | self.pin = kwargs['pin'] 32 | del kwargs['pin'] 33 | elif not self.pin: 34 | raise KineticException("This operation requires a pin.") 35 | 36 | try: 37 | f(self, *args, **kwargs) 38 | finally: 39 | self.pin = old 40 | return wrapper 41 | 42 | 43 | def requiresSsl(f): 44 | @wraps(f) 45 | def wrapper(self, *args, **kwargs): 46 | if not self.use_ssl: 47 | raise KineticException("This operation requires SSL.") 48 | f(self, *args, **kwargs) 49 | return wrapper 50 | 51 | 52 | class AdminClient(baseclient.BaseClient): 53 | 54 | def __init__(self, *args, **kwargs): 55 | if 'socket_timeout' not in kwargs: 56 | kwargs['socket_timeout'] = 60.0 57 | super(AdminClient, self).__init__(*args, **kwargs) 58 | 59 | 60 | # TODO(Nacho): this code is duplicated with client... not sure if its worth refactoring 61 | # it's pretty generic, maybe we can move it to the baseclient or something 62 | def _process(self, op, *args, **kwargs): 63 | header, value = op.build(*args, **kwargs) 64 | try: 65 | r = None 66 | with self: 67 | # update header 68 | self.update_header(header) 69 | # send message synchronously 70 | _, cmd, v = self.send(header, value) 71 | operations._check_status(cmd) 72 | return op.parse(cmd,v) 73 | except Exception as e: 74 | return op.onError(e) 75 | 76 | 77 | def getLog(self, *args, **kwargs): 78 | return self._process(operations.GetLog(), *args, **kwargs) 79 | 80 | def setClusterVersion(self, *args, **kwargs): 81 | return self._process(operations.SetClusterVersion(), *args, **kwargs) 82 | 83 | def updateFirmware(self, *args, **kwargs): 84 | return self._process(operations.UpdateFirmware(), *args, **kwargs) 85 | 86 | @withPin 87 | @requiresSsl 88 | def unlock(self, *args, **kwargs): 89 | return self._process(operations.UnlockDevice(), *args, **kwargs) 90 | 91 | @withPin 92 | @requiresSsl 93 | def lock(self, *args, **kwargs): 94 | return self._process(operations.LockDevice(), *args, **kwargs) 95 | 96 | @withPin 97 | @requiresSsl 98 | def erase(self, *args, **kwargs): 99 | return self._process(operations.EraseDevice(), *args, **kwargs) 100 | 101 | @withPin 102 | @requiresSsl 103 | def instantSecureErase(self, *args, **kwargs): 104 | return self._process(operations.SecureEraseDevice(), *args, **kwargs) 105 | 106 | @requiresSsl 107 | def setErasePin(self, *args, **kwargs): 108 | return self._process(operations.SetErasePin(), *args, **kwargs) 109 | 110 | @requiresSsl 111 | def setLockPin(self, *args, **kwargs): 112 | return self._process(operations.SetLockPin(), *args, **kwargs) 113 | 114 | @requiresSsl 115 | def setACL(self, *args, **kwargs): 116 | return self._process(operations.SetACL(), *args, **kwargs) 117 | 118 | @requiresSsl 119 | def setSecurity(self, *args, **kwargs): 120 | """ 121 | Set the access control lists to lock users out of different permissions. 122 | Arguments: aclList -> A list of ACL (Access Control List) objects. 123 | """ 124 | warnings.warn( 125 | "Shouldn't use this function anymore! Use setErasePin/setLockPin/setACL instead.", 126 | DeprecationWarning 127 | ) 128 | return self._process(operations.Security(), *args, **kwargs) 129 | 130 | -------------------------------------------------------------------------------- /kinetic/deprecated/blockingclient.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | from kinetic.baseclient import BaseClient 20 | from kinetic import operations 21 | from kinetic import kinetic_pb2 as messages 22 | from kinetic import batch 23 | import logging 24 | 25 | LOG = logging.getLogger(__name__) 26 | 27 | class BlockingClient(BaseClient): 28 | 29 | def __init__(self, *args, **kwargs): 30 | super(BlockingClient, self).__init__(*args, **kwargs) 31 | 32 | def _process(self, op, *args, **kwargs): 33 | if 'no_ack' in kwargs: 34 | send_no_ack = True 35 | del kwargs['no_ack'] 36 | else: 37 | send_no_ack = False 38 | header,value = op.build(*args, **kwargs) 39 | try: 40 | with self: 41 | # update header 42 | self.update_header(header) 43 | # send message synchronously 44 | if send_no_ack: 45 | self.send_no_ack(header, value) 46 | else: 47 | _, cmd, value = self.send(header, value) 48 | if send_no_ack: 49 | return None 50 | else: 51 | operations._check_status(cmd) 52 | return op.parse(cmd, value) 53 | except Exception as e: 54 | return op.onError(e) 55 | 56 | def noop(self, *args, **kwargs): 57 | return self._process(operations.Noop(), *args, **kwargs) 58 | 59 | def put(self, *args, **kwargs): 60 | return self._process(operations.Put(), *args, **kwargs) 61 | 62 | def get(self, *args, **kwargs): 63 | return self._process(operations.Get(), *args, **kwargs) 64 | 65 | def getMetadata(self, *args, **kwargs): 66 | return self._process(operations.GetMetadata(), *args, **kwargs) 67 | 68 | def delete(self, *args, **kwargs): 69 | return self._process(operations.Delete(), *args, **kwargs) 70 | 71 | def getNext(self, *args, **kwargs): 72 | return self._process(operations.GetNext(), *args, **kwargs) 73 | 74 | def getPrevious(self, *args, **kwargs): 75 | return self._process(operations.GetPrevious(), *args, **kwargs) 76 | 77 | def getKeyRange(self, *args, **kwargs): 78 | return self._process(operations.GetKeyRange(), *args, **kwargs) 79 | 80 | def getRange(self, startKey, endKey, startKeyInclusive=True, endKeyInclusive=True, prefetch=64): 81 | return KineticRangeIter(self, startKey, endKey, startKeyInclusive, endKeyInclusive, prefetch) 82 | 83 | def push(self, *args, **kwargs): 84 | return self._process(operations.P2pPush(), *args, **kwargs) 85 | 86 | def pipedPush(self, *args, **kwargs): 87 | return self._process(operations.P2pPipedPush(), *args, **kwargs) 88 | 89 | def getVersion(self, *args, **kwargs): 90 | """ 91 | Arguments: key -> The key you are seeking version information for. 92 | Returns a protobuf object with the version property that determines the pair's current version. 93 | """ 94 | return self._process(operations.GetVersion(), *args, **kwargs) 95 | 96 | # @RequiresProtocol('2.0.3') 97 | def flush(self, *args, **kwargs): 98 | return self._process(operations.Flush(), *args, **kwargs) 99 | 100 | # @RequiresProtocol('3.0.6') 101 | def begin_batch(self, *args, **kwargs): 102 | next_batch_id = self.next_batch_id() 103 | kwargs['batch_id'] = next_batch_id 104 | self._process(operations.StartBatch(), *args, **kwargs) 105 | return batch.Batch(self, next_batch_id) 106 | 107 | # @RequiresProtocol('3.0.0') 108 | def mediaScan(self, *args, **kwargs): 109 | return self._process(operations.MediaScan(), *args, **kwargs) 110 | 111 | # @RequiresProtocol('3.0.0') 112 | def mediaOptimize(self, *args, **kwargs): 113 | return self._process(operations.MediaOptimize(), *args, **kwargs) 114 | 115 | def getLog(self, *args, **kwargs): 116 | return self._process(operations.GetLog(), *args, **kwargs) 117 | 118 | def setClusterVersion(self, *args, **kwargs): 119 | return self._process(operations.SetClusterVersion(), *args, **kwargs) 120 | 121 | def updateFirmware(self, *args, **kwargs): 122 | return self._process(operations.UpdateFirmware(), *args, **kwargs) 123 | 124 | class KineticRangeIter(object): 125 | 126 | def __init__(self, client, startKey, endKey, startKeyInclusive,endKeyInclusive, prefetch): 127 | self.keys = [] 128 | self.nextStart = startKey 129 | self.endKey = endKey 130 | self.client = client 131 | 132 | self.startKeyInclusive = startKeyInclusive 133 | self.endKeyInclusive = endKeyInclusive 134 | self.prefetch = prefetch 135 | self.prefetchOnNext = True 136 | self.i = -1 137 | 138 | def __iter__(self): 139 | return self 140 | 141 | def next(self): 142 | 143 | if self.prefetchOnNext: self.goPrefetchKeys() 144 | 145 | if len(self.keys) > 0 : #if any keys were fetched 146 | self.i += 1 147 | 148 | if self.i == len(self.keys) - 1: #if we are on the last key prefetched 149 | self.prefetchOnNext = True #go prefetch again next time an item is asked 150 | 151 | #early exit optimization, avoids extra request to server at the end 152 | #only works with inclusive end 153 | nxt = self.keys[self.i] 154 | if nxt == self.endKey: 155 | self.keys = [] 156 | self.prefetchOnNext = False 157 | return self.client.get(nxt) 158 | 159 | return self.client.get(self.keys[self.i]) 160 | else: 161 | raise StopIteration 162 | 163 | def goPrefetchKeys(self): 164 | self.prefetchOnNext = False 165 | self.keys = self.client.getKeyRange(self.nextStart,self.endKey, self.startKeyInclusive, self.endKeyInclusive, self.prefetch) 166 | l = len(self.keys) 167 | 168 | if l > 0 : 169 | self.i = -1 170 | self.nextStart = self.keys[l-1] #move the next start to the last item received 171 | self.startKeyInclusive = False #from now on the start is exclusive 172 | -------------------------------------------------------------------------------- /kinetic/greenclient.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import logging 20 | import eventlet 21 | from eventlet.queue import Queue 22 | 23 | from eventlet.green import socket 24 | from eventlet.green.ssl import GreenSSLSocket 25 | 26 | import baseasync 27 | import common 28 | 29 | LOG = logging.getLogger(__name__) 30 | 31 | DEFAULT_POOL_SIZE = 100 32 | DEFAULT_MAX_QUEUE_SIZE = 20 33 | MAX_PENDING = 10 34 | 35 | class Client(baseasync.BaseAsync): 36 | 37 | def __init__(self, *args, **kwargs): 38 | super(Client, self).__init__(*args, **kwargs) 39 | self.pool = eventlet.greenpool.GreenPool(DEFAULT_POOL_SIZE) 40 | self.reader_thread = None 41 | self.writer_thread = None 42 | self.queue = Queue(DEFAULT_MAX_QUEUE_SIZE) 43 | self.max_pending = MAX_PENDING 44 | self.closing = False 45 | 46 | def build_socket(self, family=socket.AF_INET): 47 | return socket.socket(family) 48 | 49 | def wrap_secure_socket(self, s, ssl_version): 50 | return GreenSSLSocket(s, ssl_version=ssl_version) 51 | 52 | def connect(self): 53 | super(Client, self).connect() 54 | self.closing = False 55 | self.reader_thread = eventlet.greenthread.spawn(self._reader_run) 56 | self.writer_thread = eventlet.greenthread.spawn(self._writer_run) 57 | 58 | def dispatch(self, fn, *args, **kwargs): 59 | if LOG.isEnabledFor(logging.DEBUG): 60 | LOG.debug("Dispatching: Pending {0}".format(len(self._pending))) 61 | self.pool.spawn_n(fn,*args, **kwargs) 62 | 63 | def shutdown(self): 64 | self.closing = True 65 | if len(self._pending) + self.queue.qsize() == 0: 66 | self._end_close() 67 | 68 | def close(self): 69 | self.shutdown() 70 | self.wait() 71 | 72 | def _end_close(self): 73 | self.writer_thread.kill() 74 | self.reader_thread.kill() 75 | 76 | super(Client, self).close() 77 | 78 | self.writer_thread = None 79 | self.reader_thread = None 80 | 81 | def sendAsync(self, header, value, onSuccess, onError, no_ack=False): 82 | if self.closing: 83 | raise common.ConnectionClosed("Client is closing, can't queue more operations.") 84 | 85 | if self.faulted: 86 | self._raise(common.ConnectionFaulted("Can't send message when connection is on a faulted state."), onError) 87 | return #skip the rest 88 | 89 | # fail fast on NotConnected 90 | if not self.isConnected: 91 | self._raise(common.NotConnected("Not connected."), onError) 92 | return #skip the rest 93 | 94 | if LOG.isEnabledFor(logging.DEBUG): 95 | LOG.debug("Queue: {0}".format(self.queue.qsize())) 96 | self.queue.put((header, value, onSuccess, onError, no_ack)) 97 | eventlet.sleep(0) 98 | 99 | def wait(self): 100 | self.queue.join() 101 | 102 | def send(self, header, value): 103 | done = eventlet.event.Event() 104 | class Dummy : pass 105 | d = Dummy() 106 | d.error = None 107 | d.result = None 108 | 109 | def innerSuccess(m, r, value): 110 | d.result = (m, r, value) 111 | done.send() 112 | 113 | def innerError(e): 114 | d.error = e 115 | done.send() 116 | 117 | self.sendAsync(header, value, innerSuccess, innerError) 118 | 119 | done.wait() # TODO(Nacho): should be add a default timeout? 120 | if d.error: raise d.error 121 | return d.result 122 | 123 | def _writer_run(self): 124 | while self.isConnected and not self.faulted: 125 | try: 126 | while len(self._pending) > self.max_pending: 127 | eventlet.sleep(0) 128 | (header, value, onSuccess, onError, no_ack) = self.queue.get() 129 | super(Client, self).sendAsync(header, value, onSuccess, onError, no_ack) 130 | except common.ConnectionFaulted: pass 131 | except common.ConnectionClosed: pass 132 | except Exception as ex: 133 | self._fault_client(ex) 134 | 135 | # Yield execution, don't starve the reader 136 | eventlet.sleep(0) 137 | 138 | def _reader_run(self): 139 | while self.isConnected and not self.faulted: 140 | try: 141 | self._async_recv() 142 | self.queue.task_done() 143 | if self.closing and len(self._pending) + self.queue.qsize() == 0: 144 | self._end_close() 145 | except common.ConnectionFaulted: pass 146 | except Exception as ex: 147 | self._fault_client(ex) 148 | 149 | # Yield execution, don't starve the writer 150 | # eventlet.sleep(0) 151 | 152 | #eventlet.sleep() 153 | -------------------------------------------------------------------------------- /kinetic/operations.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | from common import Entry 20 | from common import EntryMetadata 21 | from common import KineticMessageException 22 | import common 23 | import kinetic_pb2 as messages 24 | import logging 25 | import hashlib 26 | 27 | LOG = logging.getLogger(__name__) 28 | 29 | 30 | def _check_status(command): 31 | if (command.status.code == messages.Command.Status.SUCCESS): 32 | return 33 | elif(command.status.code == messages.Command.Status.VERSION_FAILURE): 34 | raise common.ClusterVersionFailureException(command.status, command.header.clusterVersion) 35 | else: 36 | raise KineticMessageException(command.status) 37 | 38 | 39 | def _buildMessage(m, messageType, key, data=None, version='', new_version='', 40 | force=False, tag=None, algorithm=None, synchronization=None): 41 | m.header.messageType = messageType 42 | if len(key) > common.MAX_KEY_SIZE: raise common.KineticClientException("Key exceeds maximum size of {0} bytes.".format(common.MAX_KEY_SIZE)) 43 | m.body.keyValue.key = key 44 | if data: 45 | if len(data) > common.MAX_VALUE_SIZE: raise common.KineticClientException("Value exceeds maximum size of {0} bytes.".format(common.MAX_VALUE_SIZE)) 46 | 47 | if tag and algorithm: 48 | m.body.keyValue.tag = tag 49 | m.body.keyValue.algorithm = algorithm 50 | elif messageType == messages.Command.PUT: 51 | # check the data type first 52 | if data and (isinstance(data, str) or isinstance(data, bytes) or isinstance(data, bytearray)): 53 | # default to sha1 54 | m.body.keyValue.tag = hashlib.sha1(data).digest() 55 | m.body.keyValue.algorithm = common.IntegrityAlgorithms.SHA1 56 | else: 57 | m.body.keyValue.tag = 'l337' 58 | 59 | if (messageType == messages.Command.PUT or messageType == messages.Command.DELETE) and synchronization == None: 60 | synchronization = common.Synchronization.WRITEBACK 61 | 62 | if synchronization: 63 | m.body.keyValue.synchronization = synchronization 64 | if version: 65 | m.body.keyValue.dbVersion = version 66 | if new_version: 67 | m.body.keyValue.newVersion = new_version 68 | if force: 69 | m.body.keyValue.force = True 70 | 71 | return (m,data) 72 | 73 | 74 | class BaseOperation(object): 75 | 76 | def __init__(self): 77 | self.m = None 78 | 79 | def _build(): pass 80 | 81 | def build(self, *args, **kwargs): 82 | self.m = messages.Command() 83 | 84 | if 'timeout' in kwargs: 85 | self.m.header.timeout = kwargs['timeout'] 86 | del kwargs['timeout'] 87 | 88 | if 'priority' in kwargs: 89 | self.m.header.priority = kwargs['priority'] 90 | del kwargs['priority'] 91 | 92 | if 'early_exit' in kwargs: 93 | self.m.header.earlyExit = kwargs['early_exit'] 94 | del kwargs['early_exit'] 95 | 96 | if 'time_quanta' in kwargs: 97 | self.m.header.TimeQuanta = kwargs['time_quanta'] 98 | del kwargs['time_quanta'] 99 | 100 | if 'batch_id' in kwargs: 101 | self.m.header.batchID = kwargs['batch_id'] 102 | del kwargs['batch_id'] 103 | 104 | return self._build(*args, **kwargs) 105 | 106 | def parse(self, m, value): 107 | return 108 | 109 | def onError(self, e): 110 | raise e 111 | 112 | class Noop(BaseOperation): 113 | 114 | def _build(self): 115 | m = self.m 116 | m.header.messageType = messages.Command.NOOP 117 | return (m, None) 118 | 119 | 120 | class Put(BaseOperation): 121 | 122 | def _build(self, key, data, version="", new_version="", **kwargs): 123 | return _buildMessage(self.m, messages.Command.PUT, key, data, version, new_version, **kwargs) 124 | 125 | 126 | class Get(BaseOperation): 127 | 128 | def _build(self, key): 129 | return _buildMessage(self.m, messages.Command.GET, key) 130 | 131 | def parse(self, m, value): 132 | return Entry.fromResponse(m, value) 133 | 134 | def onError(self, e): 135 | if isinstance(e,KineticMessageException): 136 | if e.code and e.code == 'NOT_FOUND': 137 | return None 138 | raise e 139 | 140 | 141 | class GetMetadata(Get): 142 | 143 | def _build(self, key): 144 | (m,_) = _buildMessage(self.m, messages.Command.GET, key) 145 | m.body.keyValue.metadataOnly = True 146 | return (m, None) 147 | 148 | 149 | class Delete(BaseOperation): 150 | 151 | def _build(self, key, version="", **kwargs): 152 | return _buildMessage(self.m, messages.Command.DELETE, key, version=version, **kwargs) 153 | 154 | def parse(self, m, value): 155 | return True 156 | 157 | def onError(self, e): 158 | if isinstance(e,KineticMessageException): 159 | if e.code and e.code == 'NOT_FOUND': 160 | return False 161 | raise e 162 | 163 | 164 | class GetNext(Get): 165 | 166 | def _build(self, key): 167 | return _buildMessage(self.m, messages.Command.GETNEXT, key) 168 | 169 | 170 | class GetPrevious(Get): 171 | 172 | def _build(self, key): 173 | return _buildMessage(self.m, messages.Command.GETPREVIOUS, key) 174 | 175 | 176 | class GetKeyRange(BaseOperation): 177 | 178 | def _build(self, startKey=None, endKey=None, startKeyInclusive=True, endKeyInclusive=True, 179 | maxReturned=200, reverse=False): 180 | if not startKey: 181 | startKey = '' 182 | if not endKey: 183 | endKey = '\xFF' * common.MAX_KEY_SIZE 184 | 185 | if len(startKey) > common.MAX_KEY_SIZE: raise common.KineticClientException("Start key exceeds maximum size of {0} bytes.".format(common.MAX_KEY_SIZE)) 186 | if len(endKey) > common.MAX_KEY_SIZE: raise common.KineticClientException("End key exceeds maximum size of {0} bytes.".format(common.MAX_KEY_SIZE)) 187 | 188 | m = self.m 189 | m.header.messageType = messages.Command.GETKEYRANGE 190 | 191 | kr = m.body.range 192 | kr.startKey = startKey 193 | kr.endKey = endKey 194 | kr.startKeyInclusive = startKeyInclusive 195 | kr.endKeyInclusive = endKeyInclusive 196 | kr.maxReturned = maxReturned 197 | kr.reverse = reverse 198 | 199 | return (m, None) 200 | 201 | def parse(self, m, value): 202 | return [k for k in m.body.range.keys] # key is actually a set of keys 203 | 204 | 205 | class GetVersion(BaseOperation): 206 | 207 | def _build(self, key): 208 | (m,_) = _buildMessage(self.m, messages.Command.GETVERSION, key) 209 | return (m, None) 210 | 211 | def parse(self, m, value): 212 | return m.body.keyValue.dbVersion 213 | 214 | def onError(self, e): 215 | if isinstance(e,KineticMessageException): 216 | if e.code and e.code == 'NOT_FOUND': 217 | return None 218 | raise e 219 | 220 | class P2pPush(BaseOperation): 221 | 222 | def _build(self, keys, hostname='localhost', port=8123, tls=False): 223 | m = self.m 224 | m.header.messageType = messages.Command.PEER2PEERPUSH 225 | m.body.p2pOperation.peer.hostname = hostname 226 | m.body.p2pOperation.peer.port = port 227 | m.body.p2pOperation.peer.tls = tls 228 | 229 | operations = [] 230 | for k in keys: 231 | op = None 232 | if isinstance(k, str): 233 | op = messages.Command.P2POperation.Operation(key=k) 234 | elif isinstance(k, common.P2pOp): 235 | op = messages.Command.P2POperation.Operation(key=k.key) 236 | if k.version: 237 | op.version = k.version 238 | if k.newKey: 239 | op.newKey = k.newKey 240 | if k.force: 241 | op.force = k.force 242 | operations.append(op) 243 | 244 | m.body.p2pOperation.operation.extend(operations) 245 | 246 | return (m, None) 247 | 248 | def parse(self, m, value): 249 | return [op for op in m.body.p2pOperation.operation] 250 | 251 | 252 | class P2pPipedPush(BaseOperation): 253 | 254 | def _build(self, keys, targets): 255 | m = self.m 256 | m.header.messageType = messages.Command.PEER2PEERPUSH 257 | m.body.p2pOperation.peer.hostname = targets[0].hostname 258 | m.body.p2pOperation.peer.port = targets[0].port 259 | if targets[0].tls: 260 | m.body.p2pOperation.peer.tls = targets[0].tls 261 | 262 | 263 | def rec(targets, op): 264 | if len(targets) > 0: 265 | target = targets[0] 266 | op.p2pop.peer.hostname = target.hostname 267 | op.p2pop.peer.port = target.port 268 | if target.tls: 269 | op.p2pop.peer.tls = target.tls 270 | innerop = messages.Command.P2POperation.Operation(key=op.key) 271 | if op.version: 272 | innerop.version = op.version 273 | if op.force: 274 | innerop.force = op.force 275 | 276 | rec(targets[1:],innerop) 277 | 278 | op.p2pop.operation.extend([innerop]) 279 | 280 | warn_newKey = False 281 | operations = [] 282 | for k in keys: 283 | op = None 284 | if isinstance(k, str): 285 | op = messages.Command.P2POperation.Operation(key=k) 286 | elif isinstance(k, common.P2pOp): 287 | op = messages.Command.P2POperation.Operation(key=k.key) 288 | if k.version: 289 | op.version = k.version 290 | if k.newKey: 291 | warn_newKey = True 292 | if k.force: 293 | op.force = k.force 294 | 295 | rec(targets[1:],op) 296 | 297 | operations.append(op) 298 | 299 | if warn_newKey: 300 | LOG.warn("Setting new key on piped push is not currently supported.") 301 | 302 | m.body.p2pOperation.operation.extend(operations) 303 | 304 | return (m, None) 305 | 306 | def parse(self, m, value): 307 | return [op for op in m.body.p2pOperation.operation] 308 | 309 | 310 | class StartBatch(BaseOperation): 311 | 312 | def _build(self): 313 | self.m.header.messageType = messages.Command.START_BATCH 314 | return (self.m, None) 315 | 316 | 317 | class EndBatch(BaseOperation): 318 | 319 | def _build(self, **kwargs): 320 | # batch_op_count 321 | (m,_) = _buildMessage(self.m, messages.Command.END_BATCH, '') 322 | m.body.batch.count = kwargs['batch_op_count'] 323 | del kwargs['batch_op_count'] 324 | return (m, None) 325 | 326 | def onError(self, e): 327 | if isinstance(e,common.KineticException): 328 | if e.code and e.code == 'INVALID_BATCH': 329 | return common.BatchAbortedException(e.value) 330 | raise e 331 | 332 | class AbortBatch(BaseOperation): 333 | 334 | def _build(self): 335 | self.m.header.messageType = messages.Command.ABORT_BATCH 336 | return (self.m, None) 337 | 338 | 339 | class Flush(BaseOperation): 340 | 341 | def _build(self): 342 | m = self.m 343 | m.header.messageType = messages.Command.FLUSHALLDATA 344 | 345 | return (m, None) 346 | 347 | 348 | ### Admin Operations ### 349 | 350 | class GetLog(BaseOperation): 351 | 352 | def _build(self, types, device=None): 353 | m = self.m 354 | m.header.messageType = messages.Command.GETLOG 355 | 356 | log = m.body.getLog 357 | log.types.extend(types) #type is actually a repeatable field 358 | 359 | if device: 360 | log.device.name = device 361 | 362 | return (m, None) 363 | 364 | def parse(self, m, value): 365 | if value: 366 | return (m.body.getLog, value) 367 | else: 368 | return m.body.getLog 369 | 370 | def onError(self, e): 371 | if isinstance(e,KineticMessageException): 372 | if e.code and e.code == 'NOT_FOUND': 373 | return None 374 | raise e 375 | 376 | 377 | ###################### 378 | # Setup operations # 379 | ###################### 380 | class SetClusterVersion(BaseOperation): 381 | 382 | def _build(self, version): 383 | m = self.m 384 | m.header.messageType = messages.Command.SETUP 385 | 386 | m.body.setup.newClusterVersion = version 387 | 388 | return (m, None) 389 | 390 | 391 | class UpdateFirmware(BaseOperation): 392 | 393 | def _build(self, firmware): 394 | m = self.m 395 | m.header.messageType = messages.Command.SETUP 396 | 397 | m.body.setup.firmwareDownload = True 398 | 399 | return (m, firmware) 400 | 401 | 402 | ######################## 403 | # Security operations # 404 | ######################## 405 | class Security(BaseOperation): 406 | 407 | def _build(self, acls=None, old_erase_pin=None, new_erase_pin=None, old_lock_pin=None, new_lock_pin=None): 408 | m = self.m 409 | m.header.messageType = messages.Command.SECURITY 410 | op = m.body.security 411 | 412 | if acls: 413 | proto_acls = [] 414 | 415 | for acl in acls: 416 | proto_acl = messages.Command.Security.ACL(identity=acl.identity, 417 | key=acl.key, 418 | hmacAlgorithm=acl.hmacAlgorithm, 419 | maxPriority=acl.max_priority) 420 | 421 | proto_domains = [] 422 | 423 | for domain in acl.domains: 424 | proto_d = messages.Command.Security.ACL.Scope( 425 | TlsRequired=domain.tlsRequired) 426 | 427 | proto_d.permission.extend(domain.roles) 428 | 429 | if domain.offset: 430 | proto_d.offset = domain.offset 431 | if domain.value: 432 | proto_d.value = domain.value 433 | 434 | proto_domains.append(proto_d) 435 | 436 | proto_acl.scope.extend(proto_domains) 437 | proto_acls.append(proto_acl) 438 | 439 | op.acl.extend(proto_acls) 440 | 441 | if not old_lock_pin is None: op.oldLockPIN = old_lock_pin 442 | if not new_lock_pin is None: op.newLockPIN = new_lock_pin 443 | if not old_erase_pin is None: op.oldErasePIN = old_erase_pin 444 | if not new_erase_pin is None: op.newErasePIN = new_erase_pin 445 | 446 | return (m, None) 447 | 448 | 449 | class SetACL(Security): 450 | 451 | def _build(self, acls): 452 | return super(SetACL, self)._build(acls=acls) 453 | 454 | 455 | class SetErasePin(Security): 456 | 457 | def _build(self, new_pin, old_pin): 458 | return super(SetErasePin, self)._build(new_erase_pin=new_pin, old_erase_pin=old_pin) 459 | 460 | 461 | class SetLockPin(Security): 462 | 463 | def _build(self, new_pin, old_pin): 464 | return super(SetLockPin, self)._build(new_lock_pin=new_pin, old_lock_pin=old_pin) 465 | 466 | 467 | ########################### 468 | # Background operations # 469 | ########################### 470 | class MediaScan(BaseOperation): 471 | 472 | def _build(self, startKey=None, endKey=None, startKeyInclusive=True, endKeyInclusive=True, maxReturned=200): 473 | if not startKey: 474 | startKey = '' 475 | if not endKey: 476 | endKey = '\xFF' * common.MAX_KEY_SIZE 477 | 478 | if len(startKey) > common.MAX_KEY_SIZE: raise common.KineticClientException("Start key exceeds maximum size of {0} bytes.".format(common.MAX_KEY_SIZE)) 479 | if len(endKey) > common.MAX_KEY_SIZE: raise common.KineticClientException("End key exceeds maximum size of {0} bytes.".format(common.MAX_KEY_SIZE)) 480 | 481 | m = self.m 482 | 483 | m.header.messageType = messages.Command.MEDIASCAN 484 | 485 | kr = m.body.range 486 | kr.startKey = startKey 487 | kr.endKey = endKey 488 | kr.startKeyInclusive = startKeyInclusive 489 | kr.endKeyInclusive = endKeyInclusive 490 | kr.maxReturned = maxReturned 491 | 492 | return (m, None) 493 | 494 | def parse(self, m, value): 495 | r = m.body.range 496 | return ([k for k in r.keys], r.endKey) 497 | 498 | 499 | class MediaOptimize(BaseOperation): 500 | 501 | def _build(self, startKey=None, endKey=None, startKeyInclusive=True, endKeyInclusive=True, maxReturned=200): 502 | if not startKey: 503 | startKey = '' 504 | if not endKey: 505 | endKey = '\xFF' * common.MAX_KEY_SIZE 506 | 507 | if len(startKey) > common.MAX_KEY_SIZE: raise common.KineticClientException("Start key exceeds maximum size of {0} bytes.".format(common.MAX_KEY_SIZE)) 508 | if len(endKey) > common.MAX_KEY_SIZE: raise common.KineticClientException("End key exceeds maximum size of {0} bytes.".format(common.MAX_KEY_SIZE)) 509 | 510 | m = self.m 511 | 512 | m.header.messageType = messages.Command.MEDIAOPTIMIZE 513 | 514 | kr = m.body.range 515 | kr.startKey = startKey 516 | kr.endKey = endKey 517 | kr.startKeyInclusive = startKeyInclusive 518 | kr.endKeyInclusive = endKeyInclusive 519 | kr.maxReturned = maxReturned 520 | 521 | return (m, None) 522 | 523 | def parse(self, m, value): 524 | r = m.body.range 525 | return ([k for k in r.keys], r.endKey) 526 | 527 | 528 | #################### 529 | # Pin operations # 530 | #################### 531 | class BasePinOperation(BaseOperation): 532 | 533 | def __init__(self): 534 | super(BaseOperation, self).__init__() 535 | self.pin_op_type = None 536 | 537 | def _build(self): 538 | m = self.m 539 | m.header.messageType = messages.Command.PINOP 540 | m.body.pinOp.pinOpType = self.pin_op_type 541 | return (m, None) 542 | 543 | class UnlockDevice(BasePinOperation): 544 | 545 | def __init__(self): 546 | super(UnlockDevice, self).__init__() 547 | self.pin_op_type = messages.Command.PinOperation.UNLOCK_PINOP 548 | 549 | 550 | class LockDevice(BasePinOperation): 551 | 552 | def __init__(self): 553 | super(LockDevice, self).__init__() 554 | self.pin_op_type = messages.Command.PinOperation.LOCK_PINOP 555 | 556 | 557 | class EraseDevice(BasePinOperation): 558 | 559 | def __init__(self): 560 | super(EraseDevice, self).__init__() 561 | self.pin_op_type = messages.Command.PinOperation.ERASE_PINOP 562 | 563 | 564 | class SecureEraseDevice(BasePinOperation): 565 | 566 | def __init__(self): 567 | super(SecureEraseDevice, self).__init__() 568 | self.pin_op_type = messages.Command.PinOperation.SECURE_ERASE_PINOP 569 | -------------------------------------------------------------------------------- /kinetic/secureclient.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import logging 20 | from kinetic.deprecated import BlockingClient 21 | from kinetic import operations 22 | from kinetic.common import KineticException 23 | from functools import wraps 24 | import warnings 25 | 26 | def withPin(f): 27 | @wraps(f) 28 | def wrapper(self, *args, **kwargs): 29 | old = self.pin 30 | if 'pin' in kwargs: 31 | self.pin = kwargs['pin'] 32 | del kwargs['pin'] 33 | elif not self.pin: 34 | raise KineticException("This operation requires a pin.") 35 | 36 | try: 37 | f(self, *args, **kwargs) 38 | finally: 39 | self.pin = old 40 | return wrapper 41 | 42 | 43 | def requiresSsl(f): 44 | @wraps(f) 45 | def wrapper(self, *args, **kwargs): 46 | if not self.use_ssl: 47 | raise KineticException("This operation requires SSL.") 48 | f(self, *args, **kwargs) 49 | return wrapper 50 | 51 | 52 | class SecureClient(BlockingClient): 53 | 54 | def __init__(self, *args, **kwargs): 55 | kwargs['use_ssl'] = True 56 | # len() < 2 is because port can be positional on the baseclient 57 | if 'port' not in kwargs and len(args) < 2: 58 | kwargs['port'] = 8443 59 | 60 | super(SecureClient, self).__init__(*args, **kwargs) 61 | 62 | @withPin 63 | @requiresSsl 64 | def unlock(self, *args, **kwargs): 65 | return self._process(operations.UnlockDevice(), *args, **kwargs) 66 | 67 | @withPin 68 | @requiresSsl 69 | def lock(self, *args, **kwargs): 70 | return self._process(operations.LockDevice(), *args, **kwargs) 71 | 72 | @withPin 73 | @requiresSsl 74 | def erase(self, *args, **kwargs): 75 | return self._process(operations.EraseDevice(), *args, **kwargs) 76 | 77 | @withPin 78 | @requiresSsl 79 | def instantSecureErase(self, *args, **kwargs): 80 | return self._process(operations.SecureEraseDevice(), *args, **kwargs) 81 | 82 | @requiresSsl 83 | def setErasePin(self, *args, **kwargs): 84 | return self._process(operations.SetErasePin(), *args, **kwargs) 85 | 86 | @requiresSsl 87 | def setLockPin(self, *args, **kwargs): 88 | return self._process(operations.SetLockPin(), *args, **kwargs) 89 | 90 | @requiresSsl 91 | def setACL(self, *args, **kwargs): 92 | return self._process(operations.SetACL(), *args, **kwargs) 93 | 94 | @requiresSsl 95 | def setSecurity(self, *args, **kwargs): 96 | """ 97 | Set the access control lists to lock users out of different permissions. 98 | Arguments: aclList -> A list of ACL (Access Control List) objects. 99 | """ 100 | warnings.warn( 101 | "Shouldn't use this function anymore! Use setErasePin/setLockPin/setACL instead.", 102 | DeprecationWarning 103 | ) 104 | return self._process(operations.Security(), *args, **kwargs) 105 | 106 | -------------------------------------------------------------------------------- /kinetic/threadedclient.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import logging 20 | import thread 21 | import threading 22 | import Queue 23 | 24 | from baseasync import BaseAsync 25 | import common 26 | 27 | LOG = logging.getLogger(__name__) 28 | 29 | class ThreadedClient(BaseAsync): 30 | 31 | def __init__(self, *args, **kwargs): 32 | super(ThreadedClient, self).__init__(*args, **kwargs) 33 | self.queue = Queue.Queue() 34 | # if 'pool' in kwargs: 35 | # self.pool = kwards['pool'] 36 | # else: 37 | # self.pool=None 38 | self.pool = None 39 | 40 | def connect(self): 41 | super(ThreadedClient, self).connect() 42 | self.thread = threading.Thread(target = self._run) 43 | self.thread.daemon = True 44 | self.thread.start() 45 | 46 | self.writer_thread = threading.Thread(target = self._writer) 47 | self.writer_thread.daemon = True 48 | self.writer_thread.start() 49 | 50 | def dispatch(self, fn, *args, **kwargs): 51 | if self.pool: 52 | self.pool.submit(fn,*args,**kwargs) 53 | else: 54 | fn(*args,**kwargs) 55 | 56 | def close(self): 57 | self.dispatch(super(ThreadedClient, self).close) 58 | self.queue.put(None) 59 | self.writer_thread.join() 60 | self.thread.join() 61 | 62 | def sendAsync(self, header, value, onSuccess, onError, no_ack=False): 63 | self.queue.put((header, value, onSuccess, onError, no_ack)) 64 | 65 | def _writer(self): 66 | while self.isConnected and not self.faulted: 67 | try: 68 | item = self.queue.get() 69 | if item: 70 | (header, value, onSuccess, onError) = item 71 | super(ThreadedClient, self).sendAsync(header, value, onSuccess, onError) 72 | self.queue.task_done() 73 | except common.ConnectionFaulted: pass 74 | except common.ConnectionClosed: pass 75 | except Exception as ex: 76 | self._fault_client(ex) 77 | 78 | def _run(self): 79 | while self.isConnected and not self.faulted: 80 | try: 81 | self._async_recv() 82 | except common.ConnectionFaulted: pass 83 | except common.ConnectionClosed: pass 84 | except Exception as ex: 85 | self._fault_client(ex) 86 | -------------------------------------------------------------------------------- /kinetic/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import struct 20 | 21 | from common import KeyRange 22 | 23 | def buildRange(key): 24 | import array 25 | inBytes = map(ord,key) 26 | inBytes[len(inBytes)-1] = inBytes[len(inBytes)-1]+1 27 | keyPlus1 = array.array('B', inBytes).tostring() 28 | return KeyRange(key, keyPlus1, True, False) 29 | -------------------------------------------------------------------------------- /kinetic/zero_copy.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | 20 | import traceback 21 | import logging 22 | import os 23 | import select 24 | import fcntl 25 | import errno 26 | import os.path 27 | import socket 28 | import subprocess 29 | import ctypes 30 | import ctypes.util 31 | from eventlet.green import select as green_select 32 | 33 | LOG = logging.getLogger(__name__) 34 | 35 | 36 | def set_nonblock(fd): #pylint: disable-msg=C0103 37 | '''Set a file descriptor in non-blocking mode''' 38 | 39 | flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0) 40 | flags |= os.O_NONBLOCK 41 | fcntl.fcntl(fd, fcntl.F_SETFL, flags) 42 | 43 | 44 | def direct_transfer_select(fd_in, off_in, fd_out, off_out, length): 45 | LOG.debug("Transfering %s bytes from %s to %s" % (length, fd_in, fd_out)) 46 | (pipe_r, pipe_w) = os.pipe() 47 | pipe_r = os.fdopen(pipe_r) 48 | pipe_w = os.fdopen(pipe_w,'w') 49 | fd_pipe_r = pipe_r.fileno() 50 | fd_pipe_w = pipe_w.fileno() 51 | 52 | LOG.debug(" source(%s) -> pipe(%s, %s) -> dest(%s)" % (fd_in, fd_pipe_w, fd_pipe_r, fd_out)) 53 | 54 | set_nonblock(fd_in) 55 | set_nonblock(fd_out) 56 | set_nonblock(fd_pipe_r) 57 | set_nonblock(fd_pipe_w) 58 | 59 | flags = SPLICE_F_MOVE | SPLICE_F_MORE | SPLICE_F_NONBLOCK 60 | 61 | epoll = select.epoll() # TODO: find a proper way to check if the fd's are files 62 | fd_in_is_file = False 63 | fd_out_is_file = False 64 | 65 | try: 66 | epoll.register(fd_in, select.EPOLLIN) 67 | except: 68 | fd_in_is_file = True 69 | 70 | try: 71 | epoll.register(fd_out, select.EPOLLOUT) 72 | except: 73 | fd_out_is_file = True 74 | 75 | if fd_in_is_file: 76 | readers = [fd_pipe_r] 77 | else: 78 | readers = [fd_in, fd_pipe_r] 79 | 80 | if fd_out_is_file: 81 | writers = [fd_pipe_w] 82 | else: 83 | writers = [fd_pipe_w, fd_out] 84 | 85 | try: 86 | towrite0 = length 87 | towrite1 = length 88 | 89 | ready = [] 90 | 91 | while towrite0 > 0 or towrite1 > 0: 92 | readable_set, writable_set, _ = green_select.select(readers, writers, []) 93 | 94 | for x in readable_set: 95 | ready.append(x) 96 | readers.remove(x) 97 | 98 | for x in writable_set: 99 | ready.append(x) 100 | writers.remove(x) 101 | 102 | # two transfer options 103 | if (fd_in_is_file or fd_in in ready) and (fd_pipe_w in ready): 104 | 105 | # transfer from source to pipe 106 | try: 107 | done = splice.splice(fd_in, None, fd_pipe_w, None, towrite0, flags) 108 | LOG.debug('> Source -> pipe %s bytes' % done) 109 | towrite0 -= done 110 | except IOError as ioex: 111 | if ioex.errno in [errno.EAGAIN, errno.EWOULDBLOCK]: continue 112 | else: raise 113 | 114 | # reset 115 | if not fd_in_is_file: 116 | ready.remove(fd_in) 117 | readers.append(fd_in) 118 | 119 | ready.remove(fd_pipe_w) 120 | writers.append(fd_pipe_w) 121 | 122 | if (fd_pipe_r in ready) and (fd_out_is_file or fd_out in ready): 123 | 124 | # transfer from pipe to destination 125 | try: 126 | done = splice.splice(fd_pipe_r, None, fd_out, None, towrite1, flags) 127 | LOG.debug('> pipe -> dest %s bytes' % done) 128 | towrite1 -= done 129 | except IOError as ioex: 130 | if ioex.errno in [errno.EAGAIN, errno.EWOULDBLOCK]: continue 131 | else: raise 132 | 133 | # reset 134 | if not fd_out_is_file: 135 | ready.remove(fd_out) 136 | writers.append(fd_out) 137 | 138 | ready.remove(fd_pipe_r) 139 | readers.append(fd_pipe_r) 140 | 141 | except Exception as ex: 142 | traceback.print_exc() 143 | raise 144 | 145 | 146 | def direct_transfer_epoll(fd_in, off_in, fd_out, off_out, length): 147 | LOG.debug("Transfering %s bytes from %s to %s" % (length, fd_in, fd_out)) 148 | (pipe_r, pipe_w) = os.pipe() 149 | pipe_r = os.fdopen(pipe_r) 150 | pipe_w = os.fdopen(pipe_w,'w') 151 | fd_pipe_r = pipe_r.fileno() 152 | fd_pipe_w = pipe_w.fileno() 153 | 154 | LOG.debug(" source(%s) -> pipe(%s, %s) -> dest(%s)" % (fd_in, fd_pipe_w, fd_pipe_r, fd_out)) 155 | 156 | set_nonblock(fd_in) 157 | set_nonblock(fd_out) 158 | set_nonblock(fd_pipe_r) 159 | set_nonblock(fd_pipe_w) 160 | 161 | flags = SPLICE_F_MOVE | SPLICE_F_MORE | SPLICE_F_NONBLOCK 162 | 163 | try: 164 | towrite0 = length 165 | towrite1 = length 166 | 167 | epoll = select.epoll() 168 | fd_in_is_file = False 169 | fd_out_is_file = False 170 | 171 | try: 172 | epoll.register(fd_in, select.EPOLLIN) # handle EPOLLHUP 173 | except: 174 | fd_in_is_file = True 175 | 176 | try: 177 | epoll.register(fd_out, select.EPOLLOUT) # handle EPOLLHUP 178 | except: 179 | fd_out_is_file = True 180 | 181 | epoll.register(fd_pipe_r, select.EPOLLIN) 182 | epoll.register(fd_pipe_w, select.EPOLLOUT) 183 | 184 | 185 | while towrite0 > 0 or towrite1 > 0: 186 | events = epoll.poll() 187 | 188 | events_dict = {fd: evt for (fd, evt) in events} 189 | 190 | # two transfer options 191 | if (fd_in_is_file or fd_in in events_dict) and fd_pipe_w in events_dict: 192 | 193 | # transfer from source to pipe 194 | try: 195 | done = splice(fd_in, None, fd_pipe_w, None, towrite0, flags) 196 | LOG.debug('> Source -> pipe %s bytes' % done) 197 | towrite0 -= done 198 | except IOError as ioex: 199 | if ioex.errno in [errno.EAGAIN, errno.EWOULDBLOCK]: 200 | continue 201 | 202 | if fd_pipe_r in events_dict and (fd_out_is_file or fd_out in events_dict): 203 | 204 | # transfer from pipe to destination 205 | try: 206 | done = splice(fd_pipe_r, None, fd_out, None, towrite1, flags) 207 | LOG.debug('> pipe -> dest %s bytes' % done) 208 | towrite1 -= done 209 | except IOError as ioex: 210 | if ioex.errno in [errno.EAGAIN, errno.EWOULDBLOCK]: continue 211 | else: raise 212 | 213 | epoll.close() 214 | 215 | except Exception as ex: 216 | traceback.print_exc() 217 | raise 218 | 219 | 220 | direct_transfer = direct_transfer_epoll 221 | 222 | 223 | def forwardto(defered_value, target_fd): 224 | direct_transfer(defered_value.socket.fileno(), None, 225 | target_fd.fileno(), None, defered_value.length) 226 | defered_value.set() # signal we are done reading 227 | 228 | 229 | class ZeroCopyValue(): 230 | 231 | def __init__(self, fd, offset, length): 232 | self.fd = fd 233 | self.offset = offset 234 | self.length = length 235 | 236 | def __len__(self): return self.length 237 | 238 | def send(self, socket): 239 | # If source is a file then offset is valid, otherwise is has to be None 240 | direct_transfer(self.fd.fileno(), self.offset, socket.fileno(), None, self.length) 241 | 242 | 243 | def make_splice(): 244 | '''Set up a splice(2) wrapper''' 245 | 246 | # Load libc 247 | libc_name = ctypes.util.find_library('c') 248 | libc = ctypes.CDLL(libc_name, use_errno=True) 249 | 250 | # Get a handle to the 'splice' call 251 | c_splice = libc.splice 252 | 253 | # These should match for x86_64, might need some tweaking for other 254 | # platforms... 255 | c_loff_t = ctypes.c_uint64 256 | c_loff_t_p = ctypes.POINTER(c_loff_t) 257 | 258 | # ssize_t splice(int fd_in, loff_t *off_in, int fd_out, 259 | # loff_t *off_out, size_t len, unsigned int flags) 260 | c_splice.argtypes = [ 261 | ctypes.c_int, c_loff_t_p, 262 | ctypes.c_int, c_loff_t_p, 263 | ctypes.c_size_t, 264 | ctypes.c_uint 265 | ] 266 | c_splice.restype = ctypes.c_ssize_t 267 | 268 | # Clean-up closure names. Yup, useless nit-picking. 269 | del libc 270 | del libc_name 271 | del c_loff_t_p 272 | 273 | # pylint: disable-msg=W0621,R0913 274 | def splice(fd_in, off_in, fd_out, off_out, len_, flags): 275 | '''Wrapper for splice(2) 276 | 277 | See the syscall documentation ('man 2 splice') for more information 278 | about the arguments and return value. 279 | 280 | `off_in` and `off_out` can be `None`, which is equivalent to `NULL`. 281 | 282 | If the call to `splice` fails (i.e. returns -1), an `OSError` is raised 283 | with the appropriate `errno`, unless the error is `EINTR`, which results 284 | in the call to be retried. 285 | ''' 286 | 287 | c_off_in = \ 288 | ctypes.byref(c_loff_t(off_in)) if off_in is not None else None 289 | c_off_out = \ 290 | ctypes.byref(c_loff_t(off_out)) if off_out is not None else None 291 | 292 | # For handling EINTR... 293 | while True: 294 | res = c_splice(fd_in, c_off_in, fd_out, c_off_out, len_, flags) 295 | 296 | if res == -1: 297 | errno_ = ctypes.get_errno() 298 | 299 | # Try again on EINTR 300 | if errno_ == errno.EINTR: 301 | continue 302 | 303 | raise IOError(errno_, os.strerror(errno_)) 304 | 305 | return res 306 | 307 | return splice 308 | 309 | 310 | # Build and export wrapper 311 | splice = make_splice() #pylint: disable-msg=C0103 312 | del make_splice 313 | 314 | 315 | # From bits/fcntl.h 316 | # Values for 'flags', can be OR'ed together 317 | SPLICE_F_MOVE = 1 318 | SPLICE_F_NONBLOCK = 2 319 | SPLICE_F_MORE = 4 320 | SPLICE_F_GIFT = 8 321 | -------------------------------------------------------------------------------- /package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python setup.py bdist_egg --exclude-source-files 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | protobuf 2 | eventlet 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | 4 | [egg_info] 5 | #tag_build = dev 6 | #tag_svn_revision = true 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open('requirements.txt', 'r') as f: 4 | requires = [x.strip() for x in f if x.strip()] 5 | 6 | version = '0.9.1' 7 | 8 | setup( 9 | # overview 10 | name = 'kinetic', 11 | description = "Python client for Kinetic devices", 12 | 13 | # technical info 14 | version = version, 15 | packages=find_packages(exclude=['test']), 16 | requires = requires, 17 | install_requires=requires, 18 | 19 | # features 20 | entry_points = { 21 | 'console_scripts': [ 'kineticc = kinetic.cmd:main' ], 22 | }, 23 | 24 | # copyright 25 | author='Ignacio Corderi', 26 | license='LGPLv2.1', 27 | 28 | # more info 29 | url = 'https://github.com/Seagate/kinetic-py', 30 | 31 | # categorization 32 | keywords = ('kinetic protocol api storage key/value seagate'), 33 | classifiers = [ 34 | 'Development Status :: 4 - Beta', 35 | 'Intended Audience :: Developers', 36 | 'Intended Audience :: Information Technology', 37 | 'License :: OSI Approved :: Mozilla Public License, v. 2.0', 38 | 'Programming Language :: Python :: 2.7', 39 | 'Programming Language :: Python :: 2 :: Only', 40 | 'Topic :: Software Development :: Libraries :: Python Modules', 41 | ], 42 | ) 43 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | -------------------------------------------------------------------------------- /test/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import sys 20 | import unittest 21 | 22 | sys.argv.append('discover') 23 | 24 | unittest.TestProgram() 25 | -------------------------------------------------------------------------------- /test/base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import errno 20 | import functools 21 | import logging 22 | import os 23 | import subprocess 24 | import shutil 25 | import socket 26 | import tempfile 27 | import time 28 | import unittest 29 | 30 | from kinetic import Client 31 | from kinetic import buildRange 32 | 33 | KINETIC_JAR = os.environ.get('KINETIC_JAR') 34 | KINETIC_PORT = os.environ.get('KINETIC_PORT', 9123) 35 | KINETIC_HOST = os.environ.get('KINETIC_HOST', 'localhost') 36 | 37 | class SimulatorRuntimeError(RuntimeError): 38 | 39 | def __init__(self, stdout, stderr, returncode): 40 | super(SimulatorRuntimeError, self).__init__(stdout, stderr, returncode) 41 | # reopen file's in read mode 42 | self.stdout = open(stdout.name).read() 43 | self.stderr = open(stderr.name).read() 44 | self.returncode = returncode 45 | 46 | def __str__(self): 47 | return '\n'.join([ 48 | 'Simulator exited abnormally', 49 | 'STDOUT:\n%s' % self.stdout, 50 | 'STDERR:\n%s' % self.stderr, 51 | 'RETURNCODE: %s' % self.returncode 52 | ]) 53 | 54 | 55 | def _find_kinetic_jar(jar_path=None): 56 | if jar_path: 57 | jar_path = os.path.abspath(os.path.expanduser( 58 | os.path.expandvars(jar_path) 59 | )) 60 | else: 61 | jar_path = os.path.abspath( 62 | os.path.join( 63 | os.path.dirname(__file__), 64 | # /kinetic-py/test/. 65 | '../../kinetic-java/kinetic-simulator/target/', 66 | 'kinetic-simulator-0.6.0.3-SNAPSHOT-jar-with-dependencies.jar', 67 | ) 68 | ) 69 | if not os.path.exists(jar_path): 70 | if 'KINETIC_JAR' not in os.environ: 71 | raise KeyError('KINETIC_JAR environment variable is not set') 72 | else: 73 | msg = "%s: '%s'" % (os.strerror(errno.ENOENT), jar_path) 74 | raise IOError(errno.ENOENT, msg) 75 | return jar_path 76 | 77 | 78 | class BaseTestCase(unittest.TestCase): 79 | """ 80 | This is the BaseTestCase for running python test code against the 81 | Kinetic simulator. 82 | 83 | Each TestCase will independently verify a connection to a simulator 84 | running on localhost at the port defined by the environment variable 85 | KINETIC_PORT (default 9123) spawning an instance of the simulator if 86 | necessary. 87 | 88 | In the common case no simulator will currently be listening on port 9123 89 | and unless your override KINETIC_PORT in your environment before 90 | running tests - the TestCase will spawn an instance of the simulator using 91 | the system java runtime by pointing it at the .jar defined by the 92 | environment variable KINETIC_JAR. If KINETIC_JAR is not defined it 93 | will go looking for it relative to this file. 94 | 95 | If you want to connect to an instance of the simulator you already have 96 | running (or a development instance that hasn't yet been packaged in a jar) 97 | you can `set KINETIC_PORT=8123` (or wherever the server is). 98 | 99 | If the .jar is not readily locatable you will get an error and need to 100 | ensure that the KINETIC_JAR environment variable points to the real 101 | path for kinetic-simulator. 102 | 103 | """ 104 | 105 | @classmethod 106 | def _check_simulator(cls): 107 | if not cls.simulator: 108 | cls.datadir = tempfile.mkdtemp() 109 | cls.stdout = open(os.path.join(cls.datadir, 'simulator.log'), 'w') 110 | cls.stderr = open(os.path.join(cls.datadir, 'simulator.err'), 'w') 111 | args = ['java', '-jar', cls.jar_path, '-port', str(cls.port), '-home', cls.datadir] 112 | cls.simulator = subprocess.Popen(args, stdout=cls.stdout.fileno(), 113 | stderr=cls.stderr.fileno()) 114 | if cls.simulator.poll(): 115 | raise SimulatorRuntimeError(cls.stdout, cls.stderr, cls.simulator.returncode) 116 | 117 | @classmethod 118 | def setUpClass(cls): 119 | cls.client = None 120 | cls.baseKey = "tests/py/%s/" % cls.__name__ 121 | 122 | cls.port = int(KINETIC_PORT) 123 | cls.host = KINETIC_HOST 124 | cls.jar_path = _find_kinetic_jar(KINETIC_JAR) 125 | cls.datadir = None 126 | cls.simulator = None 127 | cls.stdout = cls.stderr = None 128 | try: 129 | backoff = 0.1 130 | while True: 131 | sock = socket.socket() 132 | try: 133 | sock.connect((cls.host, cls.port)) 134 | except socket.error: 135 | if backoff > 2: 136 | raise 137 | else: 138 | # k, we can connect 139 | sock.close() 140 | break 141 | cls._check_simulator() 142 | time.sleep(backoff) 143 | backoff *= 2 # double it! 144 | except: 145 | if hasattr(cls, 'stdout'): 146 | try: 147 | raise SimulatorRuntimeError(cls.stdout, cls.stderr, 148 | cls.simulator.returncode) 149 | except: 150 | # this is some dodgy shit to setup the re-raise at the bottom 151 | pass 152 | cls.tearDownClass() 153 | raise 154 | 155 | cls.client = Client(cls.host, cls.port) 156 | cls.client.connect() 157 | 158 | @classmethod 159 | def tearDownClass(cls): 160 | # remove all keys used by this test case 161 | r = buildRange(cls.baseKey) 162 | if cls.client: 163 | xs = cls.client.getRange(r.startKey, r.endKey, r.startKeyInclusive, r.endKeyInclusive) 164 | for x in xs: 165 | cls.client.delete(x.key, x.metadata.version) 166 | else: 167 | print 'WARNING: no cls.client' 168 | 169 | if cls.simulator: 170 | if cls.simulator.poll() is None: 171 | cls.simulator.terminate() 172 | cls.simulator.wait() 173 | [f.close() for f in (cls.stdout, cls.stderr) if f] 174 | if cls.datadir: 175 | shutil.rmtree(cls.datadir) 176 | 177 | def tearDown(self): 178 | r = buildRange(self.baseKey) 179 | xs = self.client.getRange(r.startKey, r.endKey, r.startKeyInclusive, r.endKeyInclusive) 180 | for x in xs: 181 | self.client.delete(x.key, x.metadata.version) 182 | 183 | def buildKey(self, n='test'): 184 | # self.id returns the name of the running test 185 | return self.baseKey + "%s/%s" % (self.id(), str(n)) 186 | 187 | @classmethod 188 | def debug_logging(cls, f): 189 | """ 190 | Decorator, enables logging for the test 191 | """ 192 | @functools.wraps(f) 193 | def wrapper(self, *args, **kwargs): 194 | logging.basicConfig(level=logging.DEBUG) 195 | logging.critical('DEBUG LOGGING FOR %s' % self.id()) 196 | try: 197 | logging.disable(level=logging.NOTSET) 198 | return f(self, *args, **kwargs) 199 | finally: 200 | logging.disable(level=logging.CRITICAL) 201 | return wrapper 202 | 203 | 204 | def start_simulators(jar_path, data_dir, *ports): 205 | sim_map = {} 206 | with open(os.devnull, 'w') as null: 207 | for port in ports: 208 | args = ['java', '-jar', jar_path, str(port), 209 | os.path.join(data_dir, str(port)), str(port + 443)] 210 | sim_map[port] = subprocess.Popen(args, stdout=null, stderr=null) 211 | time.sleep(1) 212 | connected = [] 213 | backoff = 0.1 214 | timeout = time.time() + 3 215 | while len(connected) < len(sim_map) and time.time() < timeout: 216 | for port in sim_map: 217 | if port in connected: 218 | continue 219 | sock = socket.socket() 220 | try: 221 | sock.connect(('localhost', port)) 222 | except socket.error: 223 | time.sleep(backoff) 224 | backoff *= 2 225 | else: 226 | connected.append(port) 227 | sock.close() 228 | if len(connected) < len(sim_map): 229 | teardown_simulators(sim_map) 230 | raise Exception('only able to connect to %r out of %r' % (connected, 231 | sim_map)) 232 | return sim_map 233 | 234 | 235 | def teardown_simulators(sim_map): 236 | for proc in sim_map.values(): 237 | try: 238 | proc.terminate() 239 | except OSError, e: 240 | if e.errno != errno.ESRCH: 241 | raise 242 | continue 243 | proc.wait() 244 | 245 | 246 | class MultiSimulatorTestCase(unittest.TestCase): 247 | 248 | PORTS = (9010, 9020) 249 | 250 | def setUp(self): 251 | self.test_dir = tempfile.mkdtemp() 252 | self.ports = self.PORTS 253 | self._sim_map = {} 254 | try: 255 | self._sim_map = start_simulators(_find_kinetic_jar(KINETIC_JAR), 256 | self.test_dir, *self.ports) 257 | self.client_map = {} 258 | for port in self.ports: 259 | self.client_map[port] = Client('localhost', port) 260 | self.client_map[port].connect() 261 | except Exception: 262 | self.tearDown() 263 | 264 | def tearDown(self): 265 | teardown_simulators(self._sim_map) 266 | shutil.rmtree(self.test_dir) 267 | 268 | def buildKey(self, key): 269 | return "tests/py/%s.%s/%s" % ( 270 | self.__class__.__name__, self.id(), str(key)) 271 | 272 | 273 | if __name__ == "__main__": 274 | # sanity 275 | print _find_kinetic_jar(KINETIC_JAR) 276 | print int(KINETIC_PORT) 277 | -------------------------------------------------------------------------------- /test/test_adminclient.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Robert Cope 18 | 19 | from kinetic import Client 20 | from kinetic import AdminClient 21 | from kinetic import KineticMessageException 22 | from base import BaseTestCase 23 | from kinetic import common 24 | from kinetic.common import KineticException 25 | import kinetic.kinetic_pb2 as messages 26 | 27 | 28 | class AdminClientTestCase(BaseTestCase): 29 | 30 | DEFAULT_CLUSTER_VERSION = 0 31 | MAX_KEY_SIZE = 4096 32 | MAX_VALUE_SIZE = 1024 * 1024 33 | MAX_VERSION_SIZE = 2048 34 | MAX_KEY_RANGE_COUNT = 200 35 | 36 | def setUp(self): 37 | super(AdminClientTestCase, self).setUp() 38 | self.adminClient = AdminClient(self.host, self.port) 39 | self.adminClient.connect() 40 | 41 | def tearDown(self): 42 | self.adminClient.close() 43 | 44 | def test_setSecurity(self): 45 | 46 | self.client.put(self.buildKey(1), "test_value_1") 47 | 48 | acl = common.ACL(identity=100) 49 | domain = common.Domain(roles=[common.Roles.READ]) 50 | acl.domains = [domain] 51 | 52 | if self.adminClient.use_ssl: 53 | self.adminClient.setSecurity([acl]) 54 | 55 | # Verify user 100 can only read 56 | read_only_client = Client(self.host, self.port, identity=100) 57 | read_only_client.get(self.buildKey(1)) # Should be OK. 58 | args = (self.buildKey(2), 'test_value_2') 59 | self.assertRaises(KineticMessageException, read_only_client.put, *args) 60 | else: 61 | try: 62 | #TODO: change this to self.assertRaises 63 | self.adminClient.setSecurity([acl]) 64 | except KineticException: 65 | pass 66 | else: 67 | self.fail('Exception should be thrown if not using SSL') 68 | 69 | def test_get_capacity(self): 70 | log = self.adminClient.getLog([messages.Command.GetLog.CAPACITIES]) 71 | self.assertIsNotNone(log) 72 | 73 | capacity = log.capacity 74 | self.assertIsNotNone(capacity) 75 | 76 | self.assertTrue(capacity.portionFull >= 0) 77 | self.assertTrue(capacity.nominalCapacityInBytes >= 0) 78 | 79 | def test_get_capacity_and_utilization(self): 80 | log = self.adminClient.getLog([messages.Command.GetLog.CAPACITIES, messages.Command.GetLog.UTILIZATIONS]) 81 | self.assertIsNotNone(log) 82 | 83 | capacity = log.capacity 84 | self.assertIsNotNone(capacity) 85 | 86 | self.assertTrue(capacity.portionFull >= 0) 87 | self.assertTrue(capacity.nominalCapacityInBytes >= 0) 88 | 89 | util_list = log.utilizations 90 | 91 | for util in util_list: 92 | self.assertTrue(util.value >= 0) 93 | 94 | def test_get_configuration(self): 95 | log = self.adminClient.getLog([messages.Command.GetLog.CONFIGURATION]) 96 | self.assertIsNotNone(log) 97 | 98 | configuration = log.configuration 99 | self.assertIsNotNone(configuration) 100 | 101 | self.assertTrue(len(configuration.compilationDate) > 0) 102 | self.assertTrue(len(configuration.model) > 0) 103 | self.assertTrue(configuration.port >= 0) 104 | self.assertTrue(configuration.tlsPort >= 0) 105 | self.assertTrue(len(configuration.serialNumber) > 0) 106 | self.assertTrue(len(configuration.sourceHash) > 0) 107 | self.assertTrue(len(configuration.vendor) > 0) 108 | self.assertTrue(len(configuration.version) > 0) 109 | 110 | for interface in configuration.interface: 111 | self.assertTrue(len(interface.name) > 0) 112 | 113 | def test_get_limits(self): 114 | log = self.adminClient.getLog([messages.Command.GetLog.LIMITS]) 115 | self.assertIsNotNone(log) 116 | 117 | limits = log.limits 118 | self.assertIsNotNone(limits) 119 | 120 | self.assertTrue(limits.maxKeySize == AdminClientTestCase.MAX_KEY_SIZE) 121 | self.assertTrue(limits.maxValueSize == AdminClientTestCase.MAX_VALUE_SIZE) 122 | self.assertTrue(limits.maxVersionSize == AdminClientTestCase.MAX_VERSION_SIZE) 123 | self.assertTrue(limits.maxKeyRangeCount == AdminClientTestCase.MAX_KEY_RANGE_COUNT) 124 | 125 | def test_get_log(self): 126 | #TODO: is there a way to specify all types without explicitly enumerating them all? 127 | log = self.adminClient.getLog([messages.Command.GetLog.TEMPERATURES, messages.Command.GetLog.UTILIZATIONS, 128 | messages.Command.GetLog.STATISTICS, messages.Command.GetLog.MESSAGES, 129 | messages.Command.GetLog.CAPACITIES, messages.Command.GetLog.LIMITS]) 130 | self.assertIsNotNone(log) 131 | 132 | self.assertTrue(len(log.temperatures) > 0) 133 | self.assertTrue(len(log.utilizations) > 0) 134 | self.assertTrue(len(log.statistics) > 0) 135 | self.assertTrue(log.messages > 0) 136 | self.assertTrue(log.capacity.portionFull >= 0) 137 | self.assertTrue(log.capacity.nominalCapacityInBytes >= 0) 138 | self.assertTrue(log.limits.maxKeySize == AdminClientTestCase.MAX_KEY_SIZE) 139 | self.assertTrue(log.limits.maxValueSize == AdminClientTestCase.MAX_VALUE_SIZE) 140 | self.assertTrue(log.limits.maxVersionSize == AdminClientTestCase.MAX_VERSION_SIZE) 141 | self.assertTrue(log.limits.maxKeyRangeCount == AdminClientTestCase.MAX_KEY_RANGE_COUNT) 142 | 143 | def test_get_temperature(self): 144 | log = self.adminClient.getLog([messages.Command.GetLog.TEMPERATURES]) 145 | self.assertIsNotNone(log) 146 | 147 | for temperature in log.temperatures: 148 | self.assertTrue(temperature.current >= 0) 149 | self.assertTrue(temperature.maximum >= 0) 150 | 151 | def test_get_temperature_and_capacity(self): 152 | log = self.adminClient.getLog([messages.Command.GetLog.TEMPERATURES, messages.Command.GetLog.CAPACITIES]) 153 | self.assertIsNotNone(log) 154 | 155 | for temperature in log.temperatures: 156 | self.assertTrue(temperature.current >= 0) 157 | self.assertTrue(temperature.maximum >= 0) 158 | 159 | capacity = log.capacity 160 | self.assertIsNotNone(capacity) 161 | 162 | self.assertTrue(capacity.portionFull >= 0) 163 | self.assertTrue(capacity.nominalCapacityInBytes >= 0) 164 | 165 | def test_get_temperature_and_capacity_and_utilization(self): 166 | log = self.adminClient.getLog([messages.Command.GetLog.TEMPERATURES, messages.Command.GetLog.CAPACITIES, 167 | messages.Command.GetLog.UTILIZATIONS]) 168 | self.assertIsNotNone(log) 169 | 170 | for temperature in log.temperatures: 171 | self.assertTrue(temperature.current >= 0) 172 | self.assertTrue(temperature.maximum >= 0) 173 | 174 | capacity = log.capacity 175 | self.assertIsNotNone(capacity) 176 | 177 | self.assertTrue(capacity.portionFull >= 0) 178 | self.assertTrue(capacity.nominalCapacityInBytes >= 0) 179 | 180 | util_list = log.utilizations 181 | 182 | for util in util_list: 183 | self.assertTrue(util.value >= 0) 184 | 185 | def test_get_temperature_and_utilization(self): 186 | log = self.adminClient.getLog([messages.Command.GetLog.TEMPERATURES, messages.Command.GetLog.UTILIZATIONS]) 187 | self.assertIsNotNone(log) 188 | 189 | for temperature in log.temperatures: 190 | self.assertTrue(temperature.current >= 0) 191 | self.assertTrue(temperature.maximum >= 0) 192 | 193 | util_list = log.utilizations 194 | 195 | for util in util_list: 196 | self.assertTrue(util.value >= 0) 197 | 198 | def test_get_utilization(self): 199 | log = self.adminClient.getLog([messages.Command.GetLog.UTILIZATIONS]) 200 | self.assertIsNotNone(log) 201 | 202 | util_list = log.utilizations 203 | 204 | for util in util_list: 205 | self.assertTrue(util.value >= 0) 206 | 207 | def reset_cluster_version_to_default(self): 208 | c = AdminClient(self.host, self.port) 209 | c.setClusterVersion(AdminClientTestCase.DEFAULT_CLUSTER_VERSION) 210 | c.close() 211 | 212 | def test_set_cluster_version(self): 213 | new_cluster_version = AdminClientTestCase.DEFAULT_CLUSTER_VERSION + 1 214 | self.adminClient.setClusterVersion(new_cluster_version) 215 | self.reset_cluster_version_to_default() 216 | 217 | def test_update_firmware(self): 218 | #TODO: implement test_update_firmware 219 | pass 220 | 221 | def test_unlock(self): 222 | #TODO: implement test_unlock 223 | if self.adminClient.use_ssl: 224 | pass 225 | else: 226 | self.assertRaises(KineticException) 227 | 228 | def test_lock(self): 229 | #TODO: implement test_lock 230 | if self.adminClient.use_ssl: 231 | pass 232 | else: 233 | self.assertRaises(KineticException) 234 | 235 | def test_erase(self): 236 | #TODO: implement test_erase 237 | if self.adminClient.use_ssl: 238 | pass 239 | else: 240 | self.assertRaises(KineticException) 241 | 242 | def test_instance_secure_erase(self): 243 | #TODO: implement test_instance_secure_erase 244 | if self.adminClient.use_ssl: 245 | pass 246 | else: 247 | self.assertRaises(KineticException) 248 | 249 | def test_set_erase_pin(self): 250 | #TODO: implement test_set_erase_pin 251 | if self.adminClient.use_ssl: 252 | pass 253 | else: 254 | self.assertRaises(KineticException) 255 | 256 | def test_set_lock_pin(self): 257 | #TODO: implement test_set_lock_pin 258 | if self.adminClient.use_ssl: 259 | pass 260 | else: 261 | self.assertRaises(KineticException) 262 | 263 | def test_set_acl(self): 264 | #TODO: implement test_set_acl 265 | if self.adminClient.use_ssl: 266 | pass 267 | else: 268 | self.assertRaises(KineticException) 269 | 270 | -------------------------------------------------------------------------------- /test/test_batch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Paul Dardeau 18 | 19 | import unittest 20 | 21 | from kinetic import Client 22 | from kinetic import batch 23 | from kinetic import common 24 | from base import BaseTestCase 25 | 26 | 27 | class BatchTestCase(BaseTestCase): 28 | 29 | def setUp(self): 30 | super(BatchTestCase, self).setUp() 31 | self.client = Client(self.host, self.port) 32 | self.client.connect() 33 | self._create_new_batch() 34 | 35 | def _create_new_batch(self): 36 | self.batch = self.client.begin_batch() 37 | if self.batch is None: 38 | raise common.KineticException("unable to create batch") 39 | 40 | def test_batch_initial_state(self): 41 | is_completed = self.batch.is_completed() 42 | op_count = len(self.batch) 43 | self.batch.abort() 44 | self.assertFalse(is_completed) 45 | self.assertEquals(op_count, 0) 46 | 47 | def test_batch_operation_count(self): 48 | key1 = self.buildKey('test_batch_operation_count_1') 49 | key2 = self.buildKey('test_batch_operation_count_2') 50 | key3 = self.buildKey('test_batch_operation_count_3') 51 | self.batch.put(key1, '') 52 | self.assertEquals(len(self.batch), 1) 53 | self.batch.put(key2, '') 54 | self.assertEquals(len(self.batch), 2) 55 | self.batch.delete(key3) 56 | self.assertEquals(len(self.batch), 3) 57 | 58 | self.batch.abort() 59 | 60 | def test_batch_commit_is_completed(self): 61 | key1 = self.buildKey('test_batch_commit_is_completed_1') 62 | key2 = self.buildKey('test_batch_commit_is_completed_2') 63 | self.assertFalse(self.batch.is_completed()) 64 | self.batch.put(key1, '') 65 | self.batch.delete(key2) 66 | self.assertFalse(self.batch.is_completed()) 67 | self.batch.commit() 68 | self.assertTrue(self.batch.is_completed()) 69 | 70 | def test_batch_abort_is_completed(self): 71 | key1 = self.buildKey('test_batch_abort_is_completed_1') 72 | key2 = self.buildKey('test_batch_abort_is_completed_2') 73 | self.assertFalse(self.batch.is_completed()) 74 | self.batch.put(key1, '') 75 | self.batch.delete(key2) 76 | self.assertFalse(self.batch.is_completed()) 77 | self.batch.abort() 78 | self.assertTrue(self.batch.is_completed()) 79 | 80 | def test_empty_batch_abort(self): 81 | # abort with no operations in batch 82 | self.assertRaises(common.BatchAbortedException, self.batch.abort()) 83 | 84 | def test_batch_abort(self): 85 | key = self.buildKey('key_should_not_exist') 86 | self.batch.put(key, '') 87 | self.batch.abort() 88 | self.assertEqual(self.client.get(key), None) 89 | 90 | def test_empty_batch_commit(self): 91 | # commit with no operations in batch 92 | self.assertRaises(common.BatchAbortedException, self.batch.commit()) 93 | 94 | def test_batch_commit(self): 95 | key = self.buildKey('key_should_exist') 96 | self.batch.put(key, '') 97 | self.batch.commit() 98 | self.assertIsNotNone(self.client.get(key)) 99 | 100 | def test_batch_delete_commit(self): 101 | # put an entry 102 | key = self.buildKey('test_batch_delete_commit') 103 | self.client.put(key, '') 104 | 105 | self.batch.delete(key) 106 | self.batch.commit() 107 | self.assertEqual(self.client.get(key), None) 108 | 109 | def test_batch_delete_abort(self): 110 | # put an entry 111 | key = self.buildKey('test_batch_delete_abort') 112 | self.client.put(key, '') 113 | 114 | self.batch.delete(key) 115 | self.batch.abort() 116 | self.assertIsNotNone(self.client.get(key)) 117 | 118 | def test_batch_put_commit(self): 119 | key = self.buildKey('test_batch_put_commit') 120 | self.batch.put(key, '') 121 | self.batch.commit() 122 | self.assertIsNotNone(self.client.get(key)) 123 | 124 | def test_batch_put_abort(self): 125 | key = self.buildKey('test_batch_put_abort') 126 | self.batch.put(key, '') 127 | self.batch.abort() 128 | self.assertEqual(self.client.get(key), None) 129 | 130 | def test_batch_multiple_put_commit(self): 131 | key1 = self.buildKey('test_batch_multiple_put_commit_1') 132 | key2 = self.buildKey('test_batch_multiple_put_commit_2') 133 | self.batch.put(key1, '') 134 | self.batch.put(key2, '') 135 | self.batch.commit() 136 | self.assertIsNotNone(self.client.get(key1)) 137 | self.assertIsNotNone(self.client.get(key2)) 138 | 139 | def test_batch_multiple_put_abort(self): 140 | key1 = self.buildKey('test_batch_multiple_put_abort_1') 141 | key2 = self.buildKey('test_batch_multiple_put_abort_2') 142 | self.batch.put(key1, '') 143 | self.batch.put(key2, '') 144 | self.batch.abort() 145 | self.assertEqual(self.client.get(key1), None) 146 | self.assertEqual(self.client.get(key2), None) 147 | 148 | def test_batch_multiple_delete_commit(self): 149 | key1 = self.buildKey('test_batch_multiple_delete_commit_1') 150 | key2 = self.buildKey('test_batch_multiple_delete_commit_2') 151 | self.client.put(key1, '') 152 | self.client.put(key2, '') 153 | 154 | self.batch.delete(key1) 155 | self.batch.delete(key2) 156 | self.batch.commit() 157 | self.assertEqual(self.client.get(key1), None) 158 | self.assertEqual(self.client.get(key2), None) 159 | 160 | def test_batch_multiple_delete_abort(self): 161 | key1 = self.buildKey('test_batch_multiple_delete_abort_1') 162 | key2 = self.buildKey('test_batch_multiple_delete_abort_2') 163 | self.client.put(key1, '') 164 | self.client.put(key2, '') 165 | 166 | self.batch.delete(key1) 167 | self.batch.delete(key2) 168 | self.batch.abort() 169 | self.assertIsNotNone(self.client.get(key1)) 170 | self.assertIsNotNone(self.client.get(key2)) 171 | 172 | def test_batch_mixed_commit(self): 173 | key1 = self.buildKey('test_batch_mixed_commit_1') 174 | key2 = self.buildKey('test_batch_mixed_commit_2') 175 | self.client.put(key1, '') 176 | 177 | self.batch.delete(key1) 178 | self.batch.put(key2, '') 179 | self.batch.commit() 180 | self.assertEqual(self.client.get(key1), None) 181 | self.assertIsNotNone(self.client.get(key2)) 182 | 183 | def test_batch_mixed_abort(self): 184 | key1 = self.buildKey('test_batch_mixed_abort_1') 185 | key2 = self.buildKey('test_batch_mixed_abort_2') 186 | self.client.put(key1, '') 187 | 188 | self.batch.delete(key1) 189 | self.batch.put(key2, '') 190 | self.batch.abort() 191 | self.assertIsNotNone(self.client.get(key1)) 192 | self.assertEqual(self.client.get(key2), None) 193 | 194 | def test_batch_reuse_after_commit(self): 195 | key1 = self.buildKey('test_batch_reuse_after_commit_1') 196 | key2 = self.buildKey('test_batch_reuse_after_commit_2') 197 | self.batch.put(key1, '') 198 | self.batch.commit() 199 | 200 | args = (key2, '') 201 | self.assertRaises(common.BatchCompletedException, self.batch.put, *args) 202 | 203 | def test_batch_reuse_after_abort(self): 204 | key = self.buildKey('test_batch_reuse_after_abort') 205 | self.batch.put(key, '') 206 | self.batch.abort() 207 | 208 | args = (key) 209 | self.assertRaises(common.BatchCompletedException, self.batch.delete, *args) 210 | 211 | 212 | if __name__ == '__main__': 213 | unittest.main() 214 | 215 | -------------------------------------------------------------------------------- /test/test_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import unittest 20 | 21 | from kinetic import Client 22 | from kinetic import KeyRange 23 | from kinetic import KineticMessageException 24 | from base import BaseTestCase 25 | from kinetic import common 26 | 27 | class KineticBasicTestCase(BaseTestCase): 28 | 29 | def setUp(self): 30 | super(KineticBasicTestCase, self).setUp() 31 | self.client = Client(self.host, self.port) 32 | self.client.connect() 33 | 34 | def test_command_put(self): 35 | self.client.put(self.buildKey(0),"test_value") 36 | 37 | def test_put_no_version_overwrite(self): 38 | key = self.buildKey() 39 | self.client.put(key, 'value') 40 | self.assertEquals('value', self.client.get(key).value) 41 | self.client.put(key, 'value1') 42 | self.assertEquals('value1', self.client.get(key).value) 43 | 44 | def test_put_no_overwrite_with_version(self): 45 | key = self.buildKey() 46 | self.client.put(key, 'value', new_version='0') 47 | self.assertEquals('value', self.client.get(key).value) 48 | # can't set it w/o any version 49 | args = (key, 'value1') 50 | self.assertRaises(KineticMessageException, self.client.put, *args) 51 | # can't set new version 52 | kwargs = dict(new_version='1') 53 | self.assertRaises(KineticMessageException, self.client.put, 54 | *args, **kwargs) 55 | # still has orig value 56 | self.assertEquals('value', self.client.get(key).value) 57 | # can overwrite with correct version 58 | self.client.put(key, 'value1', version='0') 59 | self.assertEquals('value1', self.client.get(key).value) 60 | 61 | def test_put_force_version_overwrite(self): 62 | key = self.buildKey() 63 | self.client.put(key, 'value', new_version='0') 64 | self.assertEquals('value', self.client.get(key).value) 65 | self.client.put(key, 'value1', force=True) 66 | self.assertEquals('value1', self.client.get(key).value) 67 | 68 | def test_command_get(self): 69 | self.client.get(self.buildKey(0)) 70 | 71 | def test_command_getMetadata(self): 72 | self.client.getMetadata(self.buildKey(0)) 73 | 74 | def test_command_getNext(self): 75 | self.client.put(self.buildKey(1),"test_value_1") 76 | self.client.put(self.buildKey(2),"test_value_2") 77 | self.client.getNext(self.buildKey(1)) 78 | 79 | def test_command_getPrevious(self): 80 | self.client.put(self.buildKey(1),"test_value_1") 81 | self.client.put(self.buildKey(2),"test_value_2") 82 | self.client.getPrevious(self.buildKey(2)) 83 | 84 | def test_command_delete(self): 85 | self.client.delete(self.buildKey(0)) 86 | 87 | def test_delete_existing(self): 88 | self.client.put(self.buildKey(0),"test_value") 89 | x = self.client.delete(self.buildKey(0)) 90 | self.assertTrue(x) 91 | 92 | def test_delete_non_existing(self): 93 | x = self.client.delete(self.baseKey + "none_existing_test_key") 94 | self.assertFalse(x) 95 | #selft.assertRaises(KeyNotFound) 96 | 97 | def test_put_getMetadata(self): 98 | value = "test_value" 99 | self.client.put(self.buildKey(0),value,new_version="20") 100 | x = self.client.getMetadata(self.buildKey(0)) 101 | self.assertEqual(x.metadata.version, "20") 102 | 103 | def test_put_get(self): 104 | value = "test_value" 105 | self.client.put(self.buildKey(0),value) 106 | x = self.client.get(self.buildKey(0)) 107 | self.assertEqual(value, x.value) 108 | 109 | def test_put_get_withVersion(self): 110 | value = "test_value" 111 | self.client.put(self.buildKey(0),value, new_version="1") 112 | x = self.client.get(self.buildKey(0)) 113 | self.assertEqual(x.value, value) 114 | self.assertEqual(x.metadata.version, "1") 115 | 116 | def test_delete_nonExistent(self): 117 | deleted = self.client.delete(self.buildKey(0)) 118 | self.assertEqual(deleted, False) 119 | 120 | def test_put_delete_incorrectVersion(self): 121 | value = "test_value" 122 | self.client.put(self.buildKey(0),value, new_version="1") 123 | with self.assertRaises(KineticMessageException): 124 | self.client.delete(self.buildKey(0), "2") 125 | x = self.client.get(self.buildKey(0)) 126 | self.assertEqual(x.value, value) 127 | 128 | def test_put_delete_get_withVersion(self): 129 | self.client.put(self.buildKey(0),"test_value", new_version="1") 130 | deleted = self.client.delete(self.buildKey(0), "1") 131 | self.assertEqual(deleted, True) 132 | x = self.client.get(self.buildKey(0)) 133 | self.assertEqual(x, None) 134 | 135 | def test_put_delete_get(self): 136 | self.client.put(self.buildKey(0),"test_value") 137 | deleted = self.client.delete(self.buildKey(0)) 138 | self.assertEqual(deleted, True) 139 | x = self.client.get(self.buildKey(0)) 140 | self.assertEqual(x, None) 141 | 142 | def test_getNext(self): 143 | self.client.put(self.buildKey(1),"test_value_1") 144 | yKey = self.buildKey(2) 145 | yValue = "test_value_2" 146 | self.client.put(yKey, yValue) 147 | x = self.client.getNext(self.buildKey(1)) 148 | self.assertEqual(x.key, yKey) 149 | self.assertEqual(x.value, yValue) 150 | 151 | def test_getPrevious(self): 152 | xKey = self.buildKey(1) 153 | xValue = "test_value_1" 154 | self.client.put(xKey,xValue) 155 | self.client.put(self.buildKey(2),"test_value_2") 156 | y = self.client.getPrevious(self.buildKey(2)) 157 | self.assertEqual(xKey, y.key) 158 | self.assertEqual(xValue, y.value) 159 | 160 | def setUpRangeTests(self): 161 | self.client.put(self.buildKey(1),"test_value_1") 162 | self.client.put(self.buildKey(2),"test_value_2") 163 | self.client.put(self.buildKey(3),"test_value_3") 164 | self.client.put(self.buildKey(4),"test_value_4") 165 | self.client.put(self.buildKey(5),"test_value_5") 166 | self.client.put(self.buildKey(6),"test_value_6") 167 | self.client.put(self.buildKey(7),"test_value_7") 168 | self.client.put(self.buildKey(8),"test_value_8") 169 | self.client.put(self.buildKey(9),"test_value_9") 170 | 171 | def test_getKeyRange_default(self): 172 | self.setUpRangeTests() 173 | expected = [self.buildKey(3),self.buildKey(4),self.buildKey(5),self.buildKey(6)] 174 | xs = self.client.getKeyRange(self.buildKey(3),self.buildKey(6)) 175 | self.assertEqual(xs, expected) 176 | 177 | def test_getKeyRange_empty(self): 178 | self.setUpRangeTests() 179 | xs = self.client.getKeyRange(self.baseKey + "nonexisting_test_key_1",self.baseKey + "nonexisting_test_key_99") 180 | self.assertEqual(xs, []) 181 | 182 | def test_getKeyRange_exclusiveStart(self): 183 | self.setUpRangeTests() 184 | expected = [self.buildKey(4),self.buildKey(5),self.buildKey(6)] 185 | xs = self.client.getKeyRange(self.buildKey(3),self.buildKey(6),False,True) 186 | self.assertEqual(xs, expected) 187 | 188 | def test_getKeyRange_exclusiveEnd(self): 189 | self.setUpRangeTests() 190 | expected = [self.buildKey(3),self.buildKey(4),self.buildKey(5)] 191 | xs = self.client.getKeyRange(self.buildKey(3),self.buildKey(6),True,False) 192 | self.assertEqual(xs, expected) 193 | 194 | def test_getKeyRange_bounded(self): 195 | self.setUpRangeTests() 196 | expected = [self.buildKey(4),self.buildKey(5)] 197 | xs = self.client.getKeyRange(self.buildKey(4),self.buildKey(9),maxReturned=2) 198 | self.assertEqual(xs, expected) 199 | 200 | def test_getRange_default(self): 201 | self.setUpRangeTests() 202 | expectedKeys = [self.buildKey(1),self.buildKey(2),self.buildKey(3),self.buildKey(4),self.buildKey(5),self.buildKey(6)] 203 | expectedValues = ["test_value_1","test_value_2","test_value_3","test_value_4","test_value_5","test_value_6"] 204 | xs = self.client.getRange(self.buildKey(1),self.buildKey(6)) 205 | i = 0 206 | for x in xs : 207 | self.assertEqual(x.key, expectedKeys[i]) 208 | self.assertEqual(x.value, expectedValues[i]) 209 | i += 1 210 | 211 | def test_value_too_big(self): 212 | self.assertRaises(common.KineticClientException, self.client.put, self.buildKey(1), 'x' * (common.MAX_VALUE_SIZE + 1)) 213 | 214 | def test_key_too_big(self): 215 | self.assertRaises(common.KineticClientException, self.client.put, self.buildKey('x' * (common.MAX_KEY_SIZE + 1)), 'y') 216 | 217 | def test_noop(self): 218 | self.client.noop() 219 | 220 | if __name__ == '__main__': 221 | unittest.main() 222 | -------------------------------------------------------------------------------- /test/test_cmd.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Ignacio Corderi 18 | 19 | import contextlib 20 | import StringIO 21 | import sys 22 | import unittest 23 | 24 | from kinetic import client 25 | from kinetic import cmd 26 | 27 | from base import BaseTestCase 28 | 29 | 30 | class BaseCommandTestCase(BaseTestCase): 31 | 32 | def setUp(self): 33 | super(BaseCommandTestCase, self).setUp() 34 | self.test_key = self.buildKey('test') 35 | self.client = client.Client(self.host, self.port) 36 | self.client.connect() 37 | self.conn_args = '-H %s -P %s ' % (self.host, self.port) 38 | 39 | @contextlib.contextmanager 40 | def capture_stdout(self): 41 | _orig_stdout = sys.stdout 42 | sys.stdout = StringIO.StringIO() 43 | try: 44 | yield sys.stdout 45 | finally: 46 | sys.stdout = _orig_stdout 47 | 48 | @contextlib.contextmanager 49 | def capture_stdio(self): 50 | _orig_stdout = sys.stdout 51 | sys.stdout = StringIO.StringIO() 52 | _orig_stderr = sys.stderr 53 | sys.stderr = StringIO.StringIO() 54 | try: 55 | yield sys.stdout, sys.stderr 56 | finally: 57 | sys.stdout = _orig_stdout 58 | sys.stderr = _orig_stderr 59 | 60 | def run_cmd(self, args): 61 | with self.capture_stdout() as stdout: 62 | errorcode = cmd.main(self.conn_args + args) 63 | output = stdout.getvalue() 64 | return errorcode, output 65 | 66 | class TestCommand(BaseCommandTestCase): 67 | 68 | def test_command_put(self): 69 | # make sure there's nothing there 70 | value = self.client.get(self.test_key) 71 | self.assert_(value is None) 72 | # add something from the command line 73 | args = 'put %s myvalue' % self.test_key 74 | errorcode = cmd.main(self.conn_args + args) 75 | # returns no error 76 | self.assertFalse(errorcode) 77 | # validate key is set 78 | entry = self.client.get(self.test_key) 79 | self.assertEquals('myvalue', entry.value) 80 | 81 | def test_command_get(self): 82 | # make sure there's nothing there 83 | value = self.client.get(self.test_key) 84 | self.assert_(value is None) 85 | # try to read value from command line 86 | args = 'get %s' % self.test_key 87 | with self.capture_stdout() as stdout: 88 | errorcode = cmd.main(self.conn_args + args) 89 | output = stdout.getvalue() 90 | # returns error and no output 91 | self.assert_(errorcode) 92 | self.assertEquals('', output) 93 | # put something there 94 | self.client.put(self.test_key, 'myvalue') 95 | # try to read value from command line 96 | args = 'get %s' % self.test_key 97 | with self.capture_stdout() as stdout: 98 | errorcode = cmd.main(self.conn_args + args) 99 | output = stdout.getvalue() 100 | # returns no error and value 101 | self.assertFalse(errorcode) 102 | self.assertEquals('myvalue\n', output) 103 | # and the data is in fact there 104 | entry = self.client.get(self.test_key) 105 | self.assertEquals('myvalue', entry.value) 106 | 107 | def test_command_delete(self): 108 | # make sure there's nothing there 109 | value = self.client.get(self.test_key) 110 | self.assert_(value is None) 111 | # try to remove key from command line 112 | args = 'delete %s' % self.test_key 113 | with self.capture_stdout() as stdout: 114 | errorcode = cmd.main(self.conn_args + args) 115 | output = stdout.getvalue() 116 | # returns error and no output 117 | self.assert_(errorcode) 118 | self.assertEquals('', output) 119 | # put something there 120 | self.client.put(self.test_key, 'myvalue') 121 | # try to remove key from command line 122 | args = 'delete %s' % self.test_key 123 | with self.capture_stdout() as stdout: 124 | errorcode = cmd.main(self.conn_args + args) 125 | output = stdout.getvalue() 126 | # returns no error and no output 127 | self.assertFalse(errorcode) 128 | self.assertEquals('', output) 129 | # and the data is in fact removed 130 | value = self.client.get(self.test_key) 131 | self.assert_(value is None) 132 | 133 | def test_command_list(self): 134 | # range will include test_key 135 | start = self.test_key[:-1] 136 | end = self.test_key + 'END' 137 | # range starts empty 138 | key_list = self.client.getKeyRange(start, end) 139 | self.assertEquals([], key_list) 140 | # validate empty from the command line 141 | args = 'list %s %s' % (start, end) 142 | with self.capture_stdout() as stdout: 143 | errorcode = cmd.main(self.conn_args + args) 144 | output = stdout.getvalue() 145 | # returns no error and no output 146 | self.assertFalse(errorcode) 147 | self.assertEquals('', output) 148 | # add test_key 149 | self.client.put(self.test_key, 'myvalue') 150 | # validate list from the command line 151 | args = 'list %s %s' % (start, end) 152 | with self.capture_stdout() as stdout: 153 | errorcode = cmd.main(self.conn_args + args) 154 | output = stdout.getvalue() 155 | # returns no error and list of keys 156 | self.assertFalse(errorcode) 157 | self.assertEquals('%s\n' % self.test_key, output) 158 | 159 | def test_list_prefix(self): 160 | # because the cmd output uses text/line based delimiters it's hard to 161 | # reason about keynames with a new line in them in this test 162 | bad_characters = [ord(c) for c in ('\n', '\r')] 163 | keys = [self.test_key + chr(ord_) for ord_ in range(200) if ord_ not 164 | in bad_characters] 165 | for i, key in enumerate(keys): 166 | self.client.put(key, 'myvalue.%s' % i) 167 | args = 'list %s' % self.test_key 168 | with self.capture_stdout() as stdout: 169 | errorcode = cmd.main(self.conn_args + args) 170 | output = stdout.getvalue() 171 | # returns no error and list of keys 172 | self.assertFalse(errorcode) 173 | output_keys = output.splitlines() 174 | self.assertEquals(len(keys), len(output_keys)) 175 | self.assertEquals(keys, output_keys) 176 | # add the prefix key 177 | self.client.put(self.test_key, 'mystart') 178 | with self.capture_stdout() as stdout: 179 | errorcode = cmd.main(self.conn_args + args) 180 | output = stdout.getvalue() 181 | # returns no error and list of keys 182 | self.assertFalse(errorcode) 183 | output_keys = output.splitlines() 184 | self.assertEquals(len(keys) + 1, len(output_keys)) 185 | self.assertEquals(self.test_key, output_keys[0]) 186 | # add something just "after" the prefix 187 | end_key = self.test_key[-1] + chr(ord(self.test_key[-1]) + 1) 188 | self.client.put(end_key, 'myend') 189 | with self.capture_stdout() as stdout: 190 | errorcode = cmd.main(self.conn_args + args) 191 | output = stdout.getvalue() 192 | # returns no error and list of keys 193 | self.assertFalse(errorcode) 194 | output_keys = output.splitlines() 195 | self.assertEquals(len(keys) + 1, len(output_keys)) 196 | self.assert_(end_key not in output_keys) 197 | 198 | def test_command_next(self): 199 | # make sure there's nothing there 200 | value = self.client.get(self.test_key) 201 | self.assert_(value is None) 202 | # try to read value from command line 203 | args = 'next %s' % self.test_key[:-1] 204 | with self.capture_stdout() as stdout: 205 | errorcode = cmd.main(self.conn_args + args) 206 | output = stdout.getvalue() 207 | # if simulator is empty, there's no output 208 | if not output: 209 | # returns error and no output 210 | self.assertEquals('', output) 211 | self.assertTrue(errorcode) 212 | else: 213 | self.assertFalse(errorcode) 214 | # put something there 215 | self.client.put(self.test_key, 'myvalue') 216 | # try a short offset to value from command line 217 | args = 'next %s' % self.test_key[:-1] 218 | with self.capture_stdout() as stdout: 219 | errorcode = cmd.main(self.conn_args + args) 220 | output = stdout.getvalue() 221 | # returns no error and data 222 | self.assertFalse(errorcode) 223 | self.assertEquals('myvalue\n', output) 224 | # try a longer offset to value from command line 225 | args = 'next %s' % self.test_key[0] 226 | with self.capture_stdout() as stdout: 227 | errorcode = cmd.main(self.conn_args + args) 228 | output = stdout.getvalue() 229 | # returns no error and data 230 | self.assertFalse(errorcode) 231 | self.assertEquals('myvalue\n', output) 232 | # try a next right on top of value from command line 233 | args = 'next %s' % self.test_key 234 | with self.capture_stdout() as stdout: 235 | errorcode = cmd.main(self.conn_args + args) 236 | output = stdout.getvalue() 237 | # if simulator is empty, there's no output 238 | if not output: 239 | # returns error and no output 240 | self.assert_(errorcode) 241 | self.assertEquals('', output) 242 | else: 243 | self.assertFalse(errorcode) 244 | # and past value from command line 245 | args = 'next %sXXX' % self.test_key 246 | with self.capture_stdout() as stdout: 247 | errorcode = cmd.main(self.conn_args + args) 248 | output = stdout.getvalue() 249 | # if simulator is empty, there's no output 250 | if not output: 251 | # returns error and no output 252 | self.assert_(errorcode) 253 | self.assertEquals('', output) 254 | else: 255 | self.assertFalse(errorcode) 256 | 257 | def test_command_next_verbose(self): 258 | # put something there 259 | self.client.put(self.test_key, 'myvalue') 260 | # try a short offset to value from command line 261 | args = '-vb next %s' % self.test_key[:-1] 262 | with self.capture_stdio() as stdio: 263 | errorcode = cmd.main(self.conn_args + args) 264 | stdout, stderr = stdio 265 | output = stdout.getvalue() 266 | verbose = stderr.getvalue() 267 | # returns no error and data 268 | self.assertFalse(errorcode) 269 | self.assertEquals('key: %s\n' % self.test_key, verbose) 270 | self.assertEquals('myvalue\n', output) 271 | 272 | def test_command_prev(self): 273 | # make sure there's nothing there 274 | value = self.client.get(self.test_key) 275 | self.assert_(value is None) 276 | # try to read value from command line 277 | args = 'prev %sXXX' % self.test_key 278 | with self.capture_stdout() as stdout: 279 | errorcode = cmd.main(self.conn_args + args) 280 | output = stdout.getvalue() 281 | # if simulator is empty, there's no output 282 | if not output: 283 | # returns error and no output 284 | self.assertEquals('', output) 285 | self.assertTrue(errorcode) 286 | else: 287 | self.assertFalse(errorcode) 288 | # put something there 289 | self.client.put(self.test_key, 'myvalue') 290 | # try a short offset to value from command line 291 | args = 'prev %sXXX' % self.test_key 292 | with self.capture_stdout() as stdout: 293 | errorcode = cmd.main(self.conn_args + args) 294 | output = stdout.getvalue() 295 | # returns no error and data 296 | self.assertFalse(errorcode) 297 | self.assertEquals('myvalue\n', output) 298 | # try a longer offset to value from command line 299 | args = 'prev %s~' % self.test_key.rsplit('/', 1)[0] 300 | with self.capture_stdout() as stdout: 301 | errorcode = cmd.main(self.conn_args + args) 302 | output = stdout.getvalue() 303 | # returns no error and data 304 | self.assertFalse(errorcode) 305 | self.assertEquals('myvalue\n', output) 306 | # try a prev right on top of value from command line 307 | args = 'prev %s' % self.test_key 308 | with self.capture_stdout() as stdout: 309 | errorcode = cmd.main(self.conn_args + args) 310 | output = stdout.getvalue() 311 | # if simulator is empty, there's no output 312 | if not output: 313 | # returns error and no output 314 | self.assertEquals('', output) 315 | self.assertTrue(errorcode) 316 | else: 317 | self.assertFalse(errorcode) 318 | # and past value from command line 319 | args = 'prev %s' % self.test_key[:-1] 320 | with self.capture_stdout() as stdout: 321 | errorcode = cmd.main(self.conn_args + args) 322 | output = stdout.getvalue() 323 | # if simulator is empty, there's no output 324 | if not output: 325 | # returns error and no output 326 | self.assertEquals('', output) 327 | self.assertTrue(errorcode) 328 | else: 329 | self.assertFalse(errorcode) 330 | 331 | def test_command_prev_verbose(self): 332 | # put something there 333 | self.client.put(self.test_key, 'myvalue') 334 | # try a short offset to value from command line 335 | args = '-vb prev %s~' % self.test_key 336 | with self.capture_stdio() as stdio: 337 | errorcode = cmd.main(self.conn_args + args) 338 | stdout, stderr = stdio 339 | output = stdout.getvalue() 340 | verbose = stderr.getvalue() 341 | # returns no error and data 342 | self.assertFalse(errorcode) 343 | self.assertEquals('key: %s\n' % self.test_key, verbose) 344 | self.assertEquals('myvalue\n', output) 345 | 346 | 347 | class TestGetRangeCommand(BaseCommandTestCase): 348 | 349 | def test_command_getr(self): 350 | num_keys = 3 351 | for i in range(num_keys): 352 | self.client.put(self.test_key + '.%.5d' % i, 'myvalue.%.5d' % i) 353 | args = 'getr %s' % self.test_key 354 | errorcode, output = self.run_cmd(args) 355 | self.assertFalse(errorcode) 356 | expected = ''.join(['myvalue.%.5d' % i for i in range(num_keys)]) 357 | self.assertEquals(expected + '\n', output) 358 | 359 | def test_missing_keys(self): 360 | args = 'getr %s' % self.test_key 361 | errorcode, output = self.run_cmd(args) 362 | self.assertFalse(errorcode) 363 | self.assertEquals('\n', output) 364 | 365 | def test_explicit_range(self): 366 | num_keys = 10 367 | for i in range(num_keys): 368 | self.client.put(self.test_key + '.%.5d' % i, 'myvalue.%.5d' % i) 369 | last_included_key_number = num_keys // 2 370 | args = 'getr {key} {key}.{stop:0=5}'.format(key=self.test_key, 371 | stop=(last_included_key_number)) 372 | errorcode, output = self.run_cmd(args) 373 | self.assertFalse(errorcode) 374 | expected = ''.join(['myvalue.%.5d' % i for i in 375 | range(last_included_key_number + 1)]) 376 | self.assertEquals(expected + '\n', output) 377 | 378 | 379 | class TestDeleteRangeCommand(BaseCommandTestCase): 380 | 381 | def test_command_deleter(self): 382 | num_keys = 3 383 | keys = [] 384 | for i in range(num_keys): 385 | key = self.test_key + '.%.5d' % i 386 | self.client.put(key, 'myvalue.%.5d' % i) 387 | keys.append(key) 388 | args = 'deleter %s' % self.test_key 389 | errorcode, output = self.run_cmd(args) 390 | self.assertFalse(errorcode) 391 | self.assertEquals('', output) 392 | for key in keys: 393 | self.assertEquals(None, self.client.get(key)) 394 | 395 | def test_missing_keys(self): 396 | args = 'deleter %s' % self.test_key 397 | errorcode, output = self.run_cmd(args) 398 | self.assertFalse(errorcode) 399 | self.assertEquals('', output) 400 | 401 | def test_explicit_range(self): 402 | num_keys = 10 403 | keys = [] 404 | for i in range(num_keys): 405 | key = self.test_key + '.%.5d' % i 406 | self.client.put(key, 'myvalue.%.5d' % i) 407 | keys.append(key) 408 | last_included_key_number = num_keys // 2 409 | args = 'deleter {key} {key}.{stop:0=5}'.format(key=self.test_key, 410 | stop=(last_included_key_number)) 411 | errorcode, output = self.run_cmd(args) 412 | self.assertFalse(errorcode) 413 | for i, key in enumerate(keys): 414 | if i <= last_included_key_number: 415 | # deleted 416 | self.assertEquals(None, self.client.get(key)) 417 | else: 418 | # not deleted 419 | self.assertEquals('myvalue.%.5d' % i, 420 | self.client.get(key).value) 421 | 422 | 423 | if __name__ == '__main__': 424 | unittest.main() 425 | -------------------------------------------------------------------------------- /test/test_p2p.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Seagate Technology LLC. 2 | # 3 | # This Source Code Form is subject to the terms of the Mozilla 4 | # Public License, v. 2.0. If a copy of the MPL was not 5 | # distributed with this file, You can obtain one at 6 | # https://mozilla.org/MP:/2.0/. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but is provided AS-IS, WITHOUT ANY WARRANTY; including without 10 | # the implied warranty of MERCHANTABILITY, NON-INFRINGEMENT or 11 | # FITNESS FOR A PARTICULAR PURPOSE. See the Mozilla Public 12 | # License for more details. 13 | # 14 | # See www.openkinetic.org for more project information 15 | # 16 | 17 | #@author: Clayg 18 | 19 | import unittest 20 | 21 | from kinetic import operations 22 | from kinetic import baseclient 23 | 24 | from base import MultiSimulatorTestCase 25 | 26 | class P2PTestCase(MultiSimulatorTestCase): 27 | 28 | def test_p2p_push(self): 29 | source, target = self.client_map.values() 30 | key = self.buildKey('test') 31 | source.put(key, 'value') 32 | self.assertEqual(target.get(key), None) 33 | resp = source.push([key],target.hostname, target.port) 34 | for op in resp: 35 | self.assertEquals(op.key, key) 36 | self.assertEquals(op.status.code, op.status.SUCCESS) 37 | entry = target.get(key) 38 | self.assertEquals(entry.value, 'value') 39 | 40 | 41 | if __name__ == '__main__': 42 | unittest.main() 43 | --------------------------------------------------------------------------------