├── .gitignore ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── README.rst ├── pypi-push.sh ├── requirements-test.txt ├── setup.py ├── swift_commander ├── __init__.py ├── swbundler.py ├── swc ├── swfoldersize.py ├── swhashcomp.py ├── swpget.py ├── swrm.py ├── swsearch.py └── swsymlinks.py └── tests.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # virtualenv 2 | venv 3 | venv/* 4 | 5 | .idea 6 | .idea/* 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PACKAGE_NAME=swc 2 | VENV_DIR?=venv 3 | VENV_ACTIVATE=$(VENV_DIR)/bin/activate 4 | WITH_VENV=. $(VENV_ACTIVATE); 5 | # only experimental support for 2.7 6 | # to install use VENV_PYTHON=python2.7 make teardown clean venv 7 | VENV_PYTHON?=python3 8 | # override this if you want to point to internal PyPI 9 | # this is the default for pip 10 | PIP_INDEX_URL?=https://pypi.python.org/simple 11 | 12 | .PHONY: default 13 | default: develop 14 | 15 | 16 | .PHONY: teardown 17 | teardown: 18 | rm -rf $(VENV_DIR) 19 | 20 | .PHONY: clean 21 | clean: 22 | python setup.py clean 23 | rm -rf build/ 24 | rm -rf dist/ 25 | rm -rf *.egg*/ 26 | rm -rf __pycache__/ 27 | rm -f MANIFEST 28 | find $(PACKAGE_NAME) -type f -name '*.pyc' -delete 29 | 30 | $(VENV_ACTIVATE): requirements*.txt 31 | test -f $@ || virtualenv --python=$(VENV_PYTHON) $(VENV_DIR) 32 | $(WITH_VENV) pip install -r requirements-test.txt --index-url=${PIP_INDEX_URL} 33 | $(WITH_VENV) pip install -e . --index-url=${PIP_INDEX_URL} 34 | touch $@ 35 | 36 | .PHONY: venv 37 | venv: $(VENV_ACTIVATE) 38 | 39 | .PHONY: develop 40 | # use this if you want changes in the repo to be immediately reflected in scripts 41 | develop: venv 42 | $(WITH_VENV) python setup.py develop 43 | 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | swift-commander 2 | =============== 3 | 4 | swift commander (swc) is a wrapper to various command line client tools 5 | for openstack swift cloud storage systems. The purpose of swc is 3 fold: 6 | 7 | - provide a very simple user interface to Linux users 8 | - provide a unified user interface to swiftclient, curl, etc with reasonable defaults 9 | - model commands after classic shell tools such as cd, ls, etc. 10 | 11 | # Basic Operations 12 | 13 | if swc is invoked without any options it shows a basic help page: 14 | 15 | ``` 16 | Swift Commander (swc) allows you to easily work with a swift object store. 17 | swc supports sub commands that attempt to mimic standard unix file system tools. 18 | These sub commands are currently implemented: (Arguments in square brackets are 19 | optional). 20 | 21 | swc upload - copy file / dirs from a file system to swift 22 | swc download - copy files and dirs from swift to a file system 23 | swc cd - change current folder to in swift 24 | swc ls [folder] - list contents of a folder - or the current one 25 | swc mkdir - create a folder (works only at the root) 26 | swc rm - delete all file paths that start with 27 | swc pwd - display the current swift folder name 28 | swc cat|more|less - download a file to TMPDIR and view with cat, more or less 29 | swc vi|emacs|nano - download a file to TMPDIR and edit it with vi|emacs or nano 30 | swc chgrp - grant/remove rw access to current swift account or container 31 | swc rw - add rw access to current swift account or container 32 | swc ro - add ro access to current swift account or container 33 | swc publish|hide - make root folder public (web server mode) or hide it 34 | swc list [filt] - list folder content (incl. subfolders) and filter 35 | swc search - search for a string in text files under /folder 36 | swc openwith - download a file to TMPDIR and open it with 37 | swc header - display the header of a file in swift 38 | swc meta - display custom meta data of a file in swift 39 | swc mtime - show the original mtime of a file before uploaded 40 | swc size - show the size of a swift or a local folder 41 | swc compare - compare size of a local folder with a swift folder 42 | swc hash - compare the md5sum of a local file with a swift file 43 | swc arch - create one tar archive for each folder level 44 | swc unarch - restore folders that have been archived 45 | swc auth - show current storage url and auth token 46 | swc env - show authentication env vars (ST_ and OS_) 47 | swc clean - remove current authtoken credential cache 48 | 49 | Examples: 50 | swc upload /local/folder /swift/folder 51 | swc upload --no-symlinks /local/folder /swift/folder (ignore symlinks) 52 | swc compare /local/folder /swift/folder 53 | swc download /swift/folder /scratch/folder 54 | swc download /swift/folder $TMPDIR 55 | swc rm /archive/some_prefix 56 | swc more /folder/some_file.txt 57 | swc openwith emacs /folder/some_file.txt 58 | 59 | Debugging: 60 | export OS_SWIFT_OPTS=--info 61 | ``` 62 | 63 | ## Important: What you need to know about the Swift architecture 64 | 65 | - swift does not know sub directories such as a file system. It knows containers and in containers it carries objects (which are actually files). 66 | - if you upload a path with many directory levels such as /folder1/folder2/folder3/folder4/myfile.pdf to swift it will cheat a little and put an object called `folder2/folder3/folder4/myfile.pdf` into a container called `folder1`. 67 | - the object is just like a filename that contains a number of forward slashes. Forward slashes are allowed because swift does not know any directories and can have the / character as part of a filename. These fake folders are also called `Pseudo-Hierarchical Directories` ( http://www.17od.com/2012/12/19/ten-useful-openstack-swift-features/ ) 68 | - the architecture has advantages and disadvantages. An advantage is that you can retrieve hundreds of thousands of object names in a few seconds. The disadvantage is that a single container eventually reaches a scalability limit. Currently this limit is at about 2 million objects per container. You should not put more than 2 million files into a single container or /root_folder. 69 | - swift commander (swc) allows you to ignore the fact that there are containers and pseudo folders. For the most part you can just treat them both as standard directories 70 | 71 | ## Authentication 72 | 73 | - `swc` does not implement any authentication but uses a swift authentication environment, for example as setup by `https://github.com/FredHutch/swift-switch-account` including Active Directory integration. 74 | - if a swift authentication environment is found `swc` creates swift auth_tokens on the fly and uses them with RESTful tools such as curl. 75 | 76 | ### swc upload 77 | 78 | use `swc upload /local_dir/subdir /my_swift_container/subfolder` to copy data from a local or networked posix file system to a swift object store. `swc upload` wraps `swift upload` of the standard python swift client: 79 | 80 | ``` 81 | joe@box:~/sc$ swc upload ./testing /test 82 | *** uploading ./test *** 83 | *** to Swift_Account:/test/ *** 84 | executing:swift upload --changed --segment-size=1073741824 --use-slo --segment-container=".segments_test" --header="X-Object-Meta-Uploaded-by:joe" --object-name="" "test" "./test" 85 | *** please wait... *** 86 | /fld11/file12 87 | /fld11/file11 88 | /fld11/fld2/fld3/fld4/file43 89 | /fld11/fld2/fld3/fld4/file42 90 | . 91 | 92 | ``` 93 | 94 | the swc wrapper adds the following features to `upload`: 95 | 96 | - --segment-size ensures that uploads for files > 5GB do not fail. 1073741824 = 1GB 97 | - Uploaded-by metadata keeps track of the operating system user (often Active Directory user) that upload the data 98 | - setting --segment-container ensures that containers that carry the segments for multisegment files are hidden if users access these containers with 3rd. party GUI tools (ExpanDrive, Cyberduck, FileZilla) to avoid end user confusion 99 | - --slo stands for Static Large Object and SLO's the recommended object type for large objects / files. 100 | 101 | 102 | as an addional feature you can add multiple metadata tags to each uploaded object, which is great for retrieving archived files later: 103 | 104 | ``` 105 | joe@box:~/sc$ swc upload ./test /test/example/meta project:grant-xyz collaborators:jill,joe,jim cancer:breast 106 | *** uploading ./test *** 107 | *** to Swift_Account:/test/example/meta *** 108 | executing:swift upload --changed --segment-size=1073741824 --use-slo --segment-container=".segments_test" --header="X-Object-Meta-Uploaded-by:petersen" --header=X-Object-Meta-project:grant-xyz --header=X-Object-Meta-collaborators:jill,joe,jim --header=X-Object-Meta-cancer:breast --object-name="example/meta" "test" "./test" 109 | *** please wait... *** 110 | example/meta/fld11/fld2/file21 111 | example/meta/fld11/file11 112 | . 113 | . 114 | /test/example/meta 115 | ``` 116 | 117 | These metadata tags stay in the swift object store with the data. They are stored just like other important metadata such as change data and name of the object. 118 | 119 | ``` 120 | joe@box:~/sc$ swc meta example/meta/fld11/file13 121 | Meta Cancer: breast 122 | Meta Collaborators: jill,joe,jim 123 | Meta Uploaded-By: petersen 124 | Meta Project: grant-xyz 125 | Meta Mtime: 1420047068.977197 126 | 127 | ``` 128 | if you store metadata tags you can later use an external search engine such as ElasticSearch to quickly search for metadata you populated while uploading data 129 | 130 | alias: you can use `swc up` instead of `swc upload` 131 | 132 | 133 | ### swc download 134 | 135 | use `swc download /my_swift_container/subfolder /local/subfolder` to copy data from a swift object store to local or network storage. swc download` wraps `swift download` of the standard python swift client: 136 | ``` 137 | joe@box:~/sc$ swc download /test/example/ $TMPDIR/ 138 | example/meta/fld11/fld2/file21 139 | example/meta/fld11/file11 140 | ``` 141 | 142 | alias: you can use `swc down` instead of `swc download` 143 | 144 | ### swc arch 145 | 146 | `swc arch` is a variation of `swc upload`. Instead of uploading the files as is, it creates a tar.gz archive for each directory and uploads the tar.gz archives. swc arch is different from default tar behavior because it does not create a large tar.gz file of an entire directory structure as large tar.gz files are hard to manage (as one cannot easily navigate the directory structure within or get quick access to a spcific file). Instead swc arch creates tar.gz files that do not include sub directories and it creates a separate tar.gz file for each directory and directory level. The benefit of this approach is that the entire directory structure remains intact and you can easily navigate it by using `swc cd` and `swc ls` 147 | 148 | ### swc cd, swc, ls, swc mkdir 149 | 150 | these commands are simplified versions of the equivalent standard GNU tools and should work very similar to these tools. 151 | 152 | ### swc mtime 153 | 154 | use `swc mtime /my_swift_container/subfolder/file` to see the modification time data from a swift object store to local or network storage. `swc download` wraps `swift download` of the standard python swift client: 155 | 156 | 157 | 158 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | swift-commander 2 | =============== 3 | 4 | swift commander (swc) is a wrapper to various command line client tools for openstack swift cloud 5 | storage systems. The purpose of swc is 3 fold: 6 | 7 | - provide a very simple user interface to Linux users 8 | - provide a unified user interface to swiftclient, curl, etc with reasonable defaults 9 | - model commands after classic shell tools such as cd, ls, etc. 10 | 11 | Basic Operations 12 | ================ 13 | 14 | if swc is invoked without any options it shows a basic help page: 15 | 16 | :: 17 | 18 | Swift Commander (swc) allows you to easily work with a swift object store. 19 | swc supports sub commands that attempt to mimic standard unix file system tools. 20 | These sub commands are currently implemented: (Arguments in square brackets are 21 | optional). 22 | 23 | swc upload - copy file / dirs from a file system to swift 24 | swc download - copy files and dirs from swift to a file system 25 | swc cd - change current folder to in swift 26 | swc ls [folder] - list contents of a folder - or the current one 27 | swc mkdir - create a folder (works only at the root) 28 | swc rm - delete all file paths that start with 29 | swc pwd - display the current swift folder name 30 | swc cat|more|less - download a file to TMPDIR and view with cat, more or less 31 | swc vi|emacs|nano - download a file to TMPDIR and edit it with vi|emacs or nano 32 | swc chgrp - grant/remove rw access to current swift account or container 33 | swc rw - add rw access to current swift account or container 34 | swc ro - add ro access to current swift account or container 35 | swc publish|hide - make root folder public (web server mode) or hide it 36 | swc list [filt] - list folder content (incl. subfolders) and filter 37 | swc search - search for a string in text files under /folder 38 | swc openwith - download a file to TMPDIR and open it with 39 | swc header - display the header of a file in swift 40 | swc meta - display custom meta data of a file in swift 41 | swc mtime - show the original mtime of a file before uploaded 42 | swc size - show the size of a swift or a local folder 43 | swc compare - compare size of a local folder with a swift folder 44 | swc hash - compare the md5sum of a local file with a swift file 45 | swc arch - create one tar archive for each folder level 46 | swc unarch - restore folders that have been archived 47 | swc auth - show current storage url and auth token 48 | swc env - show authentication env vars (ST_ and OS_) 49 | swc clean - remove current authtoken credential cache 50 | 51 | Examples: 52 | swc upload /local/folder /swift/folder 53 | swc upload --no-symlinks /local/folder /swift/folder (ignore symlinks) 54 | swc compare /local/folder /swift/folder 55 | swc download /swift/folder /scratch/folder 56 | swc download /swift/folder $TMPDIR 57 | swc rm /archive/some_prefix 58 | swc more /folder/some_file.txt 59 | swc openwith emacs /folder/some_file.txt 60 | 61 | Debugging: 62 | export OS_SWIFT_OPTS=--info 63 | 64 | Important: What you need to know about the Swift architecture 65 | ------------------------------------------------------------- 66 | 67 | - swift does not know sub directories such as a file system. It knows containers and in containers 68 | it carries objects (which are actually files). 69 | - if you upload a path with many directory levels such as 70 | /folder1/folder2/folder3/folder4/myfile.pdf to swift it will cheat a little and put an object 71 | called ``folder2/folder3/folder4/myfile.pdf`` into a container called ``folder1``. 72 | - the object is just like a filename that contains a number of forward slashes. Forward slashes are 73 | allowed because swift does not know any directories and can have the / character as part of a 74 | filename. These fake folders are also called ``Pseudo-Hierarchical Directories`` ( 75 | http://www.17od.com/2012/12/19/ten-useful-openstack-swift-features/ ) 76 | - the architecture has advantages and disadvantages. An advantage is that you can retrieve hundreds 77 | of thousands of object names in a few seconds. The disadvantage is that a single container 78 | eventually reaches a scalability limit. Currently this limit is at about 2 million objects per 79 | container. You should not put more than 2 million files into a single container or /root_folder. 80 | - swift commander (swc) allows you to ignore the fact that there are containers and pseudo folders. 81 | For the most part you can just treat them both as standard directories 82 | 83 | Authentication 84 | -------------- 85 | 86 | - ``swc`` does not implement any authentication but uses a swift authentication environment, for 87 | example as setup by ``https://github.com/FredHutch/swift-switch-account`` including Active 88 | Directory integration. 89 | - if a swift authentication environment is found ``swc`` creates swift auth_tokens on the fly and 90 | uses them with RESTful tools such as curl. 91 | 92 | swc upload 93 | ~~~~~~~~~~ 94 | 95 | use ``swc upload /local_dir/subdir /my_swift_container/subfolder`` to copy data from a local or 96 | networked posix file system to a swift object store. ``swc upload`` wraps ``swift upload`` of the 97 | standard python swift client: 98 | 99 | :: 100 | 101 | joe@box:~/sc$ swc upload ./testing /test 102 | *** uploading ./test *** 103 | *** to Swift_Account:/test/ *** 104 | executing:swift upload --changed --segment-size=1073741824 --use-slo --segment-container=".segments_test" --header="X-Object-Meta-Uploaded-by:joe" --object-name="" "test" "./test" 105 | *** please wait... *** 106 | /fld11/file12 107 | /fld11/file11 108 | /fld11/fld2/fld3/fld4/file43 109 | /fld11/fld2/fld3/fld4/file42 110 | . 111 | 112 | the swc wrapper adds the following features to ``upload``: 113 | 114 | - –segment-size ensures that uploads for files > 5GB do not fail. 1073741824 = 1GB 115 | - Uploaded-by metadata keeps track of the operating system user (often Active Directory user) that 116 | upload the data 117 | - setting –segment-container ensures that containers that carry the segments for multisegment files 118 | are hidden if users access these containers with 3rd. party GUI tools (ExpanDrive, Cyberduck, 119 | FileZilla) to avoid end user confusion 120 | - –slo stands for Static Large Object and SLO’s the recommended object type for large objects / 121 | files. 122 | 123 | as an addional feature you can add multiple metadata tags to each uploaded object, which is great 124 | for retrieving archived files later: 125 | 126 | :: 127 | 128 | joe@box:~/sc$ swc upload ./test /test/example/meta project:grant-xyz collaborators:jill,joe,jim cancer:breast 129 | *** uploading ./test *** 130 | *** to Swift_Account:/test/example/meta *** 131 | executing:swift upload --changed --segment-size=1073741824 --use-slo --segment-container=".segments_test" --header="X-Object-Meta-Uploaded-by:petersen" --header=X-Object-Meta-project:grant-xyz --header=X-Object-Meta-collaborators:jill,joe,jim --header=X-Object-Meta-cancer:breast --object-name="example/meta" "test" "./test" 132 | *** please wait... *** 133 | example/meta/fld11/fld2/file21 134 | example/meta/fld11/file11 135 | . 136 | . 137 | /test/example/meta 138 | 139 | These metadata tags stay in the swift object store with the data. They are stored just like other 140 | important metadata such as change data and name of the object. 141 | 142 | :: 143 | 144 | joe@box:~/sc$ swc meta example/meta/fld11/file13 145 | Meta Cancer: breast 146 | Meta Collaborators: jill,joe,jim 147 | Meta Uploaded-By: petersen 148 | Meta Project: grant-xyz 149 | Meta Mtime: 1420047068.977197 150 | 151 | if you store metadata tags you can later use an external search engine such as ElasticSearch to 152 | quickly search for metadata you populated while uploading data 153 | 154 | alias: you can use ``swc up`` instead of ``swc upload`` 155 | 156 | swc download 157 | ~~~~~~~~~~~~ 158 | 159 | use ``swc download /my_swift_container/subfolder /local/subfolder`` to copy data from a swift object 160 | store to local or network storage. swc download\ ``wraps``\ swift download\` of the standard python 161 | swift client: 162 | 163 | :: 164 | 165 | joe@box:~/sc$ swc download /test/example/ $TMPDIR/ 166 | example/meta/fld11/fld2/file21 167 | example/meta/fld11/file11 168 | 169 | alias: you can use ``swc down`` instead of ``swc download`` 170 | 171 | swc arch 172 | ~~~~~~~~ 173 | 174 | ``swc arch`` is a variation of ``swc upload``. Instead of uploading the files as is, it creates a 175 | tar.gz archive for each directory and uploads the tar.gz archives. swc arch is different from 176 | default tar behavior because it does not create a large tar.gz file of an entire directory structure 177 | as large tar.gz files are hard to manage (as one cannot easily navigate the directory structure 178 | within or get quick access to a spcific file). Instead swc arch creates tar.gz files that do not 179 | include sub directories and it creates a separate tar.gz file for each directory and directory 180 | level. The benefit of this approach is that the entire directory structure remains intact and you 181 | can easily navigate it by using ``swc cd`` and ``swc ls`` 182 | 183 | swc cd, swc, ls, swc mkdir 184 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 185 | 186 | these commands are simplified versions of the equivalent standard GNU tools and should work very 187 | similar to these tools. 188 | 189 | swc mtime 190 | ~~~~~~~~~ 191 | 192 | use ``swc mtime /my_swift_container/subfolder/file`` to see the modification time data from a swift 193 | object store to local or network storage. ``swc download`` wraps ``swift download`` of the standard 194 | python swift client: 195 | -------------------------------------------------------------------------------- /pypi-push.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | printf " loading module python3/3.5.0\n" 4 | module load python3/3.5.0 > /dev/null 2>&1 5 | 6 | version=$(grep ^__version__ setup.py | cut -d'"' -f2) 7 | 8 | pandoc --columns=100 --output=README.rst --to rst README.md 9 | git add README.rst 10 | git commit -a -m "version ${version}" 11 | git tag ${version} -m "tag for PyPI" 12 | git push --tags origin master 13 | python3 setup.py register -r pypi 14 | python3 setup.py sdist upload -r pypi 15 | 16 | echo " Done! Occasionally you may want to remove older tags:" 17 | echo "git tag 1.2.3 -d" 18 | echo "git push origin :refs/tags/1.2.3" 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | python-swiftclient 2 | python-keystoneclient 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | __version__ = "1.5.0" 4 | 5 | #try: 6 | # from pypandoc import convert 7 | # read_md = lambda f: convert(f, 'rst') 8 | #except ImportError: 9 | # print("warning: pypandoc module not found, could not convert Markdown to RST") 10 | # read_md = lambda f: open(f, 'r').read() 11 | 12 | CLASSIFIERS = [ 13 | "Development Status :: 5 - Production/Stable", 14 | "Environment :: Console", 15 | "Environment :: OpenStack", 16 | "Intended Audience :: Customer Service", 17 | "Intended Audience :: Developers", 18 | "Intended Audience :: Education", 19 | "Intended Audience :: End Users/Desktop", 20 | "Intended Audience :: Healthcare Industry", 21 | "Intended Audience :: Information Technology", 22 | "Intended Audience :: Science/Research", 23 | "Intended Audience :: System Administrators", 24 | "License :: OSI Approved :: Apache Software License", 25 | "Natural Language :: English", 26 | "Operating System :: MacOS :: MacOS X", 27 | "Operating System :: POSIX", 28 | "Operating System :: POSIX :: Linux", 29 | "Operating System :: POSIX :: Other", 30 | "Operating System :: Unix", 31 | "Programming Language :: Python :: 3", 32 | "Programming Language :: Python :: Implementation :: CPython", 33 | "Programming Language :: Python :: Implementation :: PyPy", 34 | "Programming Language :: Unix Shell", 35 | "Topic :: Desktop Environment :: File Managers", 36 | "Topic :: Internet", 37 | "Topic :: Scientific/Engineering :: Bio-Informatics", 38 | "Topic :: System :: Archiving", 39 | "Topic :: System :: Archiving :: Backup", 40 | "Topic :: System :: Filesystems", 41 | "Topic :: Software Development :: Libraries :: Python Modules", 42 | "Topic :: System :: Systems Administration", 43 | "Topic :: Utilities" 44 | ] 45 | 46 | setup( 47 | name='swift-commander', 48 | version=__version__, 49 | description='swift commander (swc) is a wrapper for curl and python-swift to access openstack swift cloud storage systems.', 50 | long_description=open('README.rst', 'r').read(), 51 | packages=['swift_commander'], 52 | scripts=['swift_commander/swc'], 53 | author = 'Dirk Petersen, Jeff Katcher', 54 | author_email = 'dp@nowhere.com', 55 | url = 'https://github.com/FredHutch/swift-commander', 56 | download_url = 'https://github.com/FredHutch/swift-commander/tarball/%s' % __version__, 57 | keywords = ['openstack', 'swift', 'cloud storage'], # arbitrary keywords 58 | classifiers = CLASSIFIERS, 59 | # 'python-swiftclient>=2.5,<3','python-keystoneclient>=1.5,<2' 60 | install_requires=[ 61 | 'psutil>=4', 62 | 'python-swiftclient>=3.2.0', 63 | 'python-keystoneclient>=2,<3' 64 | ], 65 | entry_points={ 66 | # we use console_scripts here to allow virtualenv to rewrite shebangs 67 | # to point to appropriate python and allow experimental python 2.X 68 | # support. 69 | 'console_scripts': [ 70 | 'swbundler.py=swift_commander.swbundler:main', 71 | 'swfoldersize.py=swift_commander.swfoldersize:main', 72 | 'swhashcomp.py=swift_commander.swhashcomp:main', 73 | 'swpget.py=swift_commander.swpget:main', 74 | 'swrm.py=swift_commander.swrm:main', 75 | 'swsearch.py=swift_commander.swsearch:main', 76 | 'swsymlinks.py=swift_commander.swsymlinks:main', 77 | ] 78 | } 79 | ) 80 | -------------------------------------------------------------------------------- /swift_commander/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FredHutch/swift-commander/3a25c22022d05c22f32e42a035fe0714099475b9/swift_commander/__init__.py -------------------------------------------------------------------------------- /swift_commander/swbundler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os,sys,getopt,tarfile 4 | import getpass 5 | 6 | from distutils.spawn import find_executable 7 | 8 | import time 9 | import socket 10 | import optparse 11 | import subprocess 12 | import multiprocessing 13 | 14 | from swiftclient import Connection 15 | 16 | from swiftclient import shell 17 | from swiftclient import RequestException 18 | from swiftclient.exceptions import ClientException 19 | from swiftclient.multithreading import OutputManager 20 | 21 | # swiftclient 3.2+ support 22 | import swiftclient 23 | from pkg_resources import parse_version 24 | import argparse 25 | 26 | try: 27 | from scandir import walk 28 | except: 29 | from os import walk 30 | 31 | swift_auth=os.environ.get("ST_AUTH") 32 | swift_auth_token=os.environ.get("OS_AUTH_TOKEN") 33 | storage_url=os.environ.get("OS_STORAGE_URL") 34 | haz_pigz=False 35 | 36 | # define minimum parser object(s) to allow swiftstack shell to run 37 | # old is pre swiftclient 3.1 and new is 3.1+ 38 | 39 | def shell_old_minimal_options(): 40 | global swift_auth,swift_auth_token,storage_url 41 | 42 | parser = optparse.OptionParser() 43 | 44 | parser.add_option('-A', '--auth', dest='auth', 45 | default=swift_auth) 46 | parser.add_option('-V', '--auth-version', 47 | default=os.environ.get('ST_AUTH_VERSION', 48 | (os.environ.get('OS_AUTH_VERSION','1.0')))) 49 | parser.add_option('-U', '--user', dest='user', 50 | default=os.environ.get('ST_USER')) 51 | parser.add_option('-K', '--key', dest='key', 52 | default=os.environ.get('ST_KEY')) 53 | 54 | parser.add_option('--os_auth_token',default=swift_auth_token) 55 | parser.add_option('--os_storage_url',default=storage_url) 56 | 57 | parser.add_option('--os_username') 58 | parser.add_option('--os_password') 59 | parser.add_option('--os_auth_url') 60 | 61 | parser.add_option('--os_user_id') 62 | parser.add_option('--os_user_domain_id') 63 | parser.add_option('--os_user_domain_name') 64 | parser.add_option('--os_tenant_id') 65 | parser.add_option('--os_tenant_name') 66 | parser.add_option('--os_project_id') 67 | parser.add_option('--os_project_domain_id') 68 | parser.add_option('--os_project_name') 69 | parser.add_option('--os_project_domain_name') 70 | parser.add_option('--os_service_type') 71 | parser.add_option('--os_endpoint_type') 72 | parser.add_option('--os_region_name') 73 | 74 | # new mandatory bogosity required for swiftclient >= 3.0.0 75 | parser.add_option('--debug') 76 | parser.add_option('--info') 77 | 78 | parser.add_option('-v', '--verbose', action='count', dest='verbose', 79 | default=1, help='Print more info.') 80 | 81 | return parser 82 | 83 | def shell_new_minimal_options(): 84 | global swift_auth,swift_auth_token,storage_url 85 | 86 | parser = argparse.ArgumentParser() 87 | 88 | parser.add_argument('-A', '--auth', dest='auth', 89 | default=swift_auth) 90 | parser.add_argument('-V', '--auth-version', 91 | default=os.environ.get('ST_AUTH_VERSION', 92 | (os.environ.get('OS_AUTH_VERSION','1.0')))) 93 | parser.add_argument('-U', '--user', dest='user', 94 | default=os.environ.get('ST_USER')) 95 | parser.add_argument('-K', '--key', dest='key', 96 | default=os.environ.get('ST_KEY')) 97 | 98 | parser.add_argument('--os_auth_token',default=swift_auth_token) 99 | parser.add_argument('--os_storage_url',default=storage_url) 100 | 101 | parser.add_argument('--os_username') 102 | parser.add_argument('--os_password') 103 | parser.add_argument('--os_auth_url') 104 | 105 | parser.add_argument('--os_user_id') 106 | parser.add_argument('--os_user_domain_id') 107 | parser.add_argument('--os_user_domain_name') 108 | parser.add_argument('--os_tenant_id') 109 | parser.add_argument('--os_tenant_name') 110 | parser.add_argument('--os_project_id') 111 | parser.add_argument('--os_project_domain_id') 112 | parser.add_argument('--os_project_name') 113 | parser.add_argument('--os_project_domain_name') 114 | parser.add_argument('--os_service_type') 115 | parser.add_argument('--os_endpoint_type') 116 | parser.add_argument('--os_region_name') 117 | 118 | # new mandatory bogosity required for swiftclient >= 3.0.0 119 | parser.add_argument('--debug') 120 | parser.add_argument('--info') 121 | 122 | parser.add_argument('-v', '--verbose', action='count', dest='verbose', 123 | default=1, help='Print more info.') 124 | 125 | # even more options required in later versions of swift 3.x 126 | parser.add_argument('--os_auth_type') 127 | parser.add_argument('--os_application_credential_id') 128 | parser.add_argument('--os_application_credential_secret') 129 | parser.add_argument('--prompt', action='store_true', dest='prompt', default=False) 130 | 131 | return parser 132 | 133 | # check for swiftclient version and point to appropriate shell options 134 | if parse_version(swiftclient.__version__)0: 207 | sys.stderr.write('***** TAR ERROR %s, command: %s *****\n' % 208 | (ret,tar_params)) 209 | 210 | if file_list: 211 | os.unlink(tmp_file) 212 | 213 | def upload_file_to_swift(filename,swiftname,container,meta): 214 | final=[container,filename] 215 | if meta: 216 | final=meta+final 217 | 218 | sw_upload("--object-name="+swiftname, 219 | "--segment-size=2147483648", 220 | "--use-slo", 221 | "--segment-container=.segments_"+container, 222 | "--header=X-Object-Meta-Uploaded-by:"+getpass.getuser(),*final) 223 | 224 | def archive_tar_file(src_path,file_list,container,tmp_dir,pre_path,meta, 225 | recurse=False): 226 | global tar_suffix 227 | 228 | # archive_name is name for archived object 229 | archive_name=pre_path+tar_suffix 230 | # temp_archive_name is name of local tar file 231 | temp_archive_name=unique_id()+os.path.basename(archive_name) 232 | if tmp_dir: 233 | temp_archive_name=os.path.join(tmp_dir,temp_archive_name) 234 | 235 | # Create local tar file 236 | create_tar_file(temp_archive_name,src_path,file_list,recurse) 237 | 238 | # Upload tar file to container as 'archive_name' 239 | upload_file_to_swift(temp_archive_name,archive_name,container,meta) 240 | 241 | # Delete local tar file 242 | os.unlink(temp_archive_name) 243 | 244 | def is_child_or_sib(dir_name,last_dir): 245 | dname=os.path.dirname(dir_name) 246 | return (dname==last_dir or dname==os.path.dirname(last_dir)) 247 | 248 | # param order: [src_path,file_list,container,tmp_dir,pre_path,meta] 249 | def archive_worker(item): 250 | archive_tar_file(*item) 251 | 252 | # if par==1 then multiprocessing Pool won't actually be used (debug mostly) 253 | def archive_to_swift(local_dir,container,no_hidden,tmp_dir,prefix,par,subtree, 254 | meta): 255 | last_dir="" 256 | special=['.git'] 257 | 258 | # Now updating object not container metadata 259 | #sw_post(container,*meta) 260 | sw_post(container) 261 | 262 | archive_pool=multiprocessing.Pool(par) 263 | 264 | for dir_name, subdir_list, file_list in mywalk(local_dir): 265 | rel_path=os.path.relpath(dir_name,local_dir) 266 | if (not (no_hidden and is_hidden_dir(rel_path))): 267 | # if files in root directory use basename of root 268 | if rel_path==".": 269 | rel_path=os.path.basename(dir_name)+root_id 270 | 271 | dir_t=dir_name.split('/') 272 | if dir_t[-1] in special: 273 | # special directory - archive recursively from here 274 | #print("\tlast is in special!") 275 | archive_worker([dir_name,file_list,container,tmp_dir, 276 | os.path.join(prefix,rel_path),meta,True]) 277 | elif any(item in special for item in dir_t): 278 | # assumed child of special path, ignore as archived from special 279 | #print("\tskipping child of special!") 280 | pass 281 | elif (not subtree) or (is_subtree(subtree,dir_name)): 282 | p=[dir_name,file_list,container,tmp_dir, 283 | os.path.join(prefix,rel_path),meta] 284 | if par>1: 285 | archive_pool.apply_async(archive_worker,[p]) 286 | else: 287 | archive_worker(p) 288 | 289 | last_dir=dir_name 290 | 291 | archive_pool.close() 292 | archive_pool.join() 293 | 294 | # parse name into directory tree 295 | def create_local_path(local_dir,archive_name): 296 | global tar_suffix 297 | 298 | path=os.path.join(local_dir,archive_name) 299 | if path.endswith(tar_suffix): 300 | path=path[:-len(tar_suffix)] 301 | 302 | if not os.path.exists(path): 303 | os.makedirs(path) 304 | 305 | return path 306 | 307 | def create_sw_conn(): 308 | global swift_auth,swift_auth_token,storage_url 309 | 310 | if swift_auth_token and storage_url: 311 | return Connection(preauthtoken=swift_auth_token,preauthurl=storage_url) 312 | 313 | if swift_auth: 314 | swift_user=os.environ.get("ST_USER") 315 | swift_key=os.environ.get("ST_KEY") 316 | 317 | if swift_user and swift_key: 318 | return Connection(authurl=swift_auth,user=swift_user,key=swift_key) 319 | 320 | print("Error: Swift environment not configured!") 321 | sys.exit() 322 | 323 | def extract_tar_file(tarfile,termpath): 324 | global haz_pigz 325 | 326 | tar_params=["tar","xvf",tarfile,"--directory="+termpath, 327 | '--same-permissions', '--delay-directory-restore'] 328 | # --same-owner is used when user=root 329 | if haz_pigz: 330 | tar_params+=["--use-compress-program=pigz"] 331 | 332 | ret=subprocess.call(tar_params) 333 | if ret > 0: 334 | sys.stderr.write('***** TAR ERROR %s, command: %s *****\n' % 335 | (ret,tar_params)) 336 | 337 | def retrieve_tar_file(tmp_dir,container,obj_name,local_dir,prefix): 338 | global tar_suffix 339 | global root_id 340 | 341 | # download tar file and extract into terminal directory 342 | temp_file=unique_id()+tar_suffix 343 | if tmp_dir: 344 | temp_file=os.path.join(tmp_dir,temp_file) 345 | 346 | sw_download("--output="+temp_file,container,obj_name) 347 | 348 | # strip prefix and if next char is /, strip it too 349 | if prefix and obj_name.startswith(prefix): 350 | obj_name=obj_name[len(prefix):] 351 | if obj_name[0]=='/': 352 | obj_name=obj_name[1:] 353 | 354 | # if bundle, extract using tar embedded paths 355 | if obj_name.endswith(root_id+tar_suffix): 356 | term_path=local_dir 357 | else: 358 | term_path=create_local_path(local_dir,obj_name) 359 | 360 | extract_tar_file(temp_file,term_path) 361 | 362 | os.unlink(temp_file) 363 | 364 | def extract_worker(item): 365 | retrieve_tar_file(*item) 366 | 367 | # if par==1 then multiprocessing Pool won't actually be used (debug mostly) 368 | def extract_to_local(local_dir,container,no_hidden,tmp_dir,prefix,par): 369 | global tar_suffix 370 | global root_id 371 | 372 | swift_conn=create_sw_conn() 373 | if swift_conn: 374 | try: 375 | headers,objs=swift_conn.get_container(container,prefix=prefix, 376 | full_listing=True) 377 | 378 | extract_pool=multiprocessing.Pool(par) 379 | 380 | for obj in objs: 381 | if obj['name'].endswith(tar_suffix): 382 | if no_hidden and is_hidden_dir(obj['name']): 383 | continue 384 | 385 | # param order: [tmp_dir,container,obj_name,local_dir,prefix] 386 | p=[tmp_dir,container,obj['name'],local_dir,prefix] 387 | if par>1: 388 | extract_pool.apply_async(extract_worker,[p]) 389 | else: 390 | extract_worker(p) 391 | 392 | extract_pool.close() 393 | extract_pool.join() 394 | except ClientException: 395 | print("Error: cannot access Swift container '%s'!" % container) 396 | 397 | swift_conn.close() 398 | 399 | def usage(): 400 | print("archive [parameters]") 401 | print("Parameters:") 402 | print("\t-l local_directory (default .)") 403 | print("\t-c container (required)") 404 | print("\t-x (extract from container to local directory)") 405 | print("\t-n (no hidden directories)") 406 | print("\t-t temp_dir (directory for temp files)") 407 | print("\t-a auth_token (default OS_AUTH_TOKEN)") 408 | print("\t-s storage_url (default OS_STORAGE_URL)") 409 | print("\t-p prefix") 410 | print("\t-P parallel_instances (default 3)") 411 | print("\t-m name:value (set object metadata)") 412 | 413 | # is path a child of tree? 414 | def is_subtree(tree,path): 415 | tree_sp=tree.split('/') 416 | path_sp=path.split('/') 417 | 418 | if len(path_sp)&2 21 | } 22 | pathexist(){ 23 | if ! [[ -e $1 ]]; then 24 | echoerr "Error: path '$1' does not exist\n" 25 | exit 26 | fi 27 | } 28 | printcommands(){ 29 | local pref="" 30 | local cmdlist="" 31 | pref="${scriptname} " 32 | for mycmd in $commands; do 33 | cmdlist+="$pref$mycmd, " 34 | done 35 | echo $cmdlist 36 | } 37 | argexist(){ 38 | if [[ -z $1 || "$1" == "--help" ]]; then 39 | echoerr "\nusage: $scriptname $subcmd " 40 | echoerr " -> open file from swift object store\n" 41 | echoerr "also please check these commands:" 42 | printcommands 43 | exit 44 | fi 45 | } 46 | splitpath(){ 47 | # splitting a standard path into container and object 48 | # "splitpath input:$1" 49 | local p="$1" 50 | p=${p#./} # remove leading dot + slash 51 | if [[ "$p" != /* ]]; then # if folder does not start with a slash it's relative 52 | #echo nostart 53 | local c=`cat ~/.swift/current_folder_${swcfile}` 54 | if [[ "$c" == "/" ]]; then p=$c$p; else p=$c/$p; fi 55 | #echo "rel p:$p" 56 | fi 57 | #echo "new p:$p" 58 | cont=${p#/} # remove leading slash 59 | cont=${cont%%/*} # #extract first/root dir 60 | obj=${p#/} # remove leading slash 61 | if [[ "$obj" =~ "/" ]]; then # if path contains a slash 62 | obj=${obj#*/} # remove first/root dir from string 63 | else 64 | obj="" 65 | fi 66 | #echoerr "splitpath output: cont:$cont obj:$obj" 67 | } 68 | tempset() { 69 | # custom script to create TMPDIR based on avail. of ssd, space, etc 70 | if [[ tempcfg -gt 0 ]]; then 71 | return 72 | fi 73 | if hash fhmktemp 2>/dev/null; then 74 | export TMPDIR=`fhmktemp` 75 | fi 76 | if [[ -z $TMPDIR ]]; then 77 | TMPDIR=`mktemp -d --tmpdir=/tmp $USER.XXXXX` 78 | fi 79 | tempcfg=1 80 | } 81 | urlencode_old() { 82 | local data 83 | data="$(${mycurl} -s -o /dev/null -w %{url_effective} --get --data-urlencode "$1" "")" 84 | echo "${data##/?}" 85 | } 86 | 87 | urlencode() { 88 | # urlencode 89 | old_lc_collate=$LC_COLLATE 90 | LC_COLLATE=C 91 | local length="${#1}" 92 | for (( i = 0; i < length; i++ )); do 93 | local c="${1:i:1}" 94 | case $c in 95 | [a-zA-Z0-9.~_-]) printf "$c" ;; 96 | *) printf '%%%02X' "'$c" ;; 97 | esac 98 | done 99 | LC_COLLATE=$old_lc_collate 100 | } 101 | 102 | arrayContains () { 103 | local array="$1[@]" 104 | local seeking=$2 105 | local in=1 106 | for element in "${!array}"; do 107 | if [[ $element == $seeking ]]; then 108 | in=0 109 | break 110 | fi 111 | done 112 | return $in 113 | } 114 | swiftCreds() { 115 | if [[ -n $ST_AUTH && -n $ST_USER && -n $ST_KEY ]]; then 116 | swiftauthver="v1" 117 | swifttenant="AUTH_$ST_USER" 118 | swiftaccount=$ST_USER 119 | swiftpassword=$ST_KEY 120 | swiftauthurl=$ST_AUTH 121 | elif [[ -n $OS_USERNAME ]]; then 122 | swiftauthver="v2" 123 | swifttenant=$OS_TENANT_NAME 124 | swiftaccount=$OS_USERNAME 125 | swiftpassword=$OS_PASSWORD 126 | swiftauthurl=$OS_AUTH_URL 127 | elif [[ -n $OS_AUTH_URL ]]; then 128 | swiftauthurl=$OS_AUTH_URL 129 | else 130 | swiftauthurl=$ST_AUTH 131 | fi 132 | swiftserver=$(sed -e "s/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/" <<< $swiftauthurl) 133 | } 134 | findRealUser() { 135 | # get the real username, if script is run via sudo or su - 136 | local thisPID=$$ 137 | local origUser=`whoami` 138 | local thisUser=$origUser 139 | local ARR=() 140 | local myPPid="" 141 | if [[ "$origUser" != "root" ]]; then 142 | echo $origUser 143 | return 0 144 | fi 145 | while [ "$thisUser" = "$origUser" ]; do 146 | ARR=($(ps h -p$thisPID -ouser,ppid;)) 147 | thisUser="${ARR[0]}" 148 | myPPid="${ARR[1]}" 149 | thisPID=$myPPid 150 | if [[ $thisPID == 0 ]]; then break ;fi 151 | done 152 | getent passwd "$thisUser" | cut -d: -f1 153 | } 154 | nicspeed() { 155 | local iface 156 | local interface 157 | local speed 158 | local retcode 159 | if [[ -f /sys/class/net/bonding_masters ]]; then 160 | iface=$(head -n1 /sys/class/net/bonding_masters) 161 | else 162 | for interface in $(ls /sys/class/net/ | grep -v lo); do 163 | if [[ $(cat /sys/class/net/$interface/operstate) == "up" ]]; then 164 | iface=$interface 165 | fi 166 | done 167 | fi 168 | if [[ -n $iface ]]; then 169 | speed=$(cat /sys/class/net/$iface/speed 2>/dev/null) 170 | retcode=$? 171 | if [[ $retcode = 0 ]]; then 172 | echo $speed 173 | else 174 | echo 0 175 | fi 176 | else 177 | echo 0 178 | fi 179 | } 180 | HTTPGetHeader () { 181 | local old=$(shopt -p extglob) 182 | shopt -s extglob # Required to trim whitespace; see below 183 | #OSU=$(urlencode "${OS_STORAGE_URL}${1}") 184 | #echo OSU:$OSU 185 | 186 | while IFS=':' read key value; do 187 | # trim whitespace in "value" 188 | value=${value##+([[:space:]])}; value=${value%%+([[:space:]])} 189 | case "$key" in 190 | X-Static-Large-Object) head_slo="$value" 191 | ;; 192 | Content-Type) head_ct="$value" 193 | ;; 194 | Content-Length) head_cl="$value" 195 | ;; 196 | HTTP*) read PROTO http_status MSG <<< "$key{$value:+:$value}" 197 | ;; 198 | esac 199 | done < <(${mycurl} -s --head -H "X-Auth-Token: ${OS_AUTH_TOKEN}" "${OS_STORAGE_URL}${1}") 200 | if ! [[ $? = 0 ]]; then 201 | echoerr "Error executing 'curl' command to ${1}" 202 | exit 203 | fi 204 | eval "$old" 205 | #echo $http_status 206 | #echo $head_slo 207 | #echo $head_ct 208 | #echo $head_cl 209 | } 210 | 211 | openwith () { 212 | if [[ -z $1 || -z $2 ]]; then 213 | local app="" 214 | if [[ -n $1 ]]; then 215 | app=$1 216 | fi 217 | echoerr "\nusage: $scriptname $app " 218 | echoerr " -> open file from swift object store with application\n" 219 | echoerr "also please check these commands:" 220 | printcommands 221 | exit 222 | fi 223 | splitpath "$2" 224 | tempset 225 | target_local=$(${scripname} download "$2" $TMPDIR | tail -n1) 226 | if [[ -d "$target_local" ]]; then 227 | echoerr "\n$2 is not a single file, multiple files " \ 228 | "were downloaded and saved in\n" 229 | echo $target_local 230 | else 231 | local before=$(stat -c %Y "$target_local") 232 | $1 "$target_local" 233 | local after=$(stat -c %Y "$target_local") 234 | if ! [[ "$before" == "$after" ]]; then 235 | # file was modified offer to upload to original location 236 | echoerr "\nWARNING: File \"${target_local}\" has changed, please don't forget to upload the file using this command:" 237 | echoerr "${scriptname} upload \"${target_local}\" \"/${cont}/${obj}\"" 238 | local plainfile=$(basename "${target_local}") 239 | read -p "Do you want to upload \"${plainfile}\" now? (Y/N)" -n 1 -r 240 | if [[ $REPLY =~ ^[Yy]$ ]]; then 241 | swc upload "${target_local}" "/${cont}/${obj}" 242 | else 243 | echoerr "\n" 244 | fi 245 | fi 246 | fi 247 | } 248 | 249 | refAuthToken () { 250 | local file=~/.swift/auth_token_${swcfile} 251 | eval $(swift auth) 252 | if [[ -n ${OS_AUTH_TOKEN} ]]; then 253 | echo $OS_AUTH_TOKEN > $file 254 | fi 255 | } 256 | 257 | checkAuthToken () { 258 | local file=~/.swift/auth_token_${swcfile} 259 | local maxage=$((5*24*60*60)) # seconds in a day, token is good for 7 days, we ref after 5 260 | local fileage=$(($(date +%s) - $(stat -c '%Y' "$file"))) 261 | test $fileage -ge $maxage && { 262 | ### new method using swift client 2.5 263 | if [[ -z $OS_AUTH_TOKEN || -z $OS_STORAGE_URL ]]; then 264 | if [[ -z $swiftpassword ]]; then 265 | read -t 60 -p "Enter your swift password/key for ${swifttenant}/${swiftaccount}: " -s mypass 266 | if [[ -z $mypass ]]; then 267 | echoerr "Error: Token expired or no password entered within timeout period." 268 | exit 1 269 | fi 270 | if [[ -n $OS_USERNAME ]]; then 271 | export OS_PASSWORD=$mypass 272 | elif [[ -n $ST_USER ]]; then 273 | export ST_KEY=$mypass 274 | else 275 | echoerr "no credentials set" 276 | fi 277 | fi 278 | fi 279 | echoerr "\nRefreshing credentials ....please wait" 280 | eval $(swift auth) 281 | #tokens=$(${mycurl} -sd '{"auth":{"passwordCredentials":{"username": \ 282 | # "Swift__ADM_SciComp", "password": "Rsh"}}}' \ 283 | # -H "Content-type: application/json" https://tin.fhcrc.org/v2.0/tokens) 284 | if [[ -n $OS_AUTH_TOKEN ]]; then 285 | echo $OS_AUTH_TOKEN > $file 286 | else 287 | echoerr "Environment var OS_AUTH_TOKEN could not be set." 288 | fi 289 | if [[ -n $OS_STORAGE_URL ]]; then 290 | echo $OS_STORAGE_URL > ~/.swift/storageurl_${swcfile} 291 | else 292 | echoerr "Environment var OS_STORAGE_URL could not be set." 293 | fi 294 | } 295 | } 296 | 297 | ############## main script ##############################3 298 | red=$(tput setaf 1) # set font color 299 | yellow=$(tput setaf 2) # set font color 300 | blue=$(tput setaf 4) # set font color 301 | endcolor=$(tput sgr0) # reset the foreground colour 302 | oldifs=$IFS # gets the current internal field seperator, default: $IFS=$' \t\n' 303 | if [[ -z $oldifs ]]; then 304 | oldifs=$' \t\n' 305 | fi 306 | tempcfg=0 307 | target_local="" # local file name or dir set by download function 308 | 309 | # check if swift, curl and jq exist in path 310 | dependency_msg="The following linux packages are required:\npython-swiftclient, curl & jq" 311 | if ! hash swift 2>/dev/null; then 312 | echoerr "no swift client installed, please install package 'python-swiftclient'" 313 | echoerr $dependency_msg 314 | fi 315 | if ! hash ${mycurl} 2>/dev/null; then 316 | echoerr "curl not installed, please install package 'curl'" 317 | echoerr $dependency_msg 318 | fi 319 | if ! hash jq 2>/dev/null; then 320 | echoerr "json processor jq not installed, please install package 'jq'" 321 | echoerr $dependency_msg 322 | fi 323 | #if ! hash sw2account 2>/dev/null; then 324 | # echoerr "no sw2account found, please install package 'swift-switch-account'" 325 | #fi 326 | # check if swift commander was invoked as swc with subcommands 327 | 328 | subcmd=$1 329 | selector=$subcmd 330 | shift 331 | 332 | swifttenant="" 333 | swiftaccount="" 334 | swiftpassword="" 335 | swiftauthurl="" 336 | swiftauthver="" 337 | swiftserver="" 338 | 339 | swiftCreds 340 | 341 | # get swift credentials 342 | if [[ -n ${OS_AUTH_TOKEN} && -n ${OS_STORAGE_URL} ]]; then 343 | echoerr " " 344 | elif [[ -z $swiftauthurl ]]; then 345 | echoerr " Swift Authentication URL not set. Contact your Administrator or" 346 | echoerr " put something like this into your ~/.bashrc file and login again:" 347 | echoerr "export OS_AUTH_URL=https://cluster.yourcompany.com/auth/v2.0" 348 | echoerr "export OS_USERNAME=you" 349 | echoerr "export OS_TENANT_NAME=AUTH_YourDepartment" 350 | echoerr " or, to switch frequently between multiple accounts install this:" 351 | echoerr "https://github.com/FredHutch/swift-switch-account" 352 | exit 1 353 | elif [[ -z $swiftaccount && -z $OS_STORAGE_URL ]]; then 354 | echoerr " *** Swift credentials not set. ***" 355 | echoerr "Please execute 'sw2account ' to get credentials." 356 | echoerr "Use 'sw2account --save ' to set them as default." 357 | echoerr " Example: sw2account lastname_f" 358 | echoerr "If the 'sw2account' command does not exist please ask your" 359 | echoerr "local System Administrator to install swift-switch-account." 360 | exit 1 361 | fi 362 | 363 | #echo OS_AUTH_TOKEN: ${OS_AUTH_TOKEN} 364 | 365 | # create .swift folder to persist settings / temp data on file system 366 | swcfile=${swiftserver}_${swiftauthver}_${swifttenant} 367 | if ! [[ -f ~/.swift/auth_token_${swcfile} ]]; then 368 | mkdir -p ~/.swift 369 | touch ~/.swift/storageurl_${swcfile} 370 | echo "/" > ~/.swift/current_folder_${swcfile} 371 | touch -d "7 days ago" ~/.swift/auth_token_${swcfile} 372 | chmod 600 ~/.swift/auth_token_${swcfile} 373 | fi 374 | 375 | authfromenv=0 376 | if [[ -z $OS_AUTH_TOKEN || -z $OS_STORAGE_URL ]]; then 377 | #echo "checking authtoken..." 378 | checkAuthToken 379 | export OS_AUTH_TOKEN=`cat ~/.swift/auth_token_${swcfile}` 380 | export OS_STORAGE_URL=`cat ~/.swift/storageurl_${swcfile}` 381 | else 382 | echoerr " (..using token credentials from environment)" 383 | echo "$OS_AUTH_TOKEN" > ~/.swift/auth_token_${swcfile} 384 | echo "$OS_STORAGE_URL" > ~/.swift/storageurl_${swcfile} 385 | authfromenv=1 386 | fi 387 | 388 | if [[ -z $SLURM_JOB_CPUS_PER_NODE ]]; then 389 | # here we count the cores on a single socket so we use typically half the numbers of cores per system 390 | cpusavail=$(grep -w "core id" /proc/cpuinfo | sort -u | wc -l) 391 | else 392 | # or if this is running on a cluster use all CPUs reserved for this job 393 | cpusavail=$SLURM_JOB_CPUS_PER_NODE 394 | fi 395 | 396 | # how many swift client threads do we want to use 397 | threads=10 398 | # checking if this is a 10G connection 399 | if [[ $(nicspeed) -ge 10000 ]]; then 400 | threads=25 401 | fi 402 | 403 | logger "swc ran by $USER with $@" 404 | 405 | ######### TEST ZONE ################################# 406 | 407 | #ret=`swift list something` 408 | #if ! [[ $? = 0 ]]; then 409 | # echoerr "Error executing 'swift list' command." 410 | #fi 411 | #exit 412 | #env | grep ST_ 413 | #env | grep OS_ 414 | ### 415 | if [[ "$USER" == "petersen" ]]; then 416 | dummyvar=0 417 | fi 418 | 419 | ######### END TEST ZONE ############################### 420 | 421 | case "$selector" in 422 | *upload | up) 423 | refAuthToken 424 | # need to fix upload to container root 425 | # one level too much 426 | if [[ -z $1 || -z $2 || "$1" == "--help" ]]; then 427 | echoerr "\nusage: $scriptname $subcmd " 428 | echoerr " -> uploading file or directory to swift object store\n" 429 | echoerr "also please check these commands:" 430 | printcommands 431 | exit 432 | fi 433 | dosymlinks=1 434 | if [[ "$1" == "--no-symlinks" ]]; then 435 | dosymlinks=0 436 | shift 437 | fi 438 | if [[ "$2" == "/" ]]; then 439 | echoerr "you cannot upload to the root, please upload to a folder" 440 | exit 1 441 | fi 442 | pathexist "$1" 443 | src=${1%/} #remove trailing slash from source 444 | splitpath "$2" 445 | if [[ "$cont" == "." ]]; then 446 | echoerr "$cont/$obj is not a valid target name" 447 | exit 448 | fi 449 | if [[ "$obj" == "." ]]; then 450 | obj="" 451 | fi 452 | targ="" 453 | if [[ "${2: -1}" = "/" || -z $obj ]]; then # 454 | # if target ends with a / or if prefix is empty 455 | if [[ -f "$src" ]]; then # if src is a file 456 | targ="${1##*/}" # extract filename from path $1 457 | else 458 | if [[ "$2" != "/$cont" ]]; then #only needed if targ!=container root 459 | targ=`basename "$1"` 460 | fi 461 | fi 462 | obj+=$targ # append file or base dir to target 463 | fi 464 | # check for addional meta data passed as commands 465 | cmdcnt=0 466 | metaheader='' 467 | for myarg in "$@"; do 468 | ((cmdcnt+=1)) 469 | if [[ $cmdcnt -ge 3 ]]; then 470 | myarg=${myarg// /_} 471 | metaheader+=" --header=X-Object-Meta-$myarg" 472 | fi 473 | done 474 | # a text file with all symlink info in dumped into the current directory 475 | if [[ ${dosymlinks} == 1 ]]; then 476 | swsymlinks.py --save --folder "$src" 477 | fi 478 | me=$(findRealUser) 479 | # 2GB = 2147483648; 1GB = 1073741824, 0.5 = 536870912, 0.25 = 268435456 480 | swiftcmd="swift upload ${OS_SWIFT_OPTS} --changed --segment-size=1073741824" 481 | swiftcmd+=" --use-slo --segment-container=\".segments_$cont\"" 482 | swiftcmd+=" --object-threads=$threads --segment-threads=$threads" 483 | swiftcmd+=" --header=\"X-Object-Meta-Uploaded-by:$me\"$metaheader --object-name=\"$obj\" \"$cont\" \"$src\"" 484 | # --skip-identical is better than --changed because it uses md5sum 485 | # to compare, however there is a bug: 486 | # https://bugs.launchpad.net/python-swiftclient/+bug/1379252: 487 | # --segment-size=2g is possible with latest swiftclient master 488 | # only, 2147483648 = gibibyte 489 | echoerr "*** uploading $src ***" 490 | echoerr "*** to ${yellow}${swifttenant}${endcolor}:/$cont/$obj ***" 491 | echoerr "executing:$swiftcmd" 492 | echoerr "*** please wait... ***" 493 | #ret=`swift upload --changed --segment-size=1073741824 \ 494 | #ret=`swift upload --changed --segment-size=1073741824 \ 495 | # --use-slo --segment-container=".segments_$cont" \ 496 | # --header="X-Object-Meta-Uploaded-by:$me"$metaheader --object-name="$obj" "$cont" "$src" 1>&2` 497 | swift upload ${OS_SWIFT_OPTS} --changed --segment-size=1073741824 \ 498 | --use-slo --segment-container=".segments_$cont" \ 499 | --object-threads="$threads" --segment-threads="$threads" \ 500 | --header="X-Object-Meta-Uploaded-by:$me"$metaheader --object-name="$obj" "$cont" "$src" 501 | retcode=$? 502 | if [[ $retcode = 0 ]]; then 503 | echo "/$cont/$obj" 504 | else 505 | echoerr "error $retcode uploading $src to /$cont/$obj" 506 | fi 507 | cd $cwd 508 | ;; 509 | *download | down) 510 | refAuthToken 511 | if [[ -z "$1" || "$1" == "--help" ]]; then 512 | echoerr "\nusage: $scriptname $subcmd " 513 | echoerr " -> downloading file or directory from swift object store\n" 514 | echoerr "also please check these commands:" 515 | printcommands 516 | exit 517 | fi 518 | tempset 519 | if [[ -z $2 ]]; then 520 | echoerr "\nusage: $scriptname $subcmd " 521 | echoerr " -> downloading file or directory from swift object store.\n" 522 | echoerr "You cannot omit . If you would like to download to" 523 | echoerr "the current directory, please use a single dot . as target\n" 524 | echoerr 'Another popular option is using the variable $TMPDIR as target.' 525 | echoerr "If \$TMPDIR is not set at login time, $scriptname $subcmd can generate" 526 | echoerr "a \$TMPDIR such as $TMPDIR on the fly each time" 527 | echoerr "it is run to ensure that it is empty. Use string SWCTMP as target.\n" 528 | echoerr "Examples:" 529 | echoerr "$scriptname $subcmd \"$1\" \$TMPDIR (if TMPDIR is already set)" 530 | echoerr "$scriptname $subcmd \"$1\" SWCTMP (if swc needs to set TMPDIR)" 531 | echoerr "\nalso please check these commands:" 532 | printcommands 533 | exit 534 | fi 535 | mkdirtarg=0 536 | if [[ "$1" == "--mkdir" ]]; then 537 | mkdirtarg=1 538 | shift 539 | fi 540 | splitpath "$1" 541 | dest="$2" 542 | if [[ "$2" == "SWCTMP" ]]; then 543 | dest=$TMPDIR 544 | echoerr "using TMPDIR=$dest" 545 | fi 546 | swpget="0" 547 | if hash swpget.py 2>/dev/null; then 548 | swpget="1" 549 | fi 550 | obj=${obj//"*"/""} # replace @ with whitespace 551 | url=$(urlencode "/${cont}/${obj}") 552 | HTTPGetHeader "$url" 553 | #echo http_status: $http_status 554 | #echo head_slo: $head_slo 555 | #echo head_ct: $head_ct 556 | #echo head_cl: $head_cl 557 | if [[ $http_status == "200" ]]; then 558 | #echo "this is a file" 559 | # destination must be a file 560 | #echo $http_status 561 | if [[ $mkdirtarg == 1 ]]; then 562 | mkdir -p "$dest" 563 | fi 564 | if [[ -d "$dest" ]]; then 565 | # destination is still a directory 566 | srcfile=$(basename "$1") 567 | fi 568 | workers=5 569 | # checking if this is a 10G connection 570 | if [[ $(nicspeed) -ge 10000 ]]; then 571 | workers=25 572 | fi 573 | if [[ $head_slo == "True" && $swpget == "1" && $dummy == "nottrue" ]]; then 574 | swiftcmd="swpget.py -p $workers -l \"$dest\" -c \"$cont\" \"$obj\"" 575 | else 576 | swiftcmd="swift download $cont --output=\"${dest}/${srcfile}\" --object-threads=$threads --container-threads=$threads" 577 | fi 578 | else 579 | #echo "this is a dir" 580 | # destination must be a directory 581 | #echo "directory: $http_status" 582 | if [[ "$dest" == */ ]]; then # if trailing slash 583 | dest=${dest}`basename "$1"` # add basename to target 584 | fi # just like rsync 585 | if ! [[ -d "$dest" ]]; then 586 | mkdir -p "$dest" 587 | fi 588 | swiftcmd="swift download $cont --prefix=\"$obj\" --output-dir=\"$dest\" --remove-prefix --object-threads=$threads --container-threads=$threads" 589 | #cd "$dest" 590 | fi 591 | echoerr "...executing:$swiftcmd" 592 | echoerr "...downloading $1 to $dest please wait..." 593 | if [[ $http_status == "200" ]]; then 594 | # again, it must be a file 595 | #ret=`swift download --outputdir="$dest" --remove-prefix "$cont" "$obj" 1>&2` 596 | if [[ -d $dest ]]; then 597 | cd $dest 598 | fi 599 | if [[ $head_slo == "True" && $swpget == "1" ]]; then 600 | swpget.py -p $workers -l "$dest" -c "$cont" "$obj" 601 | else 602 | swift download "$cont" "$obj" ${OS_SWIFT_OPTS} --output="${dest}/${srcfile}" --remove-prefix --object-threads=$threads --container-threads=$threads 603 | fi 604 | target_local="${dest}/${srcfile}" 605 | else 606 | # again, it must be a directory 607 | #ret=`swift download "$cont" --prefix="$obj" --output-dir="$dest" --remove-prefix 1>&2` 608 | swift download "$cont" --prefix="$obj" --output-dir="$dest" ${OS_SWIFT_OPTS} --remove-prefix --object-threads=$threads --container-threads=$threads 609 | swsymlinks.py --restore --folder "$dest" 610 | target_local="${dest}" 611 | fi 612 | if [[ $? = 0 ]]; then 613 | echo "$target_local" 614 | else 615 | echoerr "error downloading $1" 616 | fi 617 | cd "$cwd" 618 | ;; 619 | *cd) 620 | if [[ -z $1 || "$1" == "--help" ]]; then 621 | echoerr "\nusage: $scriptname $subcmd " 622 | echoerr " -> change current folder in swift object store\n" 623 | echoerr "also please check these commands:" 624 | printcommands 625 | exit 626 | fi 627 | splitpath "$1" 628 | dest="$1" 629 | if [[ "$dest" = "/" ]]; then 630 | echo "$dest" > ~/.swift/current_folder_${swcfile} 631 | echo "$dest" 632 | exit 633 | fi 634 | cur_fld=`cat ~/.swift/current_folder_${swcfile}` 635 | #echoerr "cd cur_fld:$cur_fld" 636 | if ! [[ -z $cur_fld ]]; then 637 | if [[ $dest = ".." ]]; then 638 | dest=`dirname "$cur_fld"` 639 | #echoerr "dest:$dest" 640 | if [[ -z $dest ]]; then 641 | dest="/" 642 | fi 643 | echo "$dest" > ~/.swift/current_folder_${swcfile} 644 | echo "$dest" 645 | exit 646 | fi 647 | if [[ "${dest:0:1}" != "/" ]]; then # relative cd (folder does not start with a /) 648 | dest=${cur_fld%/}/$dest 649 | #echoerr "splitpath dest:$dest" 650 | splitpath "$dest" 651 | fi 652 | fi 653 | dest="${dest%/}" # remove trailing slash 654 | obj=${obj%/} #remove trailing slash 655 | #echoerr "cont:$cont" 656 | #echoerr "obj:$obj" 657 | IFS=$'\t\n' # change internal field seperator 658 | if [[ "$obj" = "" ]]; then 659 | #objlist=(`swift list --delimiter=/ "$cont"`) 660 | objlist=(`${mycurl} -s -H "X-Auth-Token: $OS_AUTH_TOKEN" "$OS_STORAGE_URL/$cont/?delimiter=/"`) 661 | else 662 | #objlist=(`swift list --prefix="$obj/" --delimiter=/ "$cont"`) #add a trailing slash to ensure pseudo dir 663 | objlist=(`${mycurl} -s -H "X-Auth-Token: $OS_AUTH_TOKEN" "$OS_STORAGE_URL/$cont/?delimiter=/&prefix=$obj/"`) 664 | fi 665 | IFS=$oldifs 666 | if [[ $? != 0 ]]; then 667 | echoerr "error executing swift!" 668 | fi 669 | #echo "objlist:$objlist" 670 | numobj=${#objlist[@]} 671 | #echoerr "numobj2:$numobj" 672 | if [[ $numobj -gt 0 ]]; then 673 | dest="${dest%/}" # remove trailng slash 674 | echo $dest > ~/.swift/current_folder_${swcfile} 675 | #echo "wrote $dest to .swift/current_folder_${swcfile}" 676 | echo $dest 677 | else 678 | if [[ "$obj" != "" ]]; then 679 | echoerr "folder $dest does not exist." 680 | else 681 | ret=`swift stat "$cont" 2>&1` 682 | if [[ $? != 0 ]]; then 683 | echoerr "folder $dest does not exist." 684 | else 685 | echoerr "folder $dest is empty." 686 | echo $dest > ~/.swift/current_folder_${swcfile} 687 | fi 688 | fi 689 | cat ~/.swift/current_folder_${swcfile} 690 | fi 691 | ;; 692 | *pwd) 693 | cat ~/.swift/current_folder_${swcfile} 694 | ;; 695 | *ls) 696 | if [[ "$1" == "--help" ]]; then 697 | echoerr "list contents of a folder" 698 | exit 699 | fi 700 | lsbatch=0 701 | if [[ "$1" == "--batch" ]]; then 702 | echoerr "switching to batchmode" 703 | lsbatch=1 704 | shift 705 | fi 706 | dest="$1" 707 | cur_fld=`cat ~/.swift/current_folder_${swcfile}` 708 | if ! [[ -z $dest ]]; then 709 | if [[ $dest = ".." ]]; then 710 | dest=`dirname "$cur_fld"` 711 | #echoerr "dest:$dest" 712 | fi 713 | if [[ "${dest:0:1}" != "/" ]]; then # relative cd (folder does not start with a /) 714 | dest=${cur_fld%/}/$dest 715 | #echoerr "splitpath dest:$dest" 716 | splitpath "$dest" 717 | fi 718 | else 719 | dest=$cur_fld 720 | fi 721 | splitpath "$dest" 722 | #echoerr "cur_fld:$cur_fld"` 723 | if [[ -z $dest ]]; then 724 | echoerr "no current folder, use swc cd to change folder." 725 | exit 726 | fi 727 | obj=${obj%/} #remove trailing slash 728 | #objenc=${obj//" "/"%20"} #replace all whitespaces with %20 729 | #objenc=${objenc//"+"/"%2b"} #replace all + with %2b 730 | objenc=$(urlencode "${obj}") 731 | #echoerr "ls cont:$cont" 732 | #echoerr "ls obj:$obj" 733 | IFS=$'\t\n' # change internal field seperator 734 | if [[ "$obj" = "" ]]; then 735 | #objlist=(`swift list --delimiter=/ "$cont"`) 736 | objlist=(`${mycurl} -s -H "X-Auth-Token: $OS_AUTH_TOKEN" "$OS_STORAGE_URL/$cont/?delimiter=/"`) 737 | else 738 | #bjlist=(`swift list --prefix="$obj/" --delimiter=/ "$cont"`) #add a trailing slash 739 | objlist=(`${mycurl} -s -H "X-Auth-Token: $OS_AUTH_TOKEN" "$OS_STORAGE_URL/$cont/?delimiter=/&prefix=$objenc/"`) 740 | fi 741 | #echoerr $OS_AUTH_TOKEN 742 | retcode=$? 743 | if [[ $retcode != 0 ]]; then 744 | if [[ $retcode == 77 ]]; then 745 | echoerr "curl has problems reading the SSL CA cert" 746 | elif [[ $retcode == 1 ]]; then 747 | echoerr "curl error 1, unsupported protocol (e.g. https error)" 748 | elif [[ $retcode == 3 ]]; then 749 | echoerr "curl error 3, malformed URL, e.g. when no token available" 750 | else 751 | echoerr "curl error, return code:$retcode" 752 | fi 753 | echo "curl binary used:" 754 | which curl 755 | exit 1 756 | fi 757 | IFS=$oldifs 758 | if [[ "${objlist[0]}" =~ "

Unauthorized

" ]]; then 759 | rm -f ~/.swift/auth_token_${swcfile} 760 | echo "Removed invalid or outdated auth token. Please try again." 761 | exit 1 762 | fi 763 | numobj=${#objlist[@]} 764 | #echo "numobj:$numobj" 765 | if [[ $numobj -eq 0 ]]; then 766 | echoerr "folder $dest is empty." 767 | exit 768 | fi 769 | #echo "objlist:$objlist" 770 | lenprefix=${#obj} 771 | #echoerr "lenprefix:$lenprefix" 772 | declare -a myfolders=() 773 | for t in "${objlist[@]}"; do 774 | #echo "t:$t" 775 | # looping through object list, 776 | recurpath="${t:$lenprefix}" # list cur dir recursively 777 | recurpath=${recurpath#/} # remove leading slash 778 | #echoerr "recurpath:$recurpath" 779 | thislevel="${recurpath%%/*}" # extract everything left of first slash 780 | #echoerr "thislevel:$thislevel" 781 | if [[ "$recurpath" = "$thislevel" ]]; then 782 | if [[ $thislevel != *_segments && $thislevel != .* ]]; then 783 | # ignore hidden folders and folders that just contain segments 784 | if [[ $cont == "" && $lsbatch == 0 ]]; then 785 | echo "${yellow}$thislevel${endcolor}" # print containers yellow 786 | else 787 | echo "$thislevel" 788 | fi 789 | fi 790 | else 791 | #echoerr "myfolders:${myfolders[@]}" 792 | if ! ( arrayContains myfolders "$thislevel" ); then 793 | if [[ $thislevel != .* ]]; then 794 | if [[ $lsbatch == 0 ]]; then 795 | echo "${blue}$thislevel${endcolor}" 796 | else 797 | echo "$thislevel" 798 | fi 799 | fi 800 | myfolders=("${myfolders[@]}" "$thislevel") # add element to array 801 | fi 802 | fi 803 | done 804 | #IFS=$oldifs 805 | echoerr " ..current folder (${swifttenant}):" 806 | echoerr "$cur_fld" 807 | ;; 808 | *list) 809 | if [[ -z $1 || "$1" == "--help" ]]; then 810 | echoerr "\nusage: $scriptname $subcmd [search string]" 811 | echoerr " -> list all folders/files below path\n" 812 | echoerr "optionally filter result by search string" 813 | echoerr "also please check these commands:" 814 | printcommands 815 | swift list --lh | grep -v ".trash-" | grep -v "_segments" | grep -v ".segments_" 816 | exit 817 | fi 818 | dest=$1 819 | splitpath "$dest" 820 | #echoerr "obj:$obj" 821 | #echoerr "cont:$cont" 822 | refAuthToken 823 | if [[ -z $obj ]]; then 824 | if [[ -z $cont ]]; then 825 | swift list --lh 826 | else 827 | if [[ -z $2 ]]; then 828 | swift list --lh "$cont" 829 | else 830 | swift list --lh "$cont" | grep -i $2 831 | fi 832 | fi 833 | else 834 | if [[ -z $2 ]]; then 835 | swift list --lh --prefix="$obj" "$cont" 836 | else 837 | swift list --lh --prefix="$obj" "$cont" | grep -i $2 838 | fi 839 | fi 840 | ;; 841 | search) 842 | refAuthToken 843 | if [[ -z $1 || -z $2 || "$1" == "--help" ]]; then 844 | echoerr "\nusage: $scriptname $subcmd [-i] <\"search string\"> " 845 | echoerr " -> simple full text search in text files" 846 | echoerr " -> non-text files are excluded from search\n" 847 | echoerr "also please check these commands:" 848 | printcommands 849 | exit 1 850 | fi 851 | casesensitive="" 852 | if [[ "$1" == "-i" ]]; then 853 | casesensitive="-i " 854 | shift 855 | fi 856 | splitpath "$2" 857 | if ! hash swsearch.py 2>/dev/null; then 858 | echoerr "swsearch.py not found." 859 | exit 1 860 | fi 861 | # how many swift client processses do we want to use 862 | procs=10 863 | # checking if this is a 10G connection 864 | if [[ $(nicspeed) -ge 10000 ]]; then 865 | procs=30 866 | fi 867 | #obj=${obj%/} #remove trailing slash from source 868 | echoerr "searching for \"$1\" ..." 869 | swsearch.py ${casesensitive}-m $procs -c "$cont" -p "$obj" "$1" 870 | ;; 871 | *rm) 872 | refAuthToken 873 | if [[ -z $1 || "$1" == "--help" ]]; then 874 | echoerr "\nusage: $scriptname $subcmd " 875 | echoerr " -> delete files/objects from swift\n" 876 | echoerr " $scriptname $subcmd --single " 877 | echoerr " -> delete a single file using tradional method\n" 878 | echoerr "also please check these commands:" 879 | printcommands 880 | exit 1 881 | fi 882 | rmforce=0 883 | if [[ "$1" == "-rf" ]]; then 884 | echoerr "force delete" 885 | rmforce=1 886 | shift 887 | fi 888 | delsingle=0 889 | if [[ "$1" == "--single" ]]; then 890 | delsingle=1 891 | shift 892 | fi 893 | dest="$1" 894 | if [[ $dest = "/" ]]; then 895 | echoerr "cannot delete root folder" 896 | exit 1 897 | fi 898 | if [[ $dest != /* ]]; then 899 | echoerr "$scriptname $subcmd currently does not support relative paths, path must start with '/'" 900 | exit 1 901 | fi 902 | splitpath "$dest" 903 | echoerr "using account ${yellow}${swifttenant}${endcolor}:" 904 | if [[ $delsingle == 1 ]]; then 905 | echoerr "...swift delete $cont $obj" 906 | swift delete "$cont" "$obj" 907 | else 908 | if [[ $rmforce == 1 ]]; then 909 | echoerr "...swrm.py -c $cont -p $obj --force" 910 | swrm.py -c "$cont" -p "$obj" -f 911 | else 912 | echoerr "...swrm.py -c $cont -p $obj" 913 | swrm.py -c "$cont" -p "$obj" 914 | fi 915 | fi 916 | ;; 917 | *meta) 918 | if [[ -z $1 || "$1" == "--help" ]]; then 919 | echoerr "\nusage: $scriptname $subcmd " 920 | echoerr " -> show meta data of a certain file\n" 921 | echoerr "also please check these commands:" 922 | printcommands 923 | #swift list --lh | grep -v ".trash-" | grep -v "_segments" | grep -v ".segments_" 924 | exit 925 | fi 926 | dest=$1 927 | splitpath "$dest" 928 | #echoerr "obj:$obj" 929 | #echoerr "cont:$cont" 930 | swift stat $cont $obj | grep Meta 931 | ;; 932 | *header) 933 | if [[ "$1" == "--help" ]]; then 934 | echoerr "\nusage: $scriptname $subcmd " 935 | echoerr " -> show http header for this file or folder\n" 936 | echoerr "also please check these commands:" 937 | printcommands 938 | exit 939 | fi 940 | dest=$1 941 | splitpath "$dest" 942 | ${mycurl} --head -H "X-Auth-Token: $OS_AUTH_TOKEN" ${OS_STORAGE_URL}${dest} 943 | ;; 944 | mtime) 945 | if [[ "$1" == "--help" ]]; then 946 | echoerr "\nusage: $scriptname $subcmd " 947 | echoerr " -> show the original mtime of this file before uploaded\n" 948 | echoerr "also please check these commands:" 949 | printcommands 950 | exit 951 | fi 952 | dest=$1 953 | splitpath "$dest" 954 | ret=`${mycurl} -s --head -H "X-Auth-Token: $OS_AUTH_TOKEN" ${OS_STORAGE_URL}${dest} | grep -e "^X-Object-Meta-Mtime: "` 955 | mtime=${ret/"X-Object-Meta-Mtime: "/""} 956 | date -d@$mtime 957 | ;; 958 | hash) 959 | if [[ -z $1 || -z $2 || "$1" == "--help" ]]; then 960 | echoerr "\nusage: $scriptname $subcmd " 961 | echoerr " -> compare the md5sum of a local file with a file in swift" 962 | echoerr " if /local/file does not exist, swc assumes the argument" 963 | echoerr " to be an md5 hash if it is 32 chars long\n" 964 | echoerr "also please check these commands:" 965 | printcommands 966 | exit 1 967 | fi 968 | splitpath "$2" 969 | if ! hash swhashcomp.py 2>/dev/null; then 970 | echoerr "swhashcomp.py not found." 971 | exit 1 972 | fi 973 | swhashcomp.py --locfile="$1" --container=$cont --obj="$obj" 974 | ;; 975 | *compare) 976 | if [[ -z $1 || -z $2 || "$1" == "--help" ]]; then 977 | echoerr "\nusage: $scriptname $subcmd " 978 | echoerr " -> compare the size of a local folder with a swift folder\n" 979 | echoerr "also please check these commands:" 980 | printcommands 981 | exit 1 982 | fi 983 | refAuthToken 984 | objdir=${obj%/} 985 | if ! hash swfoldersize.py 2>/dev/null; then 986 | echoerr "swfoldersize.py not found." 987 | exit 1 988 | fi 989 | splitpath "$2" 990 | obj=${obj%/} # remove trailing slash 991 | if ! [[ -z $obj ]]; then 992 | obj=$obj/ 993 | fi 994 | if [[ -d "$1" && -d "$2" ]]; then 995 | swfoldersize.py --posixfolder="$1" --posixfolder2="$2" ${OS_SWIFT_OPTS} 996 | else 997 | if [[ -d "$1" ]]; then 998 | splitpath "$2" 999 | fld=$1 1000 | elif [[ -d "$2" ]]; then 1001 | fld=$2 1002 | splitpath "$1" 1003 | else 1004 | echoerr "Neither $1 nor $2 could be detected as Posix folders" 1005 | exit 1 1006 | fi 1007 | obj=${obj%/} # remove trailing slash 1008 | if ! [[ -z ${obj} ]]; then 1009 | obj=${obj}/ 1010 | fi 1011 | swfoldersize.py --posixfolder="${fld}" --container=${cont} --prefix="${obj}" ${OS_SWIFT_OPTS} 1012 | fi 1013 | ;; 1014 | size) 1015 | if [[ -z $1 || "$1" == "--help" ]]; then 1016 | echoerr "\nusage: $scriptname $subcmd " 1017 | echoerr " -> get the size of a swift folder or local folder\n" 1018 | echoerr "also please check these commands:" 1019 | printcommands 1020 | exit 1 1021 | fi 1022 | splitpath "$1" 1023 | if ! hash swfoldersize.py 2>/dev/null; then 1024 | echoerr "swfoldersize.py not found." 1025 | exit 1 1026 | fi 1027 | if [[ -d $1 ]]; then 1028 | swfoldersize.py --posixfolder="$1" ${OS_SWIFT_OPTS} 1029 | else 1030 | swfoldersize.py --container=$cont --prefix="$obj" ${OS_SWIFT_OPTS} 1031 | fi 1032 | ;; 1033 | arch | archive) 1034 | refAuthToken 1035 | if [[ -z $1 || -z $2 || "$1" == "--help" ]]; then 1036 | echoerr "\nusage: $scriptname $subcmd " 1037 | echoerr " -> archive local folder structure to swift" 1038 | echoerr " -> create one tar.gz file per folder and level\n" 1039 | echoerr "also please check these commands:" 1040 | printcommands 1041 | exit 1 1042 | fi 1043 | splitpath "$2" 1044 | if ! hash swbundler.py 2>/dev/null; then 1045 | echoerr "swbundler.py not found." 1046 | exit 1 1047 | fi 1048 | tempset 1049 | src=${1%/} #remove trailing slash from source 1050 | # check for addional meta data passed as commands 1051 | cmdcnt=0 1052 | metaheader='' 1053 | for myarg in "$@"; do 1054 | ((cmdcnt+=1)) 1055 | if [[ $cmdcnt -ge 3 ]]; then 1056 | myarg=${myarg// /_} 1057 | #metaheader+=" --header=X-Object-Meta-$myarg" 1058 | metaheader+=" -m $myarg" 1059 | fi 1060 | done 1061 | swiftcmd="swbundler.py" 1062 | swiftcmd+=" -l \"$1\" -c $cont -p \"$obj\" -t \"$TMPDIR\" -P $cpusavail $metaheader" 1063 | echoerr "*** uploading $src ***" 1064 | echoerr "*** to ${yellow}${swifttenant}${endcolor}:/$cont/$obj ***" 1065 | echoerr "executing:$swiftcmd" 1066 | echoerr "*** please wait... ***" 1067 | swbundler.py -l "$1" -c $cont -p "$obj" -t "$TMPDIR" -P $cpusavail $metaheader 1068 | retcode=$? 1069 | if [[ $retcode = 0 ]]; then 1070 | echo "/$cont/$obj" 1071 | else 1072 | echoerr "error $retcode uploading $src to /$cont/$obj" 1073 | fi 1074 | ;; 1075 | unarch | unarchive | unbundle | unbun) 1076 | refAuthToken 1077 | if [[ -z $1 || -z $2 || "$1" == "--help" ]]; then 1078 | echoerr "\nusage: $scriptname $subcmd " 1079 | echoerr " -> extract archived swift folders to local folders\n" 1080 | echoerr "also please check these commands:" 1081 | printcommands 1082 | exit 1 1083 | fi 1084 | splitpath "$1" 1085 | if ! hash swbundler.py 2>/dev/null; then 1086 | echoerr "swbundler.py not found." 1087 | exit 1 1088 | fi 1089 | tempset 1090 | mkdir -p "$2" 1091 | src=${1%/} #remove trailing slash from source 1092 | swiftcmd="swbundler.py -x" 1093 | swiftcmd+=" -l \"$2\" -c $cont -p \"$obj\" -t \"$TMPDIR\"" 1094 | echoerr "*** downloading to $2 ***" 1095 | echoerr "*** from ${yellow}$swifttenant${endcolor}:/$cont/$obj ***" 1096 | echoerr "executing:$swiftcmd" 1097 | echoerr "*** please wait... ***" 1098 | swbundler.py -x -l "$2" -c $cont -p "$obj" -t $TMPDIR -P $cpusavail 1099 | retcode=$? 1100 | if [[ $retcode = 0 ]]; then 1101 | echo "/$cont/$obj" 1102 | else 1103 | echoerr "error $retcode downloading $src to /$cont/$obj" 1104 | fi 1105 | # consider SLURM_JOB_CPUS_PER_NODE 1106 | ;; 1107 | publish | pub | hide) 1108 | if [[ "$1" == "--help" ]]; then 1109 | echoerr "\nusage: $scriptname publish [/root_folder]" 1110 | echoerr " -> allow anonymous read access to container." 1111 | echoerr " -> (basically turning swift into a web server)" 1112 | echoerr " -> $" 1113 | echoerr " -> example: $scriptname pub ${OS_STORAGE_URL}/MyCont" 1114 | echoerr " -> example: $scriptname hide ${OS_STORAGE_URL}/MyCont" 1115 | echoerr " -> query: curl -s ${OS_STORAGE_URL}/MyCont?format=json" 1116 | echoerr "also please check these commands:" 1117 | printcommands 1118 | exit 1 1119 | fi 1120 | splitpath $1 1121 | if [[ -z $cont ]]; then 1122 | echoerr "please enter a root folder / container name" 1123 | exit 1 1124 | fi 1125 | if [[ "$subcmd" == "hide" ]]; then 1126 | ${mycurl} -X POST -s -H "X-Auth-Token: ${OS_AUTH_TOKEN}" \ 1127 | -H "X-Remove-Container-Read: 0" "${OS_STORAGE_URL}/${cont}" 1128 | ${mycurl} -X POST -s -H "X-Auth-Token: ${OS_AUTH_TOKEN}" \ 1129 | -H "X-Remove-Container-Read: 0" "${OS_STORAGE_URL}/.segments_${cont}" 1130 | ${mycurl} -X POST -s -H "X-Auth-Token: ${OS_AUTH_TOKEN}" \ 1131 | -H "X-Remove-Container-Meta-Web-Listings-CSS: 0" "${OS_STORAGE_URL}/${cont}" 1132 | 1133 | else 1134 | ${mycurl} -X POST -s -H "X-Auth-Token: ${OS_AUTH_TOKEN}" \ 1135 | -H "X-Container-Read: .r:*,.rlistings" "${OS_STORAGE_URL}/${cont}" 1136 | #-H "X-Container-Meta-Web-Listings-CSS: http://123goso.gicp.net/static/listing/css/listing.css" \ 1137 | 1138 | ${mycurl} -X POST -s -H "X-Auth-Token: ${OS_AUTH_TOKEN}" \ 1139 | -H "X-Container-Read: .r:*,.rlistings" "${OS_STORAGE_URL}/.segments_${cont}" 1140 | fi 1141 | ${mycurl} --head -s -H "X-Auth-Token: $OS_AUTH_TOKEN" "${OS_STORAGE_URL}/${cont}" | grep 'X-Container-Read' 1142 | ;; 1143 | chgrp | rw | ro) 1144 | if [[ "$1" == "--help" ]]; then 1145 | echoerr "\nusage: $scriptname chgrp|rw|ro [/root_folder]" 1146 | echoerr " -> grant permissons to current swift account or container." 1147 | echoerr " -> 'chgrp' overwrites all previous permissions" 1148 | echoerr " -> 'rw' and 'ro' add to existing account permissions" 1149 | echoerr " -> example: $scriptname chgrp lastname_f_grp " 1150 | echoerr " -> example for removing permissions: " 1151 | echoerr " -> $scriptname chgrp [/root_folder] \n" 1152 | echoerr "also please check these commands:" 1153 | printcommands 1154 | exit 1 1155 | fi 1156 | if [[ -z $2 ]]; then 1157 | if [[ $1 == /* ]]; then 1158 | splitpath $1 1159 | fi 1160 | else 1161 | splitpath $2 1162 | fi 1163 | permission="read-write" 1164 | if [[ "$subcmd" == "ro" ]]; then 1165 | permission="read-only" 1166 | fi 1167 | curracl=$(${mycurl} -s --head -H "X-Auth-Token: $OS_AUTH_TOKEN" "$OS_STORAGE_URL" | grep "X-Account-Access-Control:") 1168 | curracl=${curracl//"X-Account-Access-Control: "/""} #replace X-Header with empty string 1169 | #echo "curracl: $curracl" 1170 | for grp in $(echo "$curracl" | jq -r '.["admin"] | .[]' 2>/dev/null); do 1171 | admacl+=",\"$grp\"" 1172 | done 1173 | for grp in $(echo "$curracl" | jq -r '.["read-only"] | .[]' 2>/dev/null); do 1174 | roacl+=",\"$grp\"" 1175 | done 1176 | for grp in $(echo "$curracl" | jq -r '.["read-write"] | .[]' 2>/dev/null); do 1177 | rwacl+=",\"$grp\"" 1178 | done 1179 | if [[ -n $1 ]]; then 1180 | if [[ "$subcmd" == "ro" ]]; then 1181 | if ! [[ "\"$roacl\"" =~ "\"$1\"" ]]; then 1182 | if [[ "$1" == "AUTH_"* || "$1" == "LDAP_"* ]]; then 1183 | roacl+=",\"$1\"" 1184 | else 1185 | roacl+=",\"LDAP_$1\"" 1186 | fi 1187 | fi 1188 | elif [[ "$subcmd" == "rw" ]]; then 1189 | if ! [[ "\"$rwacl\"" =~ "\"$1\"" ]]; then 1190 | if [[ "$1" == "AUTH_"* || "$1" == "LDAP_"* ]]; then 1191 | rwacl+=",\"$1\"" 1192 | else 1193 | rwacl+=",\"LDAP_$1\"" 1194 | fi 1195 | fi 1196 | elif [[ "$subcmd" == "chgrp" ]]; then 1197 | roacl="" 1198 | if [[ "$1" == "AUTH_"* || "$1" == "LDAP_"* ]]; then 1199 | rwacl="\"$1\"" 1200 | else 1201 | rwacl="\"LDAP_$1\"" 1202 | fi 1203 | fi 1204 | else 1205 | if [[ "$subcmd" == "chgrp" ]]; then 1206 | rwacl="" 1207 | roacl="" 1208 | fi 1209 | fi 1210 | admacl=${admacl#,} 1211 | roacl=${roacl#,} 1212 | rwacl=${rwacl#,} 1213 | #echo "admacl:$admacl" 1214 | #echo "roacl:$roacl" 1215 | #echo "rwacl:$rwacl" 1216 | 1217 | #scope="X-Container-Access-Control" 1218 | if [[ -z $cont ]]; then 1219 | # grant permission to account 1220 | if [[ -z $1 ]]; then 1221 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H "X-Account-Access-Control: {}" "$OS_STORAGE_URL" 1222 | echoerr "permissions reset to default" 1223 | else 1224 | rwroacl="" 1225 | if [[ $roacl != "" ]]; then 1226 | #echo "read only exists" 1227 | rwroacl+=',"read-only":['$roacl']' 1228 | fi 1229 | if [[ $rwacl != "" ]]; then 1230 | #echo "read write exists" 1231 | rwroacl+=',"read-write":['$rwacl']' 1232 | fi 1233 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H 'X-Account-Access-Control: {"admin":["'${swifttenant}'"]'$rwroacl'}' "$OS_STORAGE_URL" 1234 | fi 1235 | # verify granted permission 1236 | echoerr "New permissions for account ${swifttenant}:" 1237 | ${mycurl} -X HEAD -is -H "X-Auth-Token: $OS_AUTH_TOKEN" "$OS_STORAGE_URL" | grep 'X-Account-Access-Control' 1238 | else 1239 | # grant permission to container 1240 | if [[ -z $2 ]]; then 1241 | #perm_cont=X-Remove${perm_cont#?} 1242 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H "X-Remove-Container-Read: 0" "$OS_STORAGE_URL/$cont" 1243 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H "X-Remove-Container-Read: 0" "$OS_STORAGE_URL/.segments_$cont" 1244 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H "X-Remove-Container-Write: 0" "$OS_STORAGE_URL/$cont" 1245 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H "X-Remove-Container-Write: 0" "$OS_STORAGE_URL/.segments_$cont" 1246 | echoerr "permissions reset to default" 1247 | else 1248 | ${mycurl} -X PUT -s -o /dev/null -H "X-Auth-Token:$OS_AUTH_TOKEN" "$OS_STORAGE_URL/.segments_$cont" 1249 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H "X-Container-Read: LDAP_$1" "$OS_STORAGE_URL/$cont" 1250 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H "X-Container-Read: LDAP_$1" "$OS_STORAGE_URL/.segments_$cont" 1251 | if [[ "$subcmd" != "ro" ]]; then 1252 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H "X-Container-Write: LDAP_$1" "$OS_STORAGE_URL/$cont" 1253 | ${mycurl} -X POST -s -H "X-Auth-Token: $OS_AUTH_TOKEN" -H "X-Container-Write: LDAP_$1" "$OS_STORAGE_URL/.segments_$cont" 1254 | fi 1255 | fi 1256 | # verify granted permission 1257 | echoerr "New permissions for container ${swifttenant}/$cont:" 1258 | ${mycurl} -X HEAD -is -H "X-Auth-Token: $OS_AUTH_TOKEN" "$OS_STORAGE_URL/$cont" | grep "X-Container-Read\|X-Container-Write" 1259 | fi 1260 | ;; 1261 | more | less | cat | editor | vi | emacs | nano) 1262 | # openwith downloads to TMP and opens the file with a certain command 1263 | openwith $subcmd "$1" 1264 | ;; 1265 | openwith) 1266 | if [[ -z $1 || -z $2 ]]; then 1267 | echoerr "\nusage: $scriptname $subcmd " 1268 | echoerr " -> open file from swift object store with application\n" 1269 | echoerr "also please check these commands:" 1270 | printcommands 1271 | exit 1272 | fi 1273 | openwith "$1" "$2" 1274 | ;; 1275 | *mkdir) 1276 | if [[ -z $1 ]]; then 1277 | echoerr "\nusage: $scriptname $subcmd " 1278 | echoerr " -> creates a folder / container in swift \n" 1279 | echoerr "also please check these commands:" 1280 | printcommands 1281 | exit 1 1282 | fi 1283 | dest="$1" 1284 | if [[ $dest = "/" ]]; then 1285 | echoerr "no need to create a root folder" 1286 | exit 1 1287 | fi 1288 | if [[ $dest != /* ]]; then 1289 | echoerr "$scriptname $subcmd currently does not support relative paths, path must start with '/'" 1290 | exit 1 1291 | fi 1292 | splitpath "$dest" 1293 | ret=`swift post "$cont" 2>&1` 1294 | ;; 1295 | clean) 1296 | if [[ "$1" == "--help" ]]; then 1297 | echoerr "\nusage: $scriptname $subcmd" 1298 | echoerr " -> cleans out current settings / credentials\n" 1299 | echoerr "also please check these commands:" 1300 | printcommands 1301 | exit 1 1302 | fi 1303 | #if [[ -f ~/.swift/auth_token_${swcfile} ]]; then 1304 | rm -f ~/.swift/auth_token_${swcfile} 1305 | rm -f ~/.swift/auth_token_${swiftauthver}_${swifttenant} 1306 | rm -f ~/.swift/auth_token_v1_${swifttenant} 1307 | rm -f ~/.swift/auth_token_v2_${swifttenant} 1308 | rm -f ~/.swift/storageurl_${swcfile} 1309 | rm -f ~/.swift/storageurl_${swifttenant} 1310 | rm -f ~/.swift/current_folder_${swcfile} 1311 | rm -f ~/.swift/current_folder_${swifttenant} 1312 | echoerr "Removed settings / credentials for ${swcfile}." 1313 | #fi 1314 | ;; 1315 | env) 1316 | if [[ "$1" == "--help" ]]; then 1317 | echoerr "\nusage: $scriptname $subcmd" 1318 | echoerr " -> print all openstack env vars\n" 1319 | echoerr "also please check these commands:" 1320 | printcommands 1321 | exit 1 1322 | fi 1323 | echoerr "\n*** Swift authentication (v1) ***" 1324 | env | grep ^ST_ 1325 | echoerr "\n*** Openstack authentication (v2/v3) ***" 1326 | if [[ authfromenv == 1 ]]; then 1327 | echo "authfromenv == 1" 1328 | env | grep ^OS_ v "OS_PASSWORD" 1329 | else 1330 | env | grep ^OS_ | grep -v "OS_STORAGE_URL\|OS_AUTH_TOKEN\|OS_PASSWORD" 1331 | fi 1332 | ;; 1333 | auth) 1334 | if [[ "$1" == "--help" ]]; then 1335 | echoerr "\nusage: $scriptname $subcmd" 1336 | echoerr " -> print auth token and storageurl\n" 1337 | echoerr "also please check these commands:" 1338 | printcommands 1339 | exit 1 1340 | fi 1341 | if [[ -n $OS_AUTH_URL ]]; then 1342 | echo "export OS_AUTH_URL=${OS_AUTH_URL}" 1343 | else 1344 | echo "export ST_AUTH=${ST_AUTH}" 1345 | echo "export ST_USER=${ST_USER}" 1346 | echo "export ST_KEY=${ST_KEY}" 1347 | echo "----------------------------------------" 1348 | fi 1349 | echo "export OS_STORAGE_URL=$(cat ~/.swift/storageurl_${swcfile})" 1350 | echo "export OS_AUTH_TOKEN=$(cat ~/.swift/auth_token_${swcfile})" 1351 | ;; 1352 | *) 1353 | echoerr "\nSwift Commander ($scriptname) allows you to easily work with a swift object store." 1354 | echoerr "swc supports sub commands that attempt to mimic standard unix file system tools." 1355 | echoerr "These sub commands are currently implemented: (Arguments in square brackets are " 1356 | echoerr "optional).\n" 1357 | echoerr " $scriptname upload - copy file / dirs from a file system to swift" 1358 | echoerr " $scriptname download - copy files and dirs from swift to a file system" 1359 | echoerr " $scriptname cd - change current folder to in swift" 1360 | echoerr " $scriptname ls [folder] - list contents of a folder - or the current one" 1361 | echoerr " $scriptname mkdir - create a folder (works only at the root)" 1362 | echoerr " $scriptname rm - delete all file paths that start with " 1363 | echoerr " $scriptname pwd - display the current swift folder name" 1364 | echoerr " $scriptname cat|more|less - download a file to TMPDIR and view with cat, more or less" 1365 | echoerr " $scriptname vi|emacs|nano - download a file to TMPDIR and edit it with vi|emacs or nano" 1366 | echoerr " $scriptname chgrp - grant/remove rw access to current swift account or container" 1367 | echoerr " $scriptname rw - add rw access to current swift account or container" 1368 | echoerr " $scriptname ro - add ro access to current swift account or container" 1369 | echoerr " $scriptname publish|hide - make root folder public (web server mode) or hide it" 1370 | echoerr " $scriptname list [filt] - list folder content (incl. subfolders) and filter" 1371 | echoerr " $scriptname search - search for a string in text files under /folder" 1372 | echoerr " $scriptname openwith - download a file to TMPDIR and open it with " 1373 | echoerr " $scriptname header - display the header of a file in swift" 1374 | echoerr " $scriptname meta - display custom meta data of a file in swift" 1375 | echoerr " $scriptname mtime - show the original mtime of a file before uploaded" 1376 | echoerr " $scriptname size - show the size of a swift or a local folder" 1377 | echoerr " $scriptname compare - compare size of a local folder with a swift folder" 1378 | echoerr " $scriptname hash - compare the md5sum of a local file with a swift file" 1379 | echoerr " $scriptname arch - create one tar archive for each folder level" 1380 | echoerr " $scriptname unarch - restore folders that have been archived" 1381 | echoerr " $scriptname auth - show current storage url and auth token" 1382 | echoerr " $scriptname env - show authentication env vars (ST_ and OS_)" 1383 | echoerr " $scriptname clean - remove current authtoken credential cache" 1384 | 1385 | echoerr "\nExamples:" 1386 | echoerr " $scriptname upload /local/folder /swift/folder" 1387 | echoerr " $scriptname upload --no-symlinks /local/folder /swift/folder (don't save symlinks)" 1388 | echoerr " $scriptname compare /local/folder /swift/folder" 1389 | echoerr " $scriptname download /swift/folder /scratch/folder" 1390 | echoerr " $scriptname download /swift/folder \$TMPDIR" 1391 | echoerr " $scriptname rm /archive/some_prefix" 1392 | echoerr " $scriptname more /folder/some_file.txt" 1393 | echoerr " $scriptname openwith emacs /folder/some_file.txt" 1394 | 1395 | echoerr "\nDebugging:" 1396 | echoerr " export OS_SWIFT_OPTS=--info\n" 1397 | ;; 1398 | esac 1399 | -------------------------------------------------------------------------------- /swift_commander/swfoldersize.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # Script for comparing the size of a posix folder with the size of a swift pseudo folder 4 | # 5 | # swfoldersize.py dirkpetersen / Jan 2015 6 | # 7 | 8 | import swiftclient, sys, os, argparse, math, functools 9 | 10 | class KeyboardInterruptError(Exception): pass 11 | 12 | SizeError=False 13 | 14 | def main(): 15 | 16 | if args.container: 17 | c=create_sw_conn() 18 | sbytes=0 19 | scnt=0 20 | print (" checking swift folder %s/%s ..." % (args.container,args.prefix)) 21 | try: 22 | headers, objects = c.get_container(args.container,prefix=args.prefix,full_listing=True) 23 | for obj in objects: 24 | sbytes+=obj['bytes'] 25 | scnt+=1 26 | #print(obj['name'],obj['bytes']) 27 | if sbytes > 0: 28 | print (" %s bytes (%s) in %s/%s (swift) (average size: %s)" % (intwithcommas(sbytes),convertByteSize(sbytes),args.container,args.prefix,convertByteSize(sbytes/scnt))) 29 | else: 30 | print (" ...Error: it seems swift folder %s/%s does not contain any data." % (args.container,args.prefix)) 31 | except: 32 | print (" ...Error: it seems swift folder %s/%s does not exist" % (args.container,args.prefix)) 33 | 34 | if args.posixfolder: 35 | pbytes, exbytes, dupbytes = posixfolderprint(args.posixfolder) 36 | 37 | if sbytes == pbytes: 38 | print("OK! The size of %s and %s/%s is identical!" % \ 39 | (args.posixfolder,args.container,args.prefix)) 40 | if SizeError: 41 | print("********** WARNING !! ********** The size of at least one " + \ 42 | "file or folder could not be determined. Please check permissions, " + \ 43 | "the size comparison may not be valid") 44 | 45 | else: 46 | print("********** WARNING !! ********** The size of %s and %s/%s is NOT identical!" % \ 47 | (args.posixfolder,args.container,args.prefix)) 48 | 49 | else: 50 | if args.posixfolder: 51 | pbytes, exbytes, dupbytes = posixfolderprint(args.posixfolder) 52 | realbytes=pbytes-(exbytes+dupbytes) 53 | 54 | if args.posixfolder2: 55 | pbytes2, exbytes2, dupbytes2 = posixfolderprint(args.posixfolder2) 56 | realbytes2=pbytes2-(exbytes2+dupbytes2) 57 | 58 | if args.posixfolder and args.posixfolder2: 59 | if pbytes2 == pbytes or realbytes == realbytes2: 60 | print("OK! The size of %s and %s is identical!" % \ 61 | (args.posixfolder,args.posixfolder2)) 62 | else: 63 | print("********** WARNING !! ********** The size of %s and %s is NOT identical!" % \ 64 | (args.posixfolder,args.posixfolder2)) 65 | 66 | def posixfolderprint(path): 67 | print (" checking posix folder %s (following symlinks)..." % (path)) 68 | pbytes, exbytes, dupbytes, pcnt = getFolderSize(os.path.expanduser(path)) 69 | print (" %s bytes (%s) in %s (average file size: %s)" % (intwithcommas(pbytes),convertByteSize(pbytes),path,convertByteSize(pbytes/pcnt))) 70 | if exbytes > 0: print(" ...including %s bytes (%s) for links to outside of %s" % (intwithcommas(exbytes),convertByteSize(exbytes),path)) 71 | if dupbytes > 0: print(" ...including %s bytes (%s) for duplicate inodes." % (intwithcommas(dupbytes),convertByteSize(dupbytes))) 72 | return pbytes, exbytes, dupbytes 73 | 74 | def getFolderSize(path, externalLinks=True): # skips duplicate inodes 75 | global SizeError 76 | total_size = 0 77 | external_size = 0 78 | duplicate_inodes_size = 0 79 | total_count = 0 80 | 81 | seen = set() 82 | 83 | for dirpath, dirnames, filenames in mywalk(path): 84 | 85 | #if dirpath != path: # ignore sub directories 86 | # break 87 | 88 | for f in filenames: 89 | fp = os.path.join(dirpath, f) 90 | 91 | try: 92 | stat = os.stat(fp) 93 | except OSError as err: 94 | sys.stderr.write(str(err)) 95 | sys.stderr.write('\n') 96 | SizeError=True 97 | continue 98 | 99 | if isExternalLink(path,fp): 100 | external_size += stat.st_size 101 | if not externalLinks: 102 | if args.debug: 103 | print(' ...ignoring external link %s to %s' % (fp, os.readlink(fp))) 104 | continue 105 | else: 106 | if args.debug: 107 | print(' ...including external link %s to %s' % (fp, os.readlink(fp))) 108 | else: 109 | 110 | if stat.st_ino in seen: 111 | if args.debug: 112 | print(' ...Duplicate inode %s for %s' % (stat.st_ino, fp)) 113 | duplicate_inodes_size += stat.st_size 114 | #continue 115 | 116 | seen.add(stat.st_ino) 117 | #print(stat.st_size,fp) 118 | 119 | total_size += stat.st_size 120 | total_count += 1 121 | 122 | return total_size, external_size, duplicate_inodes_size, total_count # size in bytes 123 | 124 | def isExternalLink(root,path): 125 | if os.path.islink(path): 126 | real=os.readlink(path) 127 | if not real.startswith(root) and real.startswith('/'): 128 | return True 129 | return False 130 | 131 | def mywalk(top, skipdirs=['.snapshot',]): 132 | """ returns subset of os.walk """ 133 | for root, dirs, files in os.walk(top,topdown=True,onerror=walkerr): 134 | for skipdir in skipdirs: 135 | if skipdir in dirs: 136 | dirs.remove(skipdir) # don't visit this directory 137 | yield root, dirs, files 138 | 139 | def walkerr(oserr): 140 | sys.stderr.write(str(oserr)) 141 | sys.stderr.write('\n') 142 | return 0 143 | 144 | def getFolderSize2(path): 145 | # this is a legacy function that does not follow symlinks 146 | global SizeError 147 | if "/.snapshot/" in path: 148 | return 0 149 | if os.path.islink(path): 150 | return 0 151 | prepend = functools.partial(os.path.join, path) 152 | try: 153 | return sum([(os.path.getsize(f) if not os.path.islink(f) and os.path.isfile(f) else getFolderSize2(f)) for f in map(prepend, os.listdir(path))]) 154 | except: 155 | print(" ...Error getting size of %s" % path) 156 | SizeError=True 157 | return 0 158 | 159 | 160 | def create_sw_conn(): 161 | if args.authtoken and args.storageurl: 162 | return swiftclient.Connection(preauthtoken=args.authtoken, preauthurl=args.storageurl) 163 | else: 164 | authtoken=os.environ.get("OS_AUTH_TOKEN") 165 | storageurl=os.environ.get("OS_STORAGE_URL") 166 | if authtoken and storageurl: 167 | return swiftclient.Connection(preauthtoken=authtoken, preauthurl=storageurl) 168 | else: 169 | swift_auth=os.environ.get("ST_AUTH") 170 | swift_user=os.environ.get("ST_USER") 171 | swift_key=os.environ.get("ST_KEY") 172 | if swift_auth and swift_user and swift_key: 173 | return swiftclient.Connection(authurl=swift_auth,user=swift_user,key=swift_key) 174 | 175 | def convertByteSize(size): 176 | if size == 0: 177 | return '0 B' 178 | size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") 179 | i = int(math.floor(math.log(size,1024))) 180 | p = math.pow(1024,i) 181 | s = size/p 182 | if (s > 0): 183 | return '%0.3f %s' % (s,size_name[i]) 184 | else: 185 | return '0 B' 186 | 187 | def intwithcommas(x): 188 | result='' 189 | while x >= 1000: 190 | x,r = divmod(x, 1000) 191 | result = ",%03d%s" % (r, result) 192 | return "%d%s" % (x, result) 193 | 194 | def parse_arguments(): 195 | """ 196 | Gather command-line arguments. 197 | """ 198 | parser = argparse.ArgumentParser(prog='swfoldersize.py', 199 | description='compare the size of a posix folder with the size ' + \ 200 | 'of a swift (pseudo) folder after a data migration ' + \ 201 | '()') 202 | parser.add_argument( '--debug', '-d', dest='debug', 203 | action='store_true', 204 | help='show addional information for debug', 205 | default=False ) 206 | parser.add_argument( '--info', '-i', dest='debug', 207 | action='store_true', 208 | help='show addional information for debug', 209 | default=False ) 210 | parser.add_argument( '--posixfolder', '-p', dest='posixfolder', 211 | action='store', 212 | help='a folder on a posix file system ', 213 | default='' ) 214 | parser.add_argument( '--posixfolder2', '-2', dest='posixfolder2', 215 | action='store', 216 | help='a 2nd folder on a posix file system ', 217 | default='' ) 218 | parser.add_argument( '--container', '-c', dest='container', 219 | action='store', 220 | help='a container in the swift object store', 221 | default='' ) 222 | parser.add_argument( '--prefix', '-x', dest='prefix', 223 | action='store', 224 | help='a swift object prefix', 225 | default=None) 226 | parser.add_argument( '--proc', '-m', dest='maxproc', 227 | action='store', 228 | type=int, 229 | help='maximum number of processes to run (not yet implemented)', 230 | default=0 ) 231 | parser.add_argument( '--authtoken', '-a', dest='authtoken', 232 | action='store', 233 | help='a swift authentication token (required when storage-url is used)', 234 | default=None) 235 | parser.add_argument( '--storage-url', '-s', dest='storageurl', 236 | action='store', 237 | help='a swift storage url (required when authtoken is used)', 238 | default=None) 239 | 240 | args = parser.parse_args() 241 | return args 242 | 243 | if __name__ == '__main__': 244 | args = parse_arguments() 245 | sys.exit(main()) 246 | 247 | -------------------------------------------------------------------------------- /swift_commander/swhashcomp.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # Script for comparing the md5sum of a posix file with the md5sums of a multi chunk swift object 4 | # 5 | # swhashcomp dirkpetersen / Feb 2015 6 | # 7 | 8 | import swiftclient, sys, os, argparse, functools, hashlib, json 9 | 10 | class KeyboardInterruptError(Exception): pass 11 | 12 | def main(): 13 | 14 | c=create_sw_conn() 15 | md5all = hashlib.md5() 16 | 17 | print (" comparing swift object %s/%s with %s..." % (args.container, args.obj, args.locfile)) 18 | 19 | #headers, objects = c.get_container(args.container,prefix=args.prefix,full_listing=True) 20 | headers = c.head_object(args.container, args.obj) 21 | 22 | if 'x-static-large-object' in headers: 23 | #print(headers['x-static-large-object']) 24 | headers, body = c.get_object(args.container, args.obj, query_string='multipart-manifest=get') 25 | if not os.path.isfile(args.locfile): 26 | if 'md5sum' in headers: 27 | if args.locfile.strip() == headers['md5sum']: 28 | print(' md5sum:%s' % headers['md5sum']) 29 | is_valid = True 30 | else: 31 | is_valid = False 32 | else: 33 | is_valid = check_segments(body,args.locfile.strip(),c) 34 | else: 35 | if os.path.splitext(args.locfile)[1].lower() in ['.md5', '.sha1']: 36 | with open(args.locfile, 'r') as f: 37 | myhash = f.read().split(None,1)[0] 38 | print('md5sum from md5 file: %s' % myhash) 39 | is_valid = check_segments(body,myhash,c) 40 | else: 41 | with open(args.locfile, 'rb') as f: 42 | is_valid = check_manifest(body, f, md5all) 43 | else: 44 | is_valid=False 45 | if os.path.isfile(args.locfile): 46 | if os.path.splitext(args.locfile)[1].lower() in ['.md5', '.sha1']: 47 | with open(args.locfile, 'r') as f: 48 | myhash = f.read().split(None,1)[0] 49 | print('md5sum from md5 file: %s' % myhash) 50 | else: 51 | with open(args.locfile, 'rb') as f: 52 | hasher = hashlib.md5(f.read()) # needed for compatiblity between python3 and python2 53 | myhash = hasher.hexdigest() 54 | print('md5sum of local file: %s' % myhash) 55 | if myhash == headers['etag']: 56 | print(' md5sum:%s' % headers['etag']) 57 | is_valid = True 58 | else: 59 | if args.locfile.strip() == headers['etag']: 60 | print(' md5sum:%s' % headers['etag']) 61 | is_valid = True 62 | 63 | if is_valid: 64 | print ("object %s/%s and '%s' are identical!" % (args.container, args.obj, args.locfile)) 65 | return 0 66 | else: 67 | print ("*** WARNING ***: object %s/%s and '%s' are different!" % (args.container, args.obj, args.locfile)) 68 | return 1 69 | 70 | def check_manifest(manifest, body, md5all): 71 | """ 72 | check if a body is the same object described by the manifest 73 | 74 | :param manifest: the raw body of the manifest from swift 75 | :param body: a file like object to check against the manfiest 76 | """ 77 | manifest = json.loads(manifest.decode()) 78 | for segment in manifest: 79 | print (" testing chunk %s" % segment['name']) 80 | chunk = body.read(segment['bytes']) 81 | hasher = hashlib.md5(chunk) 82 | md5all.update(chunk) 83 | if hasher.hexdigest() != segment['hash']: 84 | print (' %s != %s' % (hasher.hexdigest(), segment['hash'])) 85 | return False 86 | print(" md5sum:%s" % md5all.hexdigest()) 87 | return True 88 | 89 | def check_segments(manifest,md5sum,c): 90 | manifest = json.loads(manifest.decode()) 91 | digest = hashlib.md5() 92 | for segment in manifest: 93 | print (" please wait ... testing chunk %s" % segment['name']) 94 | segment_container, segment_obj = parseSwiftUrl(segment['name']) 95 | attributes, content = c.get_object(segment_container, segment_obj) 96 | digest.update(content) 97 | if digest.hexdigest() != md5sum: 98 | print (' %s != %s' % (digest.hexdigest(), md5sum)) 99 | return False 100 | return True 101 | 102 | def create_sw_conn(): 103 | if args.authtoken and args.storageurl: 104 | return swiftclient.Connection(preauthtoken=args.authtoken, preauthurl=args.storageurl) 105 | else: 106 | authtoken=os.environ.get("OS_AUTH_TOKEN") 107 | storageurl=os.environ.get("OS_STORAGE_URL") 108 | if authtoken and storageurl: 109 | return swiftclient.Connection(preauthtoken=authtoken, preauthurl=storageurl) 110 | else: 111 | swift_auth=os.environ.get("ST_AUTH") 112 | swift_user=os.environ.get("ST_USER") 113 | swift_key=os.environ.get("ST_KEY") 114 | if swift_auth and swift_user and swift_key: 115 | return swiftclient.Connection(authurl=swift_auth,user=swift_user,key=swift_key) 116 | 117 | def parseSwiftUrl(path): 118 | path = path.lstrip('/') 119 | components = path.split('/'); 120 | container = components[0]; 121 | obj = '/'.join(components[1:]) 122 | return container, obj 123 | 124 | def parse_arguments(): 125 | """ 126 | Gather command-line arguments. 127 | """ 128 | 129 | parser = argparse.ArgumentParser(prog='swhashcomp', 130 | description='compare the md5sum of a local file or hash with the hash ' + \ 131 | 'of a swift object folder after a data migration ' + \ 132 | '') 133 | parser.add_argument( '--locfile', '-f', dest='locfile', 134 | action='store', 135 | help='a local or networked file to compare', 136 | default='' ) 137 | parser.add_argument( '--container', '-c', dest='container', 138 | action='store', 139 | help='a container in the swift object store', 140 | default='' ) 141 | parser.add_argument( '--obj', '-o', dest='obj', 142 | action='store', 143 | help='an object in a swift container', 144 | default=None) 145 | parser.add_argument( '--authtoken', '-a', dest='authtoken', 146 | action='store', 147 | help='a swift authentication token (required when storage-url is used)', 148 | default=None) 149 | parser.add_argument( '--storage-url', '-s', dest='storageurl', 150 | action='store', 151 | help='a swift storage url (required when authtoken is used)', 152 | default=None) 153 | args = parser.parse_args() 154 | if not args.locfile: 155 | parser.error('required option --locfile not given !') 156 | if not args.container: 157 | parser.error('required option --container not given !') 158 | if not args.obj: 159 | parser.error('required option --obj not given !') 160 | return args 161 | 162 | if __name__ == '__main__': 163 | # Parse command-line arguments 164 | args = parse_arguments() 165 | sys.exit(main()) 166 | 167 | -------------------------------------------------------------------------------- /swift_commander/swpget.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # get multisegment swift files in parallel 4 | 5 | import sys,os,getopt,json 6 | import time 7 | 8 | import multiprocessing 9 | 10 | import swiftclient 11 | 12 | def create_sparse_file(filename,length): 13 | #print("creating sparse file",filename) 14 | with open(filename, "wb") as f: 15 | f.truncate(length) 16 | 17 | swift_auth_token=os.environ.get("OS_AUTH_TOKEN") 18 | storage_url=os.environ.get("OS_STORAGE_URL") 19 | 20 | def create_sw_conn(): 21 | global swift_auth_token,storage_url 22 | 23 | if swift_auth_token and storage_url: 24 | return swiftclient.Connection(preauthtoken=swift_auth_token, 25 | preauthurl=storage_url) 26 | 27 | swift_auth=os.environ.get("ST_AUTH") 28 | swift_user=os.environ.get("ST_USER") 29 | swift_key=os.environ.get("ST_KEY") 30 | 31 | if swift_auth and swift_user and swift_key: 32 | return swiftclient.Connection(authurl=swift_auth,user=swift_user, 33 | key=swift_key) 34 | 35 | print("Error: Swift environment not configured!") 36 | sys.exit() 37 | 38 | def parseSwiftUrl(path): 39 | path = path.lstrip('/') 40 | components = path.split('/'); 41 | container = components[0]; 42 | obj = '/'.join(components[1:]) 43 | return container, obj 44 | 45 | # container, object, offset, dest 46 | def assemble_ms_object(x): 47 | #print("assembling",x) 48 | conn=create_sw_conn() 49 | 50 | headers,body=conn.get_object(x[0],x[1]) 51 | #print("body len=",len(body)) 52 | with open(x[3],"r+b") as f_out: 53 | if x[2]>0: 54 | f_out.seek(x[2]) 55 | f_out.write(bytes(body)) 56 | 57 | conn.close() 58 | 59 | def get_ms_object(sc,container,object,pool_size): 60 | #print("multisegment object",object) 61 | segments=[] 62 | segment_total=0 63 | 64 | # build segment map for parallel download 65 | headers,body=sc.get_object(container,object, 66 | query_string='multipart-manifest=get') 67 | manifest=json.loads(body.decode()) 68 | dest=os.path.basename(object) 69 | for segment in manifest: 70 | segment_container,segment_obj=parseSwiftUrl(segment['name']) 71 | # store segment container, object and offset 72 | segments.append([segment_container,segment_obj,segment_total,dest]) 73 | segment_total=segment_total+segment['bytes'] 74 | 75 | # create sparse file 76 | create_sparse_file(dest,segment_total) 77 | 78 | # sequential assembly 79 | #for seg in segments: 80 | # assemble_ms_object(seg) 81 | 82 | # parallel assembly 83 | #print("parallel assembly with",pool_size,"workers") 84 | p=multiprocessing.Pool(pool_size) 85 | p.map(assemble_ms_object,segments) 86 | 87 | def get_object(conn,container,object): 88 | headers,body=conn.get_object(container,object) 89 | with open(os.path.basename(object),"w+b") as f_out: 90 | f_out.write(bytes(body)) 91 | 92 | def set_time(headers,name): 93 | if headers: 94 | if 'x-object-meta-mtime' in headers: 95 | mmt=int(float(headers['x-object-meta-mtime'])) 96 | else: 97 | mkt=time.mktime(time.strptime(headers['last-modified'], 98 | "%a, %d %b %Y %X %Z")) 99 | mmt=int(time.mktime(time.localtime(mkt))) 100 | 101 | os.utime(os.path.basename(name),(mmt,mmt)) 102 | 103 | def get_objects(sc,container,object_list,pool_size): 104 | found=0 105 | #print("getting",object_list,"from container",container) 106 | 107 | try: 108 | headers,objs=sc.get_container(container,full_listing=True) 109 | for obj in objs: 110 | #print("found",obj['name']) 111 | if obj['name'] in object_list: 112 | #print("matched",obj['name']) 113 | found=found+1 114 | try: 115 | headers=sc.head_object(container, obj['name']) 116 | except: 117 | headers=[] 118 | 119 | if 'x-static-large-object' in headers: 120 | get_ms_object(sc,container,obj['name'],pool_size) 121 | else: 122 | get_object(sc,container,obj['name']) 123 | 124 | set_time(headers,obj['name']) 125 | 126 | if not found: 127 | print("No matching files found") 128 | 129 | except swiftclient.ClientException: 130 | print("Error: cannot access Swift container '%s'!" % container) 131 | 132 | sc.close() 133 | 134 | def validate_dir(path,param): 135 | if not os.path.isdir(path): 136 | print("Error: %s '%s' is not accessible!" % (param,path)) 137 | sys.exit() 138 | 139 | if path[-1]=='/': 140 | path=path[:-1] 141 | 142 | return(path) 143 | 144 | def usage(): 145 | print("swpget [parameters]") 146 | print("Parameters:") 147 | print("\t-l local_directory (default .)") 148 | print("\t-c container (required)") 149 | print("\t-p pool_size (default 5)") 150 | print("\t-a auth_token (default OS_AUTH_TOKEN)") 151 | print("\t-s storage_url (default OS_STORAGE_URL)") 152 | 153 | def main(argv=None): 154 | global swift_auth_token 155 | global storage_url 156 | 157 | argv = argv or sys.argv[1:] 158 | 159 | container="" 160 | pool_size=5 161 | 162 | try: 163 | opts,args=getopt.getopt(argv,"l:c:p:a:s:h") 164 | except getopt.GetoptError: 165 | usage() 166 | sys.exit() 167 | 168 | for opt,arg in opts: 169 | if opt in ("-h"): 170 | container="" 171 | break 172 | elif opt in ("-l"): # override default local directory 173 | local_dir=validate_dir(arg,"local") 174 | os.chdir(local_dir) 175 | elif opt in ("-c"): # set container 176 | container=arg 177 | elif opt in ("-p"): # parallel workers 178 | pool_size=int(arg) 179 | elif opt in ("-a"): # override swift_auth_token 180 | swift_auth_token=arg 181 | elif opt in ("-s"): # override storage URL 182 | storage_url=arg 183 | 184 | if not container or not args: 185 | usage() 186 | else: 187 | sc=create_sw_conn() 188 | if sc: 189 | get_objects(sc,container,args,pool_size) 190 | 191 | if __name__ == '__main__': 192 | main() 193 | -------------------------------------------------------------------------------- /swift_commander/swrm.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # Script to mass delete swift objects in a pseudo folder 4 | # 5 | # swrm.py dirkpetersen / Jul 2015 6 | # 7 | 8 | import swiftclient, sys, os, math, argparse, json 9 | 10 | class KeyboardInterruptError(Exception): pass 11 | 12 | def main(): 13 | global args 14 | 15 | # Parse command-line arguments 16 | args = parse_arguments() 17 | 18 | if args.container: 19 | 20 | c=create_sw_conn() 21 | 22 | storageurl=os.environ.get("OS_STORAGE_URL") 23 | if not storageurl: 24 | storageurl, authtoken = c.get_auth() 25 | 26 | swaccount=storageurl.split("/")[-1] 27 | 28 | prefix=args.prefix 29 | if prefix: 30 | if prefix.endswith('*'): 31 | prefix=args.prefix[:-1] 32 | elif not prefix.endswith('/'): 33 | # may be single object, try deleting 34 | obj = {} 35 | obj['name']=args.prefix 36 | ret=delobj(obj) 37 | if ret==200: 38 | return True 39 | elif ret==404: 40 | prefix=args.prefix+'/' 41 | else: 42 | print("Error %s deleting object %s" % (ret,args.prefix)) 43 | else: 44 | print('Warning: no prefix / pseudo folder entered - will delete container') 45 | 46 | print (" checking swift folder /%s/%s ..." % (args.container,prefix)) 47 | try: 48 | headers, objects = c.get_container(args.container,prefix=prefix,full_listing=True) 49 | c.close() 50 | except swiftclient.client.ClientException as ex: 51 | httperr=getattr(ex, 'http_status', None) 52 | if httperr == 404: 53 | print(" no objects found - error 404") 54 | else: 55 | print(" HTTP Error %s" % httperr) 56 | return False 57 | 58 | sbytes=0 59 | print(" checking size of /%s/%s ... " % (args.container,prefix)) 60 | for obj in objects: 61 | sbytes+=obj['bytes'] 62 | #print(obj['name'],obj['bytes']) 63 | if sbytes > 0: 64 | print (" found %s files with %s bytes (%s) in /%s/%s" % (len(objects),intwithcommas(sbytes), 65 | convertByteSize(sbytes),args.container,prefix)) 66 | if not args.force: 67 | if yn_choice(" Do you want to delete this data in account %s ?" % swaccount): 68 | easy_par(delobj,objects) 69 | else: 70 | easy_par(delobj,objects) 71 | else: 72 | if prefix: 73 | print (" Error: it seems swift objects at /%s/%s do not exist" % (args.container,args.prefix)) 74 | else: 75 | c=create_sw_conn() 76 | c.delete_container(args.container) 77 | c.close() 78 | else: 79 | print('no container entered - aborting') 80 | return False 81 | 82 | 83 | def delobj(obj): 84 | print(" deleting /%s/%s ... " % (args.container,obj['name'])) 85 | retcode=200 86 | c=create_sw_conn() 87 | try: 88 | headers = c.head_object(args.container, obj['name']) 89 | except: 90 | headers = [] 91 | if 'x-static-large-object' in headers: 92 | #print(headers['x-static-large-object']) 93 | headers, body = c.get_object(args.container, obj['name'], query_string='multipart-manifest=get') 94 | manifest = json.loads(body.decode()) 95 | for segment in manifest: 96 | print (" deleting segment %s" % segment['name']) 97 | segment_container, segment_object = parseSwiftUrl(segment['name']) 98 | try: 99 | c.delete_object(segment_container, segment_object) 100 | except Exception as e: 101 | print('Error %s deleting object segment %s: %r' % (e.http_status,obj['name'],e)) 102 | try: 103 | c.delete_object(args.container,obj['name']) 104 | except Exception as e: 105 | retcode=e.http_status 106 | print('Error %s deleting object %s: %r' % (e.http_status,obj['name'],e)) 107 | c.close() 108 | return retcode 109 | 110 | def parseSwiftUrl(path): 111 | path = path.lstrip('/') 112 | components = path.split('/'); 113 | container = components[0]; 114 | obj = '/'.join(components[1:]) 115 | return container, obj 116 | 117 | def yn_choice(message, default='n'): 118 | choices = 'Y/n' if default.lower() in ('y', 'yes') else 'y/N' 119 | choice = input("%s (%s) " % (message, choices)) 120 | values = ('y', 'yes', '') if default == 'y' else ('y', 'yes') 121 | return choice.strip().lower() in values 122 | 123 | def easy_par(f, sequence): 124 | from multiprocessing import Pool 125 | pool = Pool(processes=args.maxproc) 126 | try: 127 | # f is given sequence. guaranteed to be in order 128 | cleaned=False 129 | result = pool.map(f, sequence) 130 | cleaned = [x for x in result if not x is None] 131 | #cleaned = asarray(cleaned) 132 | # not optimal but safe 133 | except KeyboardInterrupt: 134 | pool.terminate() 135 | except Exception as e: 136 | print('got exception: %r' % (e,)) 137 | if not args.force: 138 | print("Terminating the pool") 139 | pool.terminate() 140 | finally: 141 | pool.close() 142 | pool.join() 143 | return cleaned 144 | 145 | def create_sw_conn(): 146 | if args.authtoken and args.storageurl: 147 | return swiftclient.Connection(preauthtoken=args.authtoken, preauthurl=args.storageurl) 148 | else: 149 | authtoken=os.environ.get("OS_AUTH_TOKEN") 150 | storageurl=os.environ.get("OS_STORAGE_URL") 151 | if authtoken and storageurl: 152 | return swiftclient.Connection(preauthtoken=authtoken, preauthurl=storageurl) 153 | else: 154 | swift_auth=os.environ.get("ST_AUTH") 155 | swift_user=os.environ.get("ST_USER") 156 | swift_key=os.environ.get("ST_KEY") 157 | if swift_auth and swift_user and swift_key: 158 | return swiftclient.Connection(authurl=swift_auth,user=swift_user,key=swift_key) 159 | 160 | def convertByteSize(size): 161 | if size == 0: 162 | return '0 B' 163 | size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") 164 | i = int(math.floor(math.log(size,1024))) 165 | p = math.pow(1024,i) 166 | s = size/p 167 | if (s > 0): 168 | return '%0.3f %s' % (s,size_name[i]) 169 | else: 170 | return '0 B' 171 | 172 | def intwithcommas(x): 173 | result='' 174 | while x >= 1000: 175 | x,r = divmod(x, 1000) 176 | result = ",%03d%s" % (r, result) 177 | return "%d%s" % (x, result) 178 | 179 | 180 | def parse_arguments(): 181 | """ 182 | Gather command-line arguments. 183 | """ 184 | parser = argparse.ArgumentParser( 185 | description='delete a pseudo folder in swift with ' + \ 186 | 'potentially many objects ' + \ 187 | '()') 188 | parser.add_argument( '--container', '-c', dest='container', 189 | action='store', 190 | help='a container in the swift object store', 191 | default='' ) 192 | parser.add_argument( '--prefix', '-p', dest='prefix', 193 | action='store', 194 | help='a swift object prefix', 195 | default=None) 196 | parser.add_argument( '--force', '-f', dest='force', 197 | action='store_true', 198 | help='force delete without prompt', 199 | default=False) 200 | parser.add_argument( '--maxproc', '-m', dest='maxproc', 201 | action='store', 202 | type=int, 203 | help='maximum number of processes to run', 204 | default=32 ) 205 | parser.add_argument( '--authtoken', '-a', dest='authtoken', 206 | action='store', 207 | help='a swift authentication token (required when storage-url is used)', 208 | default=None) 209 | parser.add_argument( '--storage-url', '-s', dest='storageurl', 210 | action='store', 211 | help='a swift storage url (required when authtoken is used)', 212 | default=None) 213 | 214 | args = parser.parse_args() 215 | return args 216 | 217 | if __name__ == '__main__': 218 | sys.exit(main()) 219 | 220 | -------------------------------------------------------------------------------- /swift_commander/swsearch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # get multisegment swift files in parallel 4 | 5 | import re 6 | import argparse 7 | import fnmatch 8 | import psutil 9 | 10 | import sys,os,getopt,json 11 | import time 12 | 13 | import multiprocessing 14 | 15 | import swiftclient 16 | 17 | def create_sw_conn(swift_auth_token,storage_url): 18 | if swift_auth_token and storage_url: 19 | return swiftclient.Connection(preauthtoken=swift_auth_token, 20 | preauthurl=storage_url) 21 | 22 | swift_auth=os.environ.get("ST_AUTH") 23 | swift_user=os.environ.get("ST_USER") 24 | swift_key=os.environ.get("ST_KEY") 25 | 26 | if swift_auth and swift_user and swift_key: 27 | return swiftclient.Connection(authurl=swift_auth,user=swift_user, 28 | key=swift_key) 29 | 30 | print("Error: Swift environment not configured!") 31 | sys.exit() 32 | 33 | textchars=bytearray({7,8,9,10,12,13,27}|set(range(0x20, 0x100))-{0x7f}) 34 | is_binary_string=lambda bytes:bool(bytes.translate(None,textchars)) 35 | 36 | def print_match(object,body,pattern,range=25): 37 | offset=body.find(pattern) 38 | if offset!=-1: 39 | print("%s: matched at offset %d" % (object,offset),flush=True) 40 | excerpt=body[max(0,offset-range):offset+len(pattern)+range] 41 | print('\t'+repr(excerpt.decode(encoding="ISO-8859-1"))) 42 | 43 | def search_object(parse_arg,object): 44 | sc=create_sw_conn(parse_arg.authtoken,parse_arg.storage_url) 45 | 46 | match=object.find(parse_arg.pattern) 47 | if match!=-1: 48 | print("%s: matched object name" % object,flush=True) 49 | 50 | headers,body=sc.get_object(parse_arg.container,object) 51 | if not parse_arg.binary or not is_binary_string(body): 52 | #print("scanning object",object,flush=True) 53 | if not parse_arg.insensitive: 54 | print_match(object,body,bytes(parse_arg.pattern,"ISO-8859-1")) 55 | else: 56 | m_o=re.search(bytes(parse_arg.pattern,"utf-8"),body,re.IGNORECASE) 57 | if m_o: 58 | print_match(object,body,m_o.group(0)) 59 | 60 | sc.close() 61 | 62 | # order is type,sc,container,object,pattern 63 | def search_worker(item): 64 | search_object(*item) 65 | 66 | skip_suffices=tuple(['.bam','.gz','.tif','.nc','.fcs','.dv','.mov','.bin',\ 67 | '.jpg','.zip','.nd2','.lsm','.bz2','.avi','.pdf','.tgz','.xls','.png',\ 68 | '.gif','.pyc','.ithmb','.wav','.pgm']) 69 | 70 | def search_container(parse_arg): 71 | global skip_suffices 72 | 73 | # changed from obsolete psutil.phymem_usage().available 74 | memavail=psutil.virtual_memory().available 75 | 76 | sc=create_sw_conn(parse_arg.authtoken,parse_arg.storage_url) 77 | 78 | try: 79 | if parse_arg.prefix: 80 | headers,objs=sc.get_container(parse_arg.container, 81 | prefix=parse_arg.prefix,full_listing=True) 82 | else: 83 | headers,objs=sc.get_container(parse_arg.container,full_listing=True) 84 | 85 | search_pool=multiprocessing.Pool(parse_arg.maxproc) 86 | 87 | for obj in objs: 88 | if obj['name'].lower().endswith(skip_suffices) or\ 89 | (parse_arg.filename and not \ 90 | fnmatch.fnmatch(obj['name'],parse_arg.filename)): 91 | continue 92 | 93 | if obj['bytes']>memavail: 94 | print("Object",obj['name'],"too large for memory!", 95 | file=sys.stderr) 96 | continue 97 | 98 | search_pool.apply_async(search_worker,[[parse_arg,obj['name']]]) 99 | #search_object(parse_arg,obj['name']) 100 | 101 | search_pool.close() 102 | search_pool.join() 103 | 104 | except swiftclient.ClientException: 105 | print("Error: cannot access Swift container '%s'!" % 106 | parse_arg.container) 107 | 108 | sc.close() 109 | 110 | def parse_arguments(): 111 | parser=argparse.ArgumentParser( 112 | description="Search text objects for pattern") 113 | parser.add_argument('-c','--container',required=True) 114 | parser.add_argument('pattern',type=str) 115 | parser.add_argument('-m','--maxproc',type=int, 116 | help="maximum number of processes to run",default=5) 117 | parser.add_argument('-a','--authtoken', 118 | default=os.environ.get("OS_AUTH_TOKEN"), 119 | help='swift authentication token (required when storage-url is used)') 120 | parser.add_argument('-s','--storage-url', 121 | default=os.environ.get("OS_STORAGE_URL"), 122 | help='swift storage url (required when authtoken is used)') 123 | parser.add_argument('-f','--filename', 124 | help='limit search to objects matching this pattern') 125 | parser.add_argument('-p','--prefix', 126 | help='limit search to objects matching this prefix') 127 | parser.add_argument('-b','--binary',action='store_true', 128 | help='try to exclude files identified as binary') 129 | parser.add_argument('-i','--insensitive',action='store_true', 130 | help='case insensitive search') 131 | 132 | return parser.parse_args() 133 | 134 | def main(): 135 | search_container(parse_arguments()) 136 | 137 | if __name__ == '__main__': 138 | main() 139 | -------------------------------------------------------------------------------- /swift_commander/swsymlinks.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # Script to save or restore all symlinks in a directory structure to a textfile 4 | # name: .symbolic-links.tree.txt 5 | # 6 | # swsymlinks dirkpetersen / Aug 2015 7 | # 8 | 9 | import sys, os, pwd, argparse, subprocess, re, time, datetime, tempfile 10 | 11 | class KeyboardInterruptError(Exception): pass 12 | 13 | def main(): 14 | global args 15 | 16 | # Parse command-line arguments 17 | args = parse_arguments() 18 | 19 | oldcurrdir = os.getcwd() 20 | os.chdir(args.folder) 21 | currdir = os.getcwd() 22 | curruser = pwd.getpwuid(os.getuid()).pw_name 23 | tmpdir = tempfile.gettempdir() 24 | 25 | if args.folder.startswith('./'): 26 | args.folder = args.folder[2:] 27 | 28 | if os.path.abspath(currdir) != os.path.abspath(args.folder): 29 | print("Current folder %s <> target folder %s, exiting....", (currdir,args.folder)) 30 | return False 31 | 32 | if args.linkdbfolder and args.restore: 33 | print("You cannot use --linkdbfolder in --restore mode.") 34 | print("Please put .symbolic-links.tree.txt in the root of --folder") 35 | return False 36 | 37 | linkdbfolder = currdir 38 | if args.linkdbfolder: 39 | linkdbfolder = args.linkdbfolder 40 | 41 | fcnt=0 42 | dcnt=0 43 | if args.save: 44 | print ('\nScanning folder %s to archive symlinks ...' % currdir) 45 | if args.single: 46 | # only create one single .symbolic-links.tree.txt at the root 47 | with open(os.path.join(linkdbfolder,".symbolic-links.tree.txt"),'w') as sfile: 48 | dcnt+=1 49 | for root, folders, files in mywalk(args.folder): 50 | items=folders+files 51 | for f in items: 52 | try: 53 | base=root.replace(currdir+'/','') 54 | p=os.path.join(base,f) 55 | if os.path.islink(p): 56 | stat=getstat(p) 57 | targ=os.readlink(p) 58 | sfile.write("%s|%s|%s\n" % (p,targ,stat.st_mtime)) 59 | fcnt+=1 60 | if args.debug: 61 | sys.stderr.write('SYMLINK:%s\n TARGET:%s\n' % (p,targ)) 62 | 63 | except Exception as err: 64 | sys.stderr.write(str(err)) 65 | sys.stderr.write('\n') 66 | 67 | else: 68 | # create a .symbolic-links.tree.txt in each folder recursively 69 | for root, folders, files in mywalk(args.folder): 70 | try: 71 | base=root.replace(currdir+'/','') 72 | treefile=os.path.join(base,'.symbolic-links.tree.txt') 73 | if os.path.exists(treefile): 74 | os.remove(treefile) 75 | oldroot='' 76 | except Exception as err: 77 | sys.stderr.write(str(err)) 78 | sys.stderr.write('\n') 79 | items=folders+files 80 | for f in items: 81 | try: 82 | p=os.path.join(base,f) 83 | if os.path.islink(p): 84 | if oldroot != root: 85 | dcnt+=1 86 | oldroot=root 87 | sys.stderr.write('Symlink Store: %s\n' % (treefile)) 88 | stat=getstat(p) 89 | targ=os.readlink(p) 90 | with open(os.path.join(root,'.symbolic-links.tree.txt'),'a') as sfile: 91 | sfile.write("%s|%s|%s\n" % (f,targ,stat.st_mtime)) 92 | fcnt+=1 93 | if args.debug: 94 | sys.stderr.write(' SYMLINK:%s\n TARGET:%s\n' % (p,targ)) 95 | 96 | except Exception as err: 97 | sys.stderr.write(str(err)) 98 | sys.stderr.write('\n') 99 | 100 | print("saved %i symlinks into %i .symbolic-links.tree.txt store(s)" % (fcnt, dcnt)) 101 | 102 | 103 | elif args.clean: 104 | # remove all .symbolic-links.tree.txt files recursively 105 | cnt=0 106 | print ('recursively clean .symbolic-links.tree.txt from %s' % currdir) 107 | for root, folders, files in mywalk(args.folder): 108 | if args.debug: 109 | print ("cleaning folder %s" % root) 110 | p=os.path.join(root,'.symbolic-links.tree.txt') 111 | if os.path.exists(p): 112 | os.remove(p) 113 | print('deleted %s' % p) 114 | cnt+=1 115 | print('%i .symbolic-links.tree.txt file(s) have been removed.' % cnt) 116 | 117 | elif args.restore: 118 | for root, folders, files in mywalk(args.folder): 119 | base=root.replace(currdir+'/','') 120 | treefile=os.path.join(base,'.symbolic-links.tree.txt') 121 | if os.path.exists(treefile): 122 | sys.stderr.write('restoring from Symlink Store: %s ...\n ' % (treefile)) 123 | dcnt+=1 124 | lines=[] 125 | with open(treefile,'r') as sfile: 126 | lines=sfile.readlines() 127 | for line in lines: 128 | try: 129 | link,targ,mtime=line.split('|') 130 | d=os.path.dirname(link) 131 | if d: 132 | #link was saved with a directory in .symbolic-links.tree.txt 133 | if not os.path.exists(d): 134 | os.makedirs(d) 135 | else: 136 | #link was saved as a file only in .symbolic-links.tree.txt 137 | link=os.path.join(base,link) 138 | if os.path.islink(link) and args.force: 139 | os.unlink(link) 140 | if not os.path.exists(link): 141 | os.symlink(targ,link) 142 | stat=getstat(link) 143 | #set both symlink atime and mtime to mtime 144 | os.utime(link,(float(mtime),float(mtime)),follow_symlinks=False) 145 | fcnt+=1 146 | if args.debug: 147 | sys.stderr.write('SYMLINK:%s\n TARGET:%s\n' % (link,targ)) 148 | else: 149 | print(' skipping restore of link %s. File already exists' % link) 150 | except Exception as err: 151 | sys.stderr.write(str(err)) 152 | sys.stderr.write('\n') 153 | print("Restored %i symlinks from %i .symbolic-links.tree.txt store(s)" % (fcnt, dcnt)) 154 | else: 155 | print("You need to use either --save, --save --single, --clean or --restore as a command option") 156 | 157 | os.chdir(oldcurrdir) 158 | 159 | 160 | def startswithpath(pathlist, pathstr): 161 | """ checks if at least one of the paths in a list of paths starts with a string """ 162 | for path in pathlist: 163 | if (os.path.join(pathstr, '')).startswith(path): 164 | return True 165 | return False 166 | 167 | def getstartpath(pathlist, pathstr): 168 | """ return the path from pathlist that is the first part of pathstr""" 169 | for path in pathlist: 170 | if (os.path.join(pathstr, '')).startswith(path): 171 | return path 172 | return '' 173 | 174 | 175 | def getstat(path): 176 | """ returns the stat information of a file""" 177 | statinfo=None 178 | try: 179 | statinfo=os.lstat(path) 180 | except (IOError, OSError) as e: # FileNotFoundError only since python 3.3 181 | if args.debug: 182 | sys.stderr.write(str(e)) 183 | except: 184 | raise 185 | return statinfo 186 | 187 | def setfiletime(path,attr="atime"): 188 | """ sets the a time of a file to the current time """ 189 | try: 190 | statinfo=getstat(path) 191 | if attr=="atime" or attr=="all": 192 | os.utime(path,(time.time(),statinfo.st_atime)) 193 | if attr=="mtime" or attr=="all": 194 | os.utime(path,(time.time(),statinfo.st_mtime)) 195 | return True 196 | except Exception as err: 197 | sys.stderr.write(str(err)) 198 | sys.stderr.write('\n') 199 | return False 200 | 201 | def uid2user(uidNumber): 202 | """ attempts to convert uidNumber to username """ 203 | import pwd 204 | try: 205 | return pwd.getpwuid(int(uidNumber)).pw_name 206 | except Exception as err: 207 | sys.stderr.write(str(err)) 208 | sys.stderr.write('\n') 209 | return str(uidNumber) 210 | 211 | def list2file(mylist,path): 212 | """ dumps a list into a text file, one line per item""" 213 | try: 214 | with open(path,'w') as f: 215 | for item in mylist: 216 | f.write("{}\r\n".format(item)) 217 | return True 218 | except Exception as err: 219 | sys.stderr.write(str(err)) 220 | sys.stderr.write('\n') 221 | return False 222 | 223 | def pathlist2file(mylist,path,root): 224 | """ dumps a list into a text file, one line per item, but removes 225 | a root folder from all paths. Used for --files-from feature in rsync""" 226 | try: 227 | with open(path,'w') as f: 228 | for item in mylist: 229 | f.write("{}\r\n".format(item[len(root):])) 230 | return True 231 | except Exception as err: 232 | sys.stderr.write(str(err)) 233 | sys.stderr.write('\n') 234 | return False 235 | 236 | def mywalk(top, skipdirs=['.snapshot',]): 237 | """ returns subset of os.walk """ 238 | for root, dirs, files in os.walk(top,topdown=True,onerror=walkerr): 239 | for skipdir in skipdirs: 240 | if skipdir in dirs: 241 | dirs.remove(skipdir) # don't visit this directory 242 | yield root, dirs, files 243 | 244 | def walkerr(oserr): 245 | sys.stderr.write(str(oserr)) 246 | sys.stderr.write('\n') 247 | return 0 248 | 249 | 250 | def parse_arguments(): 251 | """ 252 | Gather command-line arguments. 253 | """ 254 | parser = argparse.ArgumentParser(prog='swsymlinks', 255 | description='save or restore all symlinks in a directory structure' +\ 256 | 'to a textfile .symbolic-links.tree.txt') 257 | parser.add_argument( '--save', '-s', dest='save', action='store_true', 258 | help='save all symlinks to a text file', 259 | default=False ) 260 | parser.add_argument( '--single', '-n', dest='single', action='store_true', 261 | help='save all symlinks into a single file .symbolic-links.tree.txt at the root', 262 | default=False ) 263 | parser.add_argument( '--clean', '-c', dest='clean', action='store_true', 264 | help='delete all .symbolic-links.tree.txt files recursively', 265 | default=False ) 266 | parser.add_argument( '--restore', '-r', dest='restore', action='store_true', 267 | help='restore all symlinks from a text file', 268 | default=False ) 269 | parser.add_argument( '--force', '-o', dest='force', action='store_true', 270 | help='force restore, even if a symlink already exists.', 271 | default=False ) 272 | parser.add_argument( '--debug', '-g', dest='debug', action='store_true', 273 | help='print the symlink targets to STDERR', 274 | default=False ) 275 | parser.add_argument( '--linkdbfolder', '-l', dest='linkdbfolder', 276 | action='store', 277 | help='set folder location for .symbolic-links.tree.txt file') 278 | parser.add_argument( '--folder', '-f', dest='folder', 279 | action='store', 280 | help='search this folder and below for files to remove') 281 | args = parser.parse_args() 282 | if not args.folder: 283 | parser.error('required option --folder not given !') 284 | if args.debug: 285 | print('DEBUG: Arguments/Options: %s' % args) 286 | return args 287 | 288 | if __name__ == '__main__': 289 | sys.exit(main()) 290 | -------------------------------------------------------------------------------- /tests.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | PATH=swift_commander:$PATH 4 | swc clean 5 | swc env 6 | swc auth 7 | swc upload /var/log/samba /swift-commmander-tests/log 8 | swc download /swift-commmander-tests/log ./tests/log 9 | swc archive /var/log/samba /swift-commmander-tests/log.archive 10 | swc unarch /swift-commmander-tests/log.archive ./tests/log.archive 11 | swc search samba /swift-commmander-tests 12 | swc compare ./tests/log ./tests/log.archive 13 | swc rm -rf /swift-commmander-tests/log.archive 14 | swc rm -rf /swift-commmander-tests/log 15 | swc ls /swift-commmander-tests 16 | swc rm -rf /swift-commmander-tests 17 | --------------------------------------------------------------------------------