├── .coveragerc ├── .gitchangelog.rc ├── .package ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.rst ├── appveyor.yml ├── autogen.sh ├── bin └── test ├── setup.cfg ├── setup.py ├── shyaml └── shyaml.py /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc to control coverage.py 2 | [run] 3 | branch = True 4 | 5 | [report] 6 | # Regexes for lines to exclude from consideration 7 | exclude_lines = 8 | # Have to re-enable the standard pragma 9 | pragma: no cover 10 | 11 | # Don't complain about missing debug-only code: 12 | def __repr__ 13 | if self\.debug 14 | 15 | # Don't complain if tests don't hit defensive assertion code: 16 | raise AssertionError 17 | raise NotImplementedError 18 | 19 | # Don't complain if non-runnable code isn't run: 20 | if 0: 21 | 22 | ignore_errors = True 23 | [html] 24 | directory = cover 25 | -------------------------------------------------------------------------------- /.gitchangelog.rc: -------------------------------------------------------------------------------- 1 | ## 2 | ## Format 3 | ## 4 | ## ACTION: [AUDIENCE:] COMMIT_MSG [!TAG ...] 5 | ## 6 | ## Description 7 | ## 8 | ## ACTION is one of 'chg', 'fix', 'new' 9 | ## 10 | ## Is WHAT the change is about. 11 | ## 12 | ## 'chg' is for refactor, small improvement, cosmetic changes... 13 | ## 'fix' is for bug fixes 14 | ## 'new' is for new features, big improvement 15 | ## 16 | ## AUDIENCE is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc' 17 | ## 18 | ## Is WHO is concerned by the change. 19 | ## 20 | ## 'dev' is for developpers (API changes, refactors...) 21 | ## 'usr' is for final users (UI changes) 22 | ## 'pkg' is for packagers (packaging changes) 23 | ## 'test' is for testers (test only related changes) 24 | ## 'doc' is for doc guys (doc only changes) 25 | ## 26 | ## COMMIT_MSG is ... well ... the commit message itself. 27 | ## 28 | ## TAGs are additionnal adjective as 'refactor' 'minor' 'cosmetic' 29 | ## 30 | ## They are preceded with a '!' or a '@' (prefer the former, as the 31 | ## latter is wrongly interpreted in github.) Commonly used tags are: 32 | ## 33 | ## 'refactor' is obviously for refactoring code only 34 | ## 'minor' is for a very meaningless change (a typo, adding a comment) 35 | ## 'cosmetic' is for cosmetic driven change (re-indentation, 80-col...) 36 | ## 'wip' is for partial functionality but complete subfunctionality. 37 | ## 38 | ## Example: 39 | ## 40 | ## new: usr: support of bazaar implemented 41 | ## chg: re-indentend some lines !cosmetic 42 | ## new: dev: updated code to be compatible with last version of killer lib. 43 | ## fix: pkg: updated year of licence coverage. 44 | ## new: test: added a bunch of test around user usability of feature X. 45 | ## fix: typo in spelling my name in comment. !minor 46 | ## 47 | ## Please note that multi-line commit message are supported, and only the 48 | ## first line will be considered as the "summary" of the commit message. So 49 | ## tags, and other rules only applies to the summary. The body of the commit 50 | ## message will be displayed in the changelog without reformatting. 51 | 52 | 53 | ## 54 | ## ``ignore_regexps`` is a line of regexps 55 | ## 56 | ## Any commit having its full commit message matching any regexp listed here 57 | ## will be ignored and won't be reported in the changelog. 58 | ## 59 | ignore_regexps = [ 60 | r'@minor', r'!minor', 61 | r'@cosmetic', r'!cosmetic', 62 | r'@refactor', r'!refactor', 63 | r'@wip', r'!wip', 64 | r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[p|P]kg:', 65 | r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*[d|D]ev:', 66 | r'^(.{3,3}\s*:)?\s*[fF]irst commit.?\s*$', 67 | ] 68 | 69 | 70 | ## ``section_regexps`` is a list of 2-tuples associating a string label and a 71 | ## list of regexp 72 | ## 73 | ## Commit messages will be classified in sections thanks to this. Section 74 | ## titles are the label, and a commit is classified under this section if any 75 | ## of the regexps associated is matching. 76 | ## 77 | section_regexps = [ 78 | ('New', [ 79 | r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$', 80 | ]), 81 | ('Changes', [ 82 | r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$', 83 | ]), 84 | ('Fix', [ 85 | r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$', 86 | ]), 87 | 88 | ('Other', None ## Match all lines 89 | ), 90 | 91 | ] 92 | 93 | 94 | ## ``body_process`` is a callable 95 | ## 96 | ## This callable will be given the original body and result will 97 | ## be used in the changelog. 98 | ## 99 | ## Available constructs are: 100 | ## 101 | ## - any python callable that take one txt argument and return txt argument. 102 | ## 103 | ## - ReSub(pattern, replacement): will apply regexp substitution. 104 | ## 105 | ## - Indent(chars=" "): will indent the text with the prefix 106 | ## Please remember that template engines gets also to modify the text and 107 | ## will usually indent themselves the text if needed. 108 | ## 109 | ## - Wrap(regexp=r"\n\n"): re-wrap text in separate paragraph to fill 80-Columns 110 | ## 111 | ## - noop: do nothing 112 | ## 113 | ## - ucfirst: ensure the first letter is uppercase. 114 | ## (usually used in the ``subject_process`` pipeline) 115 | ## 116 | ## - final_dot: ensure text finishes with a dot 117 | ## (usually used in the ``subject_process`` pipeline) 118 | ## 119 | ## - strip: remove any spaces before or after the content of the string 120 | ## 121 | ## Additionally, you can `pipe` the provided filters, for instance: 122 | #body_process = Wrap(regexp=r'\n(?=\w+\s*:)') | Indent(chars=" ") 123 | #body_process = Wrap(regexp=r'\n(?=\w+\s*:)') 124 | body_process = noop 125 | 126 | 127 | ## ``subject_process`` is a callable 128 | ## 129 | ## This callable will be given the original subject and result will 130 | ## be used in the changelog. 131 | ## 132 | ## Available constructs are those listed in ``body_process`` doc. 133 | subject_process = (strip | 134 | ReSub(r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$', r'\4') | 135 | ucfirst | final_dot) 136 | 137 | 138 | ## ``tag_filter_regexp`` is a regexp 139 | ## 140 | ## Tags that will be used for the changelog must match this regexp. 141 | ## 142 | tag_filter_regexp = r'^[0-9]+\.[0-9]+(\.[0-9]+)?$' 143 | 144 | 145 | ## ``unreleased_version_label`` is a string 146 | ## 147 | ## This label will be used as the changelog Title of the last set of changes 148 | ## between last valid tag and HEAD if any. 149 | import os.path 150 | unreleased_version_label = lambda: swrap( 151 | (["bash"] if WIN32 else []) + 152 | [os.path.join(".", "autogen.sh"), "--get-version"], 153 | shell=False) 154 | 155 | 156 | ## ``output_engine`` is a callable 157 | ## 158 | ## This will change the output format of the generated changelog file 159 | ## 160 | ## Available choices are: 161 | ## 162 | ## - rest_py 163 | ## 164 | ## Legacy pure python engine, outputs ReSTructured text. 165 | ## This is the default. 166 | ## 167 | ## - mustache() 168 | ## 169 | ## Template name could be any of the available templates in 170 | ## ``templates/mustache/*.tpl``. 171 | ## Requires python package ``pystache``. 172 | ## Examples: 173 | ## - mustache("markdown") 174 | ## - mustache("restructuredtext") 175 | ## 176 | ## - makotemplate() 177 | ## 178 | ## Template name could be any of the available templates in 179 | ## ``templates/mako/*.tpl``. 180 | ## Requires python package ``mako``. 181 | ## Examples: 182 | ## - makotemplate("restructuredtext") 183 | ## 184 | output_engine = rest_py 185 | #output_engine = mustache("restructuredtext") 186 | #output_engine = mustache("markdown") 187 | #output_engine = makotemplate("restructuredtext") 188 | 189 | 190 | ## ``include_merges`` is a boolean 191 | ## 192 | ## This option tells git-log whether to include merge commits in the log. 193 | ## The default is to include them. 194 | include_merges = True 195 | -------------------------------------------------------------------------------- /.package: -------------------------------------------------------------------------------- 1 | NAME="shyaml" 2 | DESCRIPTION="YAML for command line" 3 | EMAIL="valentin.lab@kalysto.org" 4 | AUTHOR="Valentin Lab" 5 | AUTHOR_EMAIL="$AUTHOR <$EMAIL>" 6 | FILES="setup.cfg setup.py CHANGELOG.rst shyaml.py" 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | language: python 3 | python: 4 | - "3.9" 5 | - "3.8" 6 | - "3.7" 7 | - "2.7" 8 | install: 9 | - if [ -e requirements.txt ]; then pip install -r requirements.txt ; fi 10 | - if [ -z "$DOVIS" -o "$PKG_COVERAGE" ]; then pip install coverage; fi 11 | ## getting test deps 12 | - pip install -e .[test] 13 | script: 14 | - shyaml --version 15 | - bin/test 16 | after_success: 17 | - "bash <(curl -s https://codecov.io/bash) #dovis: ignore" 18 | - | 19 | if [ "$DOVIS" -a -d "$ARTIFACT_DIR" ]; then 20 | cp ".coverage" "$ARTIFACT_DIR" 21 | echo "$PWD" > "$ARTIFACT_DIR/cover_path" 22 | fi 23 | 24 | ## Ignored by Travis, but used internally to check packaging 25 | dist_check: 26 | options: 27 | exclude: 28 | - ["v:3.6", "pkg:old"] ## old version is breaking python 3.6 pkg_resources 29 | - ["v:3.7", "pkg:old"] ## old version is breaking python 3.7 pkg_resources 30 | tests: 31 | - label: install 32 | matrix: 33 | 'yes': 34 | - label: venv 35 | matrix: 36 | 'on': | 37 | pip install virtualenv 38 | virtualenv /tmp/virtualenv 39 | . /tmp/virtualenv/bin/activate 40 | 'off': | 41 | true 42 | - label: pkg 43 | matrix: 44 | old: | 45 | ## version 10 introduce a bug with d2to1 46 | pip install setuptools==9.1 47 | ## some ``python setup.py`` black magic do not work with d2to1 and pip ``6.0.7`` 48 | pip install pip==1.5.6 49 | docker: | 50 | ## Using the versions of python docker images 51 | true 52 | latest: | 53 | ## Using the last version of pip and setuptools 54 | pip install pip --upgrade 55 | pip install setuptools --upgrade 56 | - label: method 57 | matrix: 58 | setup: python setup.py install 59 | pip: pip install . 60 | pip+git: pip install "git+file://$PWD" 61 | dist: 62 | dist_files: 63 | pip install "$DIST_FILE" 64 | - | 65 | pip show -f $(./autogen.sh --get-name) 66 | pip list 67 | 'no': 68 | - | 69 | ln -sf $PWD/$(./autogen.sh --get-name) /usr/local/bin/ 70 | pip install pyyaml 71 | export PYTHONPATH="$PWD" 72 | touch /tmp/not-installed 73 | 74 | - | 75 | SRC=$PWD 76 | cd /tmp 77 | cat <}" 85 | python -c 'import shyaml' 86 | - | 87 | [ -e /tmp/not-installed ] || { 88 | cd "$SRC" 89 | pip uninstall -y $(./autogen.sh --get-name) 90 | } 91 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2020 Valentin Lab 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include *.rst 3 | recursive-include src *.py 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | SHYAML: YAML for the command line 3 | ================================= 4 | 5 | .. image:: https://img.shields.io/pypi/v/shyaml.svg 6 | :target: https://pypi.python.org/pypi/shyaml 7 | 8 | .. image:: https://img.shields.io/travis/0k/shyaml/master.svg?style=flat 9 | :target: https://travis-ci.com/github/0k/shyaml/ 10 | :alt: Travis CI build status 11 | 12 | .. image:: https://img.shields.io/appveyor/ci/vaab/shyaml.svg 13 | :target: https://ci.appveyor.com/project/vaab/shyaml/branch/master 14 | :alt: Appveyor CI build status 15 | 16 | .. image:: http://img.shields.io/codecov/c/github/0k/shyaml.svg?style=flat 17 | :target: https://codecov.io/gh/0k/shyaml/ 18 | :alt: Test coverage 19 | 20 | 21 | 22 | Description 23 | =========== 24 | 25 | Simple script that allow read access to YAML files through command line. 26 | 27 | This can be handy, if you want to get access to YAML data in your shell 28 | scripts. 29 | 30 | This script supports only read access and it might not support all 31 | the subtleties of YAML specification. But it should support some handy 32 | basic query of YAML file. 33 | 34 | 35 | Requirements 36 | ============ 37 | 38 | ``shyaml`` works in Linux, MacOSX, and Windows with python 2.7 and 3+. 39 | 40 | 41 | Installation 42 | ============ 43 | 44 | You don't need to download the GIT version of the code as ``shyaml`` is 45 | available on the PyPI. So you should be able to run:: 46 | 47 | pip install shyaml 48 | 49 | If you have downloaded the GIT sources, then you could add install 50 | the current version via:: 51 | 52 | pip install . 53 | 54 | And if you don't have the GIT sources but would like to get the latest 55 | master or branch from github, you could also:: 56 | 57 | pip install git+https://github.com/0k/shyaml 58 | 59 | Or even select a specific revision (branch/tag/commit):: 60 | 61 | pip install git+https://github.com/0k/shyaml@master 62 | 63 | On macOS, you can also install the latest release version via `Homebrew 64 | `_:: 65 | 66 | brew install shyaml 67 | 68 | Or to install the master branch:: 69 | 70 | brew install shyaml --HEAD 71 | 72 | 73 | Documentation 74 | ============= 75 | 76 | The following documented examples are actually tested automatically at 77 | each release for conformance on all platform and python versions. 78 | 79 | Please note that there is some subtle benign differences in some 80 | output whether ``shyaml`` is using the ``libyaml`` C implementation or 81 | the full python implementation. The documentation can be run with both 82 | implementation but some examples will fail depending on the 83 | implementation. To make things clear, I'll use some annotation and you 84 | can yourself check which version you are using with:: 85 | 86 | $ shyaml -V | grep "^libyaml used:" ## docshtest: if-success-set LIBYAML 87 | libyaml used: True 88 | 89 | 90 | Usage 91 | ===== 92 | 93 | ``shyaml`` takes its YAML input file from standard input ONLY. So let's 94 | define here a common YAML input for the next examples:: 95 | 96 | $ cat < test.yaml 97 | name: "MyName !! héhé" ## using encoding, and support comments ! 98 | subvalue: 99 | how-much: 1.1 100 | how-many: 2 101 | things: 102 | - first 103 | - second 104 | - third 105 | maintainer: "Valentin Lab" 106 | description: | 107 | Multiline description: 108 | Line 1 109 | Line 2 110 | subvalue.how-much: 1.2 111 | subvalue.how-much\more: 1.3 112 | subvalue.how-much\.more: 1.4 113 | EOF 114 | 115 | 116 | General browsing struct and displaying simple values 117 | ---------------------------------------------------- 118 | 119 | Simple query of simple attribute:: 120 | 121 | $ cat test.yaml | shyaml get-value name 122 | MyName !! héhé 123 | 124 | Query nested attributes by using '.' between key labels:: 125 | 126 | $ cat test.yaml | shyaml get-value subvalue.how-much 127 | 1.1 128 | 129 | Get type of attributes:: 130 | 131 | $ cat test.yaml | shyaml get-type name 132 | str 133 | $ cat test.yaml | shyaml get-type subvalue.how-much 134 | float 135 | 136 | Get length of structures or sequences:: 137 | 138 | $ cat test.yaml | shyaml get-length subvalue 139 | 5 140 | $ cat test.yaml | shyaml get-length subvalue.things 141 | 3 142 | 143 | But this won't work on other types:: 144 | 145 | $ cat test.yaml | shyaml get-length name 146 | Error: get-length does not support 'str' type. Please provide or select a sequence or struct. 147 | 148 | 149 | Parse structure 150 | --------------- 151 | 152 | Get sub YAML from a structure attribute:: 153 | 154 | $ cat test.yaml | shyaml get-type subvalue 155 | struct 156 | $ cat test.yaml | shyaml get-value subvalue ## docshtest: ignore-if LIBYAML 157 | how-much: 1.1 158 | how-many: 2 159 | things: 160 | - first 161 | - second 162 | - third 163 | maintainer: Valentin Lab 164 | description: 'Multiline description: 165 | 166 | Line 1 167 | 168 | Line 2 169 | 170 | ' 171 | 172 | Iteration through keys only:: 173 | 174 | $ cat test.yaml | shyaml keys 175 | name 176 | subvalue 177 | subvalue.how-much 178 | subvalue.how-much\more 179 | subvalue.how-much\.more 180 | 181 | Iteration through keys only (``\0`` terminated strings):: 182 | 183 | $ cat test.yaml | shyaml keys-0 subvalue | xargs -0 -n 1 echo "VALUE:" 184 | VALUE: how-much 185 | VALUE: how-many 186 | VALUE: things 187 | VALUE: maintainer 188 | VALUE: description 189 | 190 | Iteration through values only (``\0`` terminated string highly recommended):: 191 | 192 | $ cat test.yaml | shyaml values-0 subvalue | 193 | while IFS='' read -r -d $'\0' value; do 194 | echo "RECEIVED: '$value'" 195 | done 196 | RECEIVED: '1.1' 197 | RECEIVED: '2' 198 | RECEIVED: '- first 199 | - second 200 | - third 201 | ' 202 | RECEIVED: 'Valentin Lab' 203 | RECEIVED: 'Multiline description: 204 | Line 1 205 | Line 2 206 | ' 207 | 208 | Iteration through keys and values (``\0`` terminated string highly recommended):: 209 | 210 | $ read-0() { 211 | while [ "$1" ]; do 212 | IFS=$'\0' read -r -d '' "$1" || return 1 213 | shift 214 | done 215 | } && 216 | cat test.yaml | shyaml key-values-0 subvalue | 217 | while read-0 key value; do 218 | echo "KEY: '$key'" 219 | echo "VALUE: '$value'" 220 | echo 221 | done 222 | KEY: 'how-much' 223 | VALUE: '1.1' 224 | 225 | KEY: 'how-many' 226 | VALUE: '2' 227 | 228 | KEY: 'things' 229 | VALUE: '- first 230 | - second 231 | - third 232 | ' 233 | 234 | KEY: 'maintainer' 235 | VALUE: 'Valentin Lab' 236 | 237 | KEY: 'description' 238 | VALUE: 'Multiline description: 239 | Line 1 240 | Line 2 241 | ' 242 | 243 | 244 | Notice, that you'll get the same result using 245 | ``get-values``. ``get-values`` will support sequences and struct, 246 | and ``key-values`` support only struct. (for a complete table of 247 | which function support what you can look at the usage line) 248 | 249 | And, if you ask for keys, values, key-values on non struct like, you'll 250 | get an error:: 251 | 252 | $ cat test.yaml | shyaml keys name 253 | Error: keys does not support 'str' type. Please provide or select a struct. 254 | $ cat test.yaml | shyaml values subvalue.how-many 255 | Error: values does not support 'int' type. Please provide or select a struct. 256 | $ cat test.yaml | shyaml key-values subvalue.how-much 257 | Error: key-values does not support 'float' type. Please provide or select a struct. 258 | 259 | 260 | Parse sequence 261 | -------------- 262 | 263 | Query a sequence with ``get-value``:: 264 | 265 | $ cat test.yaml | shyaml get-value subvalue.things 266 | - first 267 | - second 268 | - third 269 | 270 | And access individual elements with python-like indexing:: 271 | 272 | $ cat test.yaml | shyaml get-value subvalue.things.0 273 | first 274 | $ cat test.yaml | shyaml get-value subvalue.things.-1 275 | third 276 | $ cat test.yaml | shyaml get-value subvalue.things.5 277 | Error: invalid path 'subvalue.things.5', index 5 is out of range (3 elements in sequence). 278 | 279 | Note that this will work only with integer (preceded or not by a minus 280 | sign):: 281 | 282 | $ cat test.yaml | shyaml get-value subvalue.things.foo 283 | Error: invalid path 'subvalue.things.foo', non-integer index 'foo' provided on a sequence. 284 | 285 | More usefull, parse a list in one go with ``get-values``:: 286 | 287 | $ cat test.yaml | shyaml get-values subvalue.things 288 | first 289 | second 290 | third 291 | 292 | Note that the action is called ``get-values``, and that output is 293 | separated by newline char(s) (which is os dependent), this can bring 294 | havoc if you are parsing values containing newlines itself. Hopefully, 295 | ``shyaml`` has a ``get-values-0`` to terminate strings by ``\0`` char, 296 | which allows complete support of any type of values, including YAML. 297 | ``get-values`` outputs key and values for ``struct`` types and only 298 | values for ``sequence`` types:: 299 | 300 | $ cat test.yaml | shyaml get-values-0 subvalue | 301 | while IFS='' read -r -d '' key && 302 | IFS='' read -r -d '' value; do 303 | echo "'$key' -> '$value'" 304 | done 305 | 'how-much' -> '1.1' 306 | 'how-many' -> '2' 307 | 'things' -> '- first 308 | - second 309 | - third 310 | ' 311 | 'maintainer' -> 'Valentin Lab' 312 | 'description' -> 'Multiline description: 313 | Line 1 314 | Line 2 315 | ' 316 | 317 | Please note that, if ``get-values{,-0}`` actually works on ``struct``, 318 | it's maybe more explicit to use the equivalent ``key-values{,0}``. It 319 | should be noted that ``key-values{,0}`` is not completly equivalent as 320 | it is meant to be used with ``struct`` only and will complain if not. 321 | 322 | You should also notice that values that are displayed are YAML compatible. So 323 | if they are complex, you can re-use ``shyaml`` on them to parse their content. 324 | 325 | Of course, ``get-values`` should only be called on sequence elements:: 326 | 327 | $ cat test.yaml | shyaml get-values name 328 | Error: get-values does not support 'str' type. Please provide or select a sequence or struct. 329 | 330 | 331 | Parse YAML document streams 332 | --------------------------- 333 | 334 | YAML input can be a stream of documents, the action will then be 335 | applied to each document:: 336 | 337 | $ i=0; while true; do 338 | ((i++)) 339 | echo "ingests:" 340 | echo " - data: xxx" 341 | echo " id: tag-$i" 342 | if ((i >= 3)); then 343 | break 344 | fi 345 | echo "---" 346 | done | shyaml get-value ingests.0.id | tr '\0' '&' 347 | tag-1&tag-2&tag-3 348 | 349 | 350 | Notice that ``NUL`` char is used by default for separating output 351 | iterations if not used in ``-y`` mode. You can use that to separate 352 | each output. ``-y`` mode will use conventional YAML way to separate 353 | documents (which is ``---``). 354 | 355 | So:: 356 | 357 | $ i=0; while true; do 358 | ((i++)) 359 | echo "ingests:" 360 | echo " - data: xxx" 361 | echo " id: tag-$i" 362 | if ((i >= 3)); then 363 | break 364 | fi 365 | echo "---" 366 | done | shyaml get-value -y ingests.0.id ## docshtest: ignore-if LIBYAML 367 | tag-1 368 | ... 369 | --- 370 | tag-2 371 | ... 372 | --- 373 | tag-3 374 | ... 375 | 376 | Notice that it is not supported to use any query that can output more than one 377 | value (like all the query that can be suffixed with ``*-0``) with a multi-document 378 | YAML:: 379 | 380 | $ i=0; while true; do 381 | ((i++)) 382 | echo "ingests:" 383 | echo " - data: xxx" 384 | echo " id: tag-$i" 385 | if ((i >= 3)); then 386 | break 387 | fi 388 | echo "---" 389 | done | shyaml keys ingests.0 >/dev/null 390 | Error: Source YAML is multi-document, which doesn't support any other action than get-type, get-length, get-value 391 | 392 | You'll probably notice also, that output seems buffered. The previous 393 | content is displayed as a whole only at the end. If you need a 394 | continuous flow of YAML document, then the command line option ``-L`` 395 | is required to force a non-buffered line-by-line reading of the file 396 | so as to ensure that each document is properly parsed as soon as 397 | possible. That means as soon as either a YAML document end is detected 398 | (``---`` or ``EOF``): 399 | 400 | Without the ``-L``, if we kill our shyaml process before the end:: 401 | 402 | $ i=0; while true; do 403 | ((i++)) 404 | echo "ingests:" 405 | echo " - data: xxx" 406 | echo " id: tag-$i" 407 | if ((i >= 2)); then 408 | break 409 | fi 410 | echo "---" 411 | sleep 10 412 | done 2>/dev/null | shyaml get-value ingests.0.id & pid=$! ; sleep 2; kill $pid 413 | 414 | 415 | With the ``-L``, if we kill our shyaml process before the end:: 416 | 417 | $ i=0; while true; do 418 | ((i++)) 419 | echo "ingests:" 420 | echo " - data: xxx" 421 | echo " id: tag-$i" 422 | if ((i >= 2)); then 423 | break 424 | fi 425 | echo "---" 426 | sleep 10 427 | done 2>/dev/null | shyaml get-value -L ingests.0.id & pid=$! ; sleep 2; kill $pid 428 | tag-1 429 | 430 | 431 | Using ``-y`` is required to force a YAML output that will be also parseable as a stream, 432 | which could help you chain shyaml calls:: 433 | 434 | $ i=0; while true; do 435 | ((i++)) 436 | echo "ingests:" 437 | echo " - data: xxx" 438 | echo " id: tag-$i" 439 | if ((i >= 3)); then 440 | break 441 | fi 442 | echo "---" 443 | sleep 0.2 444 | done | shyaml get-value ingests.0 -L -y | shyaml get-value id | tr '\0' '\n' 445 | tag-1 446 | tag-2 447 | tag-3 448 | 449 | 450 | An empty string will be still considered as an empty YAML document:: 451 | 452 | $ echo | shyaml get-value "toto" 453 | Error: invalid path 'toto', can't query subvalue 'toto' of a leaf (leaf value is None). 454 | 455 | 456 | Keys containing '.' 457 | ------------------- 458 | 459 | Use and ``\\`` to access keys with ``\`` and ``\.`` to access keys 460 | with literal ``.`` in them. Just be mindful of shell escaping (example 461 | uses single quotes):: 462 | 463 | $ cat test.yaml | shyaml get-value 'subvalue\.how-much' 464 | 1.2 465 | $ cat test.yaml | shyaml get-value 'subvalue\.how-much\\more' 466 | 1.3 467 | $ cat test.yaml | shyaml get-value 'subvalue\.how-much\\.more' default 468 | default 469 | 470 | This last one didn't escape correctly the last ``.``, this is the 471 | correct version:: 472 | 473 | $ cat test.yaml | shyaml get-value 'subvalue\.how-much\\\.more' default 474 | 1.4 475 | 476 | 477 | empty string keys 478 | ----------------- 479 | 480 | Yep, ``shyaml`` supports empty stringed keys. You might never have use 481 | for this one, but it's in YAML specification. So ``shyaml`` supports 482 | it:: 483 | 484 | $ cat < test.yaml 485 | empty-sub-key: 486 | "": 487 | a: foo 488 | "": bar 489 | "": wiz 490 | EOF 491 | 492 | $ cat test.yaml | shyaml get-value empty-sub-key.. 493 | bar 494 | $ cat test.yaml | shyaml get-value '' 495 | wiz 496 | 497 | Please notice that one empty string is different than no string at all:: 498 | 499 | $ cat < test.yaml 500 | "": 501 | a: foo 502 | b: bar 503 | "x": wiz 504 | EOF 505 | $ cat test.yaml | shyaml keys 506 | 507 | x 508 | $ cat test.yaml | shyaml keys '' 509 | a 510 | b 511 | 512 | The first asks for keys of the root YAML, the second asks for keys of the 513 | content of the empty string named element located in the root YAML. 514 | 515 | 516 | Handling missing paths 517 | ---------------------- 518 | 519 | There is a third argument on the command line of shyaml which is the 520 | DEFAULT argument. If the given KEY was not found in the YAML 521 | structure, then ``shyaml`` would return what you provided as DEFAULT. 522 | 523 | As of version < 0.3, this argument was defaulted to the empty 524 | string. For all version above 0.3 (included), if not provided, then 525 | an error message will be printed:: 526 | 527 | $ echo "a: 3" | shyaml get-value a mydefault 528 | 3 529 | 530 | $ echo "a: 3" | shyaml get-value b mydefault 531 | mydefault 532 | 533 | $ echo "a: 3" | shyaml get-value b 534 | Error: invalid path 'b', missing key 'b' in struct. 535 | 536 | You can emulate pre v0.3 behavior by specifying explicitly an empty 537 | string as third argument:: 538 | 539 | $ echo "a: 3" | shyaml get-value b '' 540 | 541 | Starting with version 0.6, you can also use the ``-q`` or ``--quiet`` to fail 542 | silently in case of KEY not found in the YAML structure:: 543 | 544 | $ echo "a: 3" | shyaml -q get-value b; echo "errlvl: $?" 545 | errlvl: 1 546 | $ echo "a: 3" | shyaml -q get-value a; echo "errlvl: $?" 547 | 3errlvl: 0 548 | 549 | 550 | Ordered mappings 551 | ---------------- 552 | 553 | Currently, using ``shyaml`` in a shell script involves happily taking 554 | YAML inputs and outputting YAML outputs that will further be processed. 555 | 556 | And this works very well. 557 | 558 | Before version ``0.4.0``, ``shyaml`` would boldly re-order (sorting them 559 | alphabetically) the keys in mappings. If this should be considered 560 | harmless per specification (mappings are indeed supposed to be 561 | unordered, this means order does not matter), in practical, YAML users 562 | could feel wronged by ``shyaml`` when there YAML got mangled and they 563 | wanted to give a meaning to the basic YAML mapping. 564 | 565 | Who am I to forbid such usage of YAML mappings ? So starting from 566 | version ``0.4.0``, ``shyaml`` will happily keep the order of your 567 | mappings:: 568 | 569 | $ cat < test.yaml 570 | mapping: 571 | a: 1 572 | c: 2 573 | b: 3 574 | EOF 575 | 576 | For ``shyaml`` version before ``0.4.0``:: 577 | 578 | # shyaml get-value mapping < test.yaml 579 | a: 1 580 | b: 3 581 | c: 2 582 | 583 | For ``shyaml`` version including and after ``0.4.0``:: 584 | 585 | $ shyaml get-value mapping < test.yaml 586 | a: 1 587 | c: 2 588 | b: 3 589 | 590 | 591 | Strict YAML for further processing 592 | ---------------------------------- 593 | 594 | Processing yaml can be done recursively and extensively through using 595 | the output of ``shyaml`` into ``shyaml``. Most of its output is itself 596 | YAML. Most ? Well, for ease of use, literal keys (string, numbers) are 597 | outputed directly without YAML quotes, which is often convenient. 598 | 599 | But this has the consequence of introducing inconsistent behavior. So 600 | when processing YAML coming out of shyaml, you should probably think 601 | about using the ``--yaml`` (or ``-y``) option to output only strict YAML. 602 | 603 | With the drawback that when you'll want to output string, you'll need to 604 | call a last time ``shyaml get-value`` to explicitly unquote the YAML. 605 | 606 | 607 | Object Tag 608 | ---------- 609 | 610 | YAML spec allows object tags which allows you to map local data to 611 | objects in your application. 612 | 613 | When using ``shyaml``, we do not want to mess with these tags, but still 614 | allow parsing their internal structure. 615 | 616 | ``get-type`` will correctly give you the type of the object:: 617 | 618 | $ cat < test.yaml 619 | %TAG !e! tag:example.com,2000:app/ 620 | --- 621 | - !e!foo "bar" 622 | EOF 623 | 624 | $ shyaml get-type 0 < test.yaml 625 | tag:example.com,2000:app/foo 626 | 627 | ``get-value`` with ``-y`` (see section Strict YAML) will give you the 628 | complete yaml tagged value:: 629 | 630 | $ shyaml get-value -y 0 < test.yaml ## docshtest: ignore-if LIBYAML 631 | ! 'bar' 632 | 633 | 634 | Another example:: 635 | 636 | $ cat < test.yaml 637 | %TAG ! tag:clarkevans.com,2002: 638 | --- !shape 639 | # Use the ! handle for presenting 640 | # tag:clarkevans.com,2002:circle 641 | - !circle 642 | center: &ORIGIN {x: 73, y: 129} 643 | radius: 7 644 | - !line 645 | start: *ORIGIN 646 | finish: { x: 89, y: 102 } 647 | - !label 648 | start: *ORIGIN 649 | color: 0xFFEEBB 650 | text: Pretty vector drawing. 651 | EOF 652 | $ shyaml get-type 2 < test.yaml 653 | tag:clarkevans.com,2002:label 654 | 655 | And you can still traverse internal value:: 656 | 657 | $ shyaml get-value -y 2.start < test.yaml 658 | x: 73 659 | y: 129 660 | 661 | 662 | Note that all global tags will be resolved and simplified (as 663 | ``!!map``, ``!!str``, ``!!seq``), but not unknown local tags:: 664 | 665 | $ cat < test.yaml 666 | %YAML 1.1 667 | --- 668 | !!map { 669 | ? !!str "sequence" 670 | : !!seq [ !!str "one", !!str "two" ], 671 | ? !!str "mapping" 672 | : !!map { 673 | ? !!str "sky" : !myobj "blue", 674 | ? !!str "sea" : !!str "green", 675 | }, 676 | } 677 | EOF 678 | 679 | $ shyaml get-value < test.yaml ## docshtest: ignore-if LIBYAML 680 | sequence: 681 | - one 682 | - two 683 | mapping: 684 | sky: !myobj 'blue' 685 | sea: green 686 | 687 | 688 | Empty documents 689 | --------------- 690 | 691 | When provided with an empty document, ``shyaml`` will consider the 692 | document to hold a ``null`` value:: 693 | 694 | $ echo | shyaml get-value -y ## docshtest: ignore-if LIBYAML 695 | null 696 | ... 697 | 698 | 699 | Usage string 700 | ------------ 701 | 702 | A quick reminder of what is available will be printed when calling 703 | ``shyaml`` without any argument:: 704 | 705 | $ shyaml 706 | Error: Bad number of arguments. 707 | Usage: 708 | 709 | shyaml {-h|--help} 710 | shyaml {-V|--version} 711 | shyaml [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] 712 | 713 | 714 | The full help is available through the usage of the standard ``-h`` or 715 | ``-help``:: 716 | 717 | 718 | $ shyaml --help 719 | 720 | Parses and output chosen subpart or values from YAML input. 721 | It reads YAML in stdin and will output on stdout it's return value. 722 | 723 | Usage: 724 | 725 | shyaml {-h|--help} 726 | shyaml {-V|--version} 727 | shyaml [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] 728 | 729 | 730 | Options: 731 | 732 | -y, --yaml 733 | Output only YAML safe value, more precisely, even 734 | literal values will be YAML quoted. This behavior 735 | is required if you want to output YAML subparts and 736 | further process it. If you know you have are dealing 737 | with safe literal value, then you don't need this. 738 | (Default: no safe YAML output) 739 | 740 | -q, --quiet 741 | In case KEY value queried is an invalid path, quiet 742 | mode will prevent the writing of an error message on 743 | standard error. 744 | (Default: no quiet mode) 745 | 746 | -L, --line-buffer 747 | Force parsing stdin line by line allowing to process 748 | streamed YAML as it is fed instead of buffering 749 | input and treating several YAML streamed document 750 | at once. This is likely to have some small performance 751 | hit if you have a huge stream of YAML document, but 752 | then you probably don't really care about the 753 | line-buffering. 754 | (Default: no line buffering) 755 | 756 | ACTION Depending on the type of data you've targetted 757 | thanks to the KEY, ACTION can be: 758 | 759 | These ACTIONs applies to any YAML type: 760 | 761 | get-type ## returns a short string 762 | get-value ## returns YAML 763 | 764 | These ACTIONs applies to 'sequence' and 'struct' YAML type: 765 | 766 | get-values{,-0} ## returns list of YAML 767 | get-length ## returns an integer 768 | 769 | These ACTION applies to 'struct' YAML type: 770 | 771 | keys{,-0} ## returns list of YAML 772 | values{,-0} ## returns list of YAML 773 | key-values,{,-0} ## returns list of YAML 774 | 775 | Note that any value returned is returned on stdout, and 776 | when returning ``list of YAML``, it'll be separated by 777 | a newline or ``NUL`` char depending of you've used the 778 | ``-0`` suffixed ACTION. 779 | 780 | KEY Identifier to browse and target subvalues into YAML 781 | structure. Use ``.`` to parse a subvalue. If you need 782 | to use a literal ``.`` or ``\``, use ``\`` to quote it. 783 | 784 | Use struct keyword to browse ``struct`` YAML data and use 785 | integers to browse ``sequence`` YAML data. 786 | 787 | DEFAULT if not provided and given KEY do not match any value in 788 | the provided YAML, then DEFAULT will be returned. If no 789 | default is provided and the KEY do not match any value 790 | in the provided YAML, shyaml will fail with an error 791 | message. 792 | 793 | Examples: 794 | 795 | ## get last grocery 796 | cat recipe.yaml | shyaml get-value groceries.-1 797 | 798 | ## get all words of my french dictionary 799 | cat dictionaries.yaml | shyaml keys-0 french.dictionary 800 | 801 | ## get YAML config part of 'myhost' 802 | cat hosts_config.yaml | shyaml get-value cfgs.myhost 803 | 804 | 805 | 806 | Using invalid keywords will issue an error and the usage message:: 807 | 808 | $ shyaml get-foo 809 | Error: 'get-foo' is not a valid action. 810 | Usage: 811 | 812 | shyaml {-h|--help} 813 | shyaml {-V|--version} 814 | shyaml [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] 815 | 816 | 817 | 818 | Version information 819 | ------------------- 820 | 821 | You can get useful information (in case of a bug) or if you want to 822 | check if shyaml is using the ``libyaml`` C bindings, thanks to 823 | ``shyaml --version`` (or ``-V``):: 824 | 825 | # shyaml -V ## Example of possible output 826 | version: unreleased 827 | PyYAML: 3.13 828 | libyaml available: 0.1.6 829 | libyaml used: True 830 | Python: 2.7.8 (default, Oct 20 2014, 15:05:19) [GCC 4.9.1] 831 | 832 | Note that you can force to use the python implementation even if 833 | ``libyaml`` is available using ``FORCE_PYTHON_YAML_IMPLEMENTATION``:: 834 | 835 | $ FORCE_PYTHON_YAML_IMPLEMENTATION=1 shyaml --version | grep "^libyaml used:" 836 | libyaml used: False 837 | 838 | 839 | Python API 840 | ========== 841 | 842 | ``shyaml`` can be used from within python if you need so:: 843 | 844 | >>> import shyaml 845 | >>> try: 846 | ... from StringIO import StringIO 847 | ... except ImportError: 848 | ... from io import StringIO 849 | 850 | >>> yaml_content = StringIO(""" 851 | ... a: 1.1 852 | ... b: 853 | ... x: foo 854 | ... y: bar 855 | ... """) 856 | 857 | >>> for out in shyaml.do(stream=yaml_content, 858 | ... action="get-type", 859 | ... key="a"): 860 | ... print(repr(out)) 861 | 'float' 862 | 863 | Please note that ``shyaml.do(..)`` outputs a generator iterating 864 | through all the yaml documents of the stream. In most usage case, 865 | you'll have only one document. 866 | 867 | You can have a peek at the code, the ``do(..)`` function has a documented 868 | prototype. 869 | 870 | 871 | Contributing 872 | ============ 873 | 874 | Any suggestion or issue is welcome. Push request are very welcome, 875 | please check out the guidelines. 876 | 877 | 878 | Push Request Guidelines 879 | ----------------------- 880 | 881 | You can send any code. I'll look at it and will integrate it myself in 882 | the code base and leave you as the author. This process can take time and 883 | it'll take less time if you follow the following guidelines: 884 | 885 | - check your code with PEP8 or pylint. Try to stick to 80 columns wide. 886 | - separate your commits per smallest concern. 887 | - each commit should pass the tests (to allow easy bisect) 888 | - each functionality/bugfix commit should contain the code, tests, 889 | and doc. 890 | - prior minor commit with typographic or code cosmetic changes are 891 | very welcome. These should be tagged in their commit summary with 892 | ``!minor``. 893 | - the commit message should follow gitchangelog rules (check the git 894 | log to get examples) 895 | - if the commit fixes an issue or finished the implementation of a 896 | feature, please mention it in the summary. 897 | 898 | If you have some questions about guidelines which is not answered here, 899 | please check the current ``git log``, you might find previous commit that 900 | would show you how to deal with your issue. 901 | 902 | 903 | License 904 | ======= 905 | 906 | Copyright (c) 2020 Valentin Lab. 907 | 908 | Licensed under the `BSD License`_. 909 | 910 | .. _BSD License: http://raw.github.com/0k/shyaml/master/LICENSE 911 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | # What Python version is installed where: 2 | # http://www.appveyor.com/docs/installed-software#python 3 | 4 | environment: 5 | global: 6 | PYTHONIOENCODING: utf-8 7 | 8 | ## We still need to force python yaml implementation because of 9 | ## small discrepancy between outputs of the C implementation and 10 | ## python implementation. 11 | 12 | FORCE_PYTHON_YAML_IMPLEMENTATION: 1 13 | matrix: 14 | - PYTHON: "C:\\Python27" 15 | - PYTHON: "C:\\Python27-x64" 16 | - PYTHON: "C:\\Python37" 17 | - PYTHON: "C:\\Python37-x64" 18 | - PYTHON: "C:\\Python38" 19 | - PYTHON: "C:\\Python38-x64" 20 | - PYTHON: "C:\\Python39" 21 | - PYTHON: "C:\\Python39-x64" 22 | 23 | ## Before repo cloning 24 | init: 25 | ## without this, temporary directory could be created in current dir 26 | ## which will make some tests fail. 27 | - mkdir C:\TMP 28 | - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% 29 | - python -V 30 | 31 | ## After repo cloning 32 | install: 33 | 34 | build: false 35 | 36 | ## Before tests 37 | before_test: 38 | - for /f %%i in ('bash .\autogen.sh --get-name') do set PACKAGE_NAME=%%i 39 | - python setup.py develop easy_install %PACKAGE_NAME%[test] 40 | - pip install coverage codecov 41 | 42 | ## Custom test script 43 | test_script: 44 | - shyaml --version 45 | - bash bin/test 46 | 47 | after_test: 48 | - "codecov & REM #dovis: ignore" 49 | - "IF DEFINED DOVIS IF DEFINED ARTIFACT_DIR IF EXIST .coverage ( 50 | cp .coverage %ARTIFACT_DIR% && 51 | echo %cd% > %ARTIFACT_DIR%/cover_path 52 | )" 53 | 54 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ## 4 | ## You can download latest version of this file: 5 | ## $ wget https://gist.github.com/vaab/9118087/raw -O autogen.sh 6 | ## $ chmod +x autogen.sh 7 | ## 8 | 9 | ## 10 | ## Functions 11 | ## 12 | 13 | exname="$(basename "$0")" 14 | long_tag="[0-9]+\.[0-9]+(\.[0-9]+)?-[0-9]+-[0-9a-f]+" 15 | short_tag="[0-9]+\.[0-9]+(\.[0-9]+)?" 16 | get_short_tag="s/^($short_tag).*\$/\1/g" 17 | 18 | get_path() { ( 19 | IFS=: 20 | for d in $PATH; do 21 | filename="$d/$1" 22 | [ -f "$filename" -a -x "$filename" ] && { 23 | echo "$d/$1" 24 | return 0 25 | } 26 | done 27 | return 1 28 | ) } 29 | 30 | print_exit() { 31 | echo "$@" 32 | exit 1 33 | } 34 | 35 | print_syntax_error() { 36 | [ "$*" ] || print_syntax_error "$FUNCNAME: no arguments" 37 | print_exit "${ERROR}script error:${NORMAL} $@" >&2 38 | } 39 | 40 | print_syntax_warning() { 41 | [ "$*" ] || print_syntax_error "$FUNCNAME: no arguments." 42 | [ "$exname" ] || print_syntax_error "$FUNCNAME: 'exname' var is null or not defined." 43 | echo "$exname: ${WARNING}script warning:${NORMAL} $@" >&2 44 | } 45 | 46 | print_error() { 47 | [ "$*" ] || print_syntax_warning "$FUNCNAME: no arguments." 48 | [ "$exname" ] || print_exit "$FUNCNAME: 'exname' var is null or not defined." >&2 49 | print_exit "$exname: ${ERROR}error:${NORMAL} $@" >&2 50 | } 51 | 52 | depends() { 53 | ## Avoid colliding with variables that are created with depends. 54 | local __i __tr __path __new_name 55 | __tr=$(get_path "tr") 56 | test "$__tr" || 57 | die "dependency check: couldn't find 'tr' command." 58 | 59 | for __i in "$@"; do 60 | if ! __path=$(get_path "$__i"); then 61 | __new_name=$(echo "$__i" | "$__tr" '_' '-') 62 | if [ "$__new_name" != "$__i" ]; then 63 | depends "$__new_name" 64 | else 65 | 66 | print_error "dependency check: couldn't find '$__i' required command." 67 | fi 68 | else 69 | if ! test -z "$__path" ; then 70 | export "$(echo "$__i" | "$__tr" -- '- ' '__')"="$__path" 71 | fi 72 | fi 73 | done 74 | } 75 | 76 | die() { 77 | [ "$*" ] || print_syntax_warning "$FUNCNAME: no arguments." 78 | [ "$exname" ] || print_exit "$FUNCNAME: 'exname' var is null or not defined." >&2 79 | print_exit "$exname: ${ERROR}error:${NORMAL}" "$@" >&2 80 | } 81 | 82 | matches() { 83 | echo "$1" | "$grep" -E "^$2\$" >/dev/null 2>&1 84 | } 85 | 86 | get_current_git_date_timestamp() { 87 | "$git" show -s --pretty=format:%ct 88 | } 89 | 90 | 91 | dev_version_tag() { 92 | compat_date "$(get_current_git_date_timestamp)" "+%Y%m%d%H%M" 93 | } 94 | 95 | 96 | get_current_version() { 97 | 98 | version=$("$git" describe --tags) 99 | if matches "$version" "$short_tag"; then 100 | echo "$version" 101 | else 102 | version=$(echo "$version" | compat_sed "$get_short_tag") 103 | echo "${version}.dev$(dev_version_tag)" 104 | fi 105 | 106 | } 107 | 108 | prepare_files() { 109 | 110 | version=$(get_current_version) 111 | short_version=$(echo "$version" | cut -f 1,2,3 -d ".") 112 | 113 | 114 | for file in $FILES; do 115 | if [ -e "$file" ]; then 116 | compat_sed_i "s#%%version%%#$version#g; 117 | s#%%short-version%%#${short_version}#g; 118 | s#%%name%%#${NAME}#g; 119 | s#%%author%%#${AUTHOR}#g; 120 | s#%%email%%#${EMAIL}#g; 121 | s#%%author-email%%#${AUTHOR_EMAIL}#g; 122 | s#%%description%%#${DESCRIPTION}#g" \ 123 | "$file" 124 | fi 125 | done 126 | 127 | echo "Version updated to $version." 128 | } 129 | 130 | ## 131 | ## LOAD CONFIG 132 | ## 133 | 134 | if [ -e ./.package ]; then 135 | . ./.package 136 | else 137 | echo "'./.package' file is missing." 138 | exit 1 139 | fi 140 | 141 | ## list of files where %%version*%% macros are to be replaced: 142 | [ -z "$FILES" ] && FILES="setup.cfg setup.py CHANGELOG.rst" 143 | 144 | [ -z "$NAME" ] && die "No \$NAME was defined in './package'." 145 | 146 | 147 | ## 148 | ## CHECK DEPS 149 | ## 150 | 151 | depends git grep 152 | 153 | ## BSD / GNU sed compatibility layer 154 | if get_path sed >/dev/null; then 155 | if sed --version >/dev/null 2>&1; then ## GNU 156 | compat_sed() { sed -r "$@"; } 157 | compat_sed_i() { sed -r -i "$@"; } 158 | else ## BSD 159 | compat_sed() { sed -E "$@"; } 160 | compat_sed_i() { sed -E -i "" "$@"; } 161 | fi 162 | else 163 | ## Look for ``gsed`` 164 | if (get_path gsed && gsed --version) >/dev/null 2>&1; then 165 | compat_sed() { gsed -r "$@"; } 166 | compat_sed_i() { gsed -r -i "$@"; } 167 | else 168 | print_error "$exname: required GNU or BSD sed not found" 169 | fi 170 | fi 171 | 172 | ## BSD / GNU date compatibility layer 173 | if get_path date >/dev/null; then 174 | if date --version >/dev/null 2>&1 ; then ## GNU 175 | compat_date() { date -d "@$1" "$2"; } 176 | else ## BSD 177 | compat_date() { date -j -f %s "$1" "$2"; } 178 | fi 179 | else 180 | if (get_path gdate && gdate --version) >/dev/null 2>&1; then 181 | compat_date() { gdate -d "@$1" "$2"; } 182 | else 183 | print_error "$exname: required GNU or BSD date not found" 184 | fi 185 | fi 186 | 187 | if ! "$git" describe --tags >/dev/null 2>&1; then 188 | die "Didn't find a git repository (or no tags found). " \ 189 | "\`\`./autogen.sh\`\` uses git to create changelog and version information." 190 | fi 191 | 192 | 193 | ## 194 | ## CODE 195 | ## 196 | 197 | if [ "$1" = "--get-version" ]; then 198 | get_current_version 199 | exit 0 200 | fi 201 | 202 | if [ "$1" = "--get-name" ]; then 203 | echo "$NAME" 204 | exit 0 205 | fi 206 | 207 | if get_path gitchangelog >/dev/null; then 208 | gitchangelog > CHANGELOG.rst 209 | if [ "$?" != 0 ]; then 210 | echo "Changelog NOT generated. An error occured while running \`\`gitchangelog\`\`." >&2 211 | else 212 | echo "Changelog generated." 213 | fi 214 | else 215 | echo "Changelog NOT generated because \`\`gitchangelog\`\` could not be found." 216 | touch CHANGELOG.rst ## create it anyway because it's required by setup.py current install 217 | fi 218 | 219 | prepare_files 220 | if [ "$?" != 0 ]; then 221 | print_error "Error while updating version information." 222 | fi 223 | -------------------------------------------------------------------------------- /bin/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docshtest_opts=() 4 | 5 | if [ -z "$DOVIS" -o "$PKG_COVERAGE" ]; then ## with coverage 6 | echo "With coverage support enabled" 7 | python="coverage run --include ./shyaml.py -a" 8 | else 9 | echo "No coverage support" 10 | python=python 11 | fi 12 | 13 | docshtest_opts+=("-r" '#\bshyaml\b#'"$python"' ./shyaml.py#') 14 | 15 | $python -m doctest shyaml.py || exit 1 16 | $python -m doctest README.rst || exit 1 17 | 18 | if python -c 'import yaml; exit(0 if yaml.__with_libyaml__ else 1)' 2>/dev/null; then 19 | echo "PyYAML has C libyaml bindings available... Testing with libyaml" 20 | export FORCE_PYTHON_YAML_IMPLEMENTATION= 21 | time docshtest README.rst "${docshtest_opts[@]}" || exit 1 22 | else 23 | echo "PyYAML has NOT any C libyaml bindings available..." 24 | fi 25 | echo "Testing with python implementation" 26 | export FORCE_PYTHON_YAML_IMPLEMENTATION=1 27 | time docshtest README.rst "${docshtest_opts[@]}" || exit 1 28 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = %%name%% 3 | version = %%version%% 4 | summary = %%description%% 5 | description-file = 6 | README.rst 7 | CHANGELOG.rst 8 | license_file = LICENSE 9 | requires-dist = 10 | pyyaml 11 | 12 | ## sdist info 13 | author = %%author%% 14 | author_email = %%email%% 15 | home_page = http://github.com/0k/%%name%% 16 | license = BSD 3-Clause License 17 | classifier = 18 | Programming Language :: Python 19 | Environment :: Console 20 | Intended Audience :: Developers 21 | License :: OSI Approved :: BSD License 22 | Topic :: Software Development 23 | Topic :: Software Development :: Libraries :: Python Modules 24 | Development Status :: 5 - Production/Stable 25 | Programming Language :: Python :: 2 26 | Programming Language :: Python :: 2.7 27 | Programming Language :: Python :: 3 28 | Programming Language :: Python :: 3.3 29 | Programming Language :: Python :: 3.4 30 | Programming Language :: Python :: 3.5 31 | Programming Language :: Python :: 3.6 32 | Programming Language :: Python :: 3.7 33 | 34 | 35 | [backwards_compat] 36 | ## without this ``pip uninstall`` fails on recent version of setuptools 37 | ## (tested failing with setuptools 34.3.3, working with setuptools 9.1) 38 | zip-safe = False 39 | 40 | 41 | [bdist_wheel] 42 | universal = 1 43 | 44 | 45 | [files] 46 | extra_files = 47 | README.rst 48 | CHANGELOG.rst 49 | setup.py 50 | ## Note: d2to1 maps ``setup.py``'s ``py_modules`` to ``modules`` here. 51 | ## This is required for single-file module, this allows the ``entry_point`` 52 | ## to reference ``shyaml``. Besides, this is needed also for direct python 53 | ## API usage. 54 | modules = 55 | shyaml 56 | 57 | ## We can't use scripts to share these simply as extension managed ``.py`` 58 | ## is not correctly handled for both windows and linux to be happy. 59 | # scripts = 60 | # shyaml 61 | 62 | [entry_points] 63 | console_scripts = 64 | ## will generate correct files with adequate extenstion in windows 65 | ## and linux, contrary to 'scripts'. Note that it requires 'shyaml' 66 | ## to be declared as a module also. 67 | shyaml = shyaml:entrypoint 68 | 69 | 70 | [flake8] 71 | ignore = E265,W391,E262,E126,E127,E266 72 | max-line-length = 80 73 | max-complexity = 10 74 | 75 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ## 4 | ## You can download latest version of this file: 5 | ## $ wget https://gist.github.com/vaab/e0eae9607ae806b662d4/raw -O setup.py 6 | ## $ chmod +x setup.py 7 | ## 8 | ## This setup.py is meant to be run along with ``./autogen.sh`` that 9 | ## you can also find here: https://gist.github.com/vaab/9118087/raw 10 | ## 11 | 12 | from setuptools import setup 13 | 14 | ## 15 | ## Ensure that ``./autogen.sh`` is run prior to using ``setup.py`` 16 | ## 17 | 18 | if "%%short-version%%".startswith("%%"): 19 | import os.path 20 | import sys 21 | WIN32 = sys.platform == 'win32' 22 | autogen = os.path.join(".", "autogen.sh") 23 | if not os.path.exists(autogen): 24 | sys.stderr.write( 25 | "This source repository was not configured.\n" 26 | "Please ensure ``./autogen.sh`` exists and that you are running " 27 | "``setup.py`` from the project root directory.\n") 28 | sys.exit(1) 29 | if os.path.exists('.autogen.sh.output'): 30 | sys.stderr.write( 31 | "It seems that ``./autogen.sh`` couldn't do its job as expected.\n" 32 | "Please try to launch ``./autogen.sh`` manualy, and send the " 33 | "results to the\nmaintainer of this package.\n" 34 | "Package will not be installed !\n") 35 | sys.exit(1) 36 | sys.stderr.write("Missing version information: " 37 | "running './autogen.sh'...\n") 38 | import os 39 | import subprocess 40 | os.system('%s%s > .autogen.sh.output' 41 | % ("bash " if WIN32 else "", 42 | autogen)) 43 | cmdline = sys.argv[:] 44 | if cmdline[0] == "-c": 45 | ## for some reason, this is needed when launched from pip 46 | cmdline[0] = "setup.py" 47 | errlvl = subprocess.call(["python", ] + cmdline) 48 | os.unlink(".autogen.sh.output") 49 | sys.exit(errlvl) 50 | 51 | 52 | ## 53 | ## Normal d2to1 setup 54 | ## 55 | 56 | setup( 57 | setup_requires=['d2to1'], 58 | extras_require={'test': [ 59 | "docshtest==0.0.3", 60 | ]}, 61 | d2to1=True 62 | ) 63 | -------------------------------------------------------------------------------- /shyaml: -------------------------------------------------------------------------------- 1 | shyaml.py -------------------------------------------------------------------------------- /shyaml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | YAML for command line. 4 | """ 5 | 6 | ## Note: to launch test, you can use: 7 | ## python -m doctest -d shyaml.py 8 | ## or 9 | ## nosetests 10 | 11 | from __future__ import print_function 12 | 13 | import sys 14 | import os.path 15 | import re 16 | import textwrap 17 | import locale 18 | 19 | import yaml 20 | 21 | 22 | __version__ = "%%version%%" ## gets filled at release time by ./autogen.sh 23 | 24 | 25 | __with_libyaml__ = False 26 | if not os.environ.get("FORCE_PYTHON_YAML_IMPLEMENTATION"): 27 | try: 28 | from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper 29 | __with_libyaml__ = True 30 | except ImportError: ## pragma: no cover 31 | pass 32 | 33 | if not __with_libyaml__: 34 | from yaml import SafeLoader, SafeDumper ## noqa: F811 35 | __with_libyaml__ = False 36 | 37 | 38 | PY3 = sys.version_info[0] >= 3 39 | WIN32 = sys.platform == 'win32' 40 | 41 | EXNAME = os.path.basename(__file__ if WIN32 else sys.argv[0]) 42 | 43 | for ext in (".py", ".pyc", ".exe", "-script.py", "-script.pyc"): ## pragma: no cover 44 | if EXNAME.endswith(ext): ## pragma: no cover 45 | EXNAME = EXNAME[:-len(ext)] 46 | break 47 | 48 | USAGE = """\ 49 | Usage: 50 | 51 | %(exname)s {-h|--help} 52 | %(exname)s {-V|--version} 53 | %(exname)s [-y|--yaml] ACTION KEY [DEFAULT] 54 | """ % {"exname": EXNAME} 55 | 56 | HELP = """ 57 | Parses and output chosen subpart or values from YAML input. 58 | It reads YAML in stdin and will output on stdout it's return value. 59 | 60 | %(usage)s 61 | 62 | Options: 63 | 64 | -y, --yaml 65 | Output only YAML safe value, more precisely, even 66 | literal values will be YAML quoted. This behavior 67 | is required if you want to output YAML subparts and 68 | further process it. If you know you have are dealing 69 | with safe literal value, then you don't need this. 70 | (Default: no safe YAML output) 71 | 72 | ACTION Depending on the type of data you've targetted 73 | thanks to the KEY, ACTION can be: 74 | 75 | These ACTIONs applies to any YAML type: 76 | 77 | get-type ## returns a short string 78 | get-value ## returns YAML 79 | 80 | These ACTIONs applies to 'sequence' and 'struct' YAML type: 81 | 82 | get-values{,-0} ## returns list of YAML 83 | get-length ## returns an integer 84 | 85 | These ACTION applies to 'struct' YAML type: 86 | 87 | keys{,-0} ## returns list of YAML 88 | values{,-0} ## returns list of YAML 89 | key-values,{,-0} ## returns list of YAML 90 | 91 | Note that any value returned is returned on stdout, and 92 | when returning ``list of YAML``, it'll be separated by 93 | a newline or ``NUL`` char depending of you've used the 94 | ``-0`` suffixed ACTION. 95 | 96 | KEY Identifier to browse and target subvalues into YAML 97 | structure. Use ``.`` to parse a subvalue. If you need 98 | to use a literal ``.`` or ``\\``, use ``\\`` to quote it. 99 | 100 | Use struct keyword to browse ``struct`` YAML data and use 101 | integers to browse ``sequence`` YAML data. 102 | 103 | DEFAULT if not provided and given KEY do not match any value in 104 | the provided YAML, then DEFAULT will be returned. If no 105 | default is provided and the KEY do not match any value 106 | in the provided YAML, %(exname)s will fail with an error 107 | message. 108 | 109 | Examples: 110 | 111 | ## get last grocery 112 | cat recipe.yaml | %(exname)s get-value groceries.-1 113 | 114 | ## get all words of my french dictionary 115 | cat dictionaries.yaml | %(exname)s keys-0 french.dictionary 116 | 117 | ## get YAML config part of 'myhost' 118 | cat hosts_config.yaml | %(exname)s get-value cfgs.myhost 119 | 120 | """ % {"exname": EXNAME, "usage": USAGE} 121 | 122 | 123 | class ShyamlSafeLoader(SafeLoader): 124 | """Shyaml specific safe loader""" 125 | 126 | 127 | class ShyamlSafeDumper(SafeDumper): 128 | """Shyaml specific safe dumper""" 129 | 130 | 131 | ## Ugly way to force both the Cython code and the normal code 132 | ## to get the output line by line. 133 | class ForcedLineStream(object): 134 | 135 | def __init__(self, fileobj): 136 | self._file = fileobj 137 | 138 | def read(self, size=-1): 139 | ## don't care about size 140 | return self._file.readline() 141 | 142 | def close(self): 143 | ## XXXvlab: for some reason, ``.close(..)`` doesn't seem to 144 | ## be used by any code. I'll keep this to avoid any bad surprise. 145 | return self._file.close() ## pragma: no cover 146 | 147 | 148 | class LineLoader(ShyamlSafeLoader): 149 | """Forcing stream in line buffer mode""" 150 | 151 | def __init__(self, stream): 152 | stream = ForcedLineStream(stream) 153 | super(LineLoader, self).__init__(stream) 154 | 155 | 156 | ## 157 | ## Keep previous order in YAML 158 | ## 159 | 160 | try: 161 | ## included in standard lib from Python 2.7 162 | from collections import OrderedDict 163 | except ImportError: ## pragma: no cover 164 | ## try importing the backported drop-in replacement 165 | ## it's available on PyPI 166 | from ordereddict import OrderedDict 167 | 168 | 169 | ## Ensure that there are no collision with legacy OrderedDict 170 | ## that could be used for omap for instance. 171 | class MyOrderedDict(OrderedDict): 172 | pass 173 | 174 | 175 | ShyamlSafeDumper.add_representer( 176 | MyOrderedDict, 177 | lambda cls, data: cls.represent_dict(data.items())) 178 | 179 | 180 | def construct_omap(cls, node): 181 | ## Force unfolding reference and merges 182 | ## otherwise it would fail on 'merge' 183 | cls.flatten_mapping(node) 184 | return MyOrderedDict(cls.construct_pairs(node)) 185 | 186 | 187 | ShyamlSafeLoader.add_constructor( 188 | yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, 189 | construct_omap) 190 | 191 | 192 | ## 193 | ## Support local and global objects 194 | ## 195 | 196 | class EncapsulatedNode(object): 197 | """Holds a yaml node""" 198 | 199 | 200 | def mk_encapsulated_node(s, node): 201 | 202 | method = "construct_%s" % (node.id, ) 203 | data = getattr(s, method)(node) 204 | 205 | class _E(data.__class__, EncapsulatedNode): 206 | pass 207 | 208 | _E.__name__ = str(node.tag) 209 | _E._node = node 210 | return _E(data) 211 | 212 | 213 | def represent_encapsulated_node(s, o): 214 | value = s.represent_data(o.__class__.__bases__[0](o)) 215 | value.tag = o.__class__.__name__ 216 | return value 217 | 218 | 219 | ShyamlSafeDumper.add_multi_representer(EncapsulatedNode, 220 | represent_encapsulated_node) 221 | ShyamlSafeLoader.add_constructor(None, mk_encapsulated_node) 222 | 223 | 224 | ## 225 | ## Key specifier 226 | ## 227 | 228 | def tokenize(s): 229 | r"""Returns an iterable through all subparts of string splitted by '.' 230 | 231 | So: 232 | 233 | >>> list(tokenize('foo.bar.wiz')) 234 | ['foo', 'bar', 'wiz'] 235 | 236 | Contrary to traditional ``.split()`` method, this function has to 237 | deal with any type of data in the string. So it actually 238 | interprets the string. Characters with meaning are '.' and '\'. 239 | Both of these can be included in a token by quoting them with '\'. 240 | 241 | So dot of slashes can be contained in token: 242 | 243 | >>> print('\n'.join(tokenize(r'foo.dot<\.>.slash<\\>'))) 244 | foo 245 | dot<.> 246 | slash<\> 247 | 248 | Notice that empty keys are also supported: 249 | 250 | >>> list(tokenize(r'foo..bar')) 251 | ['foo', '', 'bar'] 252 | 253 | Given an empty string: 254 | 255 | >>> list(tokenize(r'')) 256 | [''] 257 | 258 | And a None value: 259 | 260 | >>> list(tokenize(None)) 261 | [] 262 | 263 | """ 264 | if s is None: 265 | return 266 | tokens = (re.sub(r'\\(\\|\.)', r'\1', m.group(0)) 267 | for m in re.finditer(r'((\\.|[^.\\])*)', s)) 268 | ## an empty string superfluous token is added after all non-empty token 269 | for token in tokens: 270 | if len(token) != 0: 271 | next(tokens) 272 | yield token 273 | 274 | 275 | def mget(dct, key): 276 | r"""Allow to get values deep in recursive dict with doted keys 277 | 278 | Accessing leaf values is quite straightforward: 279 | 280 | >>> dct = {'a': {'x': 1, 'b': {'c': 2}}} 281 | >>> mget(dct, 'a.x') 282 | 1 283 | >>> mget(dct, 'a.b.c') 284 | 2 285 | 286 | But you can also get subdict if your key is not targeting a 287 | leaf value: 288 | 289 | >>> mget(dct, 'a.b') 290 | {'c': 2} 291 | 292 | As a special feature, list access is also supported by providing a 293 | (possibily signed) integer, it'll be interpreted as usual python 294 | sequence access using bracket notation: 295 | 296 | >>> mget({'a': {'x': [1, 5], 'b': {'c': 2}}}, 'a.x.-1') 297 | 5 298 | >>> mget({'a': {'x': 1, 'b': [{'c': 2}]}}, 'a.b.0.c') 299 | 2 300 | 301 | Keys that contains '.' can be accessed by escaping them: 302 | 303 | >>> dct = {'a': {'x': 1}, 'a.x': 3, 'a.y': 4} 304 | >>> mget(dct, 'a.x') 305 | 1 306 | >>> mget(dct, r'a\.x') 307 | 3 308 | >>> mget(dct, r'a.y') ## doctest: +IGNORE_EXCEPTION_DETAIL 309 | Traceback (most recent call last): 310 | ... 311 | MissingKeyError: missing key 'y' in dict. 312 | >>> mget(dct, r'a\.y') 313 | 4 314 | 315 | As a consequence, if your key contains a '\', you should also escape it: 316 | 317 | >>> dct = {r'a\x': 3, r'a\.x': 4, 'a.x': 5, 'a\\': {'x': 6}} 318 | >>> mget(dct, r'a\\x') 319 | 3 320 | >>> mget(dct, r'a\\\.x') 321 | 4 322 | >>> mget(dct, r'a\\.x') 323 | 6 324 | >>> mget({'a\\': {'b': 1}}, r'a\\.b') 325 | 1 326 | >>> mget({r'a.b\.c': 1}, r'a\.b\\\.c') 327 | 1 328 | 329 | And even empty strings key are supported: 330 | 331 | >>> dct = {r'a': {'': {'y': 3}, 'y': 4}, 'b': {'': {'': 1}}, '': 2} 332 | >>> mget(dct, r'a..y') 333 | 3 334 | >>> mget(dct, r'a.y') 335 | 4 336 | >>> mget(dct, r'') 337 | 2 338 | >>> mget(dct, r'b..') 339 | 1 340 | 341 | It will complain if you are trying to get into a leaf: 342 | 343 | >>> mget({'a': 1}, 'a.y') ## doctest: +IGNORE_EXCEPTION_DETAIL 344 | Traceback (most recent call last): 345 | ... 346 | NonDictLikeTypeError: can't query subvalue 'y' of a leaf... 347 | 348 | if the key is None, the whole dct should be sent back: 349 | 350 | >>> mget({'a': 1}, None) 351 | {'a': 1} 352 | 353 | """ 354 | return aget(dct, tokenize(key)) 355 | 356 | 357 | class MissingKeyError(KeyError): 358 | """Raised when querying a dict-like structure on non-existing keys""" 359 | 360 | def __str__(self): 361 | return self.args[0] 362 | 363 | 364 | class NonDictLikeTypeError(TypeError): 365 | """Raised when attempting to traverse non-dict like structure""" 366 | 367 | 368 | class IndexNotIntegerError(ValueError): 369 | """Raised when attempting to traverse sequence without using an integer""" 370 | 371 | 372 | class IndexOutOfRange(IndexError): 373 | """Raised when attempting to traverse sequence without using an integer""" 374 | 375 | 376 | def aget(dct, key): 377 | r"""Allow to get values deep in a dict with iterable keys 378 | 379 | Accessing leaf values is quite straightforward: 380 | 381 | >>> dct = {'a': {'x': 1, 'b': {'c': 2}}} 382 | >>> aget(dct, ('a', 'x')) 383 | 1 384 | >>> aget(dct, ('a', 'b', 'c')) 385 | 2 386 | 387 | If key is empty, it returns unchanged the ``dct`` value. 388 | 389 | >>> aget({'x': 1}, ()) 390 | {'x': 1} 391 | 392 | """ 393 | key = iter(key) 394 | try: 395 | head = next(key) 396 | except StopIteration: 397 | return dct 398 | 399 | if isinstance(dct, list): 400 | try: 401 | idx = int(head) 402 | except ValueError: 403 | raise IndexNotIntegerError( 404 | "non-integer index %r provided on a list." 405 | % head) 406 | try: 407 | value = dct[idx] 408 | except IndexError: 409 | raise IndexOutOfRange( 410 | "index %d is out of range (%d elements in list)." 411 | % (idx, len(dct))) 412 | else: 413 | try: 414 | value = dct[head] 415 | except KeyError: 416 | ## Replace with a more informative KeyError 417 | raise MissingKeyError( 418 | "missing key %r in dict." 419 | % (head, )) 420 | except Exception: 421 | raise NonDictLikeTypeError( 422 | "can't query subvalue %r of a leaf%s." 423 | % (head, 424 | (" (leaf value is %r)" % dct) 425 | if len(repr(dct)) < 15 else "")) 426 | return aget(value, key) 427 | 428 | 429 | def stderr(msg): 430 | """Convenience function to write short message to stderr.""" 431 | sys.stderr.write(msg) 432 | 433 | 434 | def stdout(value): 435 | """Convenience function to write short message to stdout.""" 436 | sys.stdout.write(value) 437 | 438 | 439 | def die(msg, errlvl=1, prefix="Error: "): 440 | """Convenience function to write short message to stderr and quit.""" 441 | stderr("%s%s\n" % (prefix, msg)) 442 | sys.exit(errlvl) 443 | 444 | 445 | SIMPLE_TYPES = (str if PY3 else basestring, int, float, type(None)) 446 | COMPLEX_TYPES = (list, dict) 447 | if PY3: 448 | STRING_TYPES = (str, ) 449 | else: 450 | STRING_TYPES = (unicode, str) 451 | 452 | ## these are not composite values 453 | ACTION_SUPPORTING_STREAMING=["get-type", "get-length", "get-value"] 454 | 455 | 456 | def magic_dump(value): 457 | """Returns a representation of values directly usable by bash. 458 | 459 | Literal types are printed as-is (avoiding quotes around string for 460 | instance). But complex type are written in a YAML useable format. 461 | 462 | """ 463 | return "%s" % value if isinstance(value, SIMPLE_TYPES) \ 464 | else yaml_dump(value) 465 | 466 | 467 | def yaml_dump(value): 468 | """Returns a representation of values directly usable by bash. 469 | 470 | Literal types are quoted and safe to use as YAML. 471 | 472 | """ 473 | return yaml.dump(value, default_flow_style=False, 474 | Dumper=ShyamlSafeDumper) 475 | 476 | 477 | def type_name(value): 478 | """Returns pseudo-YAML type name of given value.""" 479 | return type(value).__name__ if isinstance(value, EncapsulatedNode) else \ 480 | "struct" if isinstance(value, dict) else \ 481 | "sequence" if isinstance(value, (tuple, list)) else \ 482 | "str" if isinstance(value, STRING_TYPES) else \ 483 | type(value).__name__ 484 | 485 | 486 | def get_version_info(): 487 | if yaml.__with_libyaml__: 488 | import _yaml 489 | libyaml_version = _yaml.get_version_string() 490 | else: 491 | libyaml_version = False 492 | return ("unreleased" if __version__.startswith('%%') else __version__, 493 | yaml.__version__, 494 | libyaml_version, 495 | __with_libyaml__, 496 | sys.version.replace("\n", " "), 497 | ) 498 | 499 | 500 | def _parse_args(args, USAGE, HELP): 501 | opts = {} 502 | 503 | opts["dump"] = magic_dump 504 | for arg in ["-y", "--yaml"]: 505 | if arg in args: 506 | args.remove(arg) 507 | opts["dump"] = yaml_dump 508 | 509 | opts["quiet"] = False 510 | for arg in ["-q", "--quiet"]: 511 | if arg in args: 512 | args.remove(arg) 513 | opts["quiet"] = True 514 | 515 | for arg in ["-L", "--line-buffer"]: 516 | if arg not in args: 517 | continue 518 | args.remove(arg) 519 | 520 | opts["loader"] = LineLoader 521 | 522 | if len(args) == 0: 523 | stderr("Error: Bad number of arguments.\n") 524 | die(USAGE, errlvl=1, prefix="") 525 | 526 | if len(args) == 1 and args[0] in ("-h", "--help"): 527 | stdout(HELP) 528 | exit(0) 529 | 530 | if len(args) == 1 and args[0] in ("-V", "--version"): 531 | version_info = get_version_info() 532 | print("version: %s\nPyYAML: %s\nlibyaml available: %s\nlibyaml used: %s\nPython: %s" 533 | % version_info) 534 | exit(0) 535 | 536 | opts["action"] = args[0] 537 | opts["key"] = None if len(args) == 1 else args[1] 538 | opts["default"] = args[2] if len(args) > 2 else None 539 | 540 | return opts 541 | 542 | 543 | class InvalidPath(KeyError): 544 | """Invalid Path""" 545 | 546 | def __str__(self): 547 | return self.args[0] 548 | 549 | 550 | class InvalidAction(KeyError): 551 | """Invalid Action""" 552 | 553 | 554 | def traverse(contents, path, default=None): 555 | try: 556 | try: 557 | value = mget(contents, path) 558 | except (IndexOutOfRange, MissingKeyError): 559 | if default is None: 560 | raise 561 | value = default 562 | except (IndexOutOfRange, MissingKeyError, 563 | NonDictLikeTypeError, IndexNotIntegerError) as exc: 564 | msg = str(exc) 565 | raise InvalidPath( 566 | "invalid path %r, %s" 567 | % (path, msg.replace('list', 'sequence').replace('dict', 'struct'))) 568 | return value 569 | 570 | 571 | class ActionTypeError(Exception): 572 | 573 | def __init__(self, action, provided, expected): 574 | self.action = action 575 | self.provided = provided 576 | self.expected = expected 577 | 578 | def __str__(self): 579 | return ("%s does not support %r type. " 580 | "Please provide or select a %s." 581 | % (self.action, self.provided, 582 | self.expected[0] if len(self.expected) == 1 else 583 | ("%s or %s" % (", ".join(self.expected[:-1]), 584 | self.expected[-1])))) 585 | 586 | 587 | def act(action, value, dump=yaml_dump): 588 | tvalue = type_name(value) 589 | ## Note: ``\n`` will be transformed by ``universal_newlines`` mecanism for 590 | ## any platform 591 | termination = "\0" if action.endswith("-0") else "\n" 592 | 593 | if action == "get-value": 594 | return "%s" % dump(value) 595 | elif action in ("get-values", "get-values-0"): 596 | if isinstance(value, dict): 597 | return "".join("".join((dump(k), termination, 598 | dump(v), termination)) 599 | for k, v in value.items()) 600 | elif isinstance(value, list): 601 | return "".join("".join((dump(l), termination)) 602 | for l in value) 603 | else: 604 | raise ActionTypeError( 605 | action, provided=tvalue, expected=["sequence", "struct"]) 606 | elif action == "get-type": 607 | return tvalue 608 | elif action == "get-length": 609 | if isinstance(value, (dict, list)): 610 | return len(value) 611 | else: 612 | raise ActionTypeError( 613 | action, provided=tvalue, expected=["sequence", "struct"]) 614 | elif action in ("keys", "keys-0", 615 | "values", "values-0", 616 | "key-values", "key-values-0"): 617 | if isinstance(value, dict): 618 | method = value.keys if action.startswith("keys") else \ 619 | value.items if action.startswith("key-values") else \ 620 | value.values 621 | output = (lambda x: termination.join("%s" % dump(e) for e in x)) \ 622 | if action.startswith("key-values") else \ 623 | dump 624 | return "".join("".join((str(output(k)), termination)) for k in method()) 625 | else: 626 | raise ActionTypeError( 627 | action=action, provided=tvalue, expected=["struct"]) 628 | else: 629 | raise InvalidAction(action) 630 | 631 | 632 | def do(stream, action, key, default=None, dump=yaml_dump, 633 | loader=ShyamlSafeLoader): 634 | """Return string representations of target value in stream YAML 635 | 636 | The key is used for traversal of the YAML structure to target 637 | the value that will be dumped. 638 | 639 | :param stream: file like input yaml content 640 | :param action: string identifying one of the possible supported actions 641 | :param key: string dotted expression to traverse yaml input 642 | :param default: optional default value in case of missing end value when 643 | traversing input yaml. (default is ``None``) 644 | :param dump: callable that will be given python objet to dump in yaml 645 | (default is ``yaml_dump``) 646 | :param loader: PyYAML's *Loader subclass to parse YAML 647 | (default is ShyamlSafeLoader) 648 | :return: generator of string representation of target value per 649 | YAML docs in the given stream. 650 | 651 | :raises ActionTypeError: when there's a type mismatch between the 652 | action selected and the type of the targetted value. 653 | (ie: action 'key-values' on non-struct) 654 | :raises InvalidAction: when selected action is not a recognised valid 655 | action identifier. 656 | :raises InvalidPath: upon inexistent content when traversing YAML 657 | input following the key specification. 658 | 659 | """ 660 | at_least_one_content = False 661 | for content in yaml.load_all(stream, Loader=loader): 662 | at_least_one_content = True 663 | value = traverse(content, key, default=default) 664 | yield act(action, value, dump=dump) 665 | 666 | ## In case of empty stream, we consider that it is equivalent 667 | ## to one document having the ``null`` value. 668 | if at_least_one_content is False: 669 | value = traverse(None, key, default=default) 670 | yield act(action, value, dump=dump) 671 | 672 | 673 | def main(args): ## pylint: disable=too-many-branches 674 | """Entrypoint of the whole commandline application""" 675 | 676 | EXNAME = os.path.basename(__file__ if WIN32 else sys.argv[0]) 677 | 678 | for ext in (".py", ".pyc", ".exe", "-script.py", "-script.pyc"): ## pragma: no cover 679 | if EXNAME.endswith(ext): ## pragma: no cover 680 | EXNAME = EXNAME[:-len(ext)] 681 | break 682 | 683 | USAGE = """\ 684 | Usage: 685 | 686 | %(exname)s {-h|--help} 687 | %(exname)s {-V|--version} 688 | %(exname)s [-y|--yaml] [-q|--quiet] ACTION KEY [DEFAULT] 689 | """ % {"exname": EXNAME} 690 | 691 | HELP = """ 692 | Parses and output chosen subpart or values from YAML input. 693 | It reads YAML in stdin and will output on stdout it's return value. 694 | 695 | %(usage)s 696 | 697 | Options: 698 | 699 | -y, --yaml 700 | Output only YAML safe value, more precisely, even 701 | literal values will be YAML quoted. This behavior 702 | is required if you want to output YAML subparts and 703 | further process it. If you know you have are dealing 704 | with safe literal value, then you don't need this. 705 | (Default: no safe YAML output) 706 | 707 | -q, --quiet 708 | In case KEY value queried is an invalid path, quiet 709 | mode will prevent the writing of an error message on 710 | standard error. 711 | (Default: no quiet mode) 712 | 713 | -L, --line-buffer 714 | Force parsing stdin line by line allowing to process 715 | streamed YAML as it is fed instead of buffering 716 | input and treating several YAML streamed document 717 | at once. This is likely to have some small performance 718 | hit if you have a huge stream of YAML document, but 719 | then you probably don't really care about the 720 | line-buffering. 721 | (Default: no line buffering) 722 | 723 | ACTION Depending on the type of data you've targetted 724 | thanks to the KEY, ACTION can be: 725 | 726 | These ACTIONs applies to any YAML type: 727 | 728 | get-type ## returns a short string 729 | get-value ## returns YAML 730 | 731 | These ACTIONs applies to 'sequence' and 'struct' YAML type: 732 | 733 | get-values{,-0} ## returns list of YAML 734 | get-length ## returns an integer 735 | 736 | These ACTION applies to 'struct' YAML type: 737 | 738 | keys{,-0} ## returns list of YAML 739 | values{,-0} ## returns list of YAML 740 | key-values,{,-0} ## returns list of YAML 741 | 742 | Note that any value returned is returned on stdout, and 743 | when returning ``list of YAML``, it'll be separated by 744 | a newline or ``NUL`` char depending of you've used the 745 | ``-0`` suffixed ACTION. 746 | 747 | KEY Identifier to browse and target subvalues into YAML 748 | structure. Use ``.`` to parse a subvalue. If you need 749 | to use a literal ``.`` or ``\\``, use ``\\`` to quote it. 750 | 751 | Use struct keyword to browse ``struct`` YAML data and use 752 | integers to browse ``sequence`` YAML data. 753 | 754 | DEFAULT if not provided and given KEY do not match any value in 755 | the provided YAML, then DEFAULT will be returned. If no 756 | default is provided and the KEY do not match any value 757 | in the provided YAML, %(exname)s will fail with an error 758 | message. 759 | 760 | Examples: 761 | 762 | ## get last grocery 763 | cat recipe.yaml | %(exname)s get-value groceries.-1 764 | 765 | ## get all words of my french dictionary 766 | cat dictionaries.yaml | %(exname)s keys-0 french.dictionary 767 | 768 | ## get YAML config part of 'myhost' 769 | cat hosts_config.yaml | %(exname)s get-value cfgs.myhost 770 | 771 | """ % {"exname": EXNAME, "usage": USAGE} 772 | 773 | USAGE = textwrap.dedent(USAGE) 774 | HELP = textwrap.dedent(HELP) 775 | 776 | opts = _parse_args(args, USAGE, HELP) 777 | quiet = opts.pop("quiet") 778 | 779 | try: 780 | first = True 781 | for output in do(stream=sys.stdin, **opts): 782 | if first: 783 | first = False 784 | else: 785 | if opts["action"] not in ACTION_SUPPORTING_STREAMING: 786 | die("Source YAML is multi-document, " 787 | "which doesn't support any other action than %s" 788 | % ", ".join(ACTION_SUPPORTING_STREAMING)) 789 | if opts["dump"] is yaml_dump: 790 | print("---\n", end="") 791 | else: 792 | print("\0", end="") 793 | if opts.get("loader") is LineLoader: 794 | sys.stdout.flush() 795 | 796 | safe_print(output) 797 | if opts.get("loader") is LineLoader: 798 | sys.stdout.flush() 799 | except (InvalidPath, ActionTypeError) as e: 800 | if quiet: 801 | exit(1) 802 | else: 803 | die(str(e)) 804 | except InvalidAction as e: 805 | die("'%s' is not a valid action.\n%s" 806 | % (e.args[0], USAGE)) 807 | ## 808 | ## Safe print 809 | ## 810 | 811 | ## Note that locale.getpreferredencoding() does NOT follow 812 | ## PYTHONIOENCODING by default, but ``sys.stdout.encoding`` does. In 813 | ## PY2, ``sys.stdout.encoding`` without PYTHONIOENCODING set does not 814 | ## get any values set in subshells. However, if _preferred_encoding 815 | ## is not set to utf-8, it leads to encoding errors. 816 | _preferred_encoding = os.environ.get("PYTHONIOENCODING") or \ 817 | locale.getpreferredencoding() 818 | 819 | def safe_print(content): 820 | if not PY3: 821 | if isinstance(content, unicode): 822 | content = content.encode(_preferred_encoding) 823 | 824 | print(content, end='') 825 | sys.stdout.flush() 826 | 827 | 828 | 829 | 830 | def entrypoint(): 831 | sys.exit(main(sys.argv[1:])) 832 | 833 | 834 | if __name__ == "__main__": 835 | entrypoint() 836 | --------------------------------------------------------------------------------