├── .gitignore ├── hosts ├── yaml2toml_example.j2 ├── tests ├── hard_example.yaml ├── example.yaml └── influxdb.yaml ├── LICENSE.md ├── yaml2toml.py ├── README.md ├── yaml2toml_macro.j2 └── yaml2toml_playbook.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | *.toml 2 | -------------------------------------------------------------------------------- /hosts: -------------------------------------------------------------------------------- 1 | # Ansible requires Python version 2 2 | localhost ansible_python_interpreter="/usr/bin/env python2" 3 | -------------------------------------------------------------------------------- /yaml2toml_example.j2: -------------------------------------------------------------------------------- 1 | # 2 | # This file is managed by Ansible. 3 | # Do not edit this file manually. 4 | # Any changes will be automatically reverted. 5 | # 6 | 7 | {% from 'yaml2toml_macro.j2' import yaml2toml with context -%} 8 | 9 | {{ yaml2toml(yaml_data) }} 10 | -------------------------------------------------------------------------------- /tests/hard_example.yaml: -------------------------------------------------------------------------------- 1 | the: 2 | hard: 3 | another_test_string: ' Same thing, but with a string #' 4 | bit#: 5 | multi_line_array: [']'] 6 | what?: You don't think some user won't do that? 7 | harder_test_string: ' And when "''s are in the string, along with # "' 8 | test_array: ['] ', ' # '] 9 | test_array2: ['Test #11 ]proved that', 'Experiment #9 was a success'] 10 | test_string: 'You''ll hate me after this - #' 11 | -------------------------------------------------------------------------------- /tests/example.yaml: -------------------------------------------------------------------------------- 1 | title: TOML Example 2 | 3 | owner: 4 | name: Tom Preston-Werner 5 | organization: GitHub 6 | bio: "GitHub Cofounder & CEO\nLikes tater tots and beer." 7 | dob: "1979-05-27T07:32:00Z" 8 | 9 | database: 10 | server: 192.168.1.1 11 | ports: [ 8001, 8001, 8002 ] 12 | connection_max: 5000 13 | enabled: true 14 | 15 | servers: 16 | alpha: 17 | ip: 10.0.0.1 18 | dc: eqdc10 19 | beta: 20 | ip: 10.0.0.2 21 | dc: eqdc10 22 | country: "中国" 23 | 24 | clients: 25 | data: [ ["gamma", "delta"], [1, 2] ] 26 | hosts: [ "alpha", "omega" ] 27 | 28 | products: 29 | - 30 | name: Hammer 31 | sku: 738594937 32 | - 33 | name: Nail 34 | sku: 284758393 35 | color: gray 36 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Jiri Tyr 2014 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /tests/influxdb.yaml: -------------------------------------------------------------------------------- 1 | bind-address: 0.0.0.0 2 | reporting-disabled: 'true' 3 | logging: 4 | level: info 5 | file: /opt/influxdb/shared/log.txt 6 | admin: 7 | port: 8083 8 | api: 9 | port: 8086 10 | read-timeout: 5s 11 | input_plugins: 12 | graphite: 13 | enabled: 'false' 14 | collectd: 15 | enabled: 'false' 16 | udp: 17 | enabled: 'false' 18 | udp_servers: 19 | - enabled: 'false' 20 | raft: 21 | port: 8090 22 | dir: /opt/influxdb/shared/data/raft 23 | debug: 'false' 24 | storage: 25 | dir: /opt/influxdb/shared/data/db 26 | write-buffer-size: 10000 27 | default-engine: rocksdb 28 | max-open-shards: 0 29 | point-batch-size: 100 30 | write-batch-size: 5000000 31 | retention-sweep-period: 10m 32 | engines: 33 | leveldb: 34 | max-open-files: 1000 35 | lru-cache-size: 200m 36 | rocksdb: 37 | max-open-files: 1000 38 | lru-cache-size: 200m 39 | hyperleveldb: 40 | max-open-files: 1000 41 | lru-cache-size: 200m 42 | lmdb: 43 | map-size: 100g 44 | cluster: 45 | protobuf_port: 8099 46 | protobuf_timeout: 2s 47 | protobuf_heartbeat: 200ms 48 | protobuf_min_backoff: 1s 49 | protobuf_max_backoff: 10s 50 | write-buffer-size: 1000 51 | max-response-buffer-size: 100 52 | concurrent-shard-query-limit: 10 53 | wal: 54 | dir: /opt/influxdb/shared/data/wal 55 | flush-after: 1000 56 | bookmark-after: 1000 57 | index-after: 1000 58 | requests-per-logfile: 10000 59 | -------------------------------------------------------------------------------- /yaml2toml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | from jinja2 import Template 4 | from yaml import load 5 | import argparse 6 | import sys 7 | 8 | 9 | def parse_arguments(): 10 | description = 'YAML to TOML converter' 11 | 12 | parser = argparse.ArgumentParser(description=description) 13 | parser.add_argument( 14 | '--template', '-t', 15 | metavar='FILE', 16 | dest='template_fh', 17 | type=argparse.FileType('r'), 18 | default='./yaml2toml_macro.j2', 19 | help='jinja2 template (default: ./yaml2toml_macro.j2)') 20 | parser.add_argument( 21 | '--yaml', '-y', 22 | metavar='FILE', 23 | dest='yaml_fh', 24 | type=argparse.FileType('r'), 25 | default='-', 26 | help='YAML file to convert (default: -)') 27 | 28 | return (parser.parse_args(), parser) 29 | 30 | 31 | def main(): 32 | # Parse command line arguments 33 | (args, parser) = parse_arguments() 34 | 35 | # Read the content of the template file 36 | template_text = args.template_fh.read() 37 | args.template_fh.close() 38 | 39 | # Read the YAML file 40 | yaml_data = load(args.yaml_fh) 41 | args.yaml_fh.close() 42 | 43 | # Create jinja template object 44 | t = Template(template_text) 45 | # Convert the YAML data to TOML 46 | toml_output = t.module.yaml2toml(yaml_data).encode('utf8') 47 | 48 | # Print the result 49 | sys.stdout.write(toml_output) 50 | 51 | 52 | if __name__ == '__main__': 53 | main() 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | YAML to TOML converter 2 | ====================== 3 | 4 | This is a tool which helps to convert YAML data to TOML format using Jinja2 5 | template. 6 | 7 | 8 | Motivation 9 | ---------- 10 | 11 | [YAML](http://www.yaml.org/) and [TOML](https://github.com/toml-lang/toml) are 12 | languages used to describe a data structures often used in configuration files. 13 | TOML is relatively young language which is trying minimalize the configuration 14 | file format. 15 | 16 | Sometimes it's necessary to use YAML as the description of the configuration file 17 | and then it's necessary to do the conversion. For example in 18 | [Ansible](http://ansible.com), everything is defied as YAML and if a program 19 | managed by Ansible (e.g. [InfluxDB](http://influxdb.com)) needs to have config 20 | file in TOML format, it's necessay to convert the YAML data to TOML format via 21 | template. 22 | 23 | 24 | Coversion 25 | --------- 26 | 27 | A YAML file can be converted as follows: 28 | 29 | ``` 30 | $ ./yaml2toml.py -y ./tests/example.yaml 31 | ``` 32 | 33 | or 34 | 35 | ``` 36 | $ cat ./tests/example.yaml | ./yaml2toml.py 37 | ``` 38 | 39 | It is also possible to use Ansible playbook to do the conversion: 40 | 41 | ``` 42 | $ ansible-playbook --diff -i hosts ./yaml2toml_playbook.yaml 43 | ``` 44 | 45 | The output of the Ansible playbook conversion is then in the file `output.toml`. 46 | 47 | 48 | Limitation 49 | ---------- 50 | 51 | Jinja2 doesn't provide any filters for `datetime` so this converter fails if you 52 | use `datetime` types in the YAML data. The workaround is to quote the `datetime` 53 | values. 54 | 55 | 56 | Files 57 | ----- 58 | 59 | - `yaml2toml.py` - conversion script written in Python 60 | - `yaml2toml_macro.j2` - Jinja2 template with the macro which does the conversion 61 | from YAML to TOML 62 | - `yaml2toml_example.j2` - Jinja2 template calling the yaml2toml macro 63 | - `yaml2toml.yaml` - Ansible playbook doing the conversion 64 | - `hosts` - inventory file used by Ansible 65 | - `tests/*` - testing files 66 | 67 | 68 | Requirements 69 | ------------ 70 | 71 | - Python v2 72 | - `jinja` Python module 73 | - `yaml` Python module 74 | - Ansible (optional) 75 | 76 | 77 | License 78 | ------- 79 | 80 | MIT 81 | 82 | 83 | Author 84 | ------ 85 | 86 | Jiri Tyr 87 | -------------------------------------------------------------------------------- /yaml2toml_macro.j2: -------------------------------------------------------------------------------- 1 | {# 2 | # Jinja2 template which converts YAML to TOML 3 | # (https://github.com/jtyr/yaml2toml-converter) 4 | #} 5 | 6 | {%- macro yaml2toml(item, prevkey='', level=0, first=[], detect_nums=false) %} 7 | {%- if item is mapping %} 8 | {#- The item is a dict #} 9 | 10 | {#- First process all strings, numbers and lists #} 11 | {%- for key, value in item.iteritems() | sort -%} 12 | {%- if value is string or 13 | value is number or 14 | ( 15 | value is not mapping and 16 | value | length > 0 and 17 | value[0] is not mapping) %} 18 | 19 | {#- The value is a string, a number, or a list -#} 20 | {{ ' ' * level }}{{ key }}{{ " = " }}{{ yaml2toml( 21 | value, 22 | prevkey=prevkey, 23 | level=level, 24 | first=first, 25 | detect_nums=detect_nums) }} 26 | {%- if first.append(1) %}{% endif %} 27 | {%- endif %} 28 | {%- endfor %} 29 | 30 | {#- Then process all data structures #} 31 | {%- for key, value in item.iteritems() | sort %} 32 | {%- if value is not string and 33 | value is not number and 34 | ( 35 | value is mapping or 36 | value[0] is mapping) %} 37 | 38 | {%- set old_prevkey = prevkey %} 39 | {%- set old_level = level %} 40 | 41 | {%- if value is mapping %} 42 | {#- The value is a dict #} 43 | {%- if prevkey != '' and prevkey != key %} 44 | {%- set level = level + 1 %} 45 | {%- endif %} 46 | 47 | {%- set prevkey = (key if prevkey == '' else prevkey + '.' + key) %} 48 | 49 | {%- if first | length > 0 -%} 50 | {{ '\n' }} 51 | {%- endif -%} 52 | 53 | {{ ' ' * level }}[{{ prevkey }}]{{ "\n" }} 54 | {%- elif value[0] is mapping %} 55 | {#- The value is a table #} 56 | {%- set prevkey = (key if prevkey == '' else prevkey + '.' + key) %} 57 | {%- set level = level +1 %} 58 | {%- endif -%} 59 | 60 | {{ yaml2toml( 61 | value, 62 | prevkey=prevkey, 63 | level=level, 64 | first=first, 65 | detect_nums=detect_nums) }} 66 | 67 | {%- set prevkey = old_prevkey %} 68 | {%- set level = old_level %} 69 | {%- if first.append(1) %}{% endif %} 70 | {%- endif %} 71 | {%- endfor %} 72 | 73 | {%- elif item is number or 74 | (detect_nums and 75 | ( 76 | item | int | string == item or 77 | item | float | string == item)) or 78 | item == 'true' or 79 | item == 'True' or 80 | item == 'false' or 81 | item == 'False' %} 82 | 83 | {#- The item is a number or boolean -#} 84 | {{ item }}{{ "\n" }} 85 | 86 | {%- elif item is string %} 87 | {#- The item is a string -#} 88 | "{{ item | replace("\n", "\\n") | replace("\t", "\\t") }}"{{ "\n" }} 89 | 90 | {%- else %} 91 | {#- The item is a list #} 92 | {%- if item[0] is mapping %} 93 | {%- for d in item -%} 94 | {{ "\n" }}{{ ' ' * level }}[[{{ prevkey }}]]{{ "\n" }}{{ yaml2toml( 95 | d, 96 | level=level, 97 | first=first, 98 | detect_nums=detect_nums) }} 99 | {%- endfor %} 100 | {%- else -%} 101 | {{ item }}{{ "\n" }} 102 | {%- endif %} 103 | {%- endif %} 104 | {%- endmacro -%} 105 | -------------------------------------------------------------------------------- /yaml2toml_playbook.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: all 4 | connection: local 5 | gather_facts: no 6 | 7 | vars: 8 | # Example from: 9 | # https://github.com/toml-lang/toml/blob/master/tests/example.yaml 10 | test1: &test1 11 | the: 12 | hard: 13 | another_test_string: ' Same thing, but with a string #' 14 | bit#: 15 | multi_line_array: [']'] 16 | what?: You don't think some user won't do that? 17 | harder_test_string: ' And when "''s are in the string, along with # "' 18 | test_array: ['] ', ' # '] 19 | test_array2: ['Test #11 ]proved that', 'Experiment #9 was a success'] 20 | test_string: 'You''ll hate me after this - #' 21 | 22 | # Example from: 23 | # https://github.com/toml-lang/toml/blob/master/tests/hard_example.yaml 24 | test2: &test2 25 | title: TOML Example 26 | owner: 27 | name: Tom Preston-Werner 28 | organization: GitHub 29 | bio: "GitHub Cofounder & CEO\nLikes tater tots and beer." 30 | dob: '1979-05-27T07:32:00Z' 31 | database: 32 | server: 192.168.1.1 33 | ports: [ 8001, 8001, 8002 ] 34 | connection_max: 5000 35 | enabled: true 36 | servers: 37 | alpha: 38 | ip: 10.0.0.1 39 | dc: eqdc10 40 | beta: 41 | ip: 10.0.0.2 42 | dc: eqdc10 43 | country: "中国" 44 | clients: 45 | data: [ ["gamma", "delta"], [1, 2] ] 46 | hosts: [ "alpha", "omega" ] 47 | products: 48 | - name: Hammer 49 | sku: 738594937 50 | - name: Nail 51 | sku: 284758393 52 | color: gray 53 | 54 | # Example of InfluxDB configuration written in YAML: 55 | # https://github.com/influxdb/influxdb/blob/master/etc/config.sample.toml 56 | test3: &test3 57 | bind-address: 0.0.0.0 58 | reporting-disabled: 'true' 59 | logging: 60 | level: info 61 | file: /opt/influxdb/shared/log.txt 62 | admin: 63 | port: 8083 64 | api: 65 | port: 8086 66 | read-timeout: 5s 67 | input_plugins: 68 | graphite: 69 | enabled: 'false' 70 | collectd: 71 | enabled: 'false' 72 | udp: 73 | enabled: 'false' 74 | udp_servers: 75 | - enabled: 'false' 76 | raft: 77 | port: 8090 78 | dir: /opt/influxdb/shared/data/raft 79 | debug: 'false' 80 | storage: 81 | dir: /opt/influxdb/shared/data/db 82 | write-buffer-size: 10000 83 | default-engine: rocksdb 84 | max-open-shards: 0 85 | point-batch-size: 100 86 | write-batch-size: 5000000 87 | retention-sweep-period: 10m 88 | engines: 89 | leveldb: 90 | max-open-files: 1000 91 | lru-cache-size: 200m 92 | rocksdb: 93 | max-open-files: 1000 94 | lru-cache-size: 200m 95 | hyperleveldb: 96 | max-open-files: 1000 97 | lru-cache-size: 200m 98 | lmdb: 99 | map-size: 100g 100 | cluster: 101 | protobuf_port: 8099 102 | protobuf_timeout: 2s 103 | protobuf_heartbeat: 200ms 104 | protobuf_min_backoff: 1s 105 | protobuf_max_backoff: 10s 106 | write-buffer-size: 1000 107 | max-response-buffer-size: 100 108 | concurrent-shard-query-limit: 10 109 | wal: 110 | dir: /opt/influxdb/shared/data/wal 111 | flush-after: 1000 112 | bookmark-after: 1000 113 | index-after: 1000 114 | requests-per-logfile: 10000 115 | 116 | # Dereference some of the tests 117 | yaml_data: *test1 118 | 119 | tasks: 120 | # Run the conversion 121 | - name: YAML2TOML test 122 | template: 123 | src: yaml2toml_example.j2 124 | dest: ./output.toml 125 | --------------------------------------------------------------------------------