├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── setup.py ├── tests ├── __init__.py ├── example.toml ├── file1.toml ├── file2.toml ├── hard_example.toml └── test.py └── tomlpython ├── __init__.py ├── parser.py └── reader.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *~ 3 | *.swp 4 | *.pyc 5 | *.pyo 6 | 7 | __pycache__/ 8 | build/ 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Felipe Aragão Pires 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | documentation files (the "Software"), to deal in the Software without restriction, including without 5 | limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the 6 | Software, and to permit persons to whom the Software is furnished to do so, subject to the following 7 | conditions: 8 | 9 | The above copyright notice and this permission notice shall be included in all copies or substantial portions 10 | of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 13 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 14 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF 15 | CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 16 | DEALINGS IN THE SOFTWARE. 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Python parser for [TOML](https://github.com/mojombo/toml) 2 | ======================= 3 | 4 | Check out the spec here: [https://github.com/mojombo/toml](https://github.com/mojombo/toml) 5 | ### Feel free to send a pull request. 6 | 7 | ## ToDos and Features 8 | - [x] Allow multiline arrays. 9 | - [x] Disallow variable rewriting. 10 | - [x] Format to JSON. 11 | - [x] Pypi support (see [toml-python](https://pypi.python.org/pypi/toml-python)) 12 | - [x] Build unittests. 13 | - [x] Improve tests (see [toml-test](https://github.com/BurntSushi/toml-test)) 14 | - [ ] Write de-serializer 15 | - [ ] Improve debugging system. 16 | 17 | ## Installation 18 | ```bash 19 | pip install toml-python 20 | ``` 21 | 22 | ## Usage 23 | ### TOML from string 24 | ```python 25 | >>> import tomlpython 26 | >>> tomlpython.parse(""" 27 | [database] 28 | server = "192.168.1.1" 29 | ports = [ 8001, 8001, 8002 ] 30 | """) 31 | {'database': {'ports': [8001, 8001, 8002], 'server': '192.168.1.1'}} 32 | ``` 33 | 34 | ### TOML from file 35 | ```python 36 | >>> import tomlpython 37 | >>> with open('data.toml') as datafile: 38 | >>> data = tomlpython.parse(datafile) 39 | ``` 40 | 41 | ### TOML to JSON (support to prettify as in json.dumps) 42 | ```python 43 | >>> import tomlpython 44 | >>> tomlpython.toJSON(""" 45 | [database] 46 | server = "192.168.1.1" 47 | ports = [ 8001, 8001, 8002 ] 48 | """, indent=4) 49 | { 50 | "database": { 51 | "ports": [ 8001, 8001, 8002 ], 52 | "server": "192.168.1.1" 53 | } 54 | } 55 | ``` 56 | 57 | ### Testing 58 | - Use `tests/test.py` 59 | - See https://github.com/BurntSushi/toml-test 60 | 61 | ## License 62 | MIT 63 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipap/toml-python/62b249552d125d5c0d5760f1cd9bf4ed83051be7/__init__.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf8 -*- 2 | 3 | from setuptools import setup, find_packages 4 | 5 | setup( 6 | name = "toml-python", 7 | author = "Felipe Aragão Pires", 8 | author_email = "pires.a.felipe@gmail.com", 9 | version = "0.4.1", 10 | description = "TOML parser for python.", 11 | url = "https://github.com/f03lipe/toml-python", 12 | license = "MIT License", 13 | classifiers=[ 14 | 'Development Status :: 2 - Pre-Alpha', 15 | 'Intended Audience :: Developers', 16 | 'License :: OSI Approved :: MIT License', 17 | 'Programming Language :: Python :: 3', 18 | ], 19 | packages = find_packages(), 20 | ) 21 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felipap/toml-python/62b249552d125d5c0d5760f1cd9bf4ed83051be7/tests/__init__.py -------------------------------------------------------------------------------- /tests/example.toml: -------------------------------------------------------------------------------- 1 | # This is a TOML document. Boom. 2 | 3 | title = "TOML Example" 4 | 5 | [owner] 6 | name = "Tom Preston-Werner" 7 | organization = "GitHub" 8 | bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." 9 | dob = 1979-05-27T07:32:00Z # First class dates? Why not? 10 | 11 | [database] 12 | server = "192.168.1.1" 13 | ports = [ 8001, 8001, 8002 ] 14 | connection_max = 5000 15 | enabled = true 16 | 17 | [servers] 18 | 19 | # You can indent as you please. Tabs or spaces. TOML don't care. 20 | [servers.alpha] 21 | ip = "10.0.0.1" 22 | dc = "eqdc10" 23 | 24 | [servers.beta] 25 | ip = "10.0.0.2" 26 | dc = "eqdc10" 27 | 28 | [clients] 29 | data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it 30 | 31 | # Line breaks are OK when inside arrays 32 | hosts = [ 33 | "alpha", 34 | "omega" 35 | ] 36 | -------------------------------------------------------------------------------- /tests/file1.toml: -------------------------------------------------------------------------------- 1 | # This is a TOML document. Boom. 2 | 3 | title = "TOML Example" 4 | 5 | [owner] 6 | name = "Tom Preston-Werner" 7 | organization = "GitHub" 8 | bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." 9 | dob = 1979-05-27T07:32:00Z # First class dates? Why not? 10 | 11 | [database] 12 | server = "192.168.1.1" 13 | ports = [ 8001, 8001, 8002 ] 14 | connection_max = 5000 15 | enabled = true 16 | 17 | [servers] 18 | 19 | # You can indent as you please. Tabs or spaces. TOML don't care. 20 | [servers.alpha] 21 | ip = "10.0.0.1" 22 | dc = "eqdc10" 23 | 24 | [servers.beta] 25 | ip = "10.0.0.2" 26 | dc = "eqdc10" 27 | 28 | [clients] 29 | data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it 30 | -------------------------------------------------------------------------------- /tests/file2.toml: -------------------------------------------------------------------------------- 1 | # This is a TOML document. Boom. 2 | 3 | title = "TOML Example" 4 | 5 | [owner] 6 | name = "Tom Preston-Werner" 7 | organization = "GitHub" 8 | bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." 9 | dob = 1979-05-27T07:32:00Z # First class dates? Why not? 10 | 11 | [database] 12 | server = "192.168.1.1" 13 | ports = [ 8001, 8001, 8002 ] 14 | connection_max = 5000 15 | enabled = true 16 | 17 | [servers] 18 | 19 | # You can indent as you please. Tabs or spaces. TOML don't care. 20 | [servers.alpha] 21 | ip = "10.0.0.1" 22 | dc = "eqdc10" 23 | 24 | 25 | [servers.beta] 26 | ip = "10.0.0.2" 27 | dc = "eqdc10" 28 | 29 | [clients] 30 | data = [ ["gamma", "delta"], [1, 2] ] 31 | 32 | # Line breaks are OK when inside arrays 33 | hosts = [ 34 | #sdf 35 | "alpha", 36 | "omega", 37 | ] 38 | 39 | [fruit] 40 | tsype = "apple" 41 | 42 | [fruit.type.b] 43 | sapple = "yes" 44 | 45 | hosts2 = [] 46 | hosts4 = [ 47 | 48 | ] 49 | -------------------------------------------------------------------------------- /tests/hard_example.toml: -------------------------------------------------------------------------------- 1 | # Test file for TOML 2 | # Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate 3 | # This part you'll really hate 4 | 5 | [the] 6 | test_string = "You'll hate me after this - #" # " Annoying, isn't it? 7 | 8 | [the.hard] 9 | test_array = [ "] ", " # "] # ] There you go, parse this! 10 | test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ] 11 | # You didn't think it'd as easy as chucking out the last #, did you? 12 | another_test_string = " Same thing, but with a string #" 13 | harder_test_string = " And when \"'s are in the string, along with # \"" # "and comments are there too" 14 | # Things will get harder 15 | 16 | [the.hard.bit#] 17 | what? = "You don't think some user won't do that?" 18 | multi_line_array = [ 19 | "]", 20 | # ] Oh yes I did 21 | ] 22 | -------------------------------------------------------------------------------- /tests/test.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf8 -*- 2 | 3 | from os.path import abspath, dirname, join 4 | from functools import wraps 5 | from glob import glob 6 | import unittest 7 | import sys 8 | 9 | DIR = dirname(abspath(__file__)) 10 | TOMLFiles = glob(join(DIR, '*.toml')) 11 | 12 | def addSysPath(path): 13 | # Add path to sys.path during executing of routine. 14 | def decorator(func): 15 | @wraps(func) 16 | def wrapper(*args, **kwargs): 17 | sys.path.insert(0, abspath(path)) 18 | val = func(*args, **kwargs) 19 | sys.path = sys.path[1:] 20 | return val 21 | return wrapper 22 | return decorator 23 | 24 | 25 | def parseTOMLfiles(): 26 | # Parse TOML files in current dir. 27 | for filename in TOMLFiles: 28 | with open(filename) as file: 29 | print("Testing file ", filename) 30 | tomlpython.toJSON(file) 31 | tomlpython.parse(file) 32 | 33 | 34 | class Test(unittest.TestCase): 35 | 36 | def setUp(self): 37 | pass 38 | 39 | def test_toml_examples(self): 40 | # Check if Exception is not thrown 41 | parseTOMLfiles() 42 | 43 | 44 | @addSysPath(dirname(DIR)) 45 | def main(): 46 | global tomlpython # little hack? 47 | import tomlpython as tomlpython 48 | unittest.main() 49 | 50 | if __name__ == "__main__": 51 | main() 52 | 53 | 54 | -------------------------------------------------------------------------------- /tomlpython/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | __all__ = ('parse','toJSON') 4 | 5 | from tomlpython.parser import parse, toJSON 6 | -------------------------------------------------------------------------------- /tomlpython/parser.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- encoding: utf8 -*- 3 | 4 | # A TOML parser for Python3 5 | 6 | from datetime import datetime as dt 7 | import re 8 | import sys 9 | 10 | from tomlpython.reader import Reader 11 | from tomlpython.reader import pop, top, skip 12 | from tomlpython.reader import readLine, assertEOL, allownl 13 | 14 | 15 | if sys.version_info[0] == 2: 16 | from tomlpython.reader import custom_next as next 17 | 18 | 19 | class Parser(object): 20 | 21 | def __init__(self, reader, asJson=False, pedantic=True): 22 | self._asJson = asJson 23 | self._is_pedantic = pedantic 24 | self.reader = reader 25 | self.runtime = dict() 26 | self.kgObj = self.runtime 27 | self.mainLoop() 28 | 29 | def loadKeyGroup(self, keygroup): 30 | cg = self.runtime 31 | nlist = keygroup.split('.') 32 | for index, name in enumerate(nlist): 33 | if not name: 34 | raise Exception("Unexpected emtpy symbol in %s" % keygroup) 35 | elif not name in cg: 36 | cg[name] = dict() 37 | elif isinstance(cg[name], dict)\ 38 | and index == len(nlist)-1\ 39 | and self._is_pedantic: 40 | raise Exception("Duplicated keygroup definition: %s" % keygroup) 41 | cg = cg[name] 42 | self.kgObj = cg 43 | 44 | #### 45 | 46 | def parseEXP(self): 47 | # Locals are faster 48 | ISO8601 = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}Z$') 49 | FLOAT = re.compile(r'^[+-]?\d(>?\.\d+)?$') 50 | STRING = re.compile(r'(?:".*?[^\\]?")|(?:\'.*?[^\\]?\')') 51 | 52 | token = next(self.reader) 53 | if token == '[': 54 | # Array 55 | array = [] 56 | skip(self.reader, '[') 57 | while top(self.reader) != ']': 58 | array.append(self.parseEXP()) 59 | if len(array) > 1 and self._is_pedantic\ 60 | and type(array[-1]) != type(array[0]): 61 | raise Exception("Array of mixed data types.") 62 | if next(self.reader) != ',': 63 | break 64 | skip(self.reader, ",") 65 | allownl(self.reader) 66 | skip(self.reader, "]") 67 | return array 68 | elif STRING.match(token): 69 | # String 70 | return pop(self.reader)[1:-1].decode('string-escape') 71 | elif token in ('true', 'false'): 72 | # Boolean 73 | return {'true': True, 'false': False}[pop(self.reader)] 74 | elif token.isdigit() or token[1:].isdigit() and token[0] in ('+', '-'): 75 | # Integer 76 | return int(pop(self.reader)) 77 | elif FLOAT.match(token): 78 | # Float 79 | return float(pop(self.reader)) 80 | elif ISO8601.match(token): 81 | # Date 82 | date = dt.strptime(pop(self.reader), "%Y-%m-%dT%H:%M:%SZ") 83 | return date if not self._asJson else date.isoformat() 84 | raise Exception("Invalid token: %s" % token) 85 | 86 | ####### 87 | 88 | def parseCOMMENT(self): 89 | # Do nothing. 90 | # Wait loop to next line. 91 | pass 92 | 93 | def parseKEYGROUP(self): 94 | symbol = pop(self.reader)[1:-1] 95 | if not symbol or symbol.isspace(): 96 | raise Exception("Empty keygroup found.") 97 | self.loadKeyGroup(symbol) 98 | 99 | def parseASSIGN(self): 100 | # Parse an assignment 101 | # disallow variable rewriting 102 | var = pop(self.reader) # symbol 103 | pop(self.reader, expect='=') 104 | val = self.parseEXP() 105 | if self.kgObj.get(var): 106 | # Disallow variable rewriting. 107 | raise Exception("Cannot rewrite variable: %s" % var) 108 | self.kgObj[var] = val 109 | 110 | ####### 111 | 112 | def mainLoop(self): 113 | # Due to the 0-lookahead and non-recursive nature of the markup, 114 | # all expressions can be identified by the first meaninful 115 | # (non-whitespace) character of the line. 116 | 117 | while readLine(self.reader): 118 | token = next(self.reader) 119 | if token == "#": 120 | self.parseCOMMENT() 121 | elif token[0] == "[": 122 | self.parseKEYGROUP() 123 | elif re.match(r'[^\W\d_]', token, re.U): 124 | self.parseASSIGN() 125 | else: 126 | raise Exception("Unrecognized token: %s" % token) 127 | assertEOL(self.reader) 128 | # return self.runtime 129 | 130 | def parse(input): 131 | """Parse a TOML string or file.""" 132 | 133 | reader = Reader(input) 134 | parser = Parser(reader) 135 | return parser.runtime 136 | 137 | def toJSON(input, **kwargs): 138 | """Parse a TOML string or file to JSON string.""" 139 | import json 140 | reader = Reader(input) 141 | parser = Parser(reader, asJson=True) 142 | return json.dumps(parser.runtime, **kwargs) 143 | -------------------------------------------------------------------------------- /tomlpython/reader.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf8 -*- 2 | 3 | import re 4 | 5 | def assertEOL(reader): 6 | # Asserts not valid token is found until EOL. 7 | # Accepts comments. 8 | if reader.line and reader.line[0] != '#': 9 | raise Exception("EOF expected but not found.", reader.line) 10 | 11 | def pop(reader, expect=None): 12 | # Pops top token from "stack" and returns. 13 | if not reader.line: 14 | return False 15 | val = reader.line.pop(0) 16 | if expect and val != expect: 17 | raise Exception("Popped token '%s' was not expected '%s'." %\ 18 | (val, expect)) 19 | return val 20 | 21 | def top(reader): 22 | # Returns next token on STACK. Ignores comments. 23 | # If EOL, next line is loaded. 24 | rem = reader.__next__() 25 | if not rem: 26 | reader._readNextLine() 27 | return top(reader) 28 | return rem 29 | 30 | def skip(reader, *expect): 31 | # Skips next token from reader. 32 | val = pop(reader) 33 | if expect and val not in expect: 34 | raise Exception("Failed to skip token '%s': expected one in '%s,'."\ 35 | % (val, ', '.join(expect))) 36 | 37 | def readLine(reader): 38 | # Updates line on reader and returns False if EOF is found. 39 | return reader._readNextLine() 40 | 41 | def allownl(reader): 42 | # If nothing left on stack, read new line. 43 | # Used for multiline arrays and such. 44 | if not reader.__next__(): 45 | readLine(reader) 46 | 47 | def custom_next(obj): 48 | # For backward compatibility. 49 | # Imported only if python is 2.x 50 | return obj.__next__() 51 | 52 | VERBOSE = True 53 | 54 | class Reader(object): 55 | 56 | def __init__(self, input, verbose=False): 57 | """Takes as argument an object to feed lines to the Reader.""" 58 | try: 59 | # Try to use as a file. 60 | input.read(4) 61 | input.seek(0) 62 | self.lineFeeder = input 63 | except AttributeError: 64 | # Otherwise, assume it's a string. 65 | # Use string with file interface. :) 66 | from io import StringIO 67 | self.lineFeeder = StringIO(unicode(input)) 68 | global VERBOSE 69 | VERBOSE = verbose # be dragons 70 | 71 | @staticmethod 72 | def _cleverSplit(line): 73 | # Split tokens (keeping quoted strings intact). 74 | PATTERN = re.compile(r"""( 75 | ^\[.*?\] | # Match Braces 76 | ".*?[^\\]?" | '.*?[^\\]?' | # Match Single/double-quotes 77 | \# | # hash 78 | \s | \] | \[ | \, | \s= | # Whitespace, braces, comma, = 79 | )""", re.X) 80 | # Line stripping is essential for keygroup matching to work. 81 | if VERBOSE: 82 | print("token:", [p for p in PATTERN.split(line.strip()) if p.strip()]) 83 | return [p for p in PATTERN.split(line.strip()) if p.strip()] 84 | 85 | def _readNextLine(self): 86 | # Get next line from input. 87 | try: # Turn next line into a list of tokens. 88 | tline = self._cleverSplit(next(self.lineFeeder)) 89 | if not tline or tline[0] == "#": 90 | self.line = self._readNextLine() 91 | else: 92 | self.line = tline 93 | except StopIteration: 94 | self.line = None 95 | return self.line 96 | 97 | 98 | def __next__(self): 99 | # Returns next token in the current LINE. 100 | # Ignores comments. 101 | if not self.line or self.line[0] == "#": 102 | return None 103 | return self.line[0] 104 | 105 | --------------------------------------------------------------------------------