├── README.txt ├── debian ├── compat ├── rules ├── changelog ├── control └── copyright ├── flattery └── __init__.py ├── MANIFEST ├── setup.py ├── LICENSE ├── hilite_markdown.py ├── test.py ├── doc ├── pygments.css └── default.css ├── README.md └── flattery.c /README.txt: -------------------------------------------------------------------------------- 1 | README.md -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | %: 5 | dh $@ 6 | 7 | -------------------------------------------------------------------------------- /flattery/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from flattery.cext import * 4 | 5 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | python-flattery (0.1) unstable; urgency=low 2 | 3 | * Initial release 4 | 5 | -- Alan Grow Mon, 07 Mar 2011 18:28:50 -0500 6 | 7 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | build_inplace 2 | debian/changelog 3 | debian/compat 4 | debian/control 5 | debian/copyright 6 | debian/rules 7 | doc/default.css 8 | doc/pygments.css 9 | flattery.c 10 | flattery/__init__.py 11 | hilite_markdown.py 12 | LICENSE 13 | MANIFEST 14 | README.md 15 | setup.py 16 | test.py 17 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: python-flattery 2 | Section: admin 3 | Priority: extra 4 | Maintainer: Alan Grow 5 | Build-depends: debhelper (>= 7), python-support 6 | Standards-Version: 0.1.0 7 | Vcs-Git: https://acg@github.com/acg/python-flattery.git 8 | 9 | Package: python-flattery 10 | Architecture: all 11 | Section: python 12 | Build-Depends: python-support (>= 1.0.0), python (>= 2.5) 13 | Depends: ${misc:Depends}, ${python:Depends}, python (>= 2.5) 14 | Description: fast flattening and unflattening of nested Python data 15 | Flattery: fast flattening and unflattening of nested data structures. 16 | 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """flattery - fast flattening and unflattening of nested Python data 4 | 5 | This library exposes a fast C implementation for flattening and unflattening hierarchical Python data structures. A unit test suite is included. See README.md for usage examples. 6 | """ 7 | 8 | classifiers = """\ 9 | Development Status :: 4 - Beta 10 | Intended Audience :: Developers 11 | License :: OSI Approved :: BSD License 12 | Programming Language :: Python 13 | Topic :: Text Processing :: General 14 | Topic :: Software Development :: Libraries :: Python Modules 15 | Operating System :: OS Independent 16 | """ 17 | 18 | from distutils.core import setup, Extension 19 | 20 | doclines = __doc__.split("\n") 21 | 22 | setup( 23 | name='flattery', 24 | version='0.1', 25 | ext_modules=[ 26 | Extension( 27 | 'flattery.cext', 28 | [ 'flattery.c' ], 29 | extra_compile_args=['-fPIC'], 30 | define_macros=[], 31 | ), 32 | ], 33 | packages=['flattery'], 34 | maintainer="Alan Grow", 35 | maintainer_email="alangrow+flattery@gmail.com", 36 | url="https://github.com/acg/python-flattery", 37 | license = "https://github.com/acg/python-flattery/blob/master/LICENSE", 38 | platforms = ["any"], 39 | description = doclines[0], 40 | classifiers = filter(None, classifiers.split("\n")), 41 | long_description = "\n".join(doclines[2:]), 42 | ) 43 | 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The BSD License 2 | 3 | Copyright (c) 2011, Alan Grow All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | Neither the name of Alan Grow nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | 15 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | The BSD License 2 | 3 | Copyright (c) 2011, Alan Grow All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | Neither the name of Alan Grow nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | 15 | -------------------------------------------------------------------------------- /hilite_markdown.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Convert markdown to html. 5 | Use pygments to syntax highlight any code blocks that begin with a shebang. 6 | Wrap html and style it somewhat like docs.python.org. 7 | """ 8 | 9 | import sys 10 | import re 11 | from markdown import Markdown 12 | from markdown import TextPreprocessor 13 | from pygments import highlight 14 | from pygments.formatters import HtmlFormatter 15 | from pygments.lexers import get_lexer_by_name, TextLexer 16 | 17 | INLINE_STYLES = False 18 | INCLUDE_SHEBANG = False 19 | 20 | class CodeBlockPreprocessor(TextPreprocessor): 21 | 22 | pattern = re.compile( r'^(?P {4}#!(?P.*)\n(?P(( {4}.*)?\n)*))', re.M) 23 | 24 | formatter = HtmlFormatter(noclasses=INLINE_STYLES) 25 | 26 | def run(self, lines): 27 | 28 | def repl(m): 29 | 30 | # extract program from shebang 31 | argv = m.group('shebang').split() 32 | program = argv[0].split('/')[-1] 33 | if program == 'env' and len(argv) >= 2: 34 | program = argv[1] 35 | 36 | try: 37 | lexer = get_lexer_by_name(program) 38 | except ValueError: 39 | lexer = TextLexer() 40 | 41 | if INCLUDE_SHEBANG: 42 | code = m.group('codeblock') 43 | else: 44 | code = m.group('body') 45 | 46 | code = highlight(code, lexer, self.formatter) 47 | code = code.replace('\n\n', '\n \n').replace('\n', '
') 48 | return '\n\n
%s
\n\n' % code 49 | 50 | return self.pattern.sub(repl, lines) 51 | 52 | 53 | if len(sys.argv) > 1: 54 | infile = file(sys.argv[1]) 55 | else: 56 | infile = sys.stdin 57 | 58 | md = Markdown() 59 | md.textPreprocessors.insert(0, CodeBlockPreprocessor()) 60 | html = md.convert(infile.read()) 61 | 62 | template = """ 63 | 64 | 65 | 66 | 67 | 68 | 69 |
70 | %s 71 |
72 | 73 | 74 | """ 75 | 76 | print template % html 77 | 78 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import unittest 4 | from flattery import flatten, unflatten 5 | 6 | 7 | class FunctionTestCase(unittest.TestCase): 8 | 9 | def __init__(self, **keywords): 10 | unittest.TestCase.__init__(self) 11 | self.args = [] 12 | self.kwargs = {} 13 | 14 | for k, v in keywords.items(): 15 | setattr(self, k, v) 16 | 17 | def runTest(self): 18 | self.assertEqual( self.func(*self.args,**self.kwargs), self.expected ) 19 | 20 | def shortDescription(self): 21 | return self.name 22 | 23 | 24 | # TODO add more coverage 25 | # TODO add bi-directional tests of flatten() 26 | 27 | TEST_UNFLATTEN = 0x1 28 | TEST_FLATTEN = 0x2 29 | TEST_BOTH = TEST_UNFLATTEN | TEST_FLATTEN 30 | 31 | TRUTH = [ 32 | ("empty", TEST_BOTH, {}, {}), 33 | ("empty key", TEST_BOTH, {'':1}, {'':1}), 34 | ("depth 2 path x 1", TEST_BOTH, {"a.b":42}, {"a":{"b":42}}), 35 | ("depth 2 path x 2", TEST_BOTH, {"a.b":42,"c.d":"x"}, {"a":{"b":42},"c":{"d":"x"}}), 36 | ("depth 2 path x 2, overlap", TEST_BOTH, {"a.b":42,"a.c":"x"}, {"a":{"b":42,"c":"x"}}), 37 | ("simple list", TEST_BOTH, {"a.0":1,"a.1":2}, {"a":[1,2]}), 38 | ("sparse list", TEST_BOTH, {"a.0":1,"a.9":10}, {"a":[1,None,None,None,None,None,None,None,None,10]}), 39 | ("deep dict", TEST_BOTH, {"a.b.c.d.e":1}, {"a":{"b":{"c":{"d":{"e":1}}}}}), 40 | ("list under dict", TEST_BOTH, {"a.b.0":1,"a.b.1":2}, {"a":{"b":[1,2]}}), 41 | ("dict under list x 1", TEST_BOTH, {"a.0.b":1}, {"a":[{"b":1}]}), 42 | ("dict under list x 2, overlap", TEST_BOTH, {"a.0.b":1,"a.0.c":2}, {"a":[{"b":1,"c":2}]}), 43 | ("deep list", TEST_BOTH, {"a.0.0.0.0":42}, {"a":[[[[42]]]]}), 44 | ] 45 | 46 | 47 | def main(): 48 | 49 | suite = unittest.TestSuite() 50 | i = 0 51 | 52 | for name, mode, flat, unflat in TRUTH: 53 | 54 | if mode & TEST_FLATTEN: 55 | 56 | suite.addTest(FunctionTestCase( 57 | name="test %d flatten %s" % (i,name), 58 | func=flatten, 59 | args=[unflat], 60 | expected=flat, 61 | )) 62 | i += 1 63 | 64 | if mode & TEST_UNFLATTEN: 65 | 66 | suite.addTest(FunctionTestCase( 67 | name="test %d unflatten %s" % (i,name), 68 | func=unflatten, 69 | args=[flat], 70 | expected=unflat, 71 | )) 72 | i += 1 73 | 74 | runner = unittest.TextTestRunner(verbosity=2) 75 | results = runner.run(suite) 76 | 77 | if len(results.failures) or len(results.errors): 78 | return 1 79 | else: 80 | return 0 81 | 82 | 83 | if __name__ == '__main__': 84 | import sys 85 | sys.exit(main()) 86 | 87 | -------------------------------------------------------------------------------- /doc/pygments.css: -------------------------------------------------------------------------------- 1 | .hll { background-color: #ffffcc } 2 | .c { color: #408080; font-style: italic } /* Comment */ 3 | .err { border: 1px solid #FF0000 } /* Error */ 4 | .k { color: #008000; font-weight: bold } /* Keyword */ 5 | .o { color: #666666 } /* Operator */ 6 | .cm { color: #408080; font-style: italic } /* Comment.Multiline */ 7 | .cp { color: #BC7A00 } /* Comment.Preproc */ 8 | .c1 { color: #408080; font-style: italic } /* Comment.Single */ 9 | .cs { color: #408080; font-style: italic } /* Comment.Special */ 10 | .gd { color: #A00000 } /* Generic.Deleted */ 11 | .ge { font-style: italic } /* Generic.Emph */ 12 | .gr { color: #FF0000 } /* Generic.Error */ 13 | .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 14 | .gi { color: #00A000 } /* Generic.Inserted */ 15 | .go { color: #808080 } /* Generic.Output */ 16 | .gp { color: #000080; font-weight: bold } /* Generic.Prompt */ 17 | .gs { font-weight: bold } /* Generic.Strong */ 18 | .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 19 | .gt { color: #0040D0 } /* Generic.Traceback */ 20 | .kc { color: #008000; font-weight: bold } /* Keyword.Constant */ 21 | .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ 22 | .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ 23 | .kp { color: #008000 } /* Keyword.Pseudo */ 24 | .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ 25 | .kt { color: #B00040 } /* Keyword.Type */ 26 | .m { color: #666666 } /* Literal.Number */ 27 | .s { color: #BA2121 } /* Literal.String */ 28 | .na { color: #7D9029 } /* Name.Attribute */ 29 | .nb { color: #008000 } /* Name.Builtin */ 30 | .nc { color: #0000FF; font-weight: bold } /* Name.Class */ 31 | .no { color: #880000 } /* Name.Constant */ 32 | .nd { color: #AA22FF } /* Name.Decorator */ 33 | .ni { color: #999999; font-weight: bold } /* Name.Entity */ 34 | .ne { color: #D2413A; font-weight: bold } /* Name.Exception */ 35 | .nf { color: #0000FF } /* Name.Function */ 36 | .nl { color: #A0A000 } /* Name.Label */ 37 | .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ 38 | .nt { color: #008000; font-weight: bold } /* Name.Tag */ 39 | .nv { color: #19177C } /* Name.Variable */ 40 | .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ 41 | .w { color: #bbbbbb } /* Text.Whitespace */ 42 | .mf { color: #666666 } /* Literal.Number.Float */ 43 | .mh { color: #666666 } /* Literal.Number.Hex */ 44 | .mi { color: #666666 } /* Literal.Number.Integer */ 45 | .mo { color: #666666 } /* Literal.Number.Oct */ 46 | .sb { color: #BA2121 } /* Literal.String.Backtick */ 47 | .sc { color: #BA2121 } /* Literal.String.Char */ 48 | .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ 49 | .s2 { color: #BA2121 } /* Literal.String.Double */ 50 | .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */ 51 | .sh { color: #BA2121 } /* Literal.String.Heredoc */ 52 | .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */ 53 | .sx { color: #008000 } /* Literal.String.Other */ 54 | .sr { color: #BB6688 } /* Literal.String.Regex */ 55 | .s1 { color: #BA2121 } /* Literal.String.Single */ 56 | .ss { color: #19177C } /* Literal.String.Symbol */ 57 | .bp { color: #008000 } /* Name.Builtin.Pseudo */ 58 | .vc { color: #19177C } /* Name.Variable.Class */ 59 | .vg { color: #19177C } /* Name.Variable.Global */ 60 | .vi { color: #19177C } /* Name.Variable.Instance */ 61 | .il { color: #666666 } /* Literal.Number.Integer.Long */ 62 | -------------------------------------------------------------------------------- /doc/default.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Sphinx stylesheet -- default theme 3 | * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 4 | */ 5 | 6 | @import url("basic.css"); 7 | 8 | /* -- page layout ----------------------------------------------------------- */ 9 | 10 | body { 11 | font-family: sans-serif; 12 | font-size: 100%; 13 | background-color: #11303d; 14 | color: #000; 15 | margin: 0; 16 | padding: 0; 17 | } 18 | 19 | div.document { 20 | background-color: #1c4e63; 21 | } 22 | 23 | div.documentwrapper { 24 | float: left; 25 | width: 100%; 26 | } 27 | 28 | div.bodywrapper { 29 | margin: 0 0 0 230px; 30 | } 31 | 32 | div.body { 33 | background-color: #ffffff; 34 | color: #000000; 35 | padding: 0 20px 30px 20px; 36 | } 37 | 38 | div.footer { 39 | color: #ffffff; 40 | width: 100%; 41 | padding: 9px 0 9px 0; 42 | text-align: center; 43 | font-size: 75%; 44 | } 45 | 46 | div.footer a { 47 | color: #ffffff; 48 | text-decoration: underline; 49 | } 50 | 51 | div.related { 52 | background-color: #133f52; 53 | line-height: 30px; 54 | color: #ffffff; 55 | } 56 | 57 | div.related a { 58 | color: #ffffff; 59 | } 60 | 61 | div.sphinxsidebar { 62 | } 63 | 64 | div.sphinxsidebar h3 { 65 | font-family: 'Trebuchet MS', sans-serif; 66 | color: #ffffff; 67 | font-size: 1.4em; 68 | font-weight: normal; 69 | margin: 0; 70 | padding: 0; 71 | } 72 | 73 | div.sphinxsidebar h3 a { 74 | color: #ffffff; 75 | } 76 | 77 | div.sphinxsidebar h4 { 78 | font-family: 'Trebuchet MS', sans-serif; 79 | color: #ffffff; 80 | font-size: 1.3em; 81 | font-weight: normal; 82 | margin: 5px 0 0 0; 83 | padding: 0; 84 | } 85 | 86 | div.sphinxsidebar p { 87 | color: #ffffff; 88 | } 89 | 90 | div.sphinxsidebar p.topless { 91 | margin: 5px 10px 10px 10px; 92 | } 93 | 94 | div.sphinxsidebar ul { 95 | margin: 10px; 96 | padding: 0; 97 | color: #ffffff; 98 | } 99 | 100 | div.sphinxsidebar a { 101 | color: #98dbcc; 102 | } 103 | 104 | div.sphinxsidebar input { 105 | border: 1px solid #98dbcc; 106 | font-family: sans-serif; 107 | font-size: 1em; 108 | } 109 | 110 | /* -- body styles ----------------------------------------------------------- */ 111 | 112 | a { 113 | color: #355f7c; 114 | text-decoration: none; 115 | } 116 | 117 | a:hover { 118 | text-decoration: underline; 119 | } 120 | 121 | div.body p, div.body dd, div.body li { 122 | text-align: justify; 123 | line-height: 130%; 124 | } 125 | 126 | div.body h1, 127 | div.body h2, 128 | div.body h3, 129 | div.body h4, 130 | div.body h5, 131 | div.body h6 { 132 | font-family: 'Trebuchet MS', sans-serif; 133 | background-color: #f2f2f2; 134 | font-weight: normal; 135 | color: #20435c; 136 | border-bottom: 1px solid #ccc; 137 | margin: 20px -20px 10px -20px; 138 | padding: 3px 0 3px 10px; 139 | } 140 | 141 | div.body h1 { margin-top: 0; font-size: 200%; } 142 | div.body h2 { font-size: 160%; } 143 | div.body h3 { font-size: 140%; } 144 | div.body h4 { font-size: 120%; } 145 | div.body h5 { font-size: 110%; } 146 | div.body h6 { font-size: 100%; } 147 | 148 | a.headerlink { 149 | color: #c60f0f; 150 | font-size: 0.8em; 151 | padding: 0 4px 0 4px; 152 | text-decoration: none; 153 | } 154 | 155 | a.headerlink:hover { 156 | background-color: #c60f0f; 157 | color: white; 158 | } 159 | 160 | div.body p, div.body dd, div.body li { 161 | text-align: justify; 162 | line-height: 130%; 163 | } 164 | 165 | div.admonition p.admonition-title + p { 166 | display: inline; 167 | } 168 | 169 | div.admonition p { 170 | margin-bottom: 5px; 171 | } 172 | 173 | div.admonition pre { 174 | margin-bottom: 5px; 175 | } 176 | 177 | div.admonition ul, div.admonition ol { 178 | margin-bottom: 5px; 179 | } 180 | 181 | div.note { 182 | background-color: #eee; 183 | border: 1px solid #ccc; 184 | } 185 | 186 | div.seealso { 187 | background-color: #ffc; 188 | border: 1px solid #ff6; 189 | } 190 | 191 | div.topic { 192 | background-color: #eee; 193 | } 194 | 195 | div.warning { 196 | background-color: #ffe4e4; 197 | border: 1px solid #f66; 198 | } 199 | 200 | p.admonition-title { 201 | display: inline; 202 | } 203 | 204 | p.admonition-title:after { 205 | content: ":"; 206 | } 207 | 208 | pre { 209 | padding: 5px; 210 | background-color: #eeffcc; 211 | color: #333333; 212 | line-height: 120%; 213 | border: 1px solid #ac9; 214 | border-left: none; 215 | border-right: none; 216 | } 217 | 218 | tt { 219 | background-color: #ecf0f3; 220 | padding: 0 1px 0 1px; 221 | font-size: 0.95em; 222 | } 223 | 224 | .warning tt { 225 | background: #efc2c2; 226 | } 227 | 228 | .note tt { 229 | background: #d6d6d6; 230 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flattery - fast flattening and unflattening of nested Python data # 2 | 3 | This library exposes a fast C implementation for flattening and unflattening hierarchical Python data structures. A unit test suite is included. 4 | 5 | ## Examples ## 6 | 7 | ### Synopsis ### 8 | 9 | #!/usr/bin/env python 10 | from flattery import flatten, unflatten 11 | 12 | data = { 13 | "x.y.0": "zero", 14 | "x.y.1": "one", 15 | "x.z" : 42 16 | } 17 | 18 | print unflatten(data) 19 | >>> 20 | { 21 | 'x' : { 22 | 'y' : [ 'zero', 'one' ], 23 | 'z': 42 24 | } 25 | } 26 | <<< 27 | 28 | assert( data == flatten(unflatten(data)) ) 29 | 30 | ### Processing hierarchical records ### 31 | 32 | #!/usr/bin/env python 33 | import sys 34 | from flattery import unflatten 35 | 36 | cols = [ "time", "request.method", "request.uri", "response.status", "response.size" ] 37 | 38 | for line in sys.stdin: 39 | fields = line.rstrip('\r\n').split(len(cols)-1) 40 | values = dict([(cols[i],fields[i]) for i in xrange(len(cols))]) 41 | record = unflatten(values) 42 | # do something with the record... 43 | print record 44 | >>> 45 | { 'time': '12/12/2012 12:00:00', 46 | 'request': { 'method': 'GET', 'uri': '/stuff/' }, 47 | 'response': { 'status': '200', 'size': '40391' } } 48 | ... 49 | 50 | ### Web form processing: grouping data ### 51 | 52 | Or suppose you have a web form for collecting several distinct blobs of data at once, like a payment form: 53 | 54 | #!/usr/bin/html 55 |
56 |
57 | First name: 58 | Last name: 59 | Email: 60 |
61 |
62 | Credit Card Type: 63 | Number: 64 | CCV: 65 | Expiration Month: 66 | Expiration Year: 67 |
68 |
69 | 70 |
71 |
72 | 73 | In the form processing code, you can expand the key value form data pairs into a nested object: 74 | 75 | #!/usr/bin/env python 76 | from flattery import unflatten 77 | 78 | params = formdata() # however you get a dictionary of form data... 79 | data = unflatten(params) 80 | print data 81 | >>> 82 | { 'contact': 83 | { 'lastname':'Doe', 84 | 'firstname': 'John', 85 | 'email':'jdoe@example.com' }, 86 | 'payment': 87 | { 'cctype': 'amex', 88 | 'ccnumber': '4111111111111111', 89 | 'ccv': '4321', 90 | 'ccmonth': '12' , 91 | 'ccyear' : '2020' } } 92 | 93 | (But be careful with multiply-valued form data.) 94 | 95 | ### Web form processing: tabular data ### 96 | 97 | Another web example, where a user is editing tabular data: 98 | 99 | #!/usr/bin/html 100 |
101 |
102 |
    103 |
      104 |
      • 105 |
      • 106 |
      • 107 |
    108 |
      109 |
      • 110 |
      • 111 |
      • 112 |
    113 | ... 114 |
      115 |
      • 116 |
      • 117 |
      • 118 |
    119 |
120 |
121 |
122 | 123 |
124 |
125 | 126 | (You should validate that the embedded indices are below some reasonable limit to avoid a memory DoS.) 127 | 128 | In the form processing code: 129 | 130 | #!/usr/bin/env python 131 | from flattery import unflatten 132 | 133 | params = formdata() # however you get a dictionary of form data... 134 | data = unflatten(params) 135 | print data 136 | >>> 137 | { 'rows': 138 | [ 139 | 'delete': '1', 140 | 'name': 'John Doe', 141 | 'email': 'johndoe@example.com', 142 | ], 143 | [ 144 | 'delete': '0', 145 | 'name': 'Suzy Q', 146 | 'email': 'suzyq@example.com', 147 | ], 148 | ... 149 | [ 150 | 'delete': '0', 151 | 'name': 'Charlie Chaplin', 152 | 'email': 'charliechaplin@example.com', 153 | ], 154 | } 155 | 156 | ## Installation ## 157 | 158 | Ubuntu / Debian users: 159 | 160 | #!/bin/sh 161 | fakeroot ./debian/rules binary 162 | dpkg -i ../python-flattery*.deb 163 | 164 | If there's no "real" packaging for your system yet: 165 | 166 | #!/bin/sh 167 | ./setup.py build_ext --inplace 168 | ./test.py 169 | ./setup.py build 170 | ./setup.py install 171 | 172 | -------------------------------------------------------------------------------- /flattery.c: -------------------------------------------------------------------------------- 1 | #include "Python.h" 2 | #include // for isdigit() 3 | #include // for atol() 4 | 5 | 6 | static PyObject * 7 | unflatten(PyObject *ignore, PyObject *args) 8 | { 9 | PyObject *src = NULL; 10 | PyObject *dst = NULL; 11 | PyObject *nonelist = NULL; 12 | PyObject *slot = NULL; 13 | PyObject *slotvalue = NULL; 14 | PyObject *part = NULL; 15 | 16 | if (!PyArg_ParseTuple(args, "O!:unflatten", &PyDict_Type, &src)) 17 | return NULL; 18 | 19 | if (!(dst = PyDict_New())) 20 | goto error; 21 | 22 | /* Create a [None] list. Used for extending lists to higher indices. */ 23 | 24 | if (!(nonelist = PyList_New(0))) 25 | goto error; 26 | if (PyList_Append(nonelist, Py_None) < 0) 27 | goto error; 28 | 29 | /* Iterate through key value pairs in the src dict, 30 | building the nested data structure in dst as we go. */ 31 | 32 | PyObject *k, *v; 33 | Py_ssize_t pos = 0; 34 | 35 | while (PyDict_Next(src, &pos, &k, &v)) 36 | { 37 | const char *key = PyString_AsString(k); 38 | const char *p; 39 | 40 | p = key; 41 | slot = dst; 42 | Py_INCREF(slot); 43 | 44 | do 45 | { 46 | /* Extract current part of the key path. */ 47 | 48 | const char *start = p; 49 | while (*p && *p != '.') p++; 50 | part = PyString_FromStringAndSize(start, p-start); 51 | 52 | /* Advance to next part of key path, unless at the end. */ 53 | 54 | if (*p == '.') 55 | p++; 56 | 57 | /* What value should we insert under this slot? 58 | - if this is the last path part, insert the value from src. 59 | - if the next path part is numeric, insert an empty list. 60 | - otherwise, insert an empty hash. 61 | */ 62 | 63 | if (!*p) { 64 | slotvalue = v; 65 | Py_INCREF(slotvalue); 66 | } 67 | else if (isdigit(*p)) 68 | slotvalue = PyList_New(0); 69 | else 70 | slotvalue = PyDict_New(); 71 | 72 | if (!slotvalue) 73 | goto error; 74 | 75 | /* Assign to the current slot. */ 76 | 77 | if (isdigit(*start)) 78 | { 79 | /* If the current path part is numeric, index into a list. */ 80 | 81 | if (!PyList_Check(slot)) 82 | goto error; 83 | 84 | // FIXME thorough error checking here 85 | 86 | Py_ssize_t len = PyList_Size(slot); 87 | Py_ssize_t index = atol(PyString_AsString(part)); 88 | 89 | /* Extend the list with [None,None,...] if necessary. */ 90 | 91 | if (index >= len) 92 | { 93 | PyObject *tail = PySequence_Repeat(nonelist, index-len+1); 94 | PyObject *extended = PySequence_InPlaceConcat(slot, tail); 95 | Py_DECREF(tail); 96 | Py_DECREF(extended); 97 | } 98 | 99 | /* Don't clobber an existing entry. 100 | Caveat: PyList_SetItem() steals a reference to slotvalue. */ 101 | 102 | PyObject *extant = NULL; 103 | 104 | if ((extant = PyList_GetItem(slot, index)) == Py_None) { 105 | PyList_SetItem(slot, index, slotvalue); 106 | Py_INCREF(slotvalue); 107 | } 108 | else { 109 | Py_DECREF(slotvalue); 110 | slotvalue = extant; 111 | Py_INCREF(slotvalue); 112 | } 113 | } 114 | else 115 | { 116 | /* If the current path part is non-numeric, index into a dict. */ 117 | 118 | if (!PyDict_Check(slot)) 119 | goto error; 120 | 121 | /* Don't clobber an existing entry. */ 122 | 123 | PyObject *extant = NULL; 124 | 125 | if (!(extant = PyDict_GetItem(slot, part))) 126 | PyDict_SetItem(slot, part, slotvalue); 127 | else { 128 | Py_DECREF(slotvalue); 129 | slotvalue = extant; 130 | Py_INCREF(slotvalue); 131 | } 132 | } 133 | 134 | /* Descend further into the dst data structure. */ 135 | 136 | Py_DECREF(slot); 137 | slot = slotvalue; 138 | slotvalue = NULL; 139 | 140 | Py_DECREF(part); 141 | part = NULL; 142 | } 143 | while (*p); 144 | 145 | Py_DECREF(slot); 146 | slot = NULL; 147 | } 148 | 149 | Py_DECREF(nonelist); 150 | return dst; 151 | 152 | error: 153 | 154 | Py_XDECREF(dst); 155 | Py_XDECREF(nonelist); 156 | Py_XDECREF(slot); 157 | Py_XDECREF(slotvalue); 158 | Py_XDECREF(part); 159 | 160 | return NULL; 161 | } 162 | 163 | 164 | static PyObject * 165 | flatten_internal(PyObject *src) 166 | { 167 | PyObject *flat = NULL; 168 | PyObject *dst = NULL; 169 | 170 | if (PyList_Check(src)) 171 | { 172 | if (!(flat = PyDict_New())) 173 | goto error; 174 | 175 | /* Iterate through elements in the list src, recursively flattening. 176 | Skip any entries which are None -- use a sparse encoding. */ 177 | 178 | Py_ssize_t i; 179 | Py_ssize_t len = PyList_Size(src); 180 | 181 | for (i=0; i dict"}, 279 | {"flatten", flatten, METH_VARARGS, "flatten(dict) -> dict"}, 280 | {NULL, NULL} /* sentinel */ 281 | }; 282 | 283 | PyDoc_STRVAR(module_doc, "Flattery: fast flattening and unflattening of nested data structures."); 284 | 285 | /* Initialization function for the module (*must* be called initcext) */ 286 | 287 | PyMODINIT_FUNC 288 | initcext(void) 289 | { 290 | /* Create the module and add the functions */ 291 | 292 | PyObject* mod = Py_InitModule3("flattery.cext", flattery_methods, module_doc); 293 | if (mod == NULL) 294 | return; 295 | } 296 | 297 | --------------------------------------------------------------------------------