├── .gitignore ├── LICENSE ├── README.md └── data_to_py.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Peter Hinch 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # data_to_py.py 2 | 3 | A means of storing read-only data in Flash, conserving RAM. 4 | 5 | The utility runs on a PC and converts an arbitrary binary file into a Python 6 | source file. The generated Python file may be frozen as bytecode. Read-only 7 | data may thus be stored in Flash and accessed with little RAM use. 8 | 9 | Please see also [freezeFS](https://github.com/bixb922/freezeFS) - a utility by 10 | @bixb922 which allows a complete directory tree to be frozen and mounted as a 11 | filesystem. 12 | 13 | ## Arguments 14 | 15 | The utility requires two arguments, the first being the path to a file for 16 | reading and the second being the path to a file for writing. The latter must 17 | have a ``.py`` extension. If the output file exists it will be overwritten. 18 | 19 | ## The Python output file 20 | 21 | This instantiates a ``bytes`` object containing the data, which may be accessed 22 | via the file's ``data()`` function: this returns a ``memoryview`` of the data. 23 | 24 | The use of a ``memoryview`` ensures that slices of the data may be extracted 25 | without using RAM: 26 | 27 | ```python 28 | import mydata # file mydata.py (typically frozen) 29 | d = mydata.data() # d contains a memoryview 30 | s = d[1000:3000] # s contains 1000 bytes but consumes no RAM 31 | ``` 32 | 33 | A practical example is rendering a JPEG file to an LCD160CR display. The Python 34 | file ``img.py`` is generated using 35 | 36 | ``` 37 | $ ./data_to_py.py my_image.jpg img.py 38 | ``` 39 | 40 | The file ``img.py`` may then be frozen, although for test purposes it can just 41 | be copied to the target. The following code displays the image: 42 | 43 | ```python 44 | import lcd160cr 45 | import img 46 | lcd = lcd160cr.LCD160CR('Y') 47 | lcd.set_orient(lcd160cr.PORTRAIT) 48 | lcd.set_pen(lcd.rgb(0, 0, 0), lcd.rgb(0, 0, 0)) 49 | lcd.erase() 50 | lcd.set_pos(0, 0) 51 | jpeg_data = img.data() 52 | lcd.jpeg(jpeg_data) # Access the data and display it 53 | ``` 54 | 55 | ## Validation 56 | 57 | The utility can accept any file type as input, so no validation is performed. 58 | If required, this must be done by the application at runtime. For example in 59 | the above code, to be suitable for the device jpeg_data[:2] should be 60 | ``0xff, 0xd8`` and jpeg_data[-2:] should be ``0xff, 0xd9``. 61 | -------------------------------------------------------------------------------- /data_to_py.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # The MIT License (MIT) 5 | # 6 | # Copyright (c) 2016 Peter Hinch 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in 16 | # all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 | # THE SOFTWARE. 25 | 26 | import argparse 27 | import sys 28 | import os 29 | 30 | # UTILITIES FOR WRITING PYTHON SOURCECODE TO A FILE 31 | 32 | # ByteWriter takes as input a variable name and data values and writes 33 | # Python source to an output stream of the form 34 | # my_variable = b'\x01\x02\x03\x04\x05\x06\x07\x08'\ 35 | 36 | # Lines are broken with \ for readability. 37 | 38 | 39 | class ByteWriter(object): 40 | bytes_per_line = 16 41 | 42 | def __init__(self, stream, varname): 43 | self.stream = stream 44 | self.stream.write('{} =\\\n'.format(varname)) 45 | self.bytecount = 0 # For line breaks 46 | 47 | def _eol(self): 48 | self.stream.write("'\\\n") 49 | 50 | def _eot(self): 51 | self.stream.write("'\n") 52 | 53 | def _bol(self): 54 | self.stream.write("b'") 55 | 56 | # Output a single byte 57 | def obyte(self, data): 58 | if not self.bytecount: 59 | self._bol() 60 | self.stream.write('\\x{:02x}'.format(data)) 61 | self.bytecount += 1 62 | self.bytecount %= self.bytes_per_line 63 | if not self.bytecount: 64 | self._eol() 65 | 66 | # Output from a sequence 67 | def odata(self, bytelist): 68 | for byt in bytelist: 69 | self.obyte(byt) 70 | 71 | # ensure a correct final line 72 | def eot(self): # User force EOL if one hasn't occurred 73 | if self.bytecount: 74 | self._eot() 75 | self.stream.write('\n') 76 | 77 | 78 | # PYTHON FILE WRITING 79 | 80 | STR01 = """# Code generated by data_to_py.py. 81 | version = '0.1' 82 | """ 83 | 84 | STR02 = """_mvdata = memoryview(_data) 85 | 86 | def data(): 87 | return _mvdata 88 | 89 | """ 90 | 91 | def write_func(stream, name, arg): 92 | stream.write('def {}():\n return {}\n\n'.format(name, arg)) 93 | 94 | 95 | def write_data(op_path, ip_path): 96 | try: 97 | with open(ip_path, 'rb') as ip_stream: 98 | try: 99 | with open(op_path, 'w') as op_stream: 100 | write_stream(ip_stream, op_stream) 101 | except OSError: 102 | print("Can't open", op_path, 'for writing') 103 | return False 104 | except OSError: 105 | print("Can't open", ip_path) 106 | return False 107 | return True 108 | 109 | 110 | def write_stream(ip_stream, op_stream): 111 | op_stream.write(STR01) 112 | op_stream.write('\n') 113 | data = ip_stream.read() 114 | bw_data = ByteWriter(op_stream, '_data') 115 | bw_data.odata(data) 116 | bw_data.eot() 117 | op_stream.write(STR02) 118 | 119 | 120 | # PARSE COMMAND LINE ARGUMENTS 121 | 122 | def quit(msg): 123 | print(msg) 124 | sys.exit(1) 125 | 126 | DESC = """data_to_py.py 127 | Utility to convert an arbitrary binary file to Python source. 128 | Sample usage: 129 | data_to_py.py image.jpg image.py 130 | 131 | """ 132 | 133 | if __name__ == "__main__": 134 | parser = argparse.ArgumentParser(__file__, description=DESC, 135 | formatter_class=argparse.RawDescriptionHelpFormatter) 136 | parser.add_argument('infile', type=str, help='Input file path') 137 | parser.add_argument('outfile', type=str, 138 | help='Path and name of output file. Must have .py extension.') 139 | 140 | 141 | args = parser.parse_args() 142 | 143 | if not os.path.isfile(args.infile): 144 | quit("Data filename does not exist") 145 | 146 | if not os.path.splitext(args.outfile)[1].upper() == '.PY': 147 | quit('Output filename must have a .py extension.') 148 | 149 | print('Writing Python file.') 150 | if not write_data(args.outfile, args.infile): 151 | sys.exit(1) 152 | 153 | print(args.outfile, 'written successfully.') 154 | --------------------------------------------------------------------------------