├── test ├── __init__.py ├── test_timeutils.py ├── test_examples.py └── test_teafile.py ├── doc ├── _static │ └── empty.txt ├── dtheme │ ├── static │ │ ├── dtheme.css_t │ │ ├── pygments.css │ │ ├── default.css │ │ └── basic.css │ └── theme.conf ├── clockwise.jpg ├── teafiles.rst ├── index.rst ├── interactive.rst ├── clockwise.rst ├── make.bat ├── Makefile ├── conf.py └── examples.rst ├── MANIFEST.in ├── .gitignore ├── TeaFiles.Py.sln ├── teafiles ├── __init__.py ├── clockwise.py └── teafile.py ├── License.txt ├── setup.py ├── TeaFiles.pyproj ├── stopwatch.py ├── README.rst ├── README.txt ├── examples.py └── pylintrc /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/_static/empty.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/dtheme/static/dtheme.css_t: -------------------------------------------------------------------------------- 1 | @import url("default.css"); 2 | 3 | -------------------------------------------------------------------------------- /doc/dtheme/theme.conf: -------------------------------------------------------------------------------- 1 | [theme] 2 | inherit = default 3 | stylesheet = dtheme.css 4 | pygments_style = sphinx 5 | -------------------------------------------------------------------------------- /doc/clockwise.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/discretelogics/TeaFiles.Py-Time-Series-Storage-in-Files/HEAD/doc/clockwise.jpg -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # file GENERATED by distutils, do NOT edit 2 | include README.txt 3 | include LICENSE.txt 4 | include pylintrc 5 | include examples.py 6 | include stopwatch.py 7 | include setup.py 8 | recursive-include teafiles *.py 9 | recursive-include test *.py 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | 29 | # Translations 30 | *.mo 31 | 32 | # Mr Developer 33 | .mr.developer.cfg 34 | .project 35 | .pydevproject 36 | -------------------------------------------------------------------------------- /TeaFiles.Py.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 11.00 3 | # Visual Studio 2010 4 | Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "TeaFiles", "TeaFiles.pyproj", "{F7FDC9F6-8A85-47F6-85DC-B48380A978CA}" 5 | EndProject 6 | Global 7 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 8 | Debug|Any CPU = Debug|Any CPU 9 | Release|Any CPU = Release|Any CPU 10 | EndGlobalSection 11 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 12 | {F7FDC9F6-8A85-47F6-85DC-B48380A978CA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 13 | {F7FDC9F6-8A85-47F6-85DC-B48380A978CA}.Release|Any CPU.ActiveCfg = Release|Any CPU 14 | EndGlobalSection 15 | GlobalSection(SolutionProperties) = preSolution 16 | HideSolutionNode = FALSE 17 | EndGlobalSection 18 | EndGlobal 19 | -------------------------------------------------------------------------------- /teafiles/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011 discretelogics 2 | # 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | from teafiles.teafile import * 17 | __all__ = ["TeaFile"] 18 | 19 | from teafiles.clockwise import * 20 | __all__.extend(["DateTime", "Duration", "range", "rangen"]) 21 | 22 | version = "0.7.4" 23 | -------------------------------------------------------------------------------- /doc/teafiles.rst: -------------------------------------------------------------------------------- 1 | 2 | :mod:`teafile` Module 3 | --------------------- 4 | 5 | .. autoclass:: teafiles.teafile.TeaFile 6 | :members: create, openread, openwrite, read, _write, flush, seekitem, seekend, items, 7 | itemcount, close, description, getvaluestring, printitems, printsnapshot 8 | :undoc-members: 9 | :show-inheritance: 10 | 11 | .. autoclass:: teafiles.teafile.TimeScale 12 | :members: Java, wellknownname 13 | :undoc-members: 14 | :show-inheritance: 15 | 16 | .. autoclass:: teafiles.teafile.TeaFileDescription 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | 21 | .. autoclass:: teafiles.teafile.ItemDescription 22 | :members: create, getfieldbyoffset 23 | :undoc-members: 24 | :show-inheritance: 25 | 26 | .. autoclass:: teafiles.teafile.FieldType 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | .. autoclass:: teafiles.teafile.Field 33 | :members: getvalue 34 | :undoc-members: 35 | :show-inheritance: 36 | -------------------------------------------------------------------------------- /License.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2012-present Thomas Hulka 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | Python API for TeaFiles 2 | ======================= 3 | 4 | pre release information 5 | ----------------------- 6 | 7 | More information about TeaFiles and the TeaTime product line to follow soon (epected February 2012) 8 | 9 | 10 | time series simple & efficient 11 | ------------------------------ 12 | 13 | TeaFiles provide a **very simple**, yet **highly efficient** way to store **time series data** in a way that allows **data exchange** between 14 | programs written in **C++, C#** or in applications like **R, Octave, Matlab** or others, running on Linux, Unix, Mac OS X or Windows. 15 | 16 | TeaFiles store **binary data** composed from elementary data types signed and unsigned **integers, double and float** in IEEE 754 format. 17 | Data can be directly accessed via **memory mapping**. Since their header stores a description of the item structure, they relieve the opaqueness 18 | of normal binary files and remain always accessible. In other words: Knowing that a file is a TeaFile is enough to access its data. 19 | 20 | 21 | the api 22 | ------- 23 | 24 | .. toctree:: 25 | :maxdepth: 1 26 | 27 | teafiles (TeaFile) 28 | clockwise (DateTime & Duration) 29 | 30 | examples 31 | -------- 32 | 33 | .. toctree:: 34 | :maxdepth: 1 35 | 36 | programs 37 | an interactive session 38 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | name='teafiles', 5 | version='0.7.4', 6 | author='discretelogics', 7 | author_email='pythonapi@discretelogics.com', 8 | url='http://discretelogics.com', 9 | download_url='http://pypi.python.org/packages/source/t/teafiles', 10 | description='Time Series storage in flat files.', 11 | long_description=open('README.txt').read(), 12 | 13 | packages=["teafiles"], 14 | #package_dir={'': 'teafiles'}, 15 | py_modules=['examples'], 16 | 17 | keywords='timeseries time series analysis event processing teatime simulation finance', 18 | license='Creative Commons Attribution-Noncommercial-Share Alike license', 19 | classifiers= [ 20 | # 'Development Status :: 3 - Alpha' // pypi chokes on that 21 | 'Intended Audience :: Developers', 22 | 'Intended Audience :: Science/Research', 23 | 'License :: OSI Approved', 24 | 'License :: OSI Approved :: GNU General Public License (GPL)', 25 | 'Operating System :: MacOS', 26 | 'Operating System :: Microsoft :: Windows', 27 | 'Operating System :: POSIX', 28 | 'Operating System :: Unix', 29 | 'Programming Language :: Python', 30 | 'Topic :: Scientific/Engineering', 31 | 'Topic :: Software Development' 32 | 33 | ] 34 | ) 35 | -------------------------------------------------------------------------------- /test/test_timeutils.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011 discretelogics 2 | # 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | ''' pytest tests ''' 17 | 18 | import datetime 19 | 20 | from teafiles.clockwise import * 21 | 22 | 23 | def test_fromticks(): 24 | dt = DateTime(ticks=500) 25 | assert dt.ticks == 500 26 | 27 | dt = DateTime(ticks=500.4) 28 | assert dt.ticks == 500 29 | 30 | 31 | def test_equality(): 32 | assert DateTime(1970, 1, 3) == DateTime(1970, 1, 3) 33 | assert DateTime(2001, 4, 5) == DateTime(2001, 4, 5) 34 | 35 | 36 | def test_duration_ctor(): 37 | d = Duration(hours=1) 38 | assert d.ticks == 60 * 60 * 1000 39 | 40 | d = Duration(hours=1.5) 41 | assert d.ticks == 60 * 60 * 1000 * 1.5 42 | 43 | 44 | def test_duration_repr(): 45 | d = Duration(hours=1) 46 | assert d.__repr__() == "0 days 01:00:00:000" 47 | d = Duration(hours=1.5) 48 | assert d.__repr__() == "0 days 01:30:00:000" 49 | 50 | 51 | if __name__ == '__main__': 52 | test_duration_repr() 53 | test_duration_ctor() 54 | test_fromticks() 55 | test_equality() 56 | -------------------------------------------------------------------------------- /test/test_examples.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011 discretelogics 2 | # 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | ''' pytest tests ''' 17 | 18 | import unittest 19 | import os 20 | import tempfile 21 | 22 | import examples 23 | from teafiles import * 24 | 25 | 26 | def gettempfilename(): 27 | return tempfile.mktemp(".tea") 28 | 29 | 30 | def test_createticks(): 31 | filename = gettempfilename() 32 | examples.createticks(filename, 10) 33 | assert os.path.exists(filename) 34 | assert os.path.getsize(filename) > 10 * 24 35 | with TeaFile.openread(filename) as tf: 36 | assert tf.itemcount == 10 37 | 38 | 39 | def test_analyzeticks(): 40 | filename = gettempfilename() 41 | examples.createticks(filename, 10) 42 | examples.analyzeticks(filename) 43 | 44 | 45 | def test_sumprices(): 46 | filename = gettempfilename() 47 | examples.createticks(filename, 10) 48 | examples.sumprices(filename) 49 | 50 | 51 | def test_printsnapshot(): 52 | filename = gettempfilename() 53 | examples.createticks(filename, 10) 54 | TeaFile.printsnapshot(filename) 55 | 56 | 57 | if __name__ == '__main__': 58 | pass 59 | #test_createticks() 60 | #test_createticks() 61 | #test_analyzeticks() 62 | #test_sumprices() 63 | #test_printsnapshot() 64 | -------------------------------------------------------------------------------- /TeaFiles.pyproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | 2.0 6 | {f7fdc9f6-8a85-47f6-85dc-b48380a978ca} 7 | . 8 | teafiles\teafile.py 9 | . 10 | . 11 | . 12 | TeaFiles 13 | PythonApplication1 14 | False 15 | 2af0f10d-7135-4994-9156-5d01c9c11b7e 16 | Standard Python launcher 17 | 18 | 19 | False 20 | 21 | 22 | 2.7 23 | 24 | 25 | true 26 | false 27 | 28 | 29 | true 30 | false 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /stopwatch.py: -------------------------------------------------------------------------------- 1 | ''' from https://gist.github.com/1123871 2 | modifications applied 3 | ''' 4 | 5 | import time 6 | 7 | class Stopwatch(object): 8 | '''A stopwatch utility for timing execution that can be used as a regular 9 | object or as a context manager. 10 | 11 | NOTE: This should not be used an accurate benchmark of Python code, but a 12 | way to check how much time has elapsed between actions. And this does not 13 | account for changes or blips in the system clock. 14 | 15 | Instance attributes: 16 | start_time -- timestamp when the timer started 17 | stop_time -- timestamp when the timer stopped 18 | 19 | As a regular object: 20 | 21 | >>> stopwatch = Stopwatch() 22 | >>> stopwatch.start() 23 | >>> time.sleep(1) 24 | >>> 1 <= stopwatch.time_elapsed <= 2 25 | True 26 | >>> time.sleep(1) 27 | >>> stopwatch.stop() 28 | >>> 2 <= stopwatch.total_run_time 29 | True 30 | 31 | As a context manager: 32 | 33 | >>> with Stopwatch() as stopwatch: 34 | ... time.sleep(1) 35 | ... print repr(1 <= stopwatch.time_elapsed <= 2) 36 | ... time.sleep(1) 37 | True 38 | >>> 2 <= stopwatch.total_run_time 39 | True 40 | ''' 41 | 42 | def __init__(self): 43 | '''Initialize a new `Stopwatch`, but do not start timing.''' 44 | self.start_time = None 45 | self.stop_time = None 46 | 47 | def start(self): 48 | '''Start timing.''' 49 | self.start_time = time.time() 50 | 51 | def stop(self): 52 | '''Stop timing.''' 53 | self.stop_time = time.time() 54 | 55 | @property 56 | def time_elapsed(self): 57 | '''Return the number of seconds that have elapsed since this 58 | `Stopwatch` started timing. 59 | 60 | This is used for checking how much time has elapsed while the timer is 61 | still running. 62 | ''' 63 | assert not self.stop_time, \ 64 | "Can't check `time_elapsed` on an ended `Stopwatch`." 65 | return time.time() - self.start_time 66 | 67 | @property 68 | def total_run_time(self): 69 | '''Return the number of seconds that elapsed from when this `Stopwatch` 70 | started to when it ended. 71 | ''' 72 | return self.stop_time - self.start_time 73 | 74 | def __enter__(self): 75 | '''Start timing and return this `Stopwatch` instance.''' 76 | self.start() 77 | return self 78 | 79 | def __exit__(self, type_, value, traceback): 80 | '''Stop timing. 81 | 82 | If there was an exception inside the `with` block, re-raise it. 83 | 84 | >>> with Stopwatch() as stopwatch: 85 | ... raise Exception 86 | Traceback (most recent call last): 87 | ... 88 | Exception 89 | ''' 90 | self.stop() 91 | print("execution time: " + str(self.total_run_time) + " seconds") 92 | if type_: 93 | raise Exception(type_, value, traceback) 94 | -------------------------------------------------------------------------------- /doc/dtheme/static/pygments.css: -------------------------------------------------------------------------------- 1 | .highlight .hll { background-color: #ffffcc } 2 | .highlight { background: #eeffcc; } 3 | .highlight .c { color: #408090; font-style: italic } /* Comment */ 4 | .highlight .err { border: 1px solid #FF0000 } /* Error */ 5 | .highlight .k { color: #007020; font-weight: bold } /* Keyword */ 6 | .highlight .o { color: #666666 } /* Operator */ 7 | .highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ 8 | .highlight .cp { color: #007020 } /* Comment.Preproc */ 9 | .highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ 10 | .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ 11 | .highlight .gd { color: #A00000 } /* Generic.Deleted */ 12 | .highlight .ge { font-style: italic } /* Generic.Emph */ 13 | .highlight .gr { color: #FF0000 } /* Generic.Error */ 14 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 15 | .highlight .gi { color: #00A000 } /* Generic.Inserted */ 16 | .highlight .go { color: #303030 } /* Generic.Output */ 17 | .highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 18 | .highlight .gs { font-weight: bold } /* Generic.Strong */ 19 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 20 | .highlight .gt { color: #0040D0 } /* Generic.Traceback */ 21 | .highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 22 | .highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 23 | .highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 24 | .highlight .kp { color: #007020 } /* Keyword.Pseudo */ 25 | .highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 26 | .highlight .kt { color: #902000 } /* Keyword.Type */ 27 | .highlight .m { color: #208050 } /* Literal.Number */ 28 | .highlight .s { color: #4070a0 } /* Literal.String */ 29 | .highlight .na { color: #4070a0 } /* Name.Attribute */ 30 | .highlight .nb { color: #007020 } /* Name.Builtin */ 31 | .highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 32 | .highlight .no { color: #60add5 } /* Name.Constant */ 33 | .highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 34 | .highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 35 | .highlight .ne { color: #007020 } /* Name.Exception */ 36 | .highlight .nf { color: #06287e } /* Name.Function */ 37 | .highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ 38 | .highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 39 | .highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ 40 | .highlight .nv { color: #bb60d5 } /* Name.Variable */ 41 | .highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ 42 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */ 43 | .highlight .mf { color: #208050 } /* Literal.Number.Float */ 44 | .highlight .mh { color: #208050 } /* Literal.Number.Hex */ 45 | .highlight .mi { color: #208050 } /* Literal.Number.Integer */ 46 | .highlight .mo { color: #208050 } /* Literal.Number.Oct */ 47 | .highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ 48 | .highlight .sc { color: #4070a0 } /* Literal.String.Char */ 49 | .highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 50 | .highlight .s2 { color: #4070a0 } /* Literal.String.Double */ 51 | .highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 52 | .highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ 53 | .highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 54 | .highlight .sx { color: #c65d09 } /* Literal.String.Other */ 55 | .highlight .sr { color: #235388 } /* Literal.String.Regex */ 56 | .highlight .s1 { color: #4070a0 } /* Literal.String.Single */ 57 | .highlight .ss { color: #517918 } /* Literal.String.Symbol */ 58 | .highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ 59 | .highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ 60 | .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ 61 | .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ 62 | .highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ -------------------------------------------------------------------------------- /doc/interactive.rst: -------------------------------------------------------------------------------- 1 | teafiles.TeaFile in interactive use 2 | =================================== 3 | 4 | A sample interactive session using TeaFiles api for Python. :: 5 | 6 | >>> from teafiles import * 7 | >>> with TeaFile.create('acme.tea', 'Time Price Volume', 'qdq', 'acme prices', {'exchange': 'nyse', 'decimals': 2}) as tf: 8 | ... for t in range(DateTime(2000), DateTime(2001), Duration(weeks=1)): 9 | ... import random 10 | ... tf.write(t, random.random() * 100, random.randint(1, 10000)) 11 | ... 12 | >>> TeaFile.printsnapshot('acme.tea') 13 | TeaFile('acme.tea') 53 items 14 | 15 | ItemDescription 16 | Name: TPV 17 | Size: 24 18 | Fields: 19 | [Time Type: Int64 Offset: 0 IsTime:1 IsEventTime:1, 20 | Price Type: Double Offset: 8 IsTime:0 IsEventTime:0, 21 | Volume Type: Int64 Offset:16 IsTime:0 IsEventTime:0] 22 | 23 | ContentDescription 24 | acme prices 25 | 26 | NameValues 27 | {u'decimals': 2, u'exchange': u'nyse'} 28 | 29 | TimeScale 30 | Epoch: 719162 31 | Ticks per Day: 86400000 32 | Wellknown Scale: Java 33 | 34 | Items 35 | TPV(Time=2000-01-01 00:00:00:000, Price=37.579128977028674, Volume=8047) 36 | TPV(Time=2000-01-08 00:00:00:000, Price=10.618929589509186, Volume=232) 37 | TPV(Time=2000-01-15 00:00:00:000, Price=73.08506970525428, Volume=1711) 38 | TPV(Time=2000-01-22 00:00:00:000, Price=73.7749103916519, Volume=4397) 39 | TPV(Time=2000-01-29 00:00:00:000, Price=10.323610234110403, Volume=3376) 40 | >>> tf = TeaFile.openread('acme.tea') 41 | >>> from pprint import pprint 42 | >>> pprint(list(tf.items(50))) 43 | [TPV(Time=2000-12-16 00:00:00:000, Price=62.654885677497305, Volume=8097), 44 | TPV(Time=2000-12-23 00:00:00:000, Price=28.267048873450907, Volume=9226), 45 | TPV(Time=2000-12-30 00:00:00:000, Price=34.48653142780683, Volume=7790)] 46 | >>> pprint(list(tf.items(0, 10))) 47 | [TPV(Time=2000-01-01 00:00:00:000, Price=37.579128977028674, Volume=8047), 48 | TPV(Time=2000-01-08 00:00:00:000, Price=10.618929589509186, Volume=232), 49 | TPV(Time=2000-01-15 00:00:00:000, Price=73.08506970525428, Volume=1711), 50 | TPV(Time=2000-01-22 00:00:00:000, Price=73.7749103916519, Volume=4397), 51 | TPV(Time=2000-01-29 00:00:00:000, Price=10.323610234110403, Volume=3376), 52 | TPV(Time=2000-02-05 00:00:00:000, Price=5.253065316994842, Volume=3213), 53 | TPV(Time=2000-02-12 00:00:00:000, Price=77.30547456663894, Volume=1740), 54 | TPV(Time=2000-02-19 00:00:00:000, Price=70.41257838854919, Volume=3450), 55 | TPV(Time=2000-02-26 00:00:00:000, Price=26.454005280919326, Volume=8699), 56 | TPV(Time=2000-03-04 00:00:00:000, Price=18.6317489603937, Volume=8166)] 57 | 58 | >>> def showdecimals(): 59 | ... for filename in os.listdir('.'): 60 | ... with TeaFile.openread(filename) as tf: 61 | ... nvs = tf.description.namevalues 62 | ... print('{} {} {}'.format(filename, nvs.get('Decimals'), nvs.get('DisplayName'))) 63 | ... 64 | >>> showdecimals() 65 | AA.day.tea 2 Alcoa, Inc. 66 | AA.tick.tea 2 Alcoa, Inc. 67 | AXP.day.tea 2 American Express Co. 68 | BA.day.tea 2 Boeing Co. (The) 69 | BAC.day.tea 2 Bank of America Corp. 70 | CAT.day.tea 2 Caterpillar Inc. 71 | CSCO.day.tea 2 Cisco Systems, Inc. 72 | CVX.day.tea 2 Chevron Corporation 73 | DD.day.tea 2 Du Pont (E.I.) de Nemours & Co 74 | DIS.day.tea 2 WALT DISNEY-DISNEY C 75 | GE.day.tea 2 General Electric Co 76 | HD.day.tea 2 HOME DEPOT INC 77 | HPQ.day.tea 2 Hewlett-Packard Co 78 | IBM.day.tea 2 International Business Machines Corp. 79 | INTC.day.tea 2 Intel Corporation 80 | JNJ.day.tea 2 Johnson & Johnson 81 | JPM.day.tea 2 JPMorgan Chase & Co. 82 | KFT.day.tea 2 Kraft Foods, Inc. 83 | KO.day.tea 2 Coca-Cola Co (The) 84 | MCD.day.tea 2 McDonald's Corp 85 | MMM.day.tea 2 3M Co 86 | MRK.day.tea 2 Merck & Co., Inc 87 | MSFT.day.tea 2 Microsoft Corporation 88 | PFE.day.tea 2 PFIZER INC 89 | PG.day.tea 2 Procter & Gamble Co. 90 | T.day.tea 2 AT&T Inc 91 | TRV.day.tea 2 Travelers Companies Inc (The) 92 | UTX.day.tea 2 United Technologies Corp. 93 | VZ.day.tea 2 Verizon Communications Inc 94 | WMT.day.tea 2 Wal-Mart Stores, Inc. 95 | XOM.day.tea 2 Exxon Mobil Corp. 96 | >>> -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | TeaFiles.Py - Time Series Storage in Files 2 | ========================================== 3 | 4 | Use TeaFiles.Py to create, read and write files holding time series data. 5 | 6 | 7 | In Use 8 | ------ 9 | 10 | >>> tf = TeaFile.create("acme.tea", "Time Price Volume", "qdq", "ACME at NYSE", {"decimals": 2, "url": "acme.com" }) 11 | >>> tf.write(DateTime(2011, 3, 4, 9, 0), 45.11, 4500) 12 | >>> tf.write(DateTime(2011, 3, 4, 10, 0), 46.33, 1100) 13 | >>> tf.close() 14 | 15 | >>> tf = TeaFile.openread("acme.tea") 16 | >>> tf.read() 17 | TPV(Time=2011-03-04 09:00:00:000, Price=45.11, Volume=4500) 18 | >>> tf.read() 19 | TPV(Time=2011-03-04 10:00:00:000, Price=46.33, Volume=1100) 20 | >>> tf.read() 21 | >>> tf.close() 22 | 23 | 24 | Exchange Time Series across Applications / Operating Systems 25 | ------------------------------------------------------------- 26 | 27 | TeaFiles have a simple file layout, they contain raw binary data after a header that optionally holds metadata. 28 | APis abstract the header layout and make file creation as simple as 29 | 30 | >>> tf = TeaFile.create("acme.tea", "Time Price Volume", "qdq", "ACME at NYSE", {"decimals": 2, "url": "acme.com" }) 31 | 32 | At the moment, APIs exist for 33 | 34 | - C++, 35 | - C#, 36 | - Python (this one) 37 | - other APis are planned and in draft phase (R, Octave/Matlab) 38 | 39 | TeaFiles can be access from any operating system, like 40 | 41 | - Linux / Unix 42 | - Mac OS 43 | - Windows 44 | 45 | 46 | Python API Examples 47 | ------------------- 48 | - programs http://www.discretelogics.com/doc/teafiles.py/examples.html 49 | - interactive http://www.discretelogics.com/doc/teafiles.py/interactive.html 50 | - examples.py (in the package source) 51 | 52 | 53 | TeaFiles 54 | -------- 55 | Find more about TeaFiles at http://www.discretelogics.com/teafiles 56 | 57 | 58 | Documentation 59 | ------------- 60 | http://www.discretelogics.com/doc/teafiles.py/ 61 | 62 | 63 | Scope of the Python API 64 | ----------------------- 65 | The Python API makes TeaFiles accessible everywhere. It just needs a python installation on any OS to inspect the description and data 66 | of a TeaFile: 67 | 68 | 69 | >>> # Show the decimals and displayname for all files in a folder: 70 | ... 71 | >>> def showdecimals(): 72 | ... for filename in os.listdir('.'): 73 | ... with TeaFile.openread(filename) as tf: 74 | ... nvs = tf.description.namevalues 75 | ... print('{} {} {}'.format(filename, nvs.get('Decimals'), nvs.get('DisplayName'))) 76 | ... 77 | >>> showdecimals() 78 | AA.day.tea 2 Alcoa, Inc. 79 | AA.tick.tea 2 Alcoa, Inc. 80 | AXP.day.tea 2 American Express Co. 81 | ... 82 | 83 | Data download from web services for instance is a good fit. See the examples.py file in the package source for a Yahoo finance download function in about 30 lines. 84 | 85 | 86 | Limitations 87 | ----------- 88 | When it comes to high performance processing of very large time series files, this API is currently not as fast as the C++ and C# APIs. 89 | There are numerous ways to improve this if necessary, but no current plans at discretelogics to do so. Use of other languages/APIs is recommended. 90 | If you intend to make this Python API faster contact us we should be able to identify points of potential speed enhancements. 91 | 92 | 93 | Installation 94 | ------------ 95 | 96 | The package is hosted on PyPi, so installation goes by 97 | 98 | **$ pip install teafiles** 99 | 100 | package source with examples.py at https://github.com/discretelogics/TeaFiles.Py 101 | 102 | Tests 103 | ----- 104 | Run the unit tests from the package root by 105 | 106 | $ python -m pytest .\test 107 | 108 | 109 | Python 2.7 / 3.7 110 | ---------------- 111 | Package tested under CPython 2.7. 112 | 113 | A Python 3.7 version is here: https://github.com/nikhgupta/TeaFiles.Py 114 | 115 | Author 116 | ------ 117 | This API brought to you by discretelogics, company specialicing in time series analysis and event processing. 118 | http://www.discretelogics.com 119 | 120 | Version 0.7 121 | ----------- 122 | The current version is reasonably tested by doctests and some pytests. Better test coverage with unit tests (currently pytest is used) is desirable. 123 | 124 | open points towards version 1.0 125 | - pytest coverage 126 | - cleaner test runs, cleanup test files 127 | optional 128 | - enhance performance after measuring it in python 3 (struct module could play a crucial role, so results might differ considerably) 129 | 130 | License 131 | ------- 132 | This package is released under the MIT LICENSE. 133 | 134 | 135 | Feedback 136 | -------- 137 | Welcome at: office@discretelogics.com 138 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | Time Series Peristence 4 | ====================== 5 | This Python package provides Time Series storage in flat files according to the **TeaFile** file format. 6 | 7 | 8 | In Use 9 | ====== 10 | 11 | >>> tf = TeaFile.create("acme.tea", "Time Price Volume", "qdq", "ACME at NYSE", {"decimals": 2, "url": "www.acme.com" }) 12 | >>> tf.write(DateTime(2011, 3, 4, 9, 0), 45.11, 4500) 13 | >>> tf.write(DateTime(2011, 3, 4, 10, 0), 46.33, 1100) 14 | >>> tf.close() 15 | 16 | >>> tf = TeaFile.openread("acme.tea") 17 | >>> tf.read() 18 | TPV(Time=2011-03-04 09:00:00:000, Price=45.11, Volume=4500) 19 | >>> tf.read() 20 | TPV(Time=2011-03-04 10:00:00:000, Price=46.33, Volume=1100) 21 | >>> tf.read() 22 | >>> tf.close() 23 | 24 | 25 | Exchange Time Series between Apps / OS 26 | ====================================== 27 | You can create, read and write TeaFiles with 28 | 29 | - R, 30 | - C++, 31 | - C# or 32 | - other applications 33 | 34 | on 35 | 36 | - Linux, Unix, 37 | - Mac OS 38 | - Windows 39 | 40 | 41 | Python API Examples 42 | =================== 43 | - programs http://discretelogics.com/PythonAPI/examples.html 44 | - interactive http://discretelogics.com/PythonAPI/interactive.html 45 | - examples.py (available in the package source) 46 | 47 | 48 | TeaFiles 49 | ======== 50 | TeaFiles are a very **simple**, yet highly **efficient**, way to store time series data 51 | providing data exchange between programs written in C++, C# or applications like R, Octave, 52 | Matlab, running on Linux, Unix, Mac OS X or Windows. 53 | 54 | - **Binary** data composed from elementary data types **signed and unsigned integers, double and float** in IEEE 754 format is prefixed by a **header** holding a description of the item structure and the content. 55 | - Data can be directly accessed via **memory mapping**. 56 | - TeaFiles are **self describing**: Containing a description of the item structure they relieve opaqueness of straight binary files. Knowing that a file is a TeaFile is enough to access its data. 57 | 58 | link to spec http://tbd 59 | 60 | 61 | Scope of the Python API 62 | ======================= 63 | The Python API makes TeaFiles accessible everywhere. It just needs a python installation on any OS to inspect the description and data 64 | of a TeaFile: 65 | 66 | 67 | >>> # Show the decimals and displayname for all files in a folder: 68 | ... 69 | >>> def showdecimals(): 70 | ... for filename in os.listdir('.'): 71 | ... with TeaFile.openread(filename) as tf: 72 | ... nvs = tf.description.namevalues 73 | ... print('{} {} {}'.format(filename, nvs.get('Decimals'), nvs.get('DisplayName'))) 74 | ... 75 | >>> showdecimals() 76 | AA.day.tea 2 Alcoa, Inc. 77 | AA.tick.tea 2 Alcoa, Inc. 78 | AXP.day.tea 2 American Express Co. 79 | ... 80 | 81 | Data download from web services is also a good fit. See the examples.py file in the package source for a Yahoo finance download function in about 30 lines. 82 | 83 | 84 | Limitations 85 | =========== 86 | When it comes to high performance processing of very large time series files, this API is currently not as fast as the C++ and C# APIs (Numbers coming soon on http://tbd). There are numerous ways to improve this if necessary, but no current plans at discretelogics to do so. Using the other languages/APIs is recommended. If you wish the Python API to be faster or want to work on that contact us. 87 | 88 | 89 | Installation 90 | ============ 91 | 92 | **$ pip install teafiles** 93 | 94 | package source with examples.py at http://bitbucket.org/discretelogics/teafiles.py 95 | 96 | Tests 97 | ===== 98 | Run the unit tests from the package root by 99 | 100 | $ python -m pytest .\test 101 | 102 | 103 | Python 2.7 / 3.2 104 | ================ 105 | Package tested under CPython 2.7. 106 | Python 3.2 planned 107 | 108 | 109 | Author 110 | ====== 111 | This API brought to you by discretelogics, company specialicing in time series analysis and event processing. 112 | http://tbd 113 | 114 | 115 | Version 0.7 116 | =========== 117 | The current version is reasonably tested by doctests and some pytests. Better test coverage with unit tests (currently pytest is used) 118 | is desirable. 119 | 120 | tbd towards version 1.0 121 | - enhance pytest coverage 122 | - consider api feedack 123 | - cleaner test runs, cleanup test files 124 | 125 | optional 126 | - enhance performance after measuring it in python 3 (maybe struct module plays a minor role there) 127 | 128 | 129 | License 130 | ======= 131 | This package is released under the MIT LICENSE. 132 | 133 | 134 | Feedback 135 | ======== 136 | Welcome at: pythonapi@discretelogics.com 137 | -------------------------------------------------------------------------------- /doc/clockwise.rst: -------------------------------------------------------------------------------- 1 | 2 | :mod:`clockwise` module 3 | ----------------------- 4 | 5 | .. image:: clockwise.jpg 6 | 7 | right! date and time. 8 | (module name after the movie clockwise, starring monthy python john cleese) 9 | 10 | .. autoclass:: teafiles.clockwise.DateTime 11 | :members: parse, ticks, date, totimeandms 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | Supported operations: 16 | 17 | +-----------------------------------------------+----------------------------------------------------------+ 18 | | Operation | Result | 19 | +===============================================+==========================================================+ 20 | | ``t1 == t2`` | t1.ticks == t2.ticks | 21 | +-----------------------------------------------+----------------------------------------------------------+ 22 | | ``t1 != t2`` | t1.ticks != t2.ticks | 23 | +-----------------------------------------------+----------------------------------------------------------+ 24 | | ``t1 > t2`` | t1.ticks > t2.ticks | 25 | +-----------------------------------------------+----------------------------------------------------------+ 26 | | ``t1 >= t2`` | t1.ticks >= t2.ticks | 27 | +-----------------------------------------------+----------------------------------------------------------+ 28 | | ``t1 < t2`` | t1.ticks < t2.ticks | 29 | +-----------------------------------------------+----------------------------------------------------------+ 30 | | ``t1 <= t2`` | t1.ticks <= t2.ticks | 31 | +-----------------------------------------------+----------------------------------------------------------+ 32 | | ``d2 = d1 + duration`` | adds duration, returning a new DateTime | 33 | +-----------------------------------------------+----------------------------------------------------------+ 34 | | ``t += Duration`` | adds duration to t assigning new DateTime to t | 35 | +-----------------------------------------------+----------------------------------------------------------+ 36 | | ``int(DateTime)`` | returns ticks | 37 | +-----------------------------------------------+----------------------------------------------------------+ 38 | | ``math.trunc(DateTime)`` | returns itself | 39 | +-----------------------------------------------+----------------------------------------------------------+ 40 | 41 | 42 | 43 | 44 | .. autoclass:: teafiles.clockwise.Duration 45 | :members: ticks, totimedelta 46 | :undoc-members: 47 | :show-inheritance: 48 | 49 | Supported Operations: 50 | 51 | +-----------------------------------------------+--------------------------------------------------+ 52 | | Operation | Result | 53 | +===============================================+==================================================+ 54 | | ``d1 == d2`` | d1.ticks == d2.ticks | 55 | +-----------------------------------------------+--------------------------------------------------+ 56 | | ``d1 != d2`` | d1.ticks != d2.ticks | 57 | +-----------------------------------------------+--------------------------------------------------+ 58 | | ``d1 > d2`` | d1.ticks > d2.ticks | 59 | +-----------------------------------------------+--------------------------------------------------+ 60 | | ``d1 >= d2`` | d1.ticks >= d2.ticks | 61 | +-----------------------------------------------+--------------------------------------------------+ 62 | | ``d1 < d2`` | d1.ticks < d2.ticks | 63 | +-----------------------------------------------+--------------------------------------------------+ 64 | | ``d1 <= d2`` | d1.ticks <= d2.ticks | 65 | +-----------------------------------------------+--------------------------------------------------+ 66 | | ``d = d1 + 2`` | adds both durations, returning new Duration | 67 | +-----------------------------------------------+--------------------------------------------------+ 68 | | ``d += d2`` | adds d2 to d, assigning new Duration to d | 69 | +-----------------------------------------------+--------------------------------------------------+ 70 | | ``int(Duration)`` | returns ticks | 71 | +-----------------------------------------------+--------------------------------------------------+ 72 | | ``math.trunc(Duration)`` | returns itself | 73 | +-----------------------------------------------+--------------------------------------------------+ 74 | 75 | 76 | 77 | Utility functions 78 | ================= 79 | 80 | .. autofunction:: teafiles.clockwise.range(*args) 81 | .. autofunction:: teafiles.clockwise.rangen(startdate, stepduration, count) 82 | .. autofunction:: teafiles.clockwise.isdatetime(value) 83 | .. autofunction:: teafiles.clockwise.isduration(value) 84 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | 9 | echo %SPHINXBUILD% 10 | 11 | set BUILDDIR=_build 12 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 13 | set I18NSPHINXOPTS=%SPHINXOPTS% . 14 | if NOT "%PAPER%" == "" ( 15 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 16 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 17 | ) 18 | 19 | if "%1" == "" goto help 20 | 21 | if "%1" == "help" ( 22 | :help 23 | echo.Please use `make ^` where ^ is one of 24 | echo. html to make standalone HTML files 25 | echo. dirhtml to make HTML files named index.html in directories 26 | echo. singlehtml to make a single large HTML file 27 | echo. pickle to make pickle files 28 | echo. json to make JSON files 29 | echo. htmlhelp to make HTML files and a HTML help project 30 | echo. qthelp to make HTML files and a qthelp project 31 | echo. devhelp to make HTML files and a Devhelp project 32 | echo. epub to make an epub 33 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 34 | echo. text to make text files 35 | echo. man to make manual pages 36 | echo. texinfo to make Texinfo files 37 | echo. gettext to make PO message catalogs 38 | echo. changes to make an overview over all changed/added/deprecated items 39 | echo. linkcheck to check all external links for integrity 40 | echo. doctest to run all doctests embedded in the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | if "%1" == "html" ( 51 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 52 | if errorlevel 1 exit /b 1 53 | echo. 54 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 55 | goto end 56 | ) 57 | 58 | if "%1" == "dirhtml" ( 59 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 60 | if errorlevel 1 exit /b 1 61 | echo. 62 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 63 | goto end 64 | ) 65 | 66 | if "%1" == "singlehtml" ( 67 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 68 | if errorlevel 1 exit /b 1 69 | echo. 70 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 71 | goto end 72 | ) 73 | 74 | if "%1" == "pickle" ( 75 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 76 | if errorlevel 1 exit /b 1 77 | echo. 78 | echo.Build finished; now you can process the pickle files. 79 | goto end 80 | ) 81 | 82 | if "%1" == "json" ( 83 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 84 | if errorlevel 1 exit /b 1 85 | echo. 86 | echo.Build finished; now you can process the JSON files. 87 | goto end 88 | ) 89 | 90 | if "%1" == "htmlhelp" ( 91 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 92 | if errorlevel 1 exit /b 1 93 | echo. 94 | echo.Build finished; now you can run HTML Help Workshop with the ^ 95 | .hhp project file in %BUILDDIR%/htmlhelp. 96 | goto end 97 | ) 98 | 99 | if "%1" == "qthelp" ( 100 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 104 | .qhcp project file in %BUILDDIR%/qthelp, like this: 105 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\TeaFiles.qhcp 106 | echo.To view the help file: 107 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\TeaFiles.ghc 108 | goto end 109 | ) 110 | 111 | if "%1" == "devhelp" ( 112 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 113 | if errorlevel 1 exit /b 1 114 | echo. 115 | echo.Build finished. 116 | goto end 117 | ) 118 | 119 | if "%1" == "epub" ( 120 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 121 | if errorlevel 1 exit /b 1 122 | echo. 123 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 124 | goto end 125 | ) 126 | 127 | if "%1" == "latex" ( 128 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 129 | if errorlevel 1 exit /b 1 130 | echo. 131 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 132 | goto end 133 | ) 134 | 135 | if "%1" == "text" ( 136 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 137 | if errorlevel 1 exit /b 1 138 | echo. 139 | echo.Build finished. The text files are in %BUILDDIR%/text. 140 | goto end 141 | ) 142 | 143 | if "%1" == "man" ( 144 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 145 | if errorlevel 1 exit /b 1 146 | echo. 147 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 148 | goto end 149 | ) 150 | 151 | if "%1" == "texinfo" ( 152 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 153 | if errorlevel 1 exit /b 1 154 | echo. 155 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 156 | goto end 157 | ) 158 | 159 | if "%1" == "gettext" ( 160 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 161 | if errorlevel 1 exit /b 1 162 | echo. 163 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 164 | goto end 165 | ) 166 | 167 | if "%1" == "changes" ( 168 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 169 | if errorlevel 1 exit /b 1 170 | echo. 171 | echo.The overview file is in %BUILDDIR%/changes. 172 | goto end 173 | ) 174 | 175 | if "%1" == "linkcheck" ( 176 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 177 | if errorlevel 1 exit /b 1 178 | echo. 179 | echo.Link check complete; look for any errors in the above output ^ 180 | or in %BUILDDIR%/linkcheck/output.txt. 181 | goto end 182 | ) 183 | 184 | if "%1" == "doctest" ( 185 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Testing of doctests in the sources finished, look at the ^ 189 | results in %BUILDDIR%/doctest/output.txt. 190 | goto end 191 | ) 192 | 193 | :end 194 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/TeaFiles.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/TeaFiles.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/TeaFiles" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/TeaFiles" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /doc/dtheme/static/default.css: -------------------------------------------------------------------------------- 1 | /* 2 | * default.css_t 3 | * ~~~~~~~~~~~~~ 4 | * 5 | * Sphinx stylesheet -- default theme. 6 | * 7 | * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | @import url("basic.css"); 13 | 14 | /* -- page layout ----------------------------------------------------------- */ 15 | 16 | body { 17 | font-family: sans-serif; 18 | font-size: 100%; 19 | background-color: #11303d; 20 | color: #000; 21 | margin: 0; 22 | padding: 0; 23 | } 24 | 25 | div.document 26 | { 27 | background-color: #304B5A; 28 | } 29 | 30 | div.documentwrapper { 31 | float: left; 32 | width: 100%; 33 | } 34 | 35 | div.bodywrapper { 36 | margin: 0 0 0 230px; 37 | } 38 | 39 | div.body { 40 | background-color: #ffffff; 41 | color: #000000; 42 | padding: 0 20px 30px 20px; 43 | } 44 | 45 | div.footer { 46 | color: #ffffff; 47 | width: 100%; 48 | padding: 9px 0 9px 0; 49 | text-align: center; 50 | font-size: 75%; 51 | } 52 | 53 | div.footer a { 54 | color: #ffffff; 55 | text-decoration: underline; 56 | } 57 | 58 | div.related { 59 | background-color: #133f52; 60 | line-height: 30px; 61 | color: #ffffff; 62 | } 63 | 64 | div.related a { 65 | color: #ffffff; 66 | } 67 | 68 | div.sphinxsidebar { 69 | } 70 | 71 | div.sphinxsidebar h3 { 72 | font-family: 'Trebuchet MS', sans-serif; 73 | color: #ffffff; 74 | font-size: 1.4em; 75 | font-weight: normal; 76 | margin: 0; 77 | padding: 0; 78 | } 79 | 80 | div.sphinxsidebar h3 a { 81 | color: #ffffff; 82 | } 83 | 84 | div.sphinxsidebar h4 { 85 | font-family: 'Trebuchet MS', sans-serif; 86 | color: #ffffff; 87 | font-size: 1.3em; 88 | font-weight: normal; 89 | margin: 5px 0 0 0; 90 | padding: 0; 91 | } 92 | 93 | div.sphinxsidebar p { 94 | color: #ffffff; 95 | } 96 | 97 | div.sphinxsidebar p.topless { 98 | margin: 5px 10px 10px 10px; 99 | } 100 | 101 | div.sphinxsidebar ul { 102 | margin: 10px; 103 | padding: 0; 104 | color: #ffffff; 105 | } 106 | 107 | div.sphinxsidebar a { 108 | color: #98dbcc; 109 | } 110 | 111 | div.sphinxsidebar input { 112 | border: 1px solid #98dbcc; 113 | font-family: sans-serif; 114 | font-size: 1em; 115 | } 116 | 117 | 118 | 119 | /* -- hyperlink styles ------------------------------------------------------ */ 120 | 121 | a { 122 | color: #355f7c; 123 | text-decoration: none; 124 | } 125 | 126 | a:visited { 127 | color: #355f7c; 128 | text-decoration: none; 129 | } 130 | 131 | a:hover { 132 | text-decoration: underline; 133 | } 134 | 135 | 136 | 137 | /* -- body styles ----------------------------------------------------------- */ 138 | 139 | div.body h1, 140 | div.body h2, 141 | div.body h3, 142 | div.body h4, 143 | div.body h5, 144 | div.body h6 { 145 | font-family: 'Trebuchet MS', sans-serif; 146 | background-color: #f2f2f2; 147 | font-weight: normal; 148 | color: #963; 149 | __border-bottom: 1px solid #eee; 150 | margin: 20px -20px 10px 0px; 151 | } 152 | 153 | div.body h1 { margin-top: 0; 154 | margin-left:-20px; 155 | font-size: 200%; 156 | background-color: #f6f0d4; 157 | padding-left: 20px; 158 | } 159 | div.body h2 160 | { 161 | font-size: 160%; 162 | background-color: transparent; 163 | color: #333; 164 | border-bottom: dashed 1px #333; 165 | display: inline; 166 | } 167 | div.body h3 168 | { 169 | padding-left: 0.8em; 170 | font-size: 140%; 171 | background-color: transparent; 172 | color: #660066; 173 | border-bottom: dashed 1px #660066; 174 | display: inline; 175 | } 176 | div.body h4 { font-size: 120%; } 177 | div.body h5 { font-size: 110%; } 178 | div.body h6 { font-size: 100%; } 179 | 180 | a.headerlink { 181 | color: #c60f0f; 182 | font-size: 0.8em; 183 | padding: 0 4px 0 4px; 184 | text-decoration: none; 185 | } 186 | 187 | a.headerlink:hover { 188 | background-color: #c60f0f; 189 | color: white; 190 | } 191 | 192 | div.body p, div.body dd, div.body li { 193 | text-align: justify; 194 | line-height: 130%; 195 | } 196 | 197 | div.admonition p.admonition-title + p { 198 | display: inline; 199 | } 200 | 201 | div.admonition p { 202 | margin-bottom: 5px; 203 | } 204 | 205 | div.admonition pre { 206 | margin-bottom: 5px; 207 | } 208 | 209 | div.admonition ul, div.admonition ol { 210 | margin-bottom: 5px; 211 | } 212 | 213 | div.note { 214 | background-color: #eee; 215 | border: 1px solid #ccc; 216 | } 217 | 218 | div.seealso { 219 | background-color: #ffc; 220 | border: 1px solid #ff6; 221 | } 222 | 223 | div.topic { 224 | background-color: #eee; 225 | } 226 | 227 | div.warning { 228 | background-color: #ffe4e4; 229 | border: 1px solid #f66; 230 | } 231 | 232 | p.admonition-title { 233 | display: inline; 234 | } 235 | 236 | p.admonition-title:after { 237 | content: ":"; 238 | } 239 | 240 | pre { 241 | padding: 5px; 242 | background-color: #eeffcc; 243 | color: #333333; 244 | line-height: 120%; 245 | __border: 1px solid #ac9; 246 | border-left: none; 247 | border-right: none; 248 | } 249 | 250 | tt { 251 | background-color: #ecf0f3; 252 | padding: 0 1px 0 1px; 253 | font-size: 0.95em; 254 | } 255 | 256 | th { 257 | background-color: #ede; 258 | } 259 | 260 | .warning tt { 261 | background: #efc2c2; 262 | } 263 | 264 | .note tt { 265 | background: #d6d6d6; 266 | } 267 | 268 | .viewcode-back { 269 | font-family: sans-serif; 270 | } 271 | 272 | div.viewcode-block:target { 273 | background-color: #f4debf; 274 | border-top: 1px solid #ac9; 275 | border-bottom: 1px solid #ac9; 276 | } 277 | 278 | dl[class=method] > dt , 279 | dl[class=function] > dt , 280 | dl[class=staticmethod] > dt, 281 | dl[class=classmethod] > dt, 282 | dl[class=attribute] > dt 283 | { 284 | background-color: #F9D864; 285 | color: navy; 286 | display: inline; 287 | __border-bottom: dotted 1px #11303d; 288 | } 289 | 290 | dl > dt a 291 | { 292 | border: none; 293 | background-color: Orange; 294 | margin-left: -14px; 295 | } 296 | .class > dt 297 | { 298 | color: Maroon; 299 | __background-color: Lime; 300 | border-bottom: dotted 1px #11303d; 301 | margin-bottom: 1.0em; 302 | margin-top: 1.5em; 303 | } 304 | .class > dt 305 | { 306 | font-size: 135%; 307 | background-color: transparent; 308 | display: block; 309 | } 310 | 311 | table + .section 312 | { 313 | margin-top: 2em; 314 | } 315 | 316 | li > p 317 | { 318 | margin-bottom: 0; 319 | } 320 | 321 | li > blockquote 322 | { 323 | margin-top: 0; 324 | margin-bottom: 0; 325 | margin-left: 1em; 326 | } 327 | -------------------------------------------------------------------------------- /examples.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2011, DiscreteLogics (copyright@discretelogics.com) 2 | # 3 | # license: GNU LGPL 4 | # 5 | # This library is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU Lesser General Public 7 | # License as published by the Free Software Foundation; either 8 | # version 2.1 of the License, or (at your option) any later version. 9 | # 10 | # pylint: disable-msg=C0301 11 | # C0301: Line too long (104/80) - maybe trim docstrings later for terminal users 12 | 13 | ''' 14 | The examples below are commented in the package documentation. Here is the focus on the code only, 15 | so docstrings are omitted or kept essential below. 16 | ''' 17 | 18 | 19 | def teadir(): 20 | ''' this utility function lists all tea files in the current directory ''' 21 | import os 22 | print("{}:".format(os.getcwd())) 23 | for f in os.listdir('.'): 24 | if f.endswith(".tea"): 25 | print (f) 26 | 27 | 28 | import random 29 | from teafiles import * 30 | 31 | 32 | def createticks(filename, n, contentdescription=None, namevalues=None): 33 | ''' 34 | Create a TeaFile holding `n` items of random "Ticks" having fields Time, Price and Volume. 35 | ''' 36 | with TeaFile.create(filename, "Time Price Volume", "qdq", contentdescription, namevalues) as tf: 37 | for t in rangen(DateTime(2000, 1, 1), Duration(minutes=1), n): # increments n times by 1 minute 38 | r = random.random() 39 | tf.write(t, r * 100, int(r * 1000)) 40 | print(tf) 41 | 42 | 43 | def sumprices(filename): 44 | ''' 45 | Sums all prices in a teafile holding ticks. This function is used forbenchmarking. 46 | ''' 47 | with TeaFile.openread(filename) as tf: 48 | return sum(item.Price for item in tf.items()) 49 | 50 | 51 | def createsessions(filename, numberofsessions): 52 | 53 | def writedailyticks(teafile, day, isgoodday): 54 | ''' 55 | create a random series of ticks. if isgoodday is false, only 1% as much ticks will be written. 56 | ''' 57 | t = day + Duration(hours=9) # session begins at 09:00 58 | end = day + Duration(hours=17.5) # session ends at 17:30 59 | while t < end: 60 | if isgoodday or random.randint(0, 99) < 1: 61 | price = random.random() * 100 62 | tf.write(t, price, 10) 63 | t += Duration(seconds=15 + random.randint(0, 20)) 64 | 65 | with TeaFile.create(filename, "Time Price Volume", "qdq") as tf: 66 | ''' write a file with random tick, similar to ticks as they occur on a stock exchange in reality: 67 | for days we create ticks between 9:00 and 17:30. 10% of the days will 68 | create only 1% as much ticks than the other days. This simulates bad data 69 | ''' 70 | for day in rangen(DateTime(2000, 1, 1), Duration(days=1), numberofsessions): 71 | isgoodday = random.randint(1, 100) <= 90 72 | writedailyticks(tf, day, isgoodday) 73 | print(tf) 74 | 75 | 76 | def analyzeticks(filename, displayvalues=True): 77 | ''' analyze a teafile holding ticks. print elementary descriptive numbers ''' 78 | 79 | class _TradingSession: 80 | 81 | def __init__(self, begin): 82 | self.begin = begin 83 | self.end = self.begin + Duration(days=1) 84 | self.tickcount = 0 85 | 86 | def __repr__(self): 87 | return " ".join([str(self.begin), str(self.end), str(self.tickcount)]) 88 | 89 | with TeaFile.openread(filename) as tf: 90 | if tf.itemcount == 0: 91 | print("This file holds no items") 92 | 93 | tick = tf.read() 94 | session = _TradingSession(tick.Time.date) 95 | minprice = maxprice = tick.Price 96 | sessions = [session] 97 | for tick in tf.items(): 98 | if tick.Time > session.end: 99 | session = _TradingSession(tick.Time.date) 100 | sessions.append(session) 101 | session.tickcount += 1 102 | minprice = min(minprice, tick.Price) 103 | maxprice = max(maxprice, tick.Price) 104 | 105 | mintransactions = maxtransactions = session.tickcount 106 | for s in sessions: 107 | mintransactions = min(mintransactions, s.tickcount) 108 | maxtransactions = max(maxtransactions, s.tickcount) 109 | 110 | print("min price = {}".format(minprice)) 111 | print("max price = {}".format(maxprice)) 112 | print("min ticks per session = {}".format(mintransactions)) 113 | print("max ticks per session = {}".format(maxtransactions)) 114 | 115 | tickcounts = sorted([s.tickcount for s in sessions]) 116 | median = tickcounts[len(tickcounts) // 2] 117 | print("median = {}".format(median)) 118 | 119 | if displayvalues: 120 | minimumexpectedtickspersession = median / 2.0 121 | print("First 10 sessions:") 122 | for s in sessions[:15]: 123 | print("{} {}".format(s, "OK" if s.tickcount >= minimumexpectedtickspersession else "QUESTIONABLE")) 124 | 125 | 126 | def gethistoricalprices(symbol, filename, startyear, startmonth, startday, endyear, endmonth, endday): 127 | ''' fetch historical prices from Yahoo finance and store them in a file ''' 128 | from urllib import urlopen 129 | 130 | url = "http://ichart.yahoo.com/table.csv?s={0}&a={1:02}&b={2:02}&c={3:04}&d={4:02}&e={5:02}&f={6:04}&g=d&ignore=.csv" \ 131 | .format(symbol, startmonth - 1, startday, startyear, endmonth - 1, endday, endyear) 132 | response = urlopen(url) 133 | if response.code != 200: 134 | print(response.info()) 135 | raise Exception("download failed. http response code = " + response.code) 136 | headerline = response.readline().decode("utf8") 137 | print(headerline) 138 | 139 | # create the file to store the received values 140 | tf = TeaFile.create(filename, "Time Open High Low Close AdjClose Volume", "qdddddq", symbol) 141 | 142 | # values arrive in timely reversed order, so we store them in a list and ad them reversed to the file 143 | values = [] 144 | for line in response: 145 | line = line.decode("utf8") 146 | line = line.strip() 147 | parts = line.split(",") 148 | 149 | t = Time.parse(parts[0], "%Y-%m-%d") 150 | open_ = float(parts[1]) 151 | high = float(parts[2]) 152 | low = float(parts[3]) 153 | close = float(parts[4]) 154 | volume = int(parts[5]) 155 | adjclose = float(parts[6]) 156 | 157 | values.append((t, open_, high, low, close, adjclose, volume)) 158 | 159 | for item in reversed(values): 160 | tf.write(*item) # pylint:disable-msg=W0142 161 | 162 | if __name__ == '__main__': 163 | # test zone 164 | createsessions("lab.tea", 5) 165 | -------------------------------------------------------------------------------- /test/test_teafile.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011 discretelogics 2 | # 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | ''' pytest tests ''' 17 | 18 | import tempfile 19 | import os 20 | import sys 21 | from teafiles import * 22 | 23 | def setup_module(m): 24 | module = m 25 | module.testfiles = [] 26 | 27 | def gettempfilename(): 28 | filename = tempfile.mktemp(".tea") 29 | module = sys.modules[__name__] 30 | module.testfiles.append(filename) 31 | return filename 32 | 33 | 34 | def teardown_module(module): 35 | for filename in module.testfiles: 36 | os.remove(filename) 37 | 38 | 39 | def gettempfilename(): 40 | return tempfile.mktemp(".tea") 41 | 42 | 43 | def test_create_and_read(): 44 | filename = gettempfilename() 45 | with TeaFile.create(filename, "A B C", "qqq") as tf: 46 | tf.write(1, 2, 3) 47 | tf.write(21, 22, 23) 48 | with TeaFile.openread(filename) as tf: 49 | assert tf.itemcount == 2 50 | item = tf.read() 51 | assert item 52 | assert item.A == 1 53 | assert item.B == 2 54 | assert item.C == 3 55 | item = tf.read() 56 | assert item 57 | assert item.A == 21 58 | assert item.B == 22 59 | assert item.C == 23 60 | assert not tf.read() 61 | 62 | 63 | def test_itemarea_is_set_after_create(): 64 | filename = gettempfilename() 65 | with TeaFile.create(filename, "A B C", "qqq") as tf: 66 | assert tf.description.itemdescription.itemname == "ABC" 67 | assert tf.itemareastart > 32 # file holds item description, so item area starts after core header 68 | assert tf._getitemareaend() > 0 69 | assert tf._getitemareaend() == tf.itemareastart 70 | assert tf._getitemareasize() == 0 71 | assert tf.itemcount == 0 72 | 73 | 74 | def test_itemarea_is_set_after_open(): 75 | filename = gettempfilename() 76 | with TeaFile.create(filename, "A B C") as tf: 77 | pass 78 | with TeaFile.openread(filename) as tf: 79 | assert tf.description.itemdescription.itemname == "ABC" 80 | assert tf.itemareastart > 0 81 | assert tf._getitemareaend() > 0 82 | assert tf._getitemareaend() == tf.itemareastart 83 | assert tf._getitemareasize() == 0 84 | assert tf.itemcount == 0 85 | 86 | 87 | def test_itemcount(): 88 | filename = gettempfilename() 89 | with TeaFile.create(filename, "A B C", "qqq") as tf: 90 | assert tf.itemcount == 0 91 | for i in range(1, 11): 92 | tf.write(i, 22, 33) 93 | tf.flush() # required, to update the filesize correctly 94 | assert tf.itemcount == i 95 | with TeaFile.openread(filename) as tf: 96 | assert tf.itemcount == 10 97 | 98 | 99 | def test_seekitem(): 100 | filename = gettempfilename() 101 | with TeaFile.create(filename, "A B C", "qqq") as tf: 102 | for i in range(10): 103 | tf.write(i, 10 * i, 100 * i) 104 | with TeaFile.openread(filename) as tf: 105 | item = tf.read() 106 | assert len(item) == 3 107 | assert item[0] == 0 108 | assert item[1] == 0 109 | tf.seekitem(5) 110 | item = tf.read() 111 | assert item[0] == 5 112 | assert item[1] == 50 113 | tf.seekitem(2) 114 | item = tf.read() 115 | assert item[0] == 2 116 | assert item[1] == 20 117 | 118 | 119 | def test_seekitem2(): 120 | filename = gettempfilename() 121 | with TeaFile.create(filename, "A B", "qq") as tf: 122 | tf.write(1, 1) 123 | tf.write(2, 2) 124 | tf.seekitem(0) 125 | tf.write(3, 3) 126 | with TeaFile.openread(filename) as tf: 127 | assert tf.read() == (3, 3) 128 | assert tf.read() == (2, 2) 129 | with TeaFile.openwrite(filename) as tf: 130 | tf.seekend() 131 | tf.write(4, 4) 132 | tf.write(5, 5) 133 | with TeaFile.openread(filename) as tf: 134 | assert tf.read() == (3, 3) 135 | assert tf.read() == (2, 2) 136 | assert tf.read() == (4, 4) 137 | assert tf.read() == (5, 5) 138 | 139 | 140 | def test_openwrite(): 141 | filename = gettempfilename() 142 | with TeaFile.create(filename, "A B", "qq") as tf: 143 | for i in range(3): 144 | tf.write(i, i * 10) 145 | with TeaFile.openwrite(filename) as tf: 146 | tf.write(77, 770) 147 | with TeaFile.openread(filename) as tf: 148 | assert tf.read()[0] == 0 149 | assert tf.read()[0] == 1 150 | assert tf.read()[0] == 2 151 | assert tf.read()[0] == 77 152 | with TeaFile.openwrite(filename) as tf: 153 | tf.seekitem(0) 154 | tf.write(44, 440) 155 | with TeaFile.openread(filename) as tf: 156 | assert tf.read()[0] == 44 157 | assert tf.read()[0] == 1 158 | assert tf.read()[0] == 2 159 | assert tf.read()[0] == 77 160 | 161 | 162 | def test_printsnapshot(): 163 | filename = gettempfilename() 164 | with TeaFile.create(filename, "A B C", "qqq", \ 165 | "here goes the content description!", \ 166 | {"data source": "Bluum", "decimals": 4}) as tf: 167 | tf.write(1, 2, 3) 168 | tf.write(2, 2, 3) 169 | TeaFile.printsnapshot(filename) 170 | 171 | 172 | def test_namevalues(): 173 | filename = gettempfilename() 174 | with TeaFile.create(filename, "A B C", "qqq", "mycontent", {"a": 1, "bb": 22}) as tf: 175 | pass 176 | with TeaFile.openread(filename) as tf: 177 | nvs = tf.description.namevalues 178 | assert nvs["a"] == 1 179 | assert nvs["bb"] == 22 180 | assert len(nvs) == 2 181 | 182 | 183 | def test_decimals(): 184 | filename = gettempfilename() 185 | with TeaFile.create(filename, "A B C", "qqq", "mycontent", {"decimals": 3, "bb": 22}) as tf: 186 | pass 187 | with TeaFile.openread(filename) as tf: 188 | nvs = tf.description.namevalues 189 | assert tf.decimals == 3 190 | 191 | 192 | def test_items_iteration(): 193 | filename = gettempfilename() 194 | with TeaFile.create(filename, "A B C", "qqq") as tf: 195 | tf.write(1, 2, 3) 196 | tf.write(21, 22, 23) 197 | with TeaFile.openread(filename) as tf: 198 | iter = tf.items() 199 | assert len([item for item in tf.items()]) == 2 200 | 201 | 202 | if __name__ == '__main__': 203 | pass 204 | # to be run with pytest. for debugging purposes, tests may be executed here. 205 | 206 | #import sys 207 | #module = sys.modules[__name__] 208 | #setup_module(module) 209 | # 210 | #test_items_iteration() 211 | # 212 | #teardown_module(module) 213 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | 8 | # pygtk.require(). 9 | #init-hook= 10 | 11 | # Profiled execution. 12 | profile=no 13 | 14 | # Add files or directories to the blacklist. They should be base names, not 15 | 16 | # paths. 17 | ignore=CVS 18 | 19 | # Pickle collected data for later comparisons. 20 | #persistent=yes 21 | 22 | # List of plugins (as comma separated values of python modules names) to load, 23 | 24 | # usually to register additional checkers. 25 | load-plugins= 26 | 27 | 28 | [MESSAGES CONTROL] 29 | 30 | # Enable the message, report, category or checker with the given id(s). You can 31 | 32 | # either give multiple identifier separated by comma (,) or put this option 33 | 34 | # multiple time. 35 | #enable= 36 | 37 | # Disable the message, report, category or checker with the given id(s). You 38 | # can either give multiple identifier separated by comma (,) or put this option 39 | # multiple time (only on the command line, not in the configuration file where 40 | # it should appear only once). 41 | # C0301 Line too long - maybe trim docstrings later for terminal usage 42 | disable=C0301 43 | 44 | 45 | [REPORTS] 46 | 47 | # Set the output format. Available formats are text, parseable, colorized, msvs 48 | 49 | # (visual studio) and html 50 | output-format=text 51 | 52 | # Include message's id in output 53 | include-ids=no 54 | 55 | # Put messages in a separate file for each module / package specified on the 56 | 57 | # command line instead of printing them on stdout. Reports (if any) will be 58 | 59 | # written in a file name "pylint_global.[txt|html]". 60 | files-output=no 61 | 62 | # Tells whether to display a full report or only the messages 63 | reports=yes 64 | 65 | # Python expression which should return a note less than 10 (10 is the highest 66 | 67 | # note). You have access to the variables errors warning, statement which 68 | 69 | # respectively contain the number of errors / warnings messages and the total 70 | 71 | # number of statements analyzed. This is used by the global evaluation report 72 | 73 | # (RP0004). 74 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 75 | 76 | # Add a comment according to your evaluation note. This is used by the global 77 | 78 | # evaluation report (RP0004). 79 | comment=no 80 | 81 | 82 | [BASIC] 83 | 84 | # Required attributes for module, separated by a comma 85 | required-attributes= 86 | 87 | # List of builtins function names that should not be used, separated by a comma 88 | bad-functions=map,filter,apply,input 89 | 90 | # Regular expression which should only match correct module names 91 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 92 | 93 | # Regular expression which should only match correct module level names 94 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 95 | 96 | # Regular expression which should only match correct class names 97 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 98 | 99 | # Regular expression which should only match correct function names 100 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 101 | 102 | # Regular expression which should only match correct method names 103 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 104 | 105 | # Regular expression which should only match correct instance attribute names 106 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 107 | 108 | # Regular expression which should only match correct argument names 109 | argument-rgx=[a-z_][a-z0-9_]{0,30}$ 110 | 111 | # Regular expression which should only match correct variable names 112 | variable-rgx=[a-z_][a-z0-9_]{0,30}$ 113 | 114 | # Regular expression which should only match correct list comprehension / 115 | 116 | # generator expression variable names 117 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 118 | 119 | # Good variable names which should always be accepted, separated by a comma 120 | good-names=i,j,k,ex,Run,_ 121 | 122 | # Bad variable names which should always be refused, separated by a comma 123 | bad-names=foo,bar,baz,toto,tutu,tata 124 | 125 | # Regular expression which should only match functions or classes name which do 126 | 127 | # not require a docstring 128 | no-docstring-rgx=(__.*__|_.*) 129 | 130 | 131 | [FORMAT] 132 | 133 | # Maximum number of characters on a single line. 134 | max-line-length=80 135 | 136 | # Maximum number of lines in a module 137 | max-module-lines=2000 138 | 139 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 140 | 141 | # tab). 142 | indent-string=' ' 143 | 144 | 145 | [MISCELLANEOUS] 146 | 147 | # List of note tags to take in consideration, separated by a comma. 148 | notes=FIXME,XXX,TODO 149 | 150 | 151 | [SIMILARITIES] 152 | 153 | # Minimum lines number of a similarity. 154 | min-similarity-lines=4 155 | 156 | # Ignore comments when computing similarities. 157 | ignore-comments=yes 158 | 159 | # Ignore docstrings when computing similarities. 160 | ignore-docstrings=yes 161 | 162 | 163 | [TYPECHECK] 164 | 165 | # Tells whether missing members accessed in mixin class should be ignored. A 166 | 167 | # mixin class is detected if its name ends with "mixin" (case insensitive). 168 | ignore-mixin-members=yes 169 | 170 | # List of classes names for which member attributes should not be checked 171 | 172 | # (useful for classes with attributes dynamically set). 173 | ignored-classes=SQLObject 174 | 175 | # When zope mode is activated, add a predefined set of Zope acquired attributes 176 | 177 | # to generated-members. 178 | zope=no 179 | 180 | # List of members which are set dynamically and missed by pylint inference 181 | 182 | # system, and so shouldn't trigger E0201 when accessed. Python regular 183 | 184 | # expressions are accepted. 185 | generated-members=REQUEST,acl_users,aq_parent 186 | 187 | 188 | [VARIABLES] 189 | 190 | # Tells whether we should check for unused import in __init__ files. 191 | init-import=no 192 | 193 | # A regular expression matching the beginning of the name of dummy variables 194 | 195 | # (i.e. not used). 196 | dummy-variables-rgx=_|dummy 197 | 198 | # List of additional names supposed to be defined in builtins. Remember that 199 | 200 | # you should avoid to define new builtins when possible. 201 | additional-builtins= 202 | 203 | 204 | [CLASSES] 205 | 206 | # List of interface methods to ignore, separated by a comma. This is used for 207 | 208 | # instance to not check methods defines in Zope's Interface base class. 209 | ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by 210 | 211 | # List of method names used to declare (i.e. assign) instance attributes. 212 | defining-attr-methods=__init__,__new__,setUp 213 | 214 | # List of valid names for the first argument in a class method. 215 | valid-classmethod-first-arg=cls 216 | 217 | 218 | [DESIGN] 219 | 220 | # Maximum number of arguments for function / method 221 | max-args=10 222 | 223 | # Argument names that match this expression will be ignored. Default to name 224 | 225 | # with leading underscore 226 | ignored-argument-names=_.* 227 | 228 | # Maximum number of locals for function / method body 229 | max-locals=15 230 | 231 | # Maximum number of return / yield for function / method body 232 | max-returns=6 233 | 234 | # Maximum number of branch for function / method body 235 | max-branchs=12 236 | 237 | # Maximum number of statements in function / method body 238 | max-statements=50 239 | 240 | # Maximum number of parents for a class (see R0901). 241 | max-parents=7 242 | 243 | # Maximum number of attributes for a class (see R0902). 244 | max-attributes=7 245 | 246 | # Minimum number of public methods for a class (see R0903). 247 | min-public-methods=2 248 | 249 | # Maximum number of public methods for a class (see R0904). 250 | max-public-methods=20 251 | 252 | 253 | [IMPORTS] 254 | 255 | # Deprecated modules which should not be used, separated by a comma 256 | deprecated-modules=regsub,string,TERMIOS,Bastion,rexec 257 | 258 | # Create a graph of every (i.e. internal and external) dependencies in the 259 | 260 | # given file (report RP0402 must not be disabled) 261 | import-graph= 262 | 263 | # Create a graph of external dependencies in the given file (report RP0402 must 264 | 265 | # not be disabled) 266 | ext-import-graph= 267 | 268 | # Create a graph of internal dependencies in the given file (report RP0402 must 269 | 270 | # not be disabled) 271 | int-import-graph= 272 | 273 | 274 | [EXCEPTIONS] 275 | 276 | # Exceptions that will emit a warning when being caught. Defaults to 277 | 278 | # "Exception" 279 | overgeneral-exceptions=Exception 280 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # TeaFiles documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Jan 25 15:47:01 2012. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.insert(0, os.path.abspath('E:\dev\Reps\TeaTime.Py')) 20 | sys.path.insert(0, os.path.abspath('E:\dev\Reps\TeaTime.Py\teafiles')) 21 | sys.path.insert(0, os.path.abspath('E:\dev\Reps\TeaTime.Py\doc')) 22 | 23 | # -- General configuration ----------------------------------------------------- 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be extensions 29 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 30 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage'] 31 | 32 | # Add any paths that contain templates here, relative to this directory. 33 | templates_path = ['_templates'] 34 | 35 | # The suffix of source filenames. 36 | source_suffix = '.rst' 37 | 38 | # The encoding of source files. 39 | #source_encoding = 'utf-8-sig' 40 | 41 | # The master toctree document. 42 | master_doc = 'index' 43 | 44 | # General information about the project. 45 | project = u'TeaFiles' 46 | copyright = u'2012, discretelogics' 47 | 48 | # The version info for the project you're documenting, acts as replacement for 49 | # |version| and |release|, also used in various other places throughout the 50 | # built documents. 51 | # 52 | # The short X.Y version. 53 | version = '0.7.4' 54 | # The full version, including alpha/beta/rc tags. 55 | release = '0.7.4' 56 | 57 | # The language for content autogenerated by Sphinx. Refer to documentation 58 | # for a list of supported languages. 59 | #language = None 60 | 61 | # There are two options for replacing |today|: either, you set today to some 62 | # non-false value, then it is used: 63 | #today = '' 64 | # Else, today_fmt is used as the format for a strftime call. 65 | #today_fmt = '%B %d, %Y' 66 | 67 | # List of patterns, relative to source directory, that match files and 68 | # directories to ignore when looking for source files. 69 | exclude_patterns = ['_build'] 70 | 71 | # The reST default role (used for this markup: `text`) to use for all documents. 72 | #default_role = None 73 | 74 | # If true, '()' will be appended to :func: etc. cross-reference text. 75 | #add_function_parentheses = True 76 | 77 | # If true, the current module name will be prepended to all description 78 | # unit titles (such as .. function::). 79 | #add_module_names = True 80 | 81 | # If true, sectionauthor and moduleauthor directives will be shown in the 82 | # output. They are ignored by default. 83 | #show_authors = False 84 | 85 | # The name of the Pygments (syntax highlighting) style to use. 86 | pygments_style = 'sphinx' 87 | 88 | # A list of ignored prefixes for module index sorting. 89 | #modindex_common_prefix = [] 90 | 91 | 92 | # -- Options for HTML output --------------------------------------------------- 93 | 94 | # The theme to use for HTML and HTML Help pages. See the documentation for 95 | # a list of builtin themes. 96 | html_theme = 'dtheme' 97 | 98 | # Theme options are theme-specific and customize the look and feel of a theme 99 | # further. For a list of options available for each theme, see the 100 | # documentation. 101 | #html_theme_options = {} 102 | 103 | # Add any paths that contain custom themes here, relative to this directory. 104 | html_theme_path = ['.'] 105 | 106 | # The name for this set of Sphinx documents. If None, it defaults to 107 | # " v documentation". 108 | #html_title = None 109 | 110 | # A shorter title for the navigation bar. Default is the same as html_title. 111 | #html_short_title = None 112 | 113 | # The name of an image file (relative to this directory) to place at the top 114 | # of the sidebar. 115 | #html_logo = None 116 | 117 | # The name of an image file (within the static path) to use as favicon of the 118 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 119 | # pixels large. 120 | #html_favicon = None 121 | 122 | # Add any paths that contain custom static files (such as style sheets) here, 123 | # relative to this directory. They are copied after the builtin static files, 124 | # so a file named "default.css" will overwrite the builtin "default.css". 125 | html_static_path = ['_static'] 126 | 127 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 128 | # using the given strftime format. 129 | #html_last_updated_fmt = '%b %d, %Y' 130 | 131 | # If true, SmartyPants will be used to convert quotes and dashes to 132 | # typographically correct entities. 133 | #html_use_smartypants = True 134 | 135 | # Custom sidebar templates, maps document names to template names. 136 | #html_sidebars = {} 137 | 138 | # Additional templates that should be rendered to pages, maps page names to 139 | # template names. 140 | #html_additional_pages = {} 141 | 142 | # If false, no module index is generated. 143 | #html_domain_indices = True 144 | 145 | # If false, no index is generated. 146 | #html_use_index = True 147 | 148 | # If true, the index is split into individual pages for each letter. 149 | #html_split_index = False 150 | 151 | # If true, links to the reST sources are added to the pages. 152 | #html_show_sourcelink = True 153 | 154 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 155 | #html_show_sphinx = True 156 | 157 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 158 | #html_show_copyright = True 159 | 160 | # If true, an OpenSearch description file will be output, and all pages will 161 | # contain a tag referring to it. The value of this option must be the 162 | # base URL from which the finished HTML is served. 163 | #html_use_opensearch = '' 164 | 165 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 166 | #html_file_suffix = None 167 | 168 | # Output file base name for HTML help builder. 169 | htmlhelp_basename = 'TeaFilesdoc' 170 | 171 | 172 | # -- Options for LaTeX output -------------------------------------------------- 173 | 174 | latex_elements = { 175 | # The paper size ('letterpaper' or 'a4paper'). 176 | #'papersize': 'letterpaper', 177 | 178 | # The font size ('10pt', '11pt' or '12pt'). 179 | #'pointsize': '10pt', 180 | 181 | # Additional stuff for the LaTeX preamble. 182 | #'preamble': '', 183 | } 184 | 185 | # Grouping the document tree into LaTeX files. List of tuples 186 | # (source start file, target name, title, author, documentclass [howto/manual]). 187 | latex_documents = [ 188 | ('index', 'TeaFiles.tex', u'TeaFiles Documentation', u'discretelogics', 'manual'), 189 | ] 190 | 191 | # The name of an image file (relative to this directory) to place at the top of 192 | # the title page. 193 | #latex_logo = None 194 | 195 | # For "manual" documents, if this is true, then toplevel headings are parts, 196 | # not chapters. 197 | #latex_use_parts = False 198 | 199 | # If true, show page references after internal links. 200 | #latex_show_pagerefs = False 201 | 202 | # If true, show URL addresses after external links. 203 | #latex_show_urls = False 204 | 205 | # Documents to append as an appendix to all manuals. 206 | #latex_appendices = [] 207 | 208 | # If false, no module index is generated. 209 | #latex_domain_indices = True 210 | 211 | 212 | # -- Options for manual page output -------------------------------------------- 213 | 214 | # One entry per manual page. List of tuples 215 | # (source start file, name, description, authors, manual section). 216 | man_pages = [ 217 | ('index', 'teafiles', u'TeaFiles Documentation', [u'discretelogics'], 1) 218 | ] 219 | 220 | # If true, show URL addresses after external links. 221 | #man_show_urls = False 222 | 223 | 224 | # -- Options for Texinfo output ------------------------------------------------ 225 | 226 | # Grouping the document tree into Texinfo files. List of tuples 227 | # (source start file, target name, title, author, 228 | # dir menu entry, description, category) 229 | texinfo_documents = [ 230 | ('index', 'TeaFiles', u'TeaFiles Documentation', u'discretelogics', 'TeaFiles', 'One line description of project.', 231 | 'Miscellaneous'), 232 | ] 233 | 234 | # Documents to append as an appendix to all manuals. 235 | #texinfo_appendices = [] 236 | 237 | # If false, no module index is generated. 238 | #texinfo_domain_indices = True 239 | 240 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 241 | #texinfo_show_urls = 'footnote' 242 | -------------------------------------------------------------------------------- /doc/examples.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | The examples below demonstrate how to accomplish common tasks with the Python TeaFiles API. They are mostly also available 5 | using APIs for C++ and C# to which they might be compared. The package source code holds the python code in the file 6 | examples.py in the root directory. 7 | 8 | 9 | createticks 10 | ^^^^^^^^^^^ 11 | 12 | Shows how to create a TeaFile and write items into it. Useful for test file creation. :: 13 | 14 | import random 15 | from teafiles import * 16 | 17 | def createticks(filename, n, contentdescription=None, namevalues=None): 18 | with TeaFile.create(filename, "Time Price Volume", "qdq", contentdescription, namevalues) as tf: 19 | for t in rangen(DateTime(2000, 1, 1), Duration(minutes=1), n): # increments n times by 1 minute 20 | r = random.random() 21 | tf.write(t, r * 100, int(r * 1000)) 22 | 23 | 24 | This function accepts a filename, the number of random ticks to write into the file and optionally 25 | a content description and name-value collection. Details about the code 26 | 27 | :: 28 | 29 | with TeaFile.create(filename, "Time Price", "qdq", contentdescription, namevalues) as tf: 30 | 31 | Beside the filename, we pass :meth:`teafiles.TeaFile.create` the structure of our items as a list of *field names*, here 32 | **Time**, **Price** and **Volume**. 33 | 34 | We then specify the respective *field type*, by passing the format string **"qd"**, that holds one character for each field. 35 | The characters correspond to those used by the `struct module`_, so for here we defined thefields to have types int64, double and int64. 36 | 37 | Content **description** is a simple string wile the name-value dictionary can hold for more structured descriptions. We pass both 38 | to store in the file. 39 | 40 | Finally the **with** clause: TeaFile implements the context manager methods __enter__ and __exit__, guaranteeing that any open 41 | file will be closed when the with block is left, even if an exception occurs. It is highly recommended to always use TeaFile 42 | instances that way. Manual calls to :meth:`TeaFile.close()` should only be required in interactive sessions. 43 | 44 | :: 45 | 46 | tf.write(t, r * 100) 47 | 48 | writes an item into the file. Note that this write method is dynamically created in the call to create and receives argument names "Time" and 49 | "Price" which is in particular convenient when using this api from an interactive commandline. 50 | 51 | 52 | 53 | sumprices 54 | ^^^^^^^^^ 55 | 56 | Shows how to open a file and access its items. :: 57 | 58 | def sumprices(filename): 59 | with TeaFile.openread(filename) as tf: 60 | return sum(item.Price for item in tf.items()) 61 | 62 | 63 | Again we use a factory method to open the file, this time we use :meth:`teafiles.TeaFile.openread` to open the file in read only mode. The 64 | with statement ensure that the file is closed at the end. 65 | 66 | :: 67 | 68 | return sum(item.Price for item in tf.items()) 69 | 70 | We iterate over the items using items() which returns a generator (`generatoriterator`_) over all items which are named tuples like: 71 | 72 | :: 73 | 74 | >>> from pprint import pprint 75 | >>> tf = TeaFile.openread("lab.tea") 76 | >>> pprint(list(tf.items())) 77 | [TPV(Time=1970-01-01 00:00:00:000, Price=61.195988356060546, Volume=611), 78 | TPV(Time=1970-01-01 00:01:00:000, Price=56.2617855404688, Volume=562), 79 | TPV(Time=1970-01-01 00:02:00:000, Price=0.3000058069409506, Volume=3) 80 | ... 81 | ] 82 | 83 | 84 | scenario: data cleansing 85 | ^^^^^^^^^^^^^^^^^^^^^^^^ 86 | 87 | ACME company receives data from stock exchanges holding prices and volums of transactions. Each transaction is stored as a 88 | "Tick" (Time, Price, Volume), one file per financial instrument. Clara is responsible to monitor the quality of the data 89 | stored in the files. It happens that ticks come in with wrong numbers, like a price of 0, or a price obviously out of 90 | reasonable range. It even happens that ticks are missing. Each day, Clara has to check 3000 files and make sure that 91 | their data is reasonable. To quickly identify files with potentially erroneous data, she runs the following data check algorithm 92 | on each file: 93 | 94 | * for each day count the number of ticks 95 | * compute the median of all daily tick-counts 96 | * mark all those days that have a tick-count < 1/2 * median as suspect 97 | 98 | * In addition these descriptive measures are reported for each file: 99 | 100 | * minimum and maximum price 101 | * minimum and maximum tick count per session 102 | * median of tick count 103 | 104 | The python functions analyzeticks() implements the algorithm above. Before we can run it we need some test data, which we create 105 | by another function, called createsession(). 106 | 107 | createsessions 108 | ^^^^^^^^^^^^^^ 109 | 110 | For each day were trading is open at the exchange, we define a time interval from 09:00 to 17:30 were transactions occur and 111 | create randomized values and spaced ticks. The code is straight forard: :: 112 | 113 | def createsessions(filename, numberofsessions): 114 | 115 | def writedailyticks(teafile, day, isgoodday): 116 | ''' create a random series of ticks. if isgoodday is false, only 1% as much ticks will be written. ''' 117 | t = day + Duration(hours=9) # session begins at 09:00 118 | end = day + Duration(hours=17.5) # session ends at 17:30 119 | while t < end: 120 | if isgoodday or random.randint(0, 99) < 1: 121 | price = random.random() * 100 122 | tf.write(t, price, 10) 123 | t += Duration(seconds = 15 + random.randint(0, 20)) 124 | 125 | with TeaFile.create(filename, "Time Price Volume", "qdq") as tf: 126 | ''' write a file with random tick, similar to ticks as they occur on a stock exchange in reality: 127 | for days we create ticks between 9:00 and 17:30. 10% of the days will 128 | create only 1% as much ticks than the other days. This simulates bad data ''' 129 | for day in rangen(DateTime(2000, 1, 1), Duration(days=1), numberofsessions): 130 | isgoodday = random.randint(1, 100) <= 90 131 | writedailyticks(tf, day, isgoodday) 132 | print(tf) 133 | 134 | 135 | analyzeticks 136 | ^^^^^^^^^^^^ 137 | 138 | To detect days with unexpected low tick count value, we count ticks for each day. To this purpose we define a class 139 | TradingSession and create one for each day. Counting the ticks is straightfoward then: :: 140 | 141 | 142 | def analyzeticks(filename, displayvalues=True): 143 | 144 | class _TradingSession: 145 | def __init__(self, begin): 146 | self.begin = begin 147 | self.end = self.begin + Duration(days=1) 148 | self.tickcount = 0 149 | 150 | def __repr__(self): 151 | return " ".join([str(self.begin), str(self.end), str(self.tickcount)]) 152 | 153 | with TeaFile.openread(filename) as tf: 154 | 155 | if tf.itemcount == 0: 156 | print("This file holds no items") 157 | 158 | tick = tf.read() 159 | session = _TradingSession(tick.Time.date) 160 | minprice = maxprice = tick.Price 161 | sessions = [session] 162 | for tick in tf.items(): 163 | if tick.Time > session.end: 164 | session = _TradingSession(tick.Time.date) 165 | sessions.append(session) 166 | session.tickcount += 1 167 | minprice = min(minprice, tick.Price) 168 | maxprice = max(maxprice, tick.Price) 169 | 170 | mintransactions = maxtransactions = session.tickcount 171 | for s in sessions: 172 | mintransactions = min(mintransactions, s.tickcount) 173 | maxtransactions = max(maxtransactions, s.tickcount) 174 | 175 | print("min price = {}".format(minprice)) 176 | print("max price = {}".format(maxprice)) 177 | print("min ticks per session = {}".format(mintransactions)) 178 | print("max ticks per session = {}".format(maxtransactions)) 179 | 180 | tickcounts = sorted([s.tickcount for s in sessions]) 181 | median = tickcounts[len(tickcounts) // 2] 182 | print("median = {}".format(median)) 183 | 184 | if displayvalues: 185 | minimumexpectedtickspersession = median / 2.0 186 | print("First 10 sessions:") 187 | for s in sessions[:15]: 188 | print("{} {}".format(s, "OK" if s.tickcount >= minimumexpectedtickspersession else "QUESTIONABLE")) 189 | 190 | download data from Yahoo Finance 191 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 192 | 193 | Python is particularly well suited to access data like available at Yahoo finance, as changes in such public api 194 | can be quickly adopted by modifications of the script.:: 195 | 196 | from urllib import urlopen 197 | def gethistoricalprices(symbol, filename, startyear, startmonth, startday, endyear, endmonth, endday): 198 | 199 | url = "http://ichart.yahoo.com/table.csv?s={0}&a={1:02}&b={2:02}&c={3:04}&d={4:02}&e={5:02}&f={6:04}&g=d&ignore=.csv" \ 200 | .format(symbol, startmonth - 1, startday, startyear, endmonth - 1, endday, endyear) 201 | response = urlopen(url) 202 | 203 | # values arrive in timely reversed order, so we store them in a list and add them reversed to the file 204 | values = [] 205 | for line in response: 206 | line = line.decode("utf8") 207 | line = line.strip() 208 | parts = line.split(",") 209 | 210 | t = Time.parse(parts[0], "%Y-%m-%d") 211 | open_ = float(parts[1]) 212 | high = float(parts[2]) 213 | low = float(parts[3]) 214 | close = float(parts[4]) 215 | volume = int(parts[5]) 216 | adjclose = float(parts[6]) 217 | 218 | values.append((t, open_, high, low, close, adjclose, volume)) 219 | 220 | # create the file to store the received values 221 | with TeaFile.create(filename, "Time Open High Low Close AdjClose Volume", "qdddddq", symbol, {"decimals:", 2}) as tf: 222 | for item in reversed(values): 223 | tf.write(*item) 224 | 225 | 226 | .. _struct module: http://docs.python.org/library/struct.html?highlight=struct#format-characters 227 | .. _generatoriterator: http://docs.python.org/reference/simple_stmts.html#the-yield-statement -------------------------------------------------------------------------------- /doc/dtheme/static/basic.css: -------------------------------------------------------------------------------- 1 | /* 2 | * basic.css 3 | * ~~~~~~~~~ 4 | * 5 | * Sphinx stylesheet -- basic theme. 6 | * 7 | * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | /* -- main layout ----------------------------------------------------------- */ 13 | 14 | div.clearer { 15 | clear: both; 16 | } 17 | 18 | /* -- relbar ---------------------------------------------------------------- */ 19 | 20 | div.related { 21 | width: 100%; 22 | font-size: 90%; 23 | } 24 | 25 | div.related h3 { 26 | display: none; 27 | } 28 | 29 | div.related ul { 30 | margin: 0; 31 | padding: 0 0 0 10px; 32 | list-style: none; 33 | } 34 | 35 | div.related li { 36 | display: inline; 37 | } 38 | 39 | div.related li.right { 40 | float: right; 41 | margin-right: 5px; 42 | } 43 | 44 | /* -- sidebar --------------------------------------------------------------- */ 45 | 46 | div.sphinxsidebarwrapper { 47 | padding: 10px 5px 0 10px; 48 | } 49 | 50 | div.sphinxsidebar { 51 | float: left; 52 | width: 230px; 53 | margin-left: -100%; 54 | font-size: 90%; 55 | } 56 | 57 | div.sphinxsidebar ul { 58 | list-style: none; 59 | } 60 | 61 | div.sphinxsidebar ul ul, 62 | div.sphinxsidebar ul.want-points { 63 | margin-left: 20px; 64 | list-style: square; 65 | } 66 | 67 | div.sphinxsidebar ul ul { 68 | margin-top: 0; 69 | margin-bottom: 0; 70 | } 71 | 72 | div.sphinxsidebar form { 73 | margin-top: 10px; 74 | } 75 | 76 | div.sphinxsidebar input { 77 | border: 1px solid #98dbcc; 78 | font-family: sans-serif; 79 | font-size: 1em; 80 | } 81 | 82 | div.sphinxsidebar input[type="text"] { 83 | width: 170px; 84 | } 85 | 86 | div.sphinxsidebar input[type="submit"] { 87 | width: 30px; 88 | } 89 | 90 | img { 91 | border: 0; 92 | } 93 | 94 | /* -- search page ----------------------------------------------------------- */ 95 | 96 | ul.search { 97 | margin: 10px 0 0 20px; 98 | padding: 0; 99 | } 100 | 101 | ul.search li { 102 | padding: 5px 0 5px 20px; 103 | background-image: url(file.png); 104 | background-repeat: no-repeat; 105 | background-position: 0 7px; 106 | } 107 | 108 | ul.search li a { 109 | font-weight: bold; 110 | } 111 | 112 | ul.search li div.context { 113 | color: #888; 114 | margin: 2px 0 0 30px; 115 | text-align: left; 116 | } 117 | 118 | ul.keywordmatches li.goodmatch a { 119 | font-weight: bold; 120 | } 121 | 122 | /* -- index page ------------------------------------------------------------ */ 123 | 124 | table.contentstable { 125 | width: 90%; 126 | } 127 | 128 | table.contentstable p.biglink { 129 | line-height: 150%; 130 | } 131 | 132 | a.biglink { 133 | font-size: 1.3em; 134 | } 135 | 136 | span.linkdescr { 137 | font-style: italic; 138 | padding-top: 5px; 139 | font-size: 90%; 140 | } 141 | 142 | /* -- general index --------------------------------------------------------- */ 143 | 144 | table.indextable { 145 | width: 100%; 146 | } 147 | 148 | table.indextable td { 149 | text-align: left; 150 | vertical-align: top; 151 | } 152 | 153 | table.indextable dl, table.indextable dd { 154 | margin-top: 0; 155 | margin-bottom: 0; 156 | } 157 | 158 | table.indextable tr.pcap { 159 | height: 10px; 160 | } 161 | 162 | table.indextable tr.cap { 163 | margin-top: 10px; 164 | background-color: #f2f2f2; 165 | } 166 | 167 | img.toggler { 168 | margin-right: 3px; 169 | margin-top: 3px; 170 | cursor: pointer; 171 | } 172 | 173 | div.modindex-jumpbox { 174 | border-top: 1px solid #ddd; 175 | border-bottom: 1px solid #ddd; 176 | margin: 1em 0 1em 0; 177 | padding: 0.4em; 178 | } 179 | 180 | div.genindex-jumpbox { 181 | border-top: 1px solid #ddd; 182 | border-bottom: 1px solid #ddd; 183 | margin: 1em 0 1em 0; 184 | padding: 0.4em; 185 | } 186 | 187 | /* -- general body styles --------------------------------------------------- */ 188 | 189 | a.headerlink { 190 | visibility: hidden; 191 | } 192 | 193 | h1:hover > a.headerlink, 194 | h2:hover > a.headerlink, 195 | h3:hover > a.headerlink, 196 | h4:hover > a.headerlink, 197 | h5:hover > a.headerlink, 198 | h6:hover > a.headerlink, 199 | dt:hover > a.headerlink { 200 | visibility: visible; 201 | } 202 | 203 | div.body p.caption { 204 | text-align: inherit; 205 | } 206 | 207 | div.body td { 208 | text-align: left; 209 | } 210 | 211 | .field-list ul { 212 | padding-left: 1em; 213 | } 214 | 215 | .first { 216 | margin-top: 0 !important; 217 | } 218 | 219 | p.rubric { 220 | margin-top: 30px; 221 | font-weight: bold; 222 | } 223 | 224 | img.align-left, .figure.align-left, object.align-left { 225 | clear: left; 226 | float: left; 227 | margin-right: 1em; 228 | } 229 | 230 | img.align-right, .figure.align-right, object.align-right { 231 | clear: right; 232 | float: right; 233 | margin-left: 1em; 234 | } 235 | 236 | img.align-center, .figure.align-center, object.align-center { 237 | display: block; 238 | margin-left: auto; 239 | margin-right: auto; 240 | } 241 | 242 | .align-left { 243 | text-align: left; 244 | } 245 | 246 | .align-center { 247 | text-align: center; 248 | } 249 | 250 | .align-right { 251 | text-align: right; 252 | } 253 | 254 | /* -- sidebars -------------------------------------------------------------- */ 255 | 256 | div.sidebar { 257 | margin: 0 0 0.5em 1em; 258 | border: 1px solid #ddb; 259 | padding: 7px 7px 0 7px; 260 | background-color: #ffe; 261 | width: 40%; 262 | float: right; 263 | } 264 | 265 | p.sidebar-title { 266 | font-weight: bold; 267 | } 268 | 269 | /* -- topics ---------------------------------------------------------------- */ 270 | 271 | div.topic { 272 | border: 1px solid #ccc; 273 | padding: 7px 7px 0 7px; 274 | margin: 10px 0 10px 0; 275 | } 276 | 277 | p.topic-title { 278 | font-size: 1.1em; 279 | font-weight: bold; 280 | margin-top: 10px; 281 | } 282 | 283 | /* -- admonitions ----------------------------------------------------------- */ 284 | 285 | div.admonition { 286 | margin-top: 10px; 287 | margin-bottom: 10px; 288 | padding: 7px; 289 | } 290 | 291 | div.admonition dt { 292 | font-weight: bold; 293 | } 294 | 295 | div.admonition dl { 296 | margin-bottom: 0; 297 | } 298 | 299 | p.admonition-title { 300 | margin: 0px 10px 5px 0px; 301 | font-weight: bold; 302 | } 303 | 304 | div.body p.centered { 305 | text-align: center; 306 | margin-top: 25px; 307 | } 308 | 309 | /* -- tables ---------------------------------------------------------------- */ 310 | 311 | table.docutils { 312 | border: 0; 313 | border-collapse: collapse; 314 | } 315 | 316 | table.docutils td, table.docutils th { 317 | padding: 1px 8px 1px 5px; 318 | border-top: 0; 319 | border-left: 0; 320 | border-right: 0; 321 | border-bottom: 1px solid #aaa; 322 | } 323 | 324 | table.field-list td, table.field-list th { 325 | border: 0 !important; 326 | } 327 | 328 | table.footnote td, table.footnote th { 329 | border: 0 !important; 330 | } 331 | 332 | th { 333 | text-align: left; 334 | padding-right: 5px; 335 | } 336 | 337 | table.citation { 338 | border-left: solid 1px gray; 339 | margin-left: 1px; 340 | } 341 | 342 | table.citation td { 343 | border-bottom: none; 344 | } 345 | 346 | /* -- other body styles ----------------------------------------------------- */ 347 | 348 | ol.arabic { 349 | list-style: decimal; 350 | } 351 | 352 | ol.loweralpha { 353 | list-style: lower-alpha; 354 | } 355 | 356 | ol.upperalpha { 357 | list-style: upper-alpha; 358 | } 359 | 360 | ol.lowerroman { 361 | list-style: lower-roman; 362 | } 363 | 364 | ol.upperroman { 365 | list-style: upper-roman; 366 | } 367 | 368 | dl { 369 | margin-bottom: 15px; 370 | } 371 | 372 | dd p { 373 | margin-top: 0px; 374 | } 375 | 376 | dd ul, dd table { 377 | margin-bottom: 10px; 378 | } 379 | 380 | dd { 381 | margin-top: 3px; 382 | margin-bottom: 10px; 383 | margin-left: 30px; 384 | } 385 | 386 | dt:target, .highlighted { 387 | background-color: #fbe54e; 388 | } 389 | 390 | dl.glossary dt { 391 | font-weight: bold; 392 | font-size: 1.1em; 393 | } 394 | 395 | .field-list ul { 396 | margin: 0; 397 | padding-left: 1em; 398 | } 399 | 400 | .field-list p { 401 | margin: 0; 402 | } 403 | 404 | .refcount { 405 | color: #060; 406 | } 407 | 408 | .optional { 409 | font-size: 1.3em; 410 | } 411 | 412 | .versionmodified { 413 | font-style: italic; 414 | } 415 | 416 | .system-message { 417 | background-color: #fda; 418 | padding: 5px; 419 | border: 3px solid red; 420 | } 421 | 422 | .footnote:target { 423 | background-color: #ffa; 424 | } 425 | 426 | .line-block { 427 | display: block; 428 | margin-top: 1em; 429 | margin-bottom: 1em; 430 | } 431 | 432 | .line-block .line-block { 433 | margin-top: 0; 434 | margin-bottom: 0; 435 | margin-left: 1.5em; 436 | } 437 | 438 | .guilabel, .menuselection { 439 | font-family: sans-serif; 440 | } 441 | 442 | .accelerator { 443 | text-decoration: underline; 444 | } 445 | 446 | .classifier { 447 | font-style: oblique; 448 | } 449 | 450 | abbr, acronym { 451 | border-bottom: dotted 1px; 452 | cursor: help; 453 | } 454 | 455 | /* -- code displays --------------------------------------------------------- */ 456 | 457 | pre { 458 | overflow: auto; 459 | overflow-y: hidden; /* fixes display issues on Chrome browsers */ 460 | } 461 | 462 | td.linenos pre { 463 | padding: 5px 0px; 464 | border: 0; 465 | background-color: transparent; 466 | color: #aaa; 467 | } 468 | 469 | table.highlighttable { 470 | margin-left: 0.5em; 471 | } 472 | 473 | table.highlighttable td { 474 | padding: 0 0.5em 0 0.5em; 475 | } 476 | 477 | tt.descname { 478 | background-color: transparent; 479 | font-weight: bold; 480 | font-size: 1.2em; 481 | } 482 | 483 | tt.descclassname { 484 | background-color: transparent; 485 | } 486 | 487 | tt.xref, a tt { 488 | background-color: transparent; 489 | font-weight: bold; 490 | color: #963; 491 | } 492 | 493 | h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { 494 | background-color: transparent; 495 | } 496 | 497 | .viewcode-link { 498 | float: right; 499 | } 500 | 501 | .viewcode-back { 502 | float: right; 503 | font-family: sans-serif; 504 | } 505 | 506 | div.viewcode-block:target { 507 | margin: -1px -10px; 508 | padding: 0 10px; 509 | } 510 | 511 | /* -- math display ---------------------------------------------------------- */ 512 | 513 | img.math { 514 | vertical-align: middle; 515 | } 516 | 517 | div.body div.math p { 518 | text-align: center; 519 | } 520 | 521 | span.eqno { 522 | float: right; 523 | } 524 | 525 | /* -- printout stylesheet --------------------------------------------------- */ 526 | 527 | @media print { 528 | div.document, 529 | div.documentwrapper, 530 | div.bodywrapper { 531 | margin: 0 !important; 532 | width: 100%; 533 | } 534 | 535 | div.sphinxsidebar, 536 | div.related, 537 | div.footer, 538 | #top-link { 539 | display: none; 540 | } 541 | } -------------------------------------------------------------------------------- /teafiles/clockwise.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011 discretelogics 2 | # 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | 17 | ''' 18 | TeaFiles should preferaby use an integer value for time representation storing 19 | the number of milliseconds since 1.1.1970. 20 | 21 | This module provides classes DateTime and Duration that aid in arithmetic, 22 | parsing, printing and conversion to python's c-ish date and time tools. 23 | 24 | The module name lends from the movie starring John Cleese, pretty pythonesk indeed. 25 | ''' 26 | 27 | 28 | import time 29 | import datetime 30 | import calendar 31 | 32 | 33 | class DateTime: 34 | ''' Holds a date and time value measured in milliseconds since the unix 35 | epoch 1970-01-01. This value, the number of "ticks", is the only state 36 | maintained by this class. 37 | 38 | 1. Either **ticks** are passed, in which case all other arguments are ignored. 39 | 2. Otherwise all other arguments are used to compute the ticks to be stored. 40 | 41 | * *year*: beween 1 .. 9999 42 | * *month*: between 1 .. 12 43 | * *day*: between 1 .. 31 44 | * *hours*, *minutes*, *seconds* and *milliseconds* are not checked to be in any range, they 45 | are multiplied with their respective number of milliseconds and summed up to the 46 | value of _ticks. 47 | 48 | >>> DateTime() 49 | 1970-01-01 00:00:00:000 50 | >>> DateTime().ticks 51 | 0 52 | >>> DateTime(1970, 1, 1).ticks 53 | 0 54 | >>> DateTime(1970, 1, 2).ticks 55 | 86400000 56 | >>> DateTime(ticks=427).ticks 57 | 427L 58 | >>> DateTime(2000, 1, 1, 77, 88, 99, 5240000).ticks 59 | 946972619000L 60 | >>> DateTime(2000, 1, 1, 77, 88, 99, 5240000, 11).ticks 61 | 11L 62 | ''' 63 | 64 | ticksperday = 86400 * 1000 # millseconds per day 65 | 66 | def __init__(self, year=1970, month=1, day=1, hours=0, minutes=0, seconds=0, milliseconds=0, ticks=None): 67 | 68 | if ticks: 69 | self._ticks = long(ticks) 70 | else: 71 | if year > 9999: 72 | raise ValueError("Maximum value for year=9999") 73 | if year < 1: 74 | raise ValueError("Minimum value for year=0001") 75 | 76 | dt = DateTime._getticksfromdate(datetime.date(year, month, day)) + \ 77 | hours * 60 * 60 * 1000 + \ 78 | minutes * 60 * 1000 + \ 79 | seconds * 1000 + \ 80 | milliseconds 81 | self._ticks = dt 82 | 83 | @staticmethod 84 | def parse(timestring, format_): 85 | ''' 86 | Creates an instance by parsing using . 87 | 88 | >>> DateTime.parse("2007-05-07 19:22:11", "%Y-%m-%d %H:%M:%S") 89 | 2007-05-07 19:22:11:000 90 | >>> DateTime.parse("2007-04-09 09:22:11", "%Y-%m-%d %H:%M:%S") 91 | 2007-04-09 09:22:11:000 92 | >>> DateTime.parse("2007-04-09", "%Y-%m-%d") 93 | 2007-04-09 00:00:00:000 94 | ''' 95 | ts = time.strptime(timestring, format_) 96 | cticks = calendar.timegm(ts) 97 | return DateTime(ticks=cticks * 1000) 98 | 99 | @property 100 | def ticks(self): 101 | ''' 102 | Gets the internal representation of date and time, which are ticks, meaning the number of 103 | milliseconds since the epoch. 104 | 105 | >>> from datetime import date 106 | >>> DateTime().ticks 107 | 0 108 | >>> DateTime(ticks=300).ticks 109 | 300L 110 | >>> DateTime(2000, 1, 1).ticks 111 | 946684800000L 112 | >>> 113 | ''' 114 | return self._ticks 115 | 116 | @property 117 | def date(self): 118 | ''' Returns the date part, that is a DateTime with a time of 00:00:00:000. 119 | 120 | >>> t = DateTime(2000, 3, 4) + Duration(ticks=5000) 121 | >>> t 122 | 2000-03-04 00:00:05:000 123 | >>> t.date 124 | 2000-03-04 00:00:00:000 125 | >>> 126 | ''' 127 | return DateTime(ticks=self._ticks - self._ticks % DateTime.ticksperday) 128 | 129 | def totimeandms(self): 130 | ''' 131 | Gets the date and time parts as a tuple holding (time.time, milliseconds). 132 | Since this method uses `time.gmtime`, errors with negative tick values 133 | can arise when using CPython (not in IronPython). 134 | 135 | This method is used internally but can also serve as an interface to python's standard library time. 136 | 137 | >>> DateTime(2011, 4, 5, 22, 00, 14).totimeandms() 138 | (time.struct_time(tm_year=2011, tm_mon=4, tm_mday=5, tm_hour=22, tm_min=0, tm_sec=14, tm_wday=1, tm_yday=95, tm_isdst=0), 0L) 139 | >>> DateTime(2011, 4, 5, 22, 00, 14, 333).totimeandms() 140 | (time.struct_time(tm_year=2011, tm_mon=4, tm_mday=5, tm_hour=22, tm_min=0, tm_sec=14, tm_wday=1, tm_yday=95, tm_isdst=0), 333L) 141 | >>> DateTime().totimeandms() 142 | (time.struct_time(tm_year=1970, tm_mon=1, tm_mday=1, tm_hour=0, tm_min=0, tm_sec=0, tm_wday=3, tm_yday=1, tm_isdst=0), 0) 143 | ''' 144 | sec, millisec = divmod(self._ticks, 1000) 145 | return time.gmtime(sec), millisec 146 | 147 | @staticmethod 148 | def _getticksfromdate(date): 149 | ''' Gets the ticks from a datetime.date value ''' 150 | return (date.toordinal() - 719163) * DateTime.ticksperday 151 | 152 | def __repr__(self): 153 | ts, milliseconds = self.totimeandms() 154 | return '{:04}-{:02}-{:02} {:02}:{:02}:{:02}:{:03}' \ 155 | .format(ts[0], ts[1], ts[2], ts[3], ts[4], ts[5], milliseconds) 156 | 157 | # pylint: disable-msg=W0212 158 | 159 | # equality 160 | def __eq__(self, other): 161 | ''' 162 | >>> DateTime() == DateTime() 163 | True 164 | >>> DateTime(2011, 1, 2, 3, 4, 5) == DateTime(2011, 1, 2, 3, 4, 5) 165 | True 166 | >>> DateTime(2011, 1, 2, 3, 4, 999) == DateTime(2011, 1, 2, 3, 4, 5) 167 | False 168 | >>> 169 | ''' 170 | if isinstance(other, DateTime): 171 | return self._ticks == other._ticks 172 | if isinstance(other, long) or isinstance(other, int): 173 | return self._ticks == other 174 | raise ValueError('comparison of date with invalid type') 175 | 176 | def __ne__(self, other): 177 | ''' 178 | >>> DateTime() != DateTime() 179 | False 180 | >>> DateTime(2011, 1, 2, 3, 4, 5) != DateTime(2011, 1, 2, 3, 4, 5) 181 | False 182 | >>> DateTime(2011, 1, 2, 3, 4, 999) != DateTime(2011, 1, 2, 3, 4, 5) 183 | True 184 | ''' 185 | return self._ticks != other._ticks 186 | 187 | # comparison 188 | def __lt__(self, other): 189 | ''' 190 | >>> DateTime(100) < DateTime(110) 191 | True 192 | >>> DateTime(100) < DateTime(101) 193 | True 194 | >>> DateTime(100) < DateTime(100) 195 | False 196 | >>> DateTime(100) < DateTime(99) 197 | False 198 | ''' 199 | return self._ticks < other._ticks 200 | 201 | def __le__(self, other): 202 | ''' 203 | >>> DateTime(100) <= DateTime(110) 204 | True 205 | >>> DateTime(100) <= DateTime(101) 206 | True 207 | >>> DateTime(100) <= DateTime(100) 208 | True 209 | >>> DateTime(100) <= DateTime(99) 210 | False 211 | ''' 212 | return self._ticks <= other._ticks 213 | 214 | def __gt__(self, other): 215 | ''' 216 | >>> DateTime(100) > DateTime(110) 217 | False 218 | >>> DateTime(100) > DateTime(101) 219 | False 220 | >>> DateTime(100) > DateTime(100) 221 | False 222 | >>> DateTime(100) > DateTime(99) 223 | True 224 | ''' 225 | return self._ticks > other._ticks 226 | 227 | def __ge__(self, other): 228 | ''' 229 | >>> DateTime(100) >= DateTime(110) 230 | False 231 | >>> DateTime(100) >= DateTime(101) 232 | False 233 | >>> DateTime(100) >= DateTime(100) 234 | True 235 | >>> DateTime(100) >= DateTime(99) 236 | True 237 | ''' 238 | return self._ticks >= other._ticks 239 | 240 | # add 241 | def __add__(self, rhs): 242 | ''' 243 | >>> DateTime() + Duration() 244 | 1970-01-01 00:00:00:000 245 | >>> DateTime(ticks=10) + Duration() 246 | 1970-01-01 00:00:00:010 247 | >>> DateTime(ticks=10) + Duration(ticks=3) 248 | 1970-01-01 00:00:00:013 249 | >>> DateTime(ticks=10) + Duration(ticks=-17) 250 | 1969-12-31 23:59:59:993 251 | >>> t = DateTime() 252 | >>> t 253 | 1970-01-01 00:00:00:000 254 | >>> t += Duration(ticks=77) 255 | >>> t 256 | 1970-01-01 00:00:00:077 257 | ''' 258 | if isinstance(rhs, Duration): 259 | return DateTime(ticks=self._ticks + rhs._ticks) 260 | raise ValueError('only a Duration can be added to a time') 261 | 262 | # integral type 263 | def __trunc__(self): 264 | return self 265 | 266 | def __int__(self): 267 | return self._ticks 268 | 269 | 270 | class Duration: 271 | ''' 272 | Stores a duration as number of milliseconds. In combination with `DateTime` 273 | provides time arithmetic. 274 | 275 | >>> Duration(ticks=7000) # 7000 milliseconds 276 | 0 days 00:00:07:000 277 | >>> Duration(days=3) # factory functions 278 | 3 days 00:00:00:000 279 | >>> Duration(days=14) + Duration(hours=3) + 144 280 | 14 days 03:00:00:144 281 | 282 | Initialize a duration from a number holding its milliseconds. 283 | 284 | >>> from teafiles.clockwise import * 285 | >>> Duration() 286 | 0 days 00:00:00:000 287 | >>> Duration(ticks=3) 288 | 0 days 00:00:00:003 289 | >>> Duration(ticks=1003) 290 | 0 days 00:00:01:003 291 | >>> Duration(ticks=86400 + 1050) 292 | 0 days 00:01:27:450 293 | >>> Duration(ticks=86400000 + 1050) 294 | 1 days 00:00:01:050 295 | ''' 296 | 297 | MILLISECOND = 1 298 | SECOND = 1000 * MILLISECOND 299 | MINUTE = 60 * SECOND 300 | HOUR = 60 * MINUTE 301 | DAY = 24 * HOUR 302 | 303 | def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, milliseconds=0, ticks=None): 304 | 305 | if not ticks: 306 | ticks = milliseconds 307 | ticks += Duration.SECOND * seconds 308 | ticks += Duration.MINUTE * minutes 309 | ticks += Duration.HOUR * hours 310 | ticks += Duration.DAY * days 311 | ticks += 7 * Duration.DAY * weeks 312 | 313 | self._ticks = long(ticks) # ticks shall always be integers 314 | 315 | @property 316 | def ticks(self): 317 | ''' 318 | Returns the underlying number of ticks, that is the number of milliseconds. 319 | 320 | >>> Duration(ticks=1033).ticks 321 | 1033L 322 | ''' 323 | return self._ticks 324 | 325 | def totimedelta(self): 326 | ''' 327 | Converts the duration to a `datetime.timedelta` instance. 328 | 329 | >>> Duration(ticks=33).totimedelta() 330 | datetime.timedelta(0, 0, 33000) 331 | >>> Duration(seconds=33).totimedelta() 332 | datetime.timedelta(0, 33) 333 | >>> Duration(minutes=33).totimedelta() 334 | datetime.timedelta(0, 1980) 335 | >>> Duration(hours=33).totimedelta() 336 | datetime.timedelta(1, 32400) 337 | >>> Duration(days=33).totimedelta() 338 | datetime.timedelta(33) 339 | ''' 340 | return datetime.timedelta(milliseconds=self._ticks) 341 | 342 | # pylint: disable-msg=W0212 343 | 344 | def __add__(self, other): 345 | ''' 346 | Adds another duration or integer (interpreted as milliseconds). 347 | 348 | >>> Duration(days=4) + Duration(hours=7) 349 | 4 days 07:00:00:000 350 | 351 | >>> Duration(days=4) + 3000 352 | 4 days 00:00:03:000 353 | ''' 354 | if isinstance(other, Duration): 355 | return Duration(ticks=self._ticks + other._ticks) 356 | elif isinstance(other, int) or isinstance(other, long): 357 | return Duration(ticks=self._ticks + other) 358 | raise ValueError("Invalid operand: Can add only int, long or instances of Duration to Duration") 359 | 360 | def __eq__(self, other): 361 | ''' 362 | >>> Duration() == Duration() 363 | True 364 | >>> Duration() == Duration(ticks=3) 365 | False 366 | >>> Duration(ticks=3) == Duration(ticks=3) 367 | True 368 | ''' 369 | return self._ticks == other.ticks 370 | 371 | def __ne__(self, other): 372 | ''' 373 | >>> Duration() != Duration() 374 | False 375 | >>> Duration() != Duration(ticks=3) 376 | True 377 | >>> Duration(ticks=3) != Duration(ticks=3) 378 | False 379 | ''' 380 | return self._ticks != other.ticks 381 | 382 | def __gt__(self, other): 383 | ''' 384 | >>> Duration() > Duration() 385 | False 386 | >>> Duration() > Duration(ticks=3) 387 | False 388 | >>> Duration(ticks=3) > Duration(ticks=3) 389 | False 390 | >>> Duration(ticks=400) > Duration(ticks=3) 391 | True 392 | ''' 393 | return self._ticks > other.ticks 394 | 395 | def __ge__(self, other): 396 | ''' 397 | >>> Duration() >= Duration() 398 | True 399 | >>> Duration() >= Duration(ticks=3) 400 | False 401 | >>> Duration(ticks=3) >= Duration(ticks=3) 402 | True 403 | >>> Duration(ticks=400) >= Duration(ticks=3) 404 | True 405 | ''' 406 | return self._ticks >= other.ticks 407 | 408 | def __lt__(self, other): 409 | ''' 410 | >>> Duration() < Duration() 411 | False 412 | >>> Duration() < Duration(ticks=3) 413 | True 414 | >>> Duration(ticks=3) < Duration(ticks=3) 415 | False 416 | >>> Duration(ticks=400) < Duration(ticks=3) 417 | False 418 | ''' 419 | return self._ticks < other.ticks 420 | 421 | def __le__(self, other): 422 | ''' 423 | >>> Duration() <= Duration() 424 | True 425 | >>> Duration() <= Duration(ticks=3) 426 | True 427 | >>> Duration(ticks=3) <= Duration(ticks=3) 428 | True 429 | >>> Duration(ticks=400) <= Duration(ticks=3) 430 | False 431 | ''' 432 | return self._ticks <= other.ticks 433 | 434 | def __repr__(self): 435 | ''' 436 | Returns a string representation using the format "{} days {:02}:{:02}:{:02}:{:03}". 437 | ''' 438 | days, remainder = divmod(self._ticks, Duration.DAY) 439 | hours, remainder = divmod(remainder, Duration.HOUR) 440 | minutes, remainder = divmod(remainder, Duration.MINUTE) 441 | seconds, remainder = divmod(remainder, Duration.SECOND) 442 | milliseconds = remainder 443 | return "{} days {:02}:{:02}:{:02}:{:03}".format(days, hours, minutes, seconds, milliseconds) 444 | 445 | def __trunc__(self): 446 | return self 447 | 448 | def __int__(self): 449 | return self._ticks 450 | 451 | # pylint: enable-msg=W0212 452 | 453 | 454 | #utils 455 | def isdatetime(value): 456 | ''' 457 | Returns true if `value` is an instance of DateTime. 458 | ''' 459 | return isinstance(value, DateTime) 460 | 461 | 462 | def isduration(value): 463 | ''' 464 | Returns true if `value` is an instance of Duration. 465 | ''' 466 | return isinstance(value, Duration) 467 | 468 | 469 | # pimp range function 470 | from __builtin__ import range as _range 471 | 472 | 473 | def range(*args): # pylint:disable-msg=W0622 474 | ''' 475 | Overrides the range method, allowing DateTime ranges. Requires exactly 3 arguments to have effect (otherwise normal range method 476 | is called): start (DateTime), end (DateTime) and step (Duration) 477 | 478 | >>> for t in range(DateTime(2000, 9, 1), DateTime(2001, 1, 1), Duration(weeks=2)): 479 | ... print t 480 | ... 481 | 2000-09-01 00:00:00:000 482 | 2000-09-15 00:00:00:000 483 | 2000-09-29 00:00:00:000 484 | 2000-10-13 00:00:00:000 485 | 2000-10-27 00:00:00:000 486 | 2000-11-10 00:00:00:000 487 | 2000-11-24 00:00:00:000 488 | 2000-12-08 00:00:00:000 489 | 2000-12-22 00:00:00:000 490 | ''' 491 | if len(args) == 3 and \ 492 | isdatetime(args[0]) and isdatetime(args[1]) and isduration(args[2]): 493 | return _rangedate(*args) 494 | else: 495 | return _range(*args) 496 | 497 | 498 | def _rangedate(start, stop, step): 499 | ''' 500 | Returns an iterator (generator) of DateTime values. starting at `start` stopping before `stop` 501 | and a duration of `step` between. 502 | ''' 503 | t = start.ticks 504 | stop = stop.ticks 505 | step = step.ticks 506 | while t < stop: 507 | yield DateTime(ticks=t) 508 | t += step 509 | 510 | 511 | def rangen(startdate, stepduration, count): 512 | ''' 513 | Simple creation of DateTime sequences. Generate DateTimes starting 514 | with incremented by . 515 | 516 | >>> for t in rangen(DateTime(2000, 9, 1), Duration(days=1), 10): 517 | ... print t 518 | ... 519 | 2000-09-01 00:00:00:000 520 | 2000-09-02 00:00:00:000 521 | 2000-09-03 00:00:00:000 522 | 2000-09-04 00:00:00:000 523 | 2000-09-05 00:00:00:000 524 | 2000-09-06 00:00:00:000 525 | 2000-09-07 00:00:00:000 526 | 2000-09-08 00:00:00:000 527 | 2000-09-09 00:00:00:000 528 | 2000-09-10 00:00:00:000 529 | ''' 530 | t = startdate.ticks 531 | step = stepduration.ticks 532 | while count: 533 | yield DateTime(ticks=t) 534 | t += step 535 | count -= 1 536 | 537 | 538 | if __name__ == '__main__': 539 | import doctest 540 | import teafiles.clockwise 541 | doctest.testmod(teafiles.clockwise) 542 | -------------------------------------------------------------------------------- /teafiles/teafile.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011 discretelogics 2 | # 3 | # This program is free software: you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation, either version 3 of the License, or 6 | # (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program. If not, see . 15 | 16 | ''' 17 | tbd 18 | ''' 19 | 20 | # pylint: disable-msg=W0232, W0142, W0122, C0301 21 | # W0142:655,15:TeaFile.read: Used * or ** magic 22 | # W0122:730,8:TeaFile._attachwritemethod: Use of the exec statement 23 | # C0301:172,0: Line too long (104/80) - maybe trim docstrings later for terminal users 24 | 25 | import struct 26 | import uuid 27 | from io import BytesIO 28 | from collections import namedtuple 29 | from teafiles.clockwise import DateTime 30 | 31 | # if set to true, time fields are returned as instances of clockwise.DateTime, otherwise 32 | USE_TIME_DECORATION = True 33 | 34 | 35 | class TeaFile: 36 | ''' 37 | Create, write and read or just inspect a file in the TeaFile format. 38 | 39 | 1. **create** and **write** a teafile holding Time/Price/Volume 40 | items. 41 | 42 | >>> tf = TeaFile.create("acme.tea", "Time Price Volume", "qdq", "prices of acme at NYSE", {"decimals": 2, "url": "www.acme.com" }) 43 | >>> tf.write(DateTime(2011, 3, 4, 9, 0), 45.11, 4500) 44 | >>> tf.write(DateTime(2011, 3, 4, 10, 0), 46.33, 1100) 45 | >>> tf.close() 46 | 47 | Note: Arguments of tf.write show up in intellisense with their names "Time", "Price" and "Volume". 48 | 49 | 2. **read** a teafile. TeaFiles are self describung a filename is sufficient, - we might have no clue what is inside the file, due to 50 | TeaFiles 51 | 52 | >>> tf = TeaFile.openread("acme.tea") 53 | >>> tf.read() 54 | TPV(Time=2011-03-04 09:00:00:000, Price=45.11, Volume=4500) 55 | >>> tf.read() 56 | TPV(Time=2011-03-04 10:00:00:000, Price=46.33, Volume=1100) 57 | >>> tf.read() 58 | >>> tf.close() 59 | 60 | Since the item structure is described in the file, we can always open the data items in the file. 61 | We can even do so on many platforms and with many applications, like from R on Linux, Mac OS or Windows, 62 | or using proprietary C++ or C# code. 63 | 64 | 3. **describe**: See the `description` property about accessing the values passed to create. As a teaser, lets access 65 | the content description and namevalues collection for the file above: 66 | 67 | >>> tf.description.contentdescription 68 | u'prices of acme at NYSE' 69 | 70 | >>> tf.description.namevalues 71 | {u'url': u'www.acme.com', u'decimals': 2} 72 | ''' 73 | 74 | #pylint:disable-msg=R0902,W0212 75 | 76 | # the factory methods at the module level, `create`, `openread` and `openwrite` should be used to create instances of this class. 77 | def __init__(self, filename): 78 | self.decimals = -1 79 | 80 | self._filename = filename # we need the filename for the call to getsize in itemareaend() 81 | self.file = None 82 | 83 | self._description = None 84 | 85 | self.itemareastart = None 86 | self._itemareaend = None 87 | self.itemsize = None 88 | 89 | self.itemstruct = None 90 | self.nameditemtuple = None 91 | 92 | self.write = None 93 | 94 | @staticmethod 95 | def create(filename, fieldnames, fieldformat=None, contentdescription=None, namevalues=None): 96 | ''' 97 | creates a new file and writes its header based on the description passed. 98 | leaves the file open, such that items can be added immediately. the caller must close the 99 | file finally. 100 | 101 | args: 102 | * **filename**: The filename, that will internally be passed to io.open, so the same rules apply. 103 | * **fieldnames**: The name of fields passed either as a string that seperates each 104 | fieldname by whitespace or as a list ot strings 105 | * **fieldformat**: Holds a composed by the format character for each field. 106 | the format characters are those used by the struct module. 107 | *example: "qdd" means that items stored in the file have 3 fields, the first is of 108 | type int64, the second and third are double values.* 109 | If omitted, all fields are considered to be of type int64. 110 | * **contentdescription**: A teafile can store one contentdescription, a string that describes what the 111 | contents in the file is about. examples: "Weather NYC", "Network load", "ACME stock". 112 | Applications can use this string as the "title" of the time series, for instance in a chart. 113 | * **namevalues**: A collection of name-value pairs used to store descriptions about the file. 114 | Often additional properties, like the "data provider", "feed", "feed id", "ticker". 115 | By convention, the name "decimals" is used to store an integer describing how many 116 | numbers of decimals to be used to format floating point values. This api for instance makes 117 | use of this convention. Besides formatting, an application might also treat this number 118 | as the accuracy of floating point values. 119 | 120 | >>> from teafiles import * 121 | >>> tf = TeaFile.create("lab.tea", "Time Temperature Humidity", "qdd") # create a file with 3 fields of types int64, double, double 122 | >>> tf.write(DateTime(2011, 3, 1), 44.2, 33.7) 123 | >>> tf.write(DateTime(2011, 3, 2), 45.1, 31.8) 124 | >>> tf.close() 125 | >>> tf.itemcount 126 | 2L 127 | 128 | note that itemcount is still accessible, even after the file is closed. 129 | ''' 130 | tf = TeaFile(filename) 131 | 132 | # setup description 133 | tf._description = d = TeaFileDescription() 134 | id_ = ItemDescription.create(None, fieldnames, fieldformat) 135 | tf.itemstruct = id_.itemstruct 136 | d.itemdescription = id_ 137 | d.contentdescription = contentdescription 138 | d.namevalues = namevalues 139 | d.timescale = TimeScale.java() 140 | 141 | # open file and write header 142 | 143 | tf.file = open(filename, "wb") 144 | hm = _HeaderManager() 145 | fio = _FileIO(tf.file) 146 | fw = _FormattedWriter(fio) 147 | wc = hm.writeheader(fw, tf._description) 148 | tf.itemareastart = wc.itemareastart 149 | tf._itemareaend = wc.itemareaend 150 | tf.itemsize = id_.itemsize 151 | 152 | tf._attachwritemethod() # pylint:disable-msg=W0212 153 | 154 | tf.flush() 155 | return tf 156 | 157 | @staticmethod 158 | def openread(filename): 159 | ''' 160 | Open a TeaFile for read only. 161 | 162 | >>> from teafiles import * 163 | >>> with TeaFile.create("lab.tea", "Time Temperature Humidity", "qdd") as tf: 164 | ... tf.write(DateTime(2011, 3, 1), 44.2, 33.7) 165 | ... tf.write(DateTime(2011, 3, 2), 45.1, 31.8) 166 | ... 167 | >>> from pprint import pprint 168 | >>> with TeaFile.openread("lab.tea") as tf: 169 | ... pprint(list(tf.items())) 170 | ... 171 | [TTH(Time=2011-03-01 00:00:00:000, Temperature=44.2, Humidity=33.7), 172 | TTH(Time=2011-03-02 00:00:00:000, Temperature=45.1, Humidity=31.8)] 173 | 174 | The instance demonstrates that is is not writable by not having a write method at all: 175 | 176 | >>> tf.write == None 177 | True 178 | ''' 179 | tf = TeaFile._open(filename, "rb") 180 | return tf 181 | 182 | @staticmethod 183 | def openwrite(filename): 184 | ''' 185 | Open a TeaFile for read and write. 186 | 187 | The file returned will have its *filepointer set to the end of the file*, as this function 188 | calls seekend() before returning the TeaFile instance. 189 | 190 | >>> with TeaFile.create('lab.tea', 'A B') as tf: 191 | ... for i in range(3): 192 | ... tf.write(i, 10*i) 193 | ... 194 | >>> TeaFile.printitems("lab.tea") 195 | [AB(A=0, B=0), AB(A=1, B=10), AB(A=2, B=20)] 196 | >>> 197 | >>> with TeaFile.openwrite('lab.tea') as tf: # open the file to add more items 198 | ... tf.write(7, 77) 199 | ... 200 | >>> TeaFile.printitems("lab.tea") 201 | [AB(A=0, B=0), AB(A=1, B=10), AB(A=2, B=20), AB(A=7, B=77)] 202 | 203 | ''' 204 | tf = TeaFile._open(filename, "r+b") 205 | tf._attachwritemethod() # pylint: disable-msg=W0212 206 | tf.seekend() # this is what one would expect: writes append to the file 207 | return tf 208 | 209 | @staticmethod 210 | def _open(filename, mode): 211 | ''' internal open method, used by openread and openwrite ''' 212 | tf = TeaFile(filename) 213 | tf.file = open(filename, mode) 214 | fio = _FileIO(tf.file) 215 | fr = _FormattedReader(fio) 216 | hm = _HeaderManager() 217 | rc = hm.readheader(fr) 218 | tf._description = rc.description 219 | id_ = tf._description.itemdescription 220 | if id_: 221 | tf.itemsize = id_.itemsize 222 | tf.itemstruct = id_.itemstruct 223 | tf.nameditemtuple = id_.itemtype 224 | tf.itemareastart = rc.itemareastart 225 | tf._itemareaend = rc.itemareaend 226 | 227 | nvs = tf._description.namevalues 228 | if nvs and nvs.get("decimals"): 229 | tf.decimals = nvs["decimals"] 230 | return tf 231 | 232 | #pylint:enable-msg=W0212 233 | 234 | # read & write 235 | def read(self): 236 | ''' 237 | Read then next item at the position of the file pointer. If no more items exist, None is returned. 238 | 239 | >>> with TeaFile.create('lab.tea', 'A B') as tf: 240 | ... for i in range(3): 241 | ... tf.write(i, 10*i) 242 | ... 243 | >>> tf = TeaFile.openread('lab.tea') 244 | >>> tf.read() 245 | AB(A=0, B=0) 246 | >>> tf.read() 247 | AB(A=1, B=10) 248 | >>> tf.read() 249 | AB(A=2, B=20) 250 | >>> tf.read() 251 | >>> 252 | ''' 253 | itembytes = self.file.read(self.itemsize) 254 | if not itembytes: 255 | return None 256 | itemvalues = self.itemstruct.unpack(itembytes) 257 | adjusteditemvalues = [f.getvalue(itemvalues) for f in self._description.itemdescription.fields] 258 | tupelized = tuple(adjusteditemvalues) 259 | return self.nameditemtuple(*tupelized) 260 | 261 | def _write(self, *itemvalues): 262 | ''' 263 | Internal item write method accepting a value for each field. 264 | 265 | A **typed write method** will be created inside the create and openwrite methods, available 266 | as **write(field1, field2, ....)**. 267 | 268 | >>> tf = TeaFile.create("acme.tea", "Time Price Volume", "qdq", "prices of acme at NYSE", {"decimals": 2, "url": "www.acme.com" }) 269 | >>> tf.write(DateTime(2011, 3, 4, 9, 0), 45.11, 4500) 270 | >>> tf.write(DateTime(2011, 3, 4, 10, 0), 46.33, 1100) 271 | >>> tf.close() 272 | 273 | Note: Arguments of the tf.write show up in intellisense with their names "Time", "Price" and "Volume". This however works usually 274 | only in interactive shells, not in py-script editors, since they do not instantiate the class. 275 | ''' 276 | if USE_TIME_DECORATION: 277 | itemvalues = tuple([f.decoratetime(itemvalues) for f in self.description.itemdescription.fields]) 278 | bytes_ = self.itemstruct.pack(*itemvalues) 279 | self.file.write(bytes_) 280 | 281 | def flush(self): 282 | ''' 283 | Flush buffered bytes to disk. 284 | 285 | When items are written via write, they do not land directly in the file, but are buffered in memory. flush 286 | persists them on disk. Since the number of items in a TeaFile is computed from the size of the file, the 287 | `itemcount` property is accuraty only after items have been flushed. 288 | 289 | >>> with TeaFile.create('lab.tea', 'A') as tf: 290 | ... for i in range(3): 291 | ... tf.write(i) 292 | ... 293 | >>> tf = TeaFile.openwrite('lab.tea') 294 | >>> tf.itemcount 295 | 3L 296 | >>> tf.write(71) 297 | >>> tf.itemcount 298 | 3L 299 | >>> tf.flush() 300 | >>> tf.itemcount 301 | 4L 302 | >>> tf.close() 303 | ''' 304 | self.file.flush() 305 | 306 | def seekitem(self, itemindex): 307 | ''' 308 | Sets the file pointer to the item at index `temindex`. 309 | 310 | >>> with TeaFile.create("lab.tea", "A") as tf: 311 | ... for i in range(20): 312 | ... tf.write(i) 313 | ... 314 | >>> tf = TeaFile.openread('lab.tea') 315 | >>> tf.read() 316 | A(A=0) 317 | >>> tf.read() 318 | A(A=1) 319 | >>> tf.read() 320 | A(A=2) 321 | >>> tf.seekitem(7) 322 | >>> tf.read() 323 | A(A=7) 324 | >>> tf.seekitem(2) 325 | >>> tf.read() 326 | A(A=2) 327 | >>> tf.close() 328 | ''' 329 | self.file.seek(self.itemareastart + itemindex * self.itemsize) 330 | 331 | def seekend(self): 332 | ''' 333 | Sets the file pointer past the last item. 334 | 335 | >>> with TeaFile.create('lab.tea', 'A') as tf: 336 | ... for i in range(10): 337 | ... tf.write(i) 338 | ... 339 | >>> tf = TeaFile.openread('lab.tea') 340 | >>> tf.read() 341 | A(A=0) 342 | >>> tf.seekend() 343 | >>> tf.read() 344 | >>> # nothing returned, we are at the end of file 345 | >>> tf.close() 346 | ''' 347 | self.file.seek(0, 2) # SEEK_END 348 | 349 | def items(self, start=0, end=None): 350 | ''' 351 | Returns an iterator over the items in the file allowing start and end to be passed as item index. 352 | Calling this method will modify the filepointer. 353 | 354 | Optional, the range of the iterator can be returned 355 | 356 | >>> with TeaFile.create('lab.tea', 'A') as tf: 357 | ... for i in range(10): 358 | ... tf.write(i) 359 | ... 360 | >>> tf = TeaFile.openread('lab.tea') 361 | >>> tf.items() 362 | 363 | >>> list(tf.items()) 364 | [A(A=0), A(A=1), A(A=2), A(A=3), A(A=4), A(A=5), A(A=6), A(A=7), A(A=8), A(A=9)] 365 | >>> list(tf.items(2, 4)) 366 | [A(A=2), A(A=3)] 367 | >>> list(tf.items(1, 5)) 368 | [A(A=1), A(A=2), A(A=3), A(A=4)] 369 | >>> 370 | ''' 371 | self.seekitem(start) 372 | if not end: 373 | end = self.itemcount 374 | current = start 375 | while current < end: 376 | yield self.read() 377 | current += 1 378 | 379 | @property 380 | def itemcount(self): 381 | ''' The number of items in the file. ''' 382 | return self._getitemareasize() / self.itemsize 383 | 384 | def _getitemareaend(self): 385 | ''' the end of the item area, as an integer ''' 386 | import os 387 | if self._itemareaend: 388 | return self._itemareaend 389 | return os.path.getsize(self._filename) 390 | 391 | def _getitemareasize(self): 392 | ''' the item area size, as an integer ''' 393 | return self._getitemareaend() - self.itemareastart 394 | 395 | def close(self): 396 | ''' 397 | Closes the file. 398 | 399 | TeaFile implements the context manager protocol and using this protocol is prefered, so manually closing the file 400 | should be required primarily in interactive mode. 401 | ''' 402 | self.file.close() 403 | 404 | # context manager protocol 405 | def __enter__(self): 406 | return self 407 | 408 | def __exit__(self, type_, value, tb): 409 | self.close() 410 | 411 | # information about the file and its contents 412 | @property 413 | def description(self): 414 | ''' 415 | Returns the description of the file. 416 | 417 | TeaFile describe the structure of its items and annotations about its content in their header. This 418 | property returns this description which in turn (optionally) holds 419 | * the itemdescription describing field names, types and offsets 420 | * a contentdescription describing the content 421 | * a namevalue collection holding name-value pairs and 422 | * a timescale describing how time stored as numbers shall be interpreted as time.:: 423 | 424 | tf = TeaFile.create('lab.tea', 'Time Price Volume', 'qdq', 'ACME stock', {'exchange': 'nyse', 'decimals': 2}) 425 | tf.description 426 | # returns: 427 | 428 | ItemDescription 429 | Name: TPV 430 | Size: 24 431 | Fields: 432 | [Time Type: Int64 Offset: 0 IsTime:0 IsEventTime:0, 433 | Price Type: Double Offset: 8 IsTime:0 IsEventTime:0, 434 | Volume Type: Int64 Offset:16 IsTime:0 IsEventTime:0] 435 | 436 | ContentDescription 437 | ACME stock 438 | 439 | NameValues 440 | {'feed': 'bluum', 'decimals': 2, 'exchange': 'nyse'} 441 | TimeScale 442 | Epoch: 719162 443 | Ticks per Day: 86400000 444 | Wellknown Scale: Java 445 | 446 | Note that the description object remains valid even after the file is closed. 447 | 448 | ''' 449 | return self._description 450 | 451 | def __repr__(self): 452 | return "TeaFile('{}') {} items".format(self._filename, self.itemcount) 453 | 454 | def getvaluestring(self, field, item): 455 | ''' 456 | Returns the string representation of an item, considerung the number of decimals if available. 457 | 458 | >>> tf = TeaFile.create('lab.tea', 'Time Price', 'qd', 'ACME stock', {'exchange': 'nyse', 'decimals': 2}) 459 | >>> tf.write(DateTime(2010, 2, 3), 44.444444) 460 | >>> tf.write(DateTime(2010, 2, 3), 44.333333) 461 | >>> tf.close() 462 | >>> tf = TeaFile.openread('lab.tea') 463 | >>> item = tf.read() 464 | >>> item # decimals=2 is not considered 465 | TP(Time=2010-02-03 00:00:00:000, Price=44.444444) 466 | >>> pricefield = tf.description.itemdescription.fields[1] 467 | >>> pricefield 468 | Price Type: Double Offset: 8 IsTime:0 IsEventTime:0 469 | >>> tf.getvaluestring(pricefield, item) # decimals=2 is considered 470 | 44.44 471 | ''' 472 | value = field.getvalue(item) 473 | if(self.decimals != -1 and (field.fieldtype == 9 or field.fieldtype == 10)): 474 | value = round(value, self.decimals) 475 | elif(field.fieldtype == 1): 476 | value = round(value, self.decimals) 477 | return value 478 | 479 | #internals 480 | def _attachwritemethod(self): 481 | ''' generate specific write method with named arguments ''' 482 | id_ = self._description.itemdescription 483 | commafields = ",".join(id_.fieldnames) 484 | methodcode = "def customWrite(self, " + commafields + "): self._write(" + commafields + ")" 485 | import types 486 | d = {} 487 | exec(methodcode, d) 488 | func = d["customWrite"] 489 | boundmethod = types.MethodType(func, self) 490 | self.write = boundmethod 491 | 492 | @staticmethod 493 | def printitems(filename, maxnumberofitems=10): 494 | ''' 495 | Prints all items in the file. By default at most 10 items are printed. 496 | 497 | 498 | >>> with TeaFile.create("lab.tea", "A B") as tf: 499 | ... for i in range(40): 500 | ... tf.write(i, 10 * i) 501 | ... 502 | >>> TeaFile.printitems("lab.tea") 503 | [AB(A=0, B=0), AB(A=1, B=10), AB(A=2, B=20), AB(A=3, B=30), AB(A=4, B=40), AB(A=5, B=50), AB(A=6, B=60), AB(A=7, B=70), AB(A=8, B=80), AB(A=9, B=90)] 504 | 10 of 40 items 505 | >>> 506 | >>> TeaFile.printitems("lab.tea", 5) 507 | [AB(A=0, B=0), AB(A=1, B=10), AB(A=2, B=20), AB(A=3, B=30), AB(A=4, B=40)] 508 | 5 of 40 items 509 | >>> 510 | ''' 511 | with TeaFile.openread(filename) as tf: 512 | from itertools import islice 513 | print(list(islice(tf.items(), maxnumberofitems))) 514 | if tf.itemcount > maxnumberofitems: 515 | print ("{} of {} items".format(maxnumberofitems, tf.itemcount)) 516 | 517 | @staticmethod 518 | def printsnapshot(filename): 519 | ''' 520 | Prints a snapshot of an existing file, that is its complete description and the first 5 items. 521 | 522 | Example output: :: 523 | 524 | >> TeaFile.printsnapshot('lab.tea') 525 | TeaFile('lab.tea') 40 items 526 | 527 | ItemDescription 528 | Name: AB 529 | Size: 16 530 | Fields: 531 | [A Type: Int64 Offset: 0 IsTime:0 IsEventTime:0, 532 | B Type: Int64 Offset: 8 IsTime:0 IsEventTime:0] 533 | 534 | ContentDescription 535 | None 536 | 537 | NameValues 538 | None 539 | 540 | TimeScale 541 | Epoch: 719162 542 | Ticks per Day: 86400000 543 | Wellknown Scale: Java 544 | 545 | Items 546 | AB(A=0, B=0) 547 | AB(A=1, B=10) 548 | AB(A=2, B=20) 549 | AB(A=3, B=30) 550 | AB(A=4, B=40) 551 | ''' 552 | with TeaFile.openread(filename) as tf: 553 | print(tf) 554 | print("") 555 | print(tf.description) 556 | print("Items") 557 | for item in tf.items(0, 5): 558 | print(item) 559 | 560 | 561 | class _ValueKind: # pylint: disable-msg=R0903 562 | ''' enumeration type, describing the type of a value inside a name-value pair ''' 563 | Invalid, Int32, Double, Text, Uuid = [0, 1, 2, 3, 4] 564 | 565 | 566 | def _getnamevaluekind(value): 567 | ''' returns the `_ValueKind' based on the for the passed `value` ''' 568 | if isinstance(value, int): 569 | return _ValueKind.Int32 570 | if isinstance(value, float): 571 | return _ValueKind.Double 572 | if isinstance(value, basestring): 573 | return _ValueKind.Text 574 | if isinstance(value, uuid): 575 | return _ValueKind.Uuid 576 | raise ValueError("Invalid type inside NameValue") 577 | 578 | 579 | class TimeScale: 580 | ''' 581 | The TeaFile format is time format agnostic. Times in such file can be integral or float values 582 | counting seconds, milliseconds from an epoch like 0001-01-01 or 1970-01-01. The epoch together 583 | with the tick size define the `time scale` modeled by this class. These values are stored in the file. 584 | 585 | In order to support many platforms, the epoch value of 1970-01-01 and a tick size of Milliseconds is recommended. 586 | Moreover, APIs for TeaFiles should primarily support this time scale before any other, to allow exchange 587 | between applications and operating systems. In this spirit, the clockwise module in this package uses this 588 | 1970 / millisecond time scale. 589 | ''' 590 | def __init__(self, epoch, ticksperday): 591 | self._epoch = epoch 592 | self._ticksperday = ticksperday 593 | 594 | @staticmethod 595 | def java(): 596 | ''' 597 | Returns a TimeScale instance with the epoch 1970-01-01 and millisecond resolution. 598 | This time scale is that used by Java, so we call this the Java TimeScale. 599 | ''' 600 | return TimeScale(719162, 86400000) 601 | 602 | @property 603 | def wellknownname(self): 604 | ''' Returns 'Java' if epoch == 719162 (1970-01-01) and ticksperday == 86400 * 1000. 605 | Returns 'Net' if epoch == 0 (0001-01-01) and ticksperday == 86400 * 1000 * 1000 * 10. 606 | Returns None otherwise. 607 | ''' 608 | if self._epoch == 719162 and self._ticksperday == 86400000: 609 | return "Java" 610 | if self._epoch == 0 and self._ticksperday == 864000000000: 611 | return "Net" 612 | return None 613 | 614 | def __repr__(self): 615 | s = "Epoch: {:>8}\nTicks per Day: {:>8}\n" \ 616 | .format(self._epoch, self._ticksperday) 617 | wnn = self.wellknownname 618 | if wnn: 619 | s += "Wellknown Scale:{:>7}\n".format(wnn) 620 | return s 621 | 622 | 623 | class _FileIO: 624 | ''' FileIO provides the ability to read int32, int64, double and byte lists from a file ''' 625 | 626 | def __init__(self, iofile): 627 | self.file = iofile 628 | 629 | # read 630 | def readint32(self): 631 | ''' read a 32bit signed integer from the file ''' 632 | bytes_ = self.file.read(4) 633 | value = struct.unpack("i", bytes_)[0] 634 | return value 635 | 636 | def readint64(self): 637 | ''' read a 64bit signed integer from the file ''' 638 | bytes_ = self.file.read(8) 639 | value = struct.unpack("q", bytes_)[0] 640 | return value 641 | 642 | def readdouble(self): 643 | ''' read a double from the file ''' 644 | bytes_ = self.file.read(8) 645 | value = struct.unpack("d", bytes_)[0] 646 | return value 647 | 648 | def readbytes(self, n): 649 | ''' read `n` bytes from the file ''' 650 | return self.file.read(n) 651 | 652 | # write 653 | def writeint32(self, value): 654 | ''' write a 32bit signed integer to the file ''' 655 | bytes_ = struct.pack("i", value) 656 | assert len(bytes_) == 4 657 | self.file.write(bytes_) 658 | 659 | def writeint64(self, value): 660 | ''' write a 64bit signed integer to the file ''' 661 | bytes_ = struct.pack("q", value) 662 | assert len(bytes_) == 8 663 | self.file.write(bytes_) 664 | 665 | def writedouble(self, value): 666 | ''' write a double to the file ''' 667 | bytes_ = struct.pack("d", value) 668 | assert len(bytes_) == 8 669 | self.file.write(bytes_) 670 | 671 | def writebytes(self, bytes_): 672 | ''' write the list of byte to the file ''' 673 | self.file.write(bytes_) 674 | 675 | # position 676 | def skipbytes(self, bytestoskip): 677 | ''' skip `bytestoskip` in the file. increments the file pointer ''' 678 | self.file.read(bytestoskip) 679 | 680 | def position(self): 681 | ''' returns the file pointer ''' 682 | return self.file.tell() 683 | 684 | 685 | class _FormattedReader: 686 | ''' Provides formatted reading of a `_FileIO` instance.''' 687 | 688 | def __init__(self, fio): 689 | self.fio = fio 690 | 691 | def readint32(self): 692 | ''' read int32 ''' 693 | return self.fio.readint32() 694 | 695 | def readint64(self): 696 | ''' read int64 ''' 697 | return self.fio.readint64() 698 | 699 | def readdouble(self): 700 | ''' read double ''' 701 | return self.fio.readdouble() 702 | 703 | def readbytes_lengthprefixed(self): 704 | ''' read bytes, length prefixed ''' 705 | n = self.readint32() 706 | return self.fio.readbytes(n) 707 | 708 | def readtext(self): 709 | ''' read a unicode string in utf8 encoding ''' 710 | return self.readbytes_lengthprefixed().decode("utf8") 711 | 712 | def readuuid(self): 713 | ''' read a uuid ''' 714 | bytes16 = self.fio.readbytes(16) 715 | return uuid.UUID(bytes=bytes16) 716 | 717 | def skipbytes(self, bytestoskip): 718 | ''' skip `bytestoskip` bytes ''' 719 | self.fio.skipbytes(bytestoskip) 720 | 721 | def position(self): 722 | ''' returns the position of the filepointer ''' 723 | return self.fio.position() 724 | 725 | def readnamevalue(self): 726 | ''' returns a dictionary holding a single name : value pair ''' 727 | name = self.readtext() 728 | kind = self.readint32() 729 | if kind == _ValueKind.Int32: 730 | value = self.readint32() 731 | elif kind == _ValueKind.Double: 732 | value = self.readdouble() 733 | elif kind == _ValueKind.Text: 734 | value = self.readtext() 735 | elif kind == _ValueKind.Uuid: 736 | value = self.readuuid() 737 | return {name: value} 738 | 739 | 740 | class _FormattedWriter: 741 | 742 | def __init__(self, fio): 743 | self.fio = fio 744 | 745 | def writeint32(self, int32value): 746 | ''' write an int32 value ''' 747 | self.fio.writeint32(int32value) 748 | 749 | def writeint64(self, int64value): 750 | ''' write an int64 value ''' 751 | self.fio.writeint64(int64value) 752 | 753 | def writedouble(self, doublevalue): 754 | ''' write a double value ''' 755 | self.fio.writedouble(doublevalue) 756 | 757 | def writebytes_lengthprefixed(self, bytes_): 758 | ''' writes the `bytes` prefixed with their length ''' 759 | self.writeint32(len(bytes_)) 760 | self.fio.writebytes(bytes_) 761 | 762 | def writeraw(self, bytes_): 763 | ''' write `bytes` without length prefixing them ''' 764 | self.fio.writebytes(bytes_) 765 | 766 | def writetext(self, text): 767 | ''' write `text` in UTF8 encoding ''' 768 | self.writebytes_lengthprefixed(text.encode("utf8")) # todo: is this encoding right? 769 | 770 | def writeuuid(self, uuidvalue): 771 | ''' Not implemented yet. writes `uuidvalue` into the file. ''' 772 | raise Exception("cannot write uuid, feature not yet implemented. uuid={}{}".format(self, uuidvalue)) 773 | #bytes16 = self.fio.writebytes(16) 774 | #return uuid.UUID(bytes=bytes16) 775 | 776 | def skipbytes(self, bytestoskip): 777 | ''' skip `bytestoskip` ''' 778 | self.fio.skipbytes(bytestoskip) 779 | 780 | def position(self): 781 | ''' return the position (the file pointer ''' 782 | return self.fio.position() 783 | 784 | def writenamevalue(self, key, value): 785 | ''' write a name/value pair ''' 786 | kind = _getnamevaluekind(value) 787 | self.writetext(key) 788 | self.writeint32(kind) 789 | if kind == _ValueKind.Int32: 790 | self.writeint32(value) 791 | elif kind == _ValueKind.Double: 792 | self.writedouble(value) 793 | elif kind == _ValueKind.Text: 794 | self.writetext(value) 795 | elif kind == _ValueKind.Uuid: 796 | self.writeuuid(value) 797 | 798 | 799 | # descriptions 800 | class TeaFileDescription: 801 | ''' 802 | Holds the description of a time series. Its attributes are the 803 | itemdescription, describing the item's fields and layout 804 | contentdescription, a simple string describing what the time series is about 805 | namevalues, a collection of name-value pairs holding int32,double,text or uuid values and the 806 | timescale, describing the format of times inside the file 807 | ''' 808 | #pylint: disable-msg=R0903 809 | 810 | def __init__(self): 811 | self.itemdescription = None 812 | self.contentdescription = None 813 | self.namevalues = None 814 | self.timescale = None 815 | 816 | def __repr__(self): 817 | return "ItemDescription\n{}" \ 818 | "\n\nContentDescription\n{}" \ 819 | "\n\nNameValues\n{}" \ 820 | "\n\nTimeScale\n{}" \ 821 | .format(self.itemdescription, \ 822 | self.contentdescription, \ 823 | self.namevalues, \ 824 | self.timescale) 825 | 826 | 827 | class ItemDescription: 828 | ''' 829 | The item description describes the item type. 830 | Each teafile is a homogenous collection of items and an instance of this class describes 831 | the fields of this item, that is 832 | 833 | the name of each field 834 | the field's offset inside the item 835 | its type. 836 | ''' 837 | def __init__(self): 838 | self.itemsize = 0 839 | self.itemname = "" 840 | self.fields = [] 841 | self.itemstruct = None # the struct for marshalling to the file 842 | self.itemtype = None # the named tuple class used for items 843 | self.fieldnames = None 844 | 845 | def __repr__(self): 846 | from pprint import pformat 847 | return "Name:\t{}\nSize:\t{}\nFields:\n{}" \ 848 | .format(self.itemname, self.itemsize, pformat(self.fields)) 849 | 850 | @staticmethod 851 | def create(itemname, fieldnames, fieldformat): 852 | ''' 853 | Creates an ItemDescription instance to be used for the creation of a new TeaFile. 854 | 855 | itemname is the name for the items in the file (eg "Tick") 856 | fieldnames is a list of the names (eg ["Time", "Price", "Volume"]). 857 | Alternatively, fieldnames is a string that holds fieldnames separated 858 | by whitspace ("Time Price Volume") 859 | 860 | fieldformat specifies the layout of the item as used by 861 | struct.pack(fmt, ...). However the following restrictions apply: 862 | 863 | 1. the repeat oerator is not allowed. So while "4h" means the same as 864 | "hhhh" for struct.pack/unpack, this method allows only the latter 865 | without repeat number. 866 | 2. padding bytes (format character 'x') are not available. 867 | 3. Only these formats are allowed: 868 | "b", "h", "i", "q", "B", "H", "I", "Q", "f", "d". 869 | ''' 870 | 871 | id_ = ItemDescription() 872 | 873 | # prepare arguments 874 | if not isinstance(fieldnames, list): 875 | fieldnames = fieldnames.split() 876 | id_.fieldnames = fieldnames 877 | if not itemname: 878 | itemname = "".join([s[0] for s in fieldnames]) 879 | 880 | if not fieldformat: 881 | fieldformat = "q" * len(fieldnames) 882 | 883 | id_.itemtype = namedtuple(itemname, fieldnames) 884 | id_.itemstruct = struct.Struct(fieldformat) 885 | id_.itemname = itemname 886 | 887 | # ensure fieldformat has no repeat numbers 888 | if not set(fieldformat).isdisjoint("0123456789"): 889 | raise ValueError("fieldformat contains digits. change format such that no digits occur") 890 | # remove byte order specifiers used by the struct module 891 | fieldformat = "".join([c for c in fieldformat if not c in "@<>=!"]) 892 | if len(fieldformat) != len(fieldnames): 893 | raise Exception("fieldformat has different number of characters than fieldnames: " + \ 894 | fieldformat + "(" + str(len(fieldformat)) + ") vs " + \ 895 | ",".join(fieldnames) + "(" + str(len(fieldnames)) + ")") 896 | # create Fields 897 | i = 0 898 | for fname in fieldnames: 899 | f = Field() 900 | f.name = fname 901 | f.index = i 902 | f.formatchar = fieldformat[i] 903 | f.fieldtype = FieldType.getfromformatcharacter(f.formatchar) 904 | i += 1 905 | id_.fields.append(f) 906 | 907 | # analyze and assign offsets 908 | _analyzefieldoffsets(id_) 909 | id_._adjustitemstructforpadding(fieldformat) # pylint: disable-msg=W0212 910 | return id_ 911 | 912 | def _adjustitemstructforpadding(self, fieldformat): 913 | ''' we add trailing padding bytes after layout analysis, because it does not matter there ''' 914 | itemalignment = max(FieldType.getsize(f.fieldtype) for f in self.fields) 915 | rawsize = self.itemstruct.size 916 | itempadding = itemalignment - (rawsize % itemalignment) 917 | if itempadding == itemalignment: 918 | itempadding = 0 919 | self.itemsize = rawsize + itempadding 920 | if itempadding: 921 | fieldformat += str(itempadding) + "x" 922 | self.itemstruct = struct.Struct(fieldformat) 923 | 924 | def getfieldbyoffset(self, offset): 925 | ''' Returns a field given its offset ''' 926 | for f in self.fields: 927 | if f.offset == offset: 928 | return f 929 | print("field not found at offset{0}".format(offset)) 930 | raise RuntimeError() 931 | 932 | def setupfromfields(self): 933 | ''' When a TeaFile is read from file, the fields are created and appended to this instance. 934 | This method sets up all remaining fields ''' 935 | self.fieldnames = [self._getsafename(f.name) for f in self.fields] 936 | for f in self.fields: 937 | f.size = FieldType.getsize(f.fieldtype) 938 | self.itemtype = namedtuple(self._getsafename(self.itemname), self.fieldnames) 939 | fieldformat = "".join([FieldType.getformatcharacter(f.fieldtype) for f in self.fields]) 940 | self.itemstruct = struct.Struct(fieldformat) 941 | self._adjustitemstructforpadding(fieldformat) 942 | 943 | @staticmethod 944 | def _getsafename(name): 945 | ''' convert item or field name to a name valid for namedtuple ''' 946 | validchars = '_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' 947 | return ''.join(c for c in name if c in validchars) 948 | 949 | 950 | def _analyzefieldoffsets(itemdescription): 951 | ''' analyzes the itemdescription to find how it will be layouted ''' 952 | id_ = itemdescription 953 | import array 954 | buffer_ = array.array('b', [0] * id_.itemstruct.size) 955 | for f in id_.fields: 956 | fts = FieldType.getsize(f.fieldtype) 957 | for pos in range(0, id_.itemstruct.size - fts + 1): 958 | magic = FieldType.getmagicvalue(f.fieldtype) 959 | struct.pack_into(f.formatchar, buffer_, pos, magic) 960 | testitem = id_.itemstruct.unpack(buffer_) 961 | struct.pack_into(f.formatchar, buffer_, pos, 0) # reset buffer 962 | if testitem[f.index] == magic: 963 | f.offset = pos 964 | break 965 | return None 966 | 967 | 968 | class FieldType: 969 | ''' An enumeration of field types and utility functions related to. ''' 970 | 971 | Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float, Double = \ 972 | _formatNumbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 973 | _typeNames = [None, "Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64", "Float", "Double"] 974 | _typesizes = [1, 2, 4, 8, 1, 2, 4, 8, 4, 8] 975 | _formatCharacters = ["b", "h", "i", "q", "B", "H", "I", "Q", "f", "d"] 976 | _magicValues = [0, 0x71, 0x7172, 0x71727374, 0x7172737475767778, 0xa1, 0xa1a2, 0xa1a2a3a4, 0xa1a2a3a4a5a6a7a8, 1.01, 3.07] 977 | 978 | @staticmethod 979 | def getsize(fieldtype): 980 | ''' get the size of a field type ''' 981 | i = FieldType._formatNumbers.index(fieldtype) 982 | return FieldType._typesizes[i] 983 | 984 | @staticmethod 985 | def getfromformatcharacter(c): 986 | ''' get the field type given its formatting character as used by the `struct` module ''' 987 | try: 988 | i = FieldType._formatCharacters.index(c) 989 | return FieldType._formatNumbers[i] 990 | except: 991 | raise ValueError("Invalid format character: " + c) 992 | 993 | @staticmethod 994 | def getformatcharacter(fieldtype): 995 | ''' get the formatting character of a field type, as used by the `struct` module ''' 996 | try: 997 | i = FieldType._formatNumbers.index(fieldtype) 998 | return FieldType._formatCharacters[i] 999 | except: 1000 | raise ValueError("Invalid fieldtype: " + fieldtype) 1001 | 1002 | @staticmethod 1003 | def getmagicvalue(fieldtype): 1004 | ''' given a fieldtype, get a magic value. This is used for analyzing the item layout. ''' 1005 | return FieldType._magicValues[fieldtype] 1006 | 1007 | @staticmethod 1008 | def getname(fieldtype): 1009 | ''' get the string representation of `fieldtype`` ''' 1010 | return FieldType._typeNames[fieldtype] 1011 | 1012 | 1013 | class Field: 1014 | ''' 1015 | Describes a field inside an item. 1016 | 1017 | Attributes are: 1018 | * name 1019 | * offset 1020 | * istime 1021 | * iseventtime 1022 | * index 1023 | * formatchar 1024 | ''' 1025 | #pylint:disable-msg=R0903 1026 | def __init__(self): 1027 | self.name = "" 1028 | self.offset = None 1029 | self.fieldtype = None 1030 | self.istime = False 1031 | self.iseventtime = False 1032 | self.index = None 1033 | self.formatchar = None 1034 | 1035 | def getvalue(self, item): 1036 | ''' Given a field and an item, returns the value of this field. 1037 | 1038 | If the field is a time field, the value is packed into a `Time`, unless 1039 | configured otherwise by setting `use_time_decoration` to False. 1040 | ''' 1041 | value = item[self.index] 1042 | if self.istime: 1043 | value = DateTime(ticks=value) 1044 | return value 1045 | 1046 | def decoratetime(self, item): 1047 | value = item[self.index] 1048 | if isinstance(value, DateTime): 1049 | value = value.ticks # unpack time into its ticks 1050 | return value 1051 | 1052 | def __repr__(self): 1053 | return "{:10} Type:{:>7} Offset:{:>2} IsTime:{} IsEventTime:{}".format(self.name, FieldType.getname(self.fieldtype), self.offset, int(self.istime), int(self.iseventtime)) 1054 | 1055 | 1056 | # context, section formatters 1057 | class _ReadContext: 1058 | ''' context used for header reading ''' 1059 | #pylint:disable-msg=R0903 1060 | def __init__(self, formattedreader): 1061 | self.reader = formattedreader 1062 | self.description = TeaFileDescription() 1063 | self.itemareastart = None 1064 | self.itemareaend = None 1065 | self.sectioncount = None 1066 | 1067 | 1068 | class _WriteContext: 1069 | ''' context used for header writing ''' 1070 | #pylint:disable-msg=R0903 1071 | def __init__(self, formattedwriter): 1072 | self.writer = formattedwriter 1073 | self.itemareastart = None 1074 | self.itemareaend = None 1075 | self.sectioncount = None 1076 | self.description = None 1077 | 1078 | 1079 | # disable method could be function warnings for the formatters read/write methods. check back later if and how this could be done more pythonesk 1080 | #pylint:disable-msg=R0201 1081 | 1082 | class _ItemSectionFormatter: 1083 | ''' reads and writes the itemdescription into / from the file ''' 1084 | id = 10 1085 | 1086 | def read(self, rc): 1087 | ''' read the section ''' 1088 | id_ = ItemDescription() 1089 | r = rc.reader 1090 | id_.itemsize = r.readint32() 1091 | id_.itemname = r.readtext() 1092 | fieldcount = r.readint32() 1093 | i = 0 1094 | for _ in range(fieldcount): 1095 | f = Field() 1096 | f.index = i 1097 | f.fieldtype = r.readint32() 1098 | f.offset = r.readint32() 1099 | f.name = r.readtext() 1100 | id_.fields.append(f) 1101 | i += 1 1102 | id_.setupfromfields() 1103 | rc.description.itemdescription = id_ 1104 | 1105 | def write(self, wc): 1106 | ''' writes the section ''' 1107 | id_ = wc.description.itemdescription 1108 | w = wc.writer 1109 | w.writeint32(id_.itemsize) 1110 | w.writetext(id_.itemname) 1111 | w.writeint32(len(id_.fields)) 1112 | for f in id_.fields: 1113 | w.writeint32(f.fieldtype) 1114 | w.writeint32(f.offset) 1115 | w.writetext(f.name) 1116 | 1117 | 1118 | class _ContentSectionFormatter: 1119 | ''' reads and writes the contentdescription into / from the file ''' 1120 | id = 0x80 1121 | 1122 | def read(self, rc): 1123 | ''' read the section ''' 1124 | r = rc.reader 1125 | rc.description.contentdescription = r.readtext() 1126 | 1127 | def write(self, wc): 1128 | ''' writes the section ''' 1129 | cd = wc.description.contentdescription 1130 | if cd: 1131 | wc.writer.writetext(cd) 1132 | 1133 | 1134 | class _NameValueSectionFormatter: 1135 | ''' reads and writes the namevalue-description into / from the file ''' 1136 | id = 0x81 1137 | 1138 | def read(self, rc): 1139 | ''' read the section ''' 1140 | r = rc.reader 1141 | n = r.readint32() 1142 | if n == 0: 1143 | return 1144 | nvc = {} 1145 | while n > 0: 1146 | nv = r.readnamevalue() 1147 | nvc.update(nv) 1148 | n = n - 1 1149 | rc.description.namevalues = nvc 1150 | 1151 | def write(self, wc): 1152 | ''' writes the section ''' 1153 | nvs = wc.description.namevalues 1154 | if not nvs: 1155 | return 1156 | w = wc.writer 1157 | w.writeint32(len(nvs)) 1158 | for key, value in nvs.items(): 1159 | w.writenamevalue(key, value) 1160 | 1161 | 1162 | class _TimeSectionFormatter: 1163 | ''' reads and writes the timescale and description of time fields into / from the file ''' 1164 | id = 0x40 1165 | 1166 | def read(self, rc): 1167 | ''' read the section ''' 1168 | r = rc.reader 1169 | 1170 | # time scale 1171 | epoch = r.readint64() 1172 | ticksperday = r.readint64() 1173 | rc.description.timescale = TimeScale(epoch, ticksperday) 1174 | 1175 | # time fields 1176 | timefieldcount = r.readint32() 1177 | if timefieldcount == 0: 1178 | return 1179 | 1180 | id_ = rc.description.itemdescription 1181 | isfirsttimefield = True 1182 | for _ in range(timefieldcount): 1183 | o = r.readint32() 1184 | f = id_.getfieldbyoffset(o) 1185 | f.istime = True 1186 | f.iseventtime = isfirsttimefield 1187 | isfirsttimefield = False 1188 | 1189 | def write(self, wc): 1190 | ''' writes the section ''' 1191 | w = wc.writer 1192 | # this api restricts time formats to JavaTime 1193 | # in addition, the first field named "time" is considered the EventTime 1194 | w.writeint64(719162) # days between 0001-01-01 and 1970-01-01 1195 | w.writeint64(86400 * 1000) # millisecond resolution 1196 | id_ = wc.description.itemdescription 1197 | timefields = [f for f in id_.fields if f.name.lower() == "time"] 1198 | w.writeint32(len(timefields)) # will be 0 or 1 1199 | for f in timefields: 1200 | w.writeint32(f.offset) 1201 | 1202 | 1203 | class _HeaderManager: 1204 | ''' reads and writes the file header, delegating the formatting of sections to the sectionformatters. ''' 1205 | def __init__(self): 1206 | self.sectionformatters = ([ 1207 | _ItemSectionFormatter(), 1208 | _ContentSectionFormatter(), 1209 | _NameValueSectionFormatter(), 1210 | _TimeSectionFormatter()]) 1211 | 1212 | def getformatter(self, id_): 1213 | ''' the a formatter given its id ''' 1214 | for f in self.sectionformatters: 1215 | if f.id == id_: 1216 | return f 1217 | raise RuntimeError() 1218 | 1219 | def readheader(self, r): 1220 | ''' read the file header ''' 1221 | rc = _ReadContext(r) 1222 | bom = r.readint64() 1223 | if bom != 0x0d0e0a0402080500: 1224 | print("Byteordermark mismatch: ", bom) 1225 | raise RuntimeError() 1226 | rc.itemareastart = r.readint64() 1227 | rc.itemareaend = r.readint64() 1228 | rc.sectioncount = r.readint64() 1229 | n = rc.sectioncount 1230 | while n > 0: 1231 | self.readsection(rc) 1232 | n = n - 1 1233 | bytestoskip = rc.itemareastart - r.position() # padding bytes between header and item area 1234 | r.skipbytes(bytestoskip) 1235 | return rc 1236 | 1237 | def readsection(self, rc): 1238 | ''' read a section ''' 1239 | r = rc.reader 1240 | sectionid = r.readint32() 1241 | nextsectionoffset = r.readint32() 1242 | beforesection = r.position() 1243 | f = self.getformatter(sectionid) 1244 | f.read(rc) 1245 | aftersection = r.position() 1246 | if (aftersection - beforesection) > nextsectionoffset: 1247 | print("section reads too many bytes") 1248 | raise RuntimeError() 1249 | 1250 | def writeheader(self, fw, description): 1251 | ''' write the file header ''' 1252 | wc = _WriteContext(fw) 1253 | wc.itemareastart = 32 1254 | wc.itemareaend = 0 # no preallocation 1255 | wc.description = description 1256 | wc.sectioncount = 0 1257 | sectionbytes = self.createsections(wc) 1258 | 1259 | fw.writeint64(0x0d0e0a0402080500) 1260 | fw.writeint64(wc.itemareastart) 1261 | fw.writeint64(wc.itemareaend) 1262 | fw.writeint64(wc.sectioncount) 1263 | wc.writer.writeraw(sectionbytes) 1264 | 1265 | return wc 1266 | 1267 | def createsections(self, wc): 1268 | ''' writing the sections into the file raises a small challange: 1269 | before writing the first section, we need to know how many sections will follow, as the 1270 | file format prefixes the section count before the actual sections. 1271 | This implementation accomplishes this by writing the header first into memory, afterwards 1272 | the sectioncount and the sections. Alternatives would be to move the file pointer or 1273 | to enhance the sectionformatters such that they provide the section length without 1274 | writing the section. 1275 | ''' 1276 | saved = wc.writer 1277 | sectionstream = BytesIO() 1278 | sectionwriter = _FormattedWriter(_FileIO(sectionstream)) 1279 | pos = 32 # sections start at byte position 32 1280 | for formatter in self.sectionformatters: 1281 | payloadstream = BytesIO() 1282 | wc.writer = _FormattedWriter(_FileIO(payloadstream)) 1283 | formatter.write(wc) 1284 | payload = payloadstream.getvalue() 1285 | size = len(payload) 1286 | if size > 0: 1287 | # section id 1288 | sectionwriter.writeint32(formatter.id) 1289 | pos += 4 1290 | 1291 | # nextSectionOffset 1292 | sectionwriter.writeint32(size) 1293 | pos += 4 1294 | 1295 | # payload 1296 | sectionwriter.writeraw(payloadstream.getvalue()) 1297 | pos += size # no padding or spacing done here 1298 | 1299 | wc.sectioncount += 1 1300 | 1301 | # padding 1302 | paddingbytes = 8 - pos % 8 1303 | if paddingbytes == 8: 1304 | paddingbytes = 0 1305 | if paddingbytes: 1306 | padding = b"\0" * paddingbytes 1307 | sectionwriter.writeraw(padding) 1308 | wc.itemareastart = pos + paddingbytes # first item starts padded on 8 byte boundary. 1309 | 1310 | wc.writer = saved 1311 | return sectionstream.getvalue() 1312 | 1313 | if __name__ == '__main__': 1314 | import doctest, teafiles.teafile 1315 | doctest.testmod(teafiles.teafile, optionflags = doctest.ELLIPSIS) 1316 | --------------------------------------------------------------------------------