├── .gitignore ├── acme.tea ├── teafiles.nimble ├── README.md ├── test.nim ├── LICENSE └── teafiles.nim /.gitignore: -------------------------------------------------------------------------------- 1 | nimcache 2 | /test 3 | acme2.tea -------------------------------------------------------------------------------- /acme.tea: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andreaferretti/nim-teafiles/HEAD/acme.tea -------------------------------------------------------------------------------- /teafiles.nimble: -------------------------------------------------------------------------------- 1 | # Copyright 2017 UniCredit S.p.A. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | mode = ScriptMode.Verbose 16 | 17 | packageName = "teafiles" 18 | version = "0.1.5" 19 | author = "Andrea Ferretti" 20 | description = "TeaFiles for Nim" 21 | license = "Apache2" 22 | skipFiles = @["test.nim", "acme.tea"] 23 | 24 | requires "nim >= 0.18.0" 25 | 26 | --forceBuild 27 | 28 | task test, "run tests": 29 | --hints: off 30 | --linedir: on 31 | --stacktrace: on 32 | --linetrace: on 33 | --debuginfo 34 | --run 35 | setCommand "c", "test.nim" 36 | 37 | task gendoc, "generate documentation": 38 | --docSeeSrcUrl: https://github.com/unicredit/nim-teafiles/blob/master 39 | setCommand "doc2", "teafiles.nim" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Nim TeaFiles 2 | ============ 3 | 4 | [![nimble](https://raw.githubusercontent.com/yglukhov/nimble-tag/master/nimble.png)](https://github.com/yglukhov/nimble-tag) 5 | 6 | This is a Nim library to read and write [TeaFiles](http://discretelogics.com/teafiles/). 7 | 8 | TeaFiles provide fast read/write access to time series data from any software 9 | package on any platform. Time Series are considered homogeneous collections of 10 | items, ordered by their timestamp. Items are stored in raw binary format, such 11 | that data can be memory mapped for fast read/write access. In order to ensure 12 | correct data interpretation when data is exchanged between multiple applications, 13 | TeaFiles optionally embedd a description of the data layout in the file header, 14 | along with other optional description of the file's contents. 15 | 16 | Documentation 17 | ------------- 18 | 19 | There is not much in terms of documentation yet, but you can have a look at 20 | 21 | * the [tests usage](https://github.com/unicredit/nim-teafiles/blob/master/test.nim) 22 | * the [API docs](http://unicredit.github.io/nim-teafiles/api.html) 23 | * the [original TeaFiles spec](http://discretelogics.com/resources/teafilespec/) 24 | 25 | License 26 | ------- 27 | 28 | Unlike the TeaFiles packages provided for other languages, this library is 29 | provided under the [Apache2 license](http://www.apache.org/licenses/LICENSE-2.0). 30 | This is possible because it is a clean room implementation entirely derived 31 | from the [TeaFiles specification](http://discretelogics.com/resources/teafilespec/). 32 | 33 | Credits 34 | ------- 35 | 36 | The TeaFiles format was originally designed and developed by [DiscreteLogics](http://discretelogics.com). 37 | TeaFiles is a copyright by DiscreteLogics. -------------------------------------------------------------------------------- /test.nim: -------------------------------------------------------------------------------- 1 | # Copyright 2016 UniCredit S.p.A. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import unittest, teafiles, os, tables 16 | 17 | type Tick = object 18 | date: int64 19 | price: float64 20 | volume: int64 21 | 22 | suite "reading files": 23 | test "reading a file while knowing the type": 24 | var f = teafile[Tick]("acme.tea") 25 | echo f 26 | f.close() 27 | 28 | test "reading an untyped file": 29 | var f = dynteafile("acme.tea") 30 | echo f 31 | echo "We can read fields dynamically, knowing the type: ", f[0]["Time", int64] 32 | echo "...or even not knowing it: ", f[0]["Time"] 33 | echo "We can also iterate over a single field:" 34 | for price in f.col("Price", float64): 35 | echo price 36 | echo "Even without knowing the exact type:" 37 | for price in f.floatcol("Price"): 38 | echo price 39 | f.close() 40 | 41 | suite "writing files": 42 | test "writing a file and reading it back": 43 | var f = teafile[Tick]("acme.tea") 44 | var w = create("acme2.tea", meta(f)) 45 | for tick in f: 46 | append[Tick](w, tick) 47 | w.close() 48 | f.close() 49 | var g = dynteafile("acme2.tea") 50 | echo g 51 | g.close() 52 | removeFile("acme2.tea") 53 | 54 | suite "metadata": 55 | test "reading metadata from a file": 56 | var f = teafile[Tick]("acme.tea") 57 | let metadata = meta(f) 58 | check metadata.content.description == "ACME at NYSE" 59 | check metadata.namevalues.ints["decimals"] == 2 60 | check metadata.time.ticksPerDay == 86400000 61 | check metadata.items.fields[0].name == "Time" 62 | f.close() -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /teafiles.nim: -------------------------------------------------------------------------------- 1 | # Copyright 2016 UniCredit S.p.A. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import os, memfiles, streams, tables 17 | 18 | const teaForTwoAtFive = 0x0d0e0a0402080500 19 | 20 | ### TYPES ### 21 | 22 | type 23 | Uuid = array[16, byte] 24 | Section = enum 25 | itemSection = 0x0a.int32 26 | timeSection = 0x40.int32 27 | contentSection = 0x80.int32 28 | nameValueSection = 0x81.int32 29 | FieldType* = enum 30 | DoesNotExist = 0 # just because it is used as discriminator 31 | Int8 = 1 32 | Int16 = 2 33 | Int32 = 3 34 | Int64 = 4 35 | UInt8 = 5 36 | UInt16 = 6 37 | UInt32 = 7 38 | UInt64 = 8 39 | Float = 9 40 | Double = 10 41 | ValueType = enum 42 | intType = 1 43 | floatType = 2 44 | stringType = 3 45 | uuidType = 4 46 | Field* = object 47 | tp*: FieldType 48 | offset*: int32 49 | name*: string 50 | Header = object 51 | itemStart: int64 52 | itemEnd: int64 53 | sectionCount: int64 54 | ContentSection* = object 55 | description*: string 56 | TimeSection* = object 57 | epoch*: int64 58 | ticksPerDay*: int64 59 | timeFields*: seq[int32] 60 | NameValueSection* = object 61 | ints*: Table[string, int32] 62 | floats*: Table[string, float64] 63 | strings*: Table[string, string] 64 | uuids*: Table[string, Uuid] 65 | ItemSection* = object 66 | size*: int32 67 | name*: string 68 | fields*: seq[Field] 69 | TeaFile*[T] = object 70 | header: Header 71 | content: ContentSection 72 | time: TimeSection 73 | namevalues: NameValueSection 74 | items: ItemSection 75 | data: ptr[UncheckedArray[T]] 76 | underlying: MemFile 77 | Dyn* = object 78 | layout: seq[Field] 79 | position: pointer 80 | DynTeaFile* = object 81 | header: Header 82 | content: ContentSection 83 | time: TimeSection 84 | namevalues: NameValueSection 85 | items: ItemSection 86 | data: ptr[UncheckedArray[byte]] 87 | underlying: MemFile 88 | AnyTeaFile = TeaFile or DynTeaFile 89 | Number* = object 90 | case kind*: FieldType 91 | of DoesNotExist: 92 | discard 93 | of Int8: 94 | int8value*: int8 95 | of Int16: 96 | int16value*: int16 97 | of Int32: 98 | int32value*: int32 99 | of Int64: 100 | int64value*: int64 101 | of UInt8: 102 | uint8value*: uint8 103 | of UInt16: 104 | uint16value*: uint16 105 | of UInt32: 106 | uint32value*: uint32 107 | of UInt64: 108 | uint64value*: uint64 109 | of Float: 110 | float32value*: float32 111 | of Double: 112 | float64value*: float64 113 | Meta* = object 114 | content*: ContentSection 115 | time*: TimeSection 116 | namevalues*: NameValueSection 117 | items*: ItemSection 118 | WritableTeaFile = object 119 | meta: Meta 120 | stream: Stream 121 | 122 | ### POINTER HELPERS ### 123 | 124 | template advance(p: pointer, b: int) = 125 | p = cast[pointer](cast[int](p) + b) 126 | 127 | template next(p: pointer, T: typedesc): auto = 128 | let x = cast[ptr T](p)[] 129 | when T is Section or T is ValueType or T is FieldType: 130 | advance(p, 4) 131 | else: 132 | advance(p, sizeof(T)) 133 | x 134 | 135 | template nextString(p: pointer): auto = 136 | # let 137 | # length = p.next(int32) 138 | # bytes = cast[ptr UncheckedArray[char]](p) 139 | let length = cast[ptr int32](p)[] 140 | advance(p, sizeof(int32)) 141 | let bytes = cast[ptr UncheckedArray[char]](p) 142 | var s = newString(length) 143 | for i in 0'i64 ..< length: 144 | s[i.int] = bytes[i.int] 145 | advance(p, length.int) 146 | s 147 | 148 | ### READING PREAMBLE ### 149 | 150 | proc readHeader(tea: var AnyTeaFile) = 151 | var cursor = tea.underlying.mem 152 | let magic = cursor.next(int64) 153 | assert magic == teaForTwoAtFive 154 | tea.header.itemStart = cursor.next(int64) 155 | tea.header.itemEnd = cursor.next(int64) 156 | tea.header.sectionCount = cursor.next(int64) 157 | assert tea.header.sectionCount >= 0 and tea.header.sectionCount <= 4 158 | assert tea.header.itemStart > 0 159 | assert tea.header.itemEnd >= 0 160 | 161 | proc readContentSection(tea: var AnyTeaFile, cursor: pointer) = 162 | var c = cursor 163 | tea.content.description = c.nextString() 164 | 165 | proc readTimeSection(tea: var AnyTeaFile, cursor: pointer) = 166 | var c = cursor 167 | tea.time.epoch = c.next(int64) 168 | tea.time.ticksPerDay = c.next(int64) 169 | let L = c.next(int32) 170 | tea.time.timeFields = newSeq[int32](L) 171 | for i in 0 ..< L: 172 | tea.time.timeFields[i] = c.next(int32) 173 | 174 | proc readNameValueSection(tea: var AnyTeaFile, cursor: pointer) = 175 | tea.namevalues.ints = initTable[string, int32]() 176 | tea.namevalues.floats = initTable[string, float64]() 177 | tea.namevalues.strings = initTable[string, string]() 178 | tea.namevalues.uuids = initTable[string, Uuid]() 179 | var c = cursor 180 | let length = c.next(int32) 181 | for _ in 1 .. length: 182 | let key = c.nextString() 183 | case c.next(ValueType): 184 | of intType: 185 | tea.namevalues.ints.add(key, c.next(int32)) 186 | of floatType: 187 | tea.namevalues.floats.add(key, c.next(float64)) 188 | of stringType: 189 | tea.namevalues.strings.add(key, c.nextString()) 190 | of uuidType: 191 | tea.namevalues.uuids.add(key, c.next(Uuid)) 192 | 193 | proc readItemSection(tea: var AnyTeaFile, cursor: pointer) = 194 | var c = cursor 195 | tea.items.size = c.next(int32) 196 | tea.items.name = c.nextString() 197 | let length = c.next(int32) 198 | # assert length >= 1 199 | tea.items.fields = newSeq[Field](length) 200 | for i in 0 ..< length: 201 | tea.items.fields[i] = Field( 202 | tp: c.next(FieldType), 203 | offset: c.next(int32), 204 | name: c.nextString() 205 | ) 206 | 207 | proc readSection(tea: var AnyTeaFile, cursor: var pointer) = 208 | let id = cursor.next(Section) 209 | let offset = cursor.next(int32) 210 | case id: 211 | of contentSection: 212 | readContentSection(tea, cursor) 213 | of timeSection: 214 | readTimeSection(tea, cursor) 215 | of nameValueSection: 216 | readNameValueSection(tea, cursor) 217 | of itemSection: 218 | readItemSection(tea, cursor) 219 | cursor.advance(offset.int) 220 | 221 | proc readSections(tea: var AnyTeaFile) = 222 | var cursor = tea.underlying.mem 223 | cursor.advance(32) 224 | for _ in 1 .. tea.header.sectionCount: 225 | readSection(tea, cursor) 226 | 227 | proc readStart[T](tea: var TeaFile[T]) = 228 | var cursor = tea.underlying.mem 229 | cursor.advance(tea.header.itemStart.int) 230 | tea.data = cast[ptr[UncheckedArray[T]]](cursor) 231 | 232 | proc readStart(tea: var DynTeaFile) = 233 | var cursor = tea.underlying.mem 234 | cursor.advance(tea.header.itemStart.int) 235 | tea.data = cast[ptr[UncheckedArray[byte]]](cursor) 236 | 237 | ### WRITING PREAMBLE ### 238 | 239 | proc len(s: ContentSection): int32 = 240 | (4 + s.description.len).int32 241 | 242 | proc len(s: TimeSection): int32 = 243 | (8 + 8 + 4 + 4 * s.timeFields.len).int32 244 | 245 | proc len(s: NameValueSection): int32 = 246 | result = 12'i32 247 | for k, v in s.ints: 248 | result += (8 + len(k)).int32 249 | for k, v in s.floats: 250 | result += (12 + len(k)).int32 251 | for k, v in s.strings: 252 | result += (8 + len(v) + len(k)).int32 253 | for k, v in s.uuids: 254 | result += (16 + len(k)).int32 255 | 256 | proc len(s: ItemSection): int32 = 257 | result = (12 + s.name.len).int32 258 | for field in s.fields: 259 | result += (12 + field.name.len).int32 260 | 261 | proc lengthAndPadding(meta: Meta): tuple[length: int64, padding: int] = 262 | let 263 | length = (64 + meta.content.len + meta.time.len + 264 | meta.namevalues.len + meta.items.len).int64 265 | rem = length mod 8 266 | padding = if rem == 0: 0 else: 8 - rem.int 267 | return (length, padding) 268 | 269 | proc len(meta: Meta): int64 = 270 | let (length, padding) = lengthAndPadding(meta) 271 | return length + padding 272 | 273 | proc writeString(s: var Stream, text: string) = 274 | s.write(text.len.int32) 275 | s.write(text) 276 | 277 | proc writeHeader(s: var Stream, meta: Meta) = 278 | s.write(teaForTwoAtFive) 279 | s.write(meta.len) # itemStart 280 | s.write(0'i64) # itemEnd 281 | s.write(4'i64) # sectionCount 282 | 283 | proc writeContent(s: var Stream, meta: Meta) = 284 | s.write(contentSection.int32) 285 | s.write(meta.content.len) 286 | s.writeString(meta.content.description) 287 | 288 | proc writeTimeSection(s: var Stream, meta: Meta) = 289 | s.write(timeSection.int32) 290 | s.write(meta.time.len) 291 | s.write(meta.time.epoch) 292 | s.write(meta.time.ticksPerDay) 293 | s.write(meta.time.timeFields.len.int32) 294 | for t in meta.time.timeFields: 295 | s.write(t) 296 | 297 | proc writeItemSection(s: var Stream, meta: Meta) = 298 | s.write(itemSection.int32) 299 | s.write(meta.items.len) 300 | s.write(meta.items.size.int32) 301 | s.writeString(meta.items.name) 302 | s.write(meta.items.fields.len.int32) 303 | for field in meta.items.fields: 304 | s.write(field.tp.int32) 305 | s.write(field.offset) 306 | s.writeString(field.name) 307 | 308 | proc writeNameValueSection(s: var Stream, meta: Meta) = 309 | s.write(nameValueSection.int32) 310 | s.write(meta.namevalues.len) 311 | let length = len(meta.namevalues.ints) + len(meta.namevalues.floats) + 312 | len(meta.namevalues.strings) + len(meta.namevalues.uuids) 313 | s.write(length.int32) 314 | for k, v in meta.namevalues.ints: 315 | s.writeString(k) 316 | s.write(intType.int32) 317 | s.write(v) 318 | for k, v in meta.namevalues.floats: 319 | s.writeString(k) 320 | s.write(floatType.int32) 321 | s.write(v) 322 | for k, v in meta.namevalues.strings: 323 | s.writeString(k) 324 | s.write(stringType.int32) 325 | s.writeString(v) 326 | for k, v in meta.namevalues.uuids: 327 | s.writeString(k) 328 | s.write(uuidType.int32) 329 | s.write(v) 330 | 331 | proc writePadding(s: var Stream, meta: Meta) = 332 | let (_, padding) = lengthAndPadding(meta) 333 | for _ in 1.. padding: 334 | s.write(0'i8) 335 | 336 | proc create*(path: string, meta: Meta): WritableTeaFile = 337 | result.meta = meta 338 | result.stream = newFileStream(path, fmWrite) 339 | result.stream.writeHeader(meta) 340 | result.stream.writeItemSection(meta) 341 | result.stream.writeContent(meta) 342 | result.stream.writeNameValueSection(meta) 343 | result.stream.writeTimeSection(meta) 344 | result.stream.writePadding(meta) 345 | 346 | ### OPENING AND CLOSING TEAFILES ### 347 | 348 | proc teafile*[T](path: string): TeaFile[T] = 349 | let f = memfiles.open(path, mode = fmReadWrite) 350 | result.underlying = f 351 | readHeader(result) 352 | readSections(result) 353 | readStart(result) 354 | 355 | proc dynteafile*(path: string): DynTeaFile = 356 | let f = memfiles.open(path, mode = fmRead) 357 | result.underlying = f 358 | readHeader(result) 359 | readSections(result) 360 | readStart(result) 361 | 362 | proc close*(tea: var AnyTeaFile) = close(tea.underlying) 363 | 364 | proc close*(tea: var WritableTeaFile) = close(tea.stream) 365 | 366 | ### ACCESSORS ### 367 | 368 | proc len*[T](tea: TeaFile[T]): int64 = 369 | if tea.header.itemEnd == 0: 370 | (tea.underlying.size - tea.header.itemStart) div sizeof(T) 371 | else: 372 | (tea.header.itemEnd - tea.header.itemStart) div sizeof(T) 373 | 374 | proc len*(tea: DynTeaFile): int64 = 375 | if tea.header.itemEnd == 0: 376 | (tea.underlying.size - tea.header.itemStart) div tea.items.size 377 | else: 378 | (tea.header.itemEnd - tea.header.itemStart) div tea.items.size 379 | 380 | proc `[]`*[T](tea: TeaFile[T], i: int): T = 381 | assert i >= 0 and i < len(tea) 382 | tea.data[i] 383 | 384 | proc `[]`*(tea: DynTeaFile, i: int): Dyn = 385 | assert i >= 0 and i < len(tea) 386 | result.layout = tea.items.fields 387 | result.position = tea.data 388 | advance(result.position, i * tea.items.size) 389 | 390 | proc `[]=`*[T](tea: var TeaFile[T], i: int, val: T) = 391 | assert i >= 0 and i < len(tea) 392 | tea.data[i] = val 393 | 394 | proc append*[T](tea: var WritableTeaFile, val: T) = 395 | tea.stream.write(val) 396 | 397 | ### ITERATORS ### 398 | 399 | iterator items*[T](tea: TeaFile[T]): T {.inline.} = 400 | for i in 0'i64 ..< len(tea): 401 | yield tea.data[i] 402 | 403 | iterator items*(tea: DynTeaFile): Dyn {.inline.} = 404 | var p = cast[pointer](tea.data) 405 | for _ in 0'i64 ..< len(tea): 406 | yield Dyn(layout: tea.items.fields, position: cast[ptr UncheckedArray[byte]](p)) 407 | advance(p, tea.items.size) 408 | 409 | iterator pairs*[T](tea: TeaFile[T]): tuple[key: int, val: T] {.inline.} = 410 | for i in 0'i64 ..< len(tea): 411 | yield (i, tea.data[i]) 412 | 413 | iterator pairs*(tea: DynTeaFile): tuple[key: int64, val: Dyn] {.inline.} = 414 | var p = cast[pointer](tea.data) 415 | for i in 0'i64 ..< len(tea): 416 | yield (i, Dyn(layout: tea.items.fields, position: cast[ptr UncheckedArray[byte]](p))) 417 | advance(p, tea.items.size) 418 | 419 | iterator col*(tea: AnyTeaFile, key: string, T: typedesc): T = 420 | var 421 | offset: int32 422 | tp: FieldType 423 | found = false 424 | for f in tea.items.fields: 425 | if f.name == key: 426 | offset = f.offset 427 | tp = f.tp 428 | found = true 429 | break 430 | if not found: 431 | assert false 432 | when T is int8: assert tp == Int8 433 | when T is int16: assert tp == Int16 434 | when T is int32: assert tp == Int32 435 | when T is int64: assert tp == Int64 436 | when T is uint8: assert tp == UInt8 437 | when T is uint16: assert tp == UInt16 438 | when T is uint32: assert tp == UInt32 439 | when T is uint64: assert tp == UInt64 440 | when T is float32: assert tp == Float 441 | when T is float64: assert tp == Double 442 | var p = cast[pointer](tea.data) 443 | advance(p, offset) 444 | for i in 0'i64 ..< len(tea): 445 | yield cast[ptr T](p)[] 446 | advance(p, tea.items.size) 447 | 448 | iterator intcol*(tea: AnyTeaFile, key: string): int = 449 | var 450 | offset: int32 451 | tp: FieldType 452 | found = false 453 | for f in tea.items.fields: 454 | if f.name == key: 455 | offset = f.offset 456 | tp = f.tp 457 | found = true 458 | break 459 | if not found: 460 | assert false 461 | var p = cast[pointer](tea.data) 462 | advance(p, offset) 463 | for i in 0'i64 ..< len(tea): 464 | case tp: 465 | of DoesNotExist: 466 | assert false 467 | of Int8: 468 | yield int(cast[ptr int8](p)[]) 469 | of Int16: 470 | yield int(cast[ptr int16](p)[]) 471 | of Int32: 472 | yield int(cast[ptr int32](p)[]) 473 | of Int64: 474 | yield int(cast[ptr int64](p)[]) 475 | of UInt8: 476 | yield int(cast[ptr uint8](p)[]) 477 | of UInt16: 478 | yield int(cast[ptr uint16](p)[]) 479 | of UInt32: 480 | yield int(cast[ptr uint32](p)[]) 481 | of UInt64: 482 | yield int(cast[ptr uint64](p)[]) 483 | of Float: 484 | yield int(cast[ptr float32](p)[]) 485 | of Double: 486 | yield int(cast[ptr float64](p)[]) 487 | advance(p, tea.items.size) 488 | 489 | iterator floatcol*(tea: AnyTeaFile, key: string): float = 490 | var 491 | offset: int32 492 | tp: FieldType 493 | found = false 494 | for f in tea.items.fields: 495 | if f.name == key: 496 | offset = f.offset 497 | tp = f.tp 498 | found = true 499 | break 500 | if not found: 501 | assert false 502 | var p = cast[pointer](tea.data) 503 | advance(p, offset) 504 | for i in 0'i64 ..< len(tea): 505 | case tp: 506 | of DoesNotExist: 507 | assert false 508 | of Int8: 509 | yield float(cast[ptr int8](p)[]) 510 | of Int16: 511 | yield float(cast[ptr int16](p)[]) 512 | of Int32: 513 | yield float(cast[ptr int32](p)[]) 514 | of Int64: 515 | yield float(cast[ptr int64](p)[]) 516 | of UInt8: 517 | yield float(cast[ptr uint8](p)[]) 518 | of UInt16: 519 | yield float(cast[ptr uint16](p)[]) 520 | of UInt32: 521 | yield float(cast[ptr uint32](p)[]) 522 | of UInt64: 523 | yield float(cast[ptr uint64](p)[]) 524 | of Float: 525 | yield float(cast[ptr float32](p)[]) 526 | of Double: 527 | yield float(cast[ptr float64](p)[]) 528 | advance(p, tea.items.size) 529 | 530 | ### ACCESS TO DYNAMIC FIELDS ### 531 | 532 | proc `[]`*(d: Dyn, key: string): Number = 533 | var 534 | offset: int32 535 | tp: FieldType 536 | found = false 537 | for f in d.layout: 538 | if f.name == key: 539 | offset = f.offset 540 | tp = f.tp 541 | found = true 542 | break 543 | if not found: 544 | assert false 545 | var p = d.position 546 | advance(p, offset) 547 | case tp: 548 | of DoesNotExist: 549 | assert false 550 | of Int8: 551 | result = Number(kind: Int8, int8value: cast[ptr int8](p)[]) 552 | of Int16: 553 | result = Number(kind: Int16, int16value: cast[ptr int16](p)[]) 554 | of Int32: 555 | result = Number(kind: Int32, int32value: cast[ptr int32](p)[]) 556 | of Int64: 557 | result = Number(kind: Int64, int64value: cast[ptr int64](p)[]) 558 | of UInt8: 559 | result = Number(kind: UInt8, uint8value: cast[ptr uint8](p)[]) 560 | of UInt16: 561 | result = Number(kind: UInt16, uint16value: cast[ptr uint16](p)[]) 562 | of UInt32: 563 | result = Number(kind: UInt32, uint32value: cast[ptr uint32](p)[]) 564 | of UInt64: 565 | result = Number(kind: UInt64, uint64value: cast[ptr uint64](p)[]) 566 | of Float: 567 | result = Number(kind: Float, float32value: cast[ptr float32](p)[]) 568 | of Double: 569 | result = Number(kind: Double, float64value: cast[ptr float64](p)[]) 570 | 571 | proc `[]`*(d: Dyn, key: string, T: typedesc): T = 572 | var 573 | offset: int32 574 | tp: FieldType 575 | found = false 576 | for f in d.layout: 577 | if f.name == key: 578 | offset = f.offset 579 | tp = f.tp 580 | found = true 581 | break 582 | if not found: 583 | assert false 584 | var p = d.position 585 | advance(p, offset) 586 | when T is int8: assert tp == Int8 587 | when T is int16: assert tp == Int16 588 | when T is int32: assert tp == Int32 589 | when T is int64: assert tp == Int64 590 | when T is uint8: assert tp == UInt8 591 | when T is uint16: assert tp == UInt16 592 | when T is uint32: assert tp == UInt32 593 | when T is uint64: assert tp == UInt64 594 | when T is float32: assert tp == Float 595 | when T is float64: assert tp == Double 596 | return cast[ptr T](p)[] 597 | 598 | ### PRINTING ### 599 | 600 | proc `$`*(d: Dyn): string = 601 | result = "" 602 | for f in d.layout: 603 | result &= f.name & ": " 604 | var p = d.position 605 | advance(p, f.offset) 606 | case f.tp: 607 | of DoesNotExist: 608 | assert false 609 | of Int8: 610 | result &= $(cast[ptr int8](p)[]) & " " 611 | of Int16: 612 | result &= $(cast[ptr int16](p)[]) & " " 613 | of Int32: 614 | result &= $(cast[ptr int32](p)[]) & " " 615 | of Int64: 616 | result &= $(cast[ptr int64](p)[]) & " " 617 | of UInt8: 618 | result &= $(cast[ptr uint8](p)[]) & " " 619 | of UInt16: 620 | result &= $(cast[ptr uint16](p)[]) & " " 621 | of UInt32: 622 | result &= $(cast[ptr uint32](p)[]) & " " 623 | of UInt64: 624 | result &= $(cast[ptr uint64](p)[]) & " " 625 | of Float: 626 | result &= $(cast[ptr float32](p)[]) & " " 627 | of Double: 628 | result &= $(cast[ptr float64](p)[]) & " " 629 | 630 | proc `$`(u: Uuid): string = 631 | result = newString(16) 632 | for i in 1 .. 16: 633 | result[i] = u[i].char 634 | 635 | proc `$`(field: Field): string = 636 | field.name & " [" & $(field.tp) & "] -> " & $(field.offset) 637 | 638 | proc printDescription*(tea: AnyTeaFile): string = 639 | result = "" 640 | if tea.content.description != "": 641 | result &= "\nDescription:\n" 642 | result &= "------------\n" 643 | result &= tea.content.description & "\n" 644 | 645 | proc printItems*(tea: AnyTeaFile): string = 646 | result = "\nItems:\n" 647 | result &= "------\n" 648 | result &= "Record name: " & tea.items.name & "\n" 649 | result &= "Record length: " & $(tea.items.size) & "\n" 650 | result &= "Field offsets:\n" 651 | for f in tea.items.fields: 652 | result &= " " & $(f) & "\n" 653 | 654 | proc printNameValue*(tea: AnyTeaFile): string = 655 | result = "\nName/Value pairs:\n" 656 | result &= "-----------------\n" 657 | for k, v in tea.namevalues.ints: 658 | result &= k & ": " & $(v) & "\n" 659 | for k, v in tea.namevalues.floats: 660 | result &= k & ": " & $(v) & "\n" 661 | for k, v in tea.namevalues.strings: 662 | result &= k & ": " & v & "\n" 663 | for k, v in tea.namevalues.uuids: 664 | result &= k & ": " & $(v) & "\n" 665 | 666 | proc printTime*(tea: AnyTeaFile): string = 667 | result = "\nTime section:\n" 668 | result &= "-------------\n" 669 | result &= "Epoch: " & $(tea.time.epoch) & "\n" 670 | result &= "Ticks/day: " & $(tea.time.ticksPerDay) & "\n" 671 | result &= "Time fields: " & $(tea.time.timeFields) & "\n" 672 | 673 | proc printExcerpt*(tea: AnyTeaFile, maxItems: int = 5): string = 674 | let length = len(tea) 675 | result = "\nLength: " & $(length) & "\n" 676 | result &= "\nExcerpt:\n" 677 | result &= "--------\n" 678 | for i in 0'i64 ..< min(maxItems, length): 679 | result &= $(tea[i.int]) & "\n" 680 | if length > maxItems: 681 | result &= "...\n" 682 | 683 | proc print(tea: AnyTeaFile): string = 684 | result = printDescription(tea) 685 | result &= printItems(tea) 686 | result &= printNameValue(tea) 687 | result &= printTime(tea) 688 | result &= printExcerpt(tea) 689 | 690 | proc `$`*[T](tea: TeaFile[T]): string = print(tea) 691 | 692 | proc `$`*(tea: DynTeaFile): string = print(tea) 693 | 694 | ### CREATE METAS ### 695 | 696 | proc meta*(t: AnyTeaFile): Meta = 697 | result.content = t.content 698 | result.time = t.time 699 | result.namevalues = t.namevalues 700 | result.items = t.items 701 | 702 | proc meta*(description: string): Meta = 703 | result.content.description = description 704 | result.namevalues.ints = initTable[string, int32]() 705 | result.namevalues.floats = initTable[string, float64]() 706 | result.namevalues.strings = initTable[string, string]() 707 | result.namevalues.uuids = initTable[string, Uuid]() 708 | 709 | proc `[]=`*(meta: var Meta, key: string, val: int32) = 710 | meta.namevalues.ints[key] = val 711 | 712 | proc `[]=`*(meta: var Meta, key: string, val: float64) = 713 | meta.namevalues.floats[key] = val 714 | 715 | proc `[]=`*(meta: var Meta, key: string, val: string) = 716 | meta.namevalues.strings[key] = val 717 | 718 | proc `[]=`*(meta: var Meta, key: string, val: Uuid) = 719 | meta.namevalues.uuids[key] = val 720 | 721 | ### EXPORT TABLE FUNCTIONS NEEDED BY CLIENTS ### 722 | 723 | export tables.pairs --------------------------------------------------------------------------------