├── .github └── workflows │ ├── rust.yml │ └── security_audit.yml ├── .gitignore ├── .todo ├── Cargo.lock ├── Cargo.toml ├── README.md └── src ├── engine.rs ├── engine ├── array_vec.rs └── btree.rs ├── lib.rs ├── main.rs ├── query_engine.rs └── server.rs /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Install latest nightly 13 | uses: actions-rs/toolchain@v1 14 | with: 15 | toolchain: nightly 16 | profile: minimal 17 | override: true 18 | - name: Build 19 | run: cargo build --verbose 20 | - name: Run tests 21 | run: cargo test --verbose 22 | -------------------------------------------------------------------------------- /.github/workflows/security_audit.yml: -------------------------------------------------------------------------------- 1 | name: Security audit 2 | on: 3 | push: 4 | paths: 5 | - '**/Cargo.toml' 6 | - '**/Cargo.lock' 7 | jobs: 8 | security_audit: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Install latest nightly 13 | uses: actions-rs/toolchain@v1 14 | with: 15 | toolchain: nightly 16 | profile: minimal 17 | override: true 18 | - uses: actions-rs/audit-check@v1 19 | with: 20 | token: ${{ secrets.GITHUB_TOKEN }} 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | -------------------------------------------------------------------------------- /.todo: -------------------------------------------------------------------------------- 1 | [h] - multiple aggregates 2 | - last-block cache smart evicting 3 | - fsync policies 4 | - multiple codecs 5 | - text query parser 6 | - index based selects 7 | - group by 8 | - support tuples as value 9 | - support named tuples as values 10 | 11 | Timestamp(u64) - Type representing an UNIX timestamp 12 | 13 | Point - Type representing one data-point 14 | + timestamp: Timestamp 15 | + value: T 16 | 17 | Schema - Trait representing any schema (method of encoding the data) 18 | + type EncState: Default 19 | + type DecState: Default 20 | + encode(state: &mut EncState, entry: Point) -> SmallVec 21 | + decode(state: &mut DecState, buff: &[u8]) -> (Point, usize) 22 | 23 | F32 : Schema - One realization of Schema 24 | 25 | BlockHeader - Type representing metadata about the Block. 26 | + free_bytes 27 | 28 | Block - Type owning data of block 29 | + header: BlockHeader 30 | + data: [u8] 31 | 32 | BlockSpec - Type that represents a "handle" to potentially unloaded Block 33 | + block_id 34 | + series_id 35 | 36 | BlockCache - Type that owns loaded Blocks 37 | + blocks_cache: Map 38 | + lru: Vec> 39 | + lru_idx 40 | + contains_block(spec: BlockSpec) -> bool 41 | + get_mut(spec: BlockSpec) -> &mut Block 42 | + insert(spec: BlockSpec, block: Block) 43 | 44 | FsyncPolicy - Policy of sync-ing data to disk 45 | - Immediate 46 | - Never 47 | - Every10s 48 | 49 | BlockIO - load & saving block files to disk 50 | + open_files: Map 51 | + fsync_policy: FsyncPolicy 52 | + create_load_block(spec: BlockSpec) -> Block 53 | + write_block(spec: BlockSpec, block: &mut Block) 54 | 55 | BlocksInfo - Information about block storage structure. 56 | + block_count: usize 57 | + block_size: usize 58 | + last_block_unused_bytes 59 | 60 | TimestampIndex 61 | + file: File 62 | + fsync_policy: FsyncPolicy 63 | + data: Vec 64 | + dirty_from_idx 65 | + create_or_load() -> TimestampIndex 66 | + write_index() 67 | 68 | Series - Type that contains metadata about series 69 | + name: String 70 | + schema: F32 71 | + blocks_info: BlocksInfo 72 | + enc_state: 73 | + timestamp_index: TimestampIndex 74 | 75 | Clock - Source of time that provides timestamps that do not go backwards in time 76 | + now() -> Timestamp 77 | 78 | Server 79 | + clock: Clock 80 | + series: Map 81 | + cache: BlockCache 82 | + block_io: BlockIO 83 | + simple_commands: SimpleCommands 84 | + query_engine: QueryEngine 85 | 86 | SimpleCommands - Type that provides simple commands of DB for storing and retrieving data 87 | + create_series(series_name: String) 88 | + insert_point(series_name: String, point: Point) 89 | + retrieve_points(series_name: String, from: Timestamp, to: Timestamp) -> Iter> 90 | 91 | 92 | QueryEngine - Type that provides advanced querying capabilities (filtering, grouping, aggregate functions, sub-queries) 93 | + handle_query(query: Query) -> Result 94 | 95 | Selectable 96 | - Timestamp 97 | - Value 98 | - AggFn 99 | 100 | From 101 | - Series(String) 102 | - Temporary(Result) 103 | 104 | GroupBy 105 | - Minute 106 | - Hour 107 | - Day 108 | - Week 109 | - Month 110 | - Year 111 | 112 | Between 113 | + min_timestamp: Option 114 | + max_timestamp: Option 115 | 116 | Cond 117 | - Lt(T) 118 | - Gt(T) 119 | - Le(T) 120 | - Ge(T) 121 | - Eq(T) 122 | - Ne(T) 123 | 124 | Query 125 | + from: From 126 | + select: Vec 127 | + between: Option 128 | + where: Vec 129 | + group_by: GroupBy 130 | 131 | Result 132 | + points: Vec> ????? 133 | 134 | 135 | SELECT timestamp, value FROM outdoor_sunlight AFTER '01-01-2019 20:00:05' 136 | SELECT timestamp, value FROM outdoor_sunlight BETWEEN 01-01-2019 AND 02-01-2019 137 | Select([S::Timestamp, S::Value], Retrieve(Timestamp(), Timestamp(), "outdoor_sunlight")) 138 | 139 | SELECT AVG(value), MIN(value), MAX(value) FROM outdoor_sunlight BETWEEN 01-01-2019 AND 02-01-2019 GROUP BY HOUR 140 | Aggregate([Value(), Min(), Max()], A:HOUR, Retrieve(Timestamp(), Timestamp(), "outdoor_sunlight")) 141 | 142 | SELECT SUM(value), COUNT(value) FROM outdoor_sunlight AFTER NOW() - 1 HOUR 143 | SELECT timestamp, value FROM outdoor_sunlight WHERE value > 500 144 | 145 | SELECT AVG(a.value + b.value) FROM bikesharing_station01 a, bikesharing_station01 b GROUP BY DAY 146 | 147 | 148 | 0.dat (series file) 149 | 0.tsidx (main timestamp index file) 150 | 151 | Point 152 | + value: V 153 | + timestamp: Timestamp 154 | + tags: Vec 155 | 156 | Series 157 | + id: usize 158 | + timestamp_index: Index 159 | + indices: Vec> 160 | + encoder_state: V::EncState 161 | + schema: V::Schema 162 | 163 | 164 | 165 | 166 | ArrayVec - stack-allocated Vec 167 | 168 | Storage - provides fd pooling, loading/saving parts of files, fsync (periodic) 169 | 170 | Block - provides format of storing the data 171 | 172 | Index - provides indices (btree, bitmap) based on Storage & Block 173 | 174 | Cache - provides LRU cache with unevictable entries 175 | 176 | Engine - glues components together to allow storing and retrieval of data 177 | 178 | QueryEngine - provides support for advanced queries 179 | 180 | Protocol - defines how to talk to Server, parser 181 | 182 | Auth - defines authentication methods of Server 183 | 184 | Server - uses tokio to provides Engine over network, service 185 | 186 | 187 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | [[package]] 4 | name = "arc-swap" 5 | version = "0.4.4" 6 | source = "registry+https://github.com/rust-lang/crates.io-index" 7 | 8 | [[package]] 9 | name = "atty" 10 | version = "0.2.13" 11 | source = "registry+https://github.com/rust-lang/crates.io-index" 12 | checksum = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" 13 | dependencies = [ 14 | "libc", 15 | "winapi", 16 | ] 17 | 18 | [[package]] 19 | name = "autocfg" 20 | version = "0.1.7" 21 | source = "registry+https://github.com/rust-lang/crates.io-index" 22 | checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" 23 | 24 | [[package]] 25 | name = "bitflags" 26 | version = "1.2.1" 27 | source = "registry+https://github.com/rust-lang/crates.io-index" 28 | 29 | [[package]] 30 | name = "byteio" 31 | version = "0.2.3" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "1628aaab25b23803453d80873600ebffe6326577c1db81f5a3d123290e6287af" 34 | 35 | [[package]] 36 | name = "bytes" 37 | version = "0.5.3" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | 40 | [[package]] 41 | name = "cfg-if" 42 | version = "0.1.10" 43 | source = "registry+https://github.com/rust-lang/crates.io-index" 44 | checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" 45 | 46 | [[package]] 47 | name = "chrono" 48 | version = "0.4.9" 49 | source = "registry+https://github.com/rust-lang/crates.io-index" 50 | checksum = "e8493056968583b0193c1bb04d6f7684586f3726992d6c573261941a895dbd68" 51 | dependencies = [ 52 | "libc", 53 | "num-integer", 54 | "num-traits", 55 | "time", 56 | ] 57 | 58 | [[package]] 59 | name = "colored" 60 | version = "1.9.0" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "433e7ac7d511768127ed85b0c4947f47a254131e37864b2dc13f52aa32cd37e5" 63 | dependencies = [ 64 | "atty", 65 | "lazy_static", 66 | "winapi", 67 | ] 68 | 69 | [[package]] 70 | name = "fnv" 71 | version = "1.0.6" 72 | source = "registry+https://github.com/rust-lang/crates.io-index" 73 | 74 | [[package]] 75 | name = "fuchsia-cprng" 76 | version = "0.1.1" 77 | source = "registry+https://github.com/rust-lang/crates.io-index" 78 | 79 | [[package]] 80 | name = "fuchsia-zircon" 81 | version = "0.3.3" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | dependencies = [ 84 | "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", 85 | "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", 86 | ] 87 | 88 | [[package]] 89 | name = "fuchsia-zircon-sys" 90 | version = "0.3.3" 91 | source = "registry+https://github.com/rust-lang/crates.io-index" 92 | 93 | [[package]] 94 | name = "futures" 95 | version = "0.3.1" 96 | source = "registry+https://github.com/rust-lang/crates.io-index" 97 | dependencies = [ 98 | "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 99 | "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 100 | "futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 101 | "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 102 | "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 103 | "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 104 | "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 105 | ] 106 | 107 | [[package]] 108 | name = "futures-channel" 109 | version = "0.3.1" 110 | source = "registry+https://github.com/rust-lang/crates.io-index" 111 | dependencies = [ 112 | "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 113 | "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 114 | ] 115 | 116 | [[package]] 117 | name = "futures-core" 118 | version = "0.3.1" 119 | source = "registry+https://github.com/rust-lang/crates.io-index" 120 | 121 | [[package]] 122 | name = "futures-executor" 123 | version = "0.3.1" 124 | source = "registry+https://github.com/rust-lang/crates.io-index" 125 | dependencies = [ 126 | "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 127 | "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 128 | "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 129 | ] 130 | 131 | [[package]] 132 | name = "futures-io" 133 | version = "0.3.1" 134 | source = "registry+https://github.com/rust-lang/crates.io-index" 135 | 136 | [[package]] 137 | name = "futures-macro" 138 | version = "0.3.1" 139 | source = "registry+https://github.com/rust-lang/crates.io-index" 140 | dependencies = [ 141 | "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", 142 | "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", 143 | "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", 144 | "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", 145 | ] 146 | 147 | [[package]] 148 | name = "futures-sink" 149 | version = "0.3.1" 150 | source = "registry+https://github.com/rust-lang/crates.io-index" 151 | 152 | [[package]] 153 | name = "futures-task" 154 | version = "0.3.1" 155 | source = "registry+https://github.com/rust-lang/crates.io-index" 156 | 157 | [[package]] 158 | name = "futures-util" 159 | version = "0.3.1" 160 | source = "registry+https://github.com/rust-lang/crates.io-index" 161 | dependencies = [ 162 | "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 163 | "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 164 | "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 165 | "futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 166 | "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 167 | "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 168 | "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", 169 | "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", 170 | "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", 171 | "proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", 172 | "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", 173 | ] 174 | 175 | [[package]] 176 | name = "hermit-abi" 177 | version = "0.1.5" 178 | source = "registry+https://github.com/rust-lang/crates.io-index" 179 | dependencies = [ 180 | "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", 181 | ] 182 | 183 | [[package]] 184 | name = "iovec" 185 | version = "0.1.4" 186 | source = "registry+https://github.com/rust-lang/crates.io-index" 187 | dependencies = [ 188 | "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", 189 | ] 190 | checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" 191 | 192 | [[package]] 193 | name = "itoa" 194 | version = "0.4.4" 195 | source = "registry+https://github.com/rust-lang/crates.io-index" 196 | checksum = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" 197 | 198 | [[package]] 199 | name = "kernel32-sys" 200 | version = "0.2.2" 201 | source = "registry+https://github.com/rust-lang/crates.io-index" 202 | dependencies = [ 203 | "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", 204 | "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", 205 | ] 206 | 207 | [[package]] 208 | name = "lazy_static" 209 | version = "1.4.0" 210 | source = "registry+https://github.com/rust-lang/crates.io-index" 211 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 212 | 213 | [[package]] 214 | name = "libc" 215 | version = "0.2.65" 216 | source = "registry+https://github.com/rust-lang/crates.io-index" 217 | checksum = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" 218 | 219 | [[package]] 220 | name = "log" 221 | version = "0.4.13" 222 | source = "registry+https://github.com/rust-lang/crates.io-index" 223 | checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" 224 | dependencies = [ 225 | "cfg-if", 226 | ] 227 | 228 | [[package]] 229 | name = "memchr" 230 | version = "2.2.1" 231 | source = "registry+https://github.com/rust-lang/crates.io-index" 232 | 233 | [[package]] 234 | name = "mio" 235 | version = "0.6.21" 236 | source = "registry+https://github.com/rust-lang/crates.io-index" 237 | dependencies = [ 238 | "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", 239 | "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", 240 | "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", 241 | "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", 242 | "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", 243 | "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", 244 | "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", 245 | "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", 246 | "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", 247 | "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", 248 | "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", 249 | ] 250 | 251 | [[package]] 252 | name = "mio-named-pipes" 253 | version = "0.1.6" 254 | source = "registry+https://github.com/rust-lang/crates.io-index" 255 | dependencies = [ 256 | "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", 257 | "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", 258 | "miow 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", 259 | "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", 260 | ] 261 | 262 | [[package]] 263 | name = "mio-uds" 264 | version = "0.6.7" 265 | source = "registry+https://github.com/rust-lang/crates.io-index" 266 | dependencies = [ 267 | "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", 268 | "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", 269 | "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", 270 | ] 271 | 272 | [[package]] 273 | name = "miow" 274 | version = "0.2.1" 275 | source = "registry+https://github.com/rust-lang/crates.io-index" 276 | dependencies = [ 277 | "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", 278 | "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", 279 | "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", 280 | "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", 281 | ] 282 | 283 | [[package]] 284 | name = "miow" 285 | version = "0.3.3" 286 | source = "registry+https://github.com/rust-lang/crates.io-index" 287 | dependencies = [ 288 | "socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", 289 | "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", 290 | ] 291 | 292 | [[package]] 293 | name = "nano-leb128" 294 | version = "0.1.0" 295 | source = "registry+https://github.com/rust-lang/crates.io-index" 296 | checksum = "f778a7433cee0fc957551fe7e8bb7abb5caa7ea5d5ed52f9fabb63873415728e" 297 | dependencies = [ 298 | "byteio", 299 | ] 300 | 301 | [[package]] 302 | name = "net2" 303 | version = "0.2.33" 304 | source = "registry+https://github.com/rust-lang/crates.io-index" 305 | dependencies = [ 306 | "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", 307 | "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", 308 | "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", 309 | ] 310 | 311 | [[package]] 312 | name = "num-integer" 313 | version = "0.1.41" 314 | source = "registry+https://github.com/rust-lang/crates.io-index" 315 | checksum = "b85e541ef8255f6cf42bbfe4ef361305c6c135d10919ecc26126c4e5ae94bc09" 316 | dependencies = [ 317 | "autocfg", 318 | "num-traits", 319 | ] 320 | 321 | [[package]] 322 | name = "num-traits" 323 | version = "0.2.9" 324 | source = "registry+https://github.com/rust-lang/crates.io-index" 325 | checksum = "443c53b3c3531dfcbfa499d8893944db78474ad7a1d87fa2d94d1a2231693ac6" 326 | dependencies = [ 327 | "autocfg", 328 | ] 329 | 330 | [[package]] 331 | name = "num_cpus" 332 | version = "1.11.1" 333 | source = "registry+https://github.com/rust-lang/crates.io-index" 334 | dependencies = [ 335 | "hermit-abi 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", 336 | "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", 337 | ] 338 | 339 | [[package]] 340 | name = "pin-project-lite" 341 | version = "0.1.1" 342 | source = "registry+https://github.com/rust-lang/crates.io-index" 343 | 344 | [[package]] 345 | name = "pin-utils" 346 | version = "0.1.0-alpha.4" 347 | source = "registry+https://github.com/rust-lang/crates.io-index" 348 | 349 | [[package]] 350 | name = "proc-macro-hack" 351 | version = "0.5.11" 352 | source = "registry+https://github.com/rust-lang/crates.io-index" 353 | dependencies = [ 354 | "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", 355 | "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", 356 | "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", 357 | ] 358 | 359 | [[package]] 360 | name = "proc-macro-nested" 361 | version = "0.1.3" 362 | source = "registry+https://github.com/rust-lang/crates.io-index" 363 | 364 | [[package]] 365 | name = "proc-macro2" 366 | version = "1.0.24" 367 | source = "registry+https://github.com/rust-lang/crates.io-index" 368 | checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" 369 | dependencies = [ 370 | "unicode-xid", 371 | ] 372 | 373 | [[package]] 374 | name = "quote" 375 | version = "1.0.2" 376 | source = "registry+https://github.com/rust-lang/crates.io-index" 377 | checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" 378 | dependencies = [ 379 | "proc-macro2", 380 | ] 381 | 382 | [[package]] 383 | name = "rand" 384 | version = "0.4.6" 385 | source = "registry+https://github.com/rust-lang/crates.io-index" 386 | checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" 387 | dependencies = [ 388 | "fuchsia-cprng", 389 | "libc", 390 | "rand_core 0.3.1", 391 | "rdrand", 392 | "winapi", 393 | ] 394 | 395 | [[package]] 396 | name = "rand_core" 397 | version = "0.3.1" 398 | source = "registry+https://github.com/rust-lang/crates.io-index" 399 | checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" 400 | dependencies = [ 401 | "rand_core 0.4.2", 402 | ] 403 | 404 | [[package]] 405 | name = "rand_core" 406 | version = "0.4.2" 407 | source = "registry+https://github.com/rust-lang/crates.io-index" 408 | checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" 409 | 410 | [[package]] 411 | name = "rdrand" 412 | version = "0.4.0" 413 | source = "registry+https://github.com/rust-lang/crates.io-index" 414 | checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" 415 | dependencies = [ 416 | "rand_core 0.3.1", 417 | ] 418 | 419 | [[package]] 420 | name = "redox_syscall" 421 | version = "0.1.56" 422 | source = "registry+https://github.com/rust-lang/crates.io-index" 423 | checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" 424 | 425 | [[package]] 426 | name = "remove_dir_all" 427 | version = "0.5.2" 428 | source = "registry+https://github.com/rust-lang/crates.io-index" 429 | checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" 430 | dependencies = [ 431 | "winapi", 432 | ] 433 | 434 | [[package]] 435 | name = "ryu" 436 | version = "1.0.2" 437 | source = "registry+https://github.com/rust-lang/crates.io-index" 438 | checksum = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" 439 | 440 | [[package]] 441 | name = "serde" 442 | version = "1.0.125" 443 | source = "registry+https://github.com/rust-lang/crates.io-index" 444 | checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" 445 | dependencies = [ 446 | "serde_derive", 447 | ] 448 | 449 | [[package]] 450 | name = "serde_derive" 451 | version = "1.0.125" 452 | source = "registry+https://github.com/rust-lang/crates.io-index" 453 | checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" 454 | dependencies = [ 455 | "proc-macro2", 456 | "quote", 457 | "syn", 458 | ] 459 | 460 | [[package]] 461 | name = "serde_json" 462 | version = "1.0.59" 463 | source = "registry+https://github.com/rust-lang/crates.io-index" 464 | checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" 465 | dependencies = [ 466 | "itoa", 467 | "ryu", 468 | "serde", 469 | ] 470 | 471 | [[package]] 472 | name = "signal-hook-registry" 473 | version = "1.2.0" 474 | source = "registry+https://github.com/rust-lang/crates.io-index" 475 | dependencies = [ 476 | "arc-swap 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", 477 | "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", 478 | ] 479 | 480 | [[package]] 481 | name = "simple_logger" 482 | version = "1.11.0" 483 | source = "registry+https://github.com/rust-lang/crates.io-index" 484 | checksum = "cd57f17c093ead1d4a1499dc9acaafdd71240908d64775465543b8d9a9f1d198" 485 | dependencies = [ 486 | "atty", 487 | "chrono", 488 | "colored", 489 | "log", 490 | "winapi", 491 | ] 492 | 493 | [[package]] 494 | name = "slab" 495 | version = "0.4.2" 496 | source = "registry+https://github.com/rust-lang/crates.io-index" 497 | 498 | [[package]] 499 | name = "socket2" 500 | version = "0.3.11" 501 | source = "registry+https://github.com/rust-lang/crates.io-index" 502 | dependencies = [ 503 | "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", 504 | "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", 505 | "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", 506 | "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", 507 | ] 508 | 509 | [[package]] 510 | name = "static_assertions" 511 | version = "1.1.0" 512 | source = "registry+https://github.com/rust-lang/crates.io-index" 513 | checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" 514 | 515 | [[package]] 516 | name = "syn" 517 | version = "1.0.62" 518 | source = "registry+https://github.com/rust-lang/crates.io-index" 519 | checksum = "123a78a3596b24fee53a6464ce52d8ecbf62241e6294c7e7fe12086cd161f512" 520 | dependencies = [ 521 | "proc-macro2", 522 | "quote", 523 | "unicode-xid", 524 | ] 525 | 526 | [[package]] 527 | name = "tempdir" 528 | version = "0.3.7" 529 | source = "registry+https://github.com/rust-lang/crates.io-index" 530 | checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" 531 | dependencies = [ 532 | "rand", 533 | "remove_dir_all", 534 | ] 535 | 536 | [[package]] 537 | name = "time" 538 | version = "0.1.42" 539 | source = "registry+https://github.com/rust-lang/crates.io-index" 540 | checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" 541 | dependencies = [ 542 | "libc", 543 | "redox_syscall", 544 | "winapi", 545 | ] 546 | 547 | [[package]] 548 | name = "tokio" 549 | version = "0.2.4" 550 | source = "registry+https://github.com/rust-lang/crates.io-index" 551 | dependencies = [ 552 | "bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", 553 | "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", 554 | "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 555 | "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", 556 | "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", 557 | "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", 558 | "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", 559 | "mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", 560 | "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", 561 | "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", 562 | "pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", 563 | "signal-hook-registry 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", 564 | "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", 565 | "tokio-macros 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", 566 | "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", 567 | ] 568 | 569 | [[package]] 570 | name = "tokio-macros" 571 | version = "0.2.0" 572 | source = "registry+https://github.com/rust-lang/crates.io-index" 573 | dependencies = [ 574 | "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", 575 | "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", 576 | ] 577 | 578 | [[package]] 579 | name = "tsdb" 580 | version = "0.1.0" 581 | dependencies = [ 582 | "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", 583 | "log", 584 | "nano-leb128", 585 | "serde", 586 | "serde_json", 587 | "simple_logger", 588 | "static_assertions", 589 | "tempdir", 590 | "tokio 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", 591 | ] 592 | 593 | [[package]] 594 | name = "unicode-xid" 595 | version = "0.2.0" 596 | source = "registry+https://github.com/rust-lang/crates.io-index" 597 | checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" 598 | 599 | [[package]] 600 | name = "winapi" 601 | version = "0.2.8" 602 | source = "registry+https://github.com/rust-lang/crates.io-index" 603 | 604 | [[package]] 605 | name = "winapi" 606 | version = "0.3.8" 607 | source = "registry+https://github.com/rust-lang/crates.io-index" 608 | checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" 609 | dependencies = [ 610 | "winapi-i686-pc-windows-gnu", 611 | "winapi-x86_64-pc-windows-gnu", 612 | ] 613 | 614 | [[package]] 615 | name = "winapi-build" 616 | version = "0.1.1" 617 | source = "registry+https://github.com/rust-lang/crates.io-index" 618 | 619 | [[package]] 620 | name = "winapi-i686-pc-windows-gnu" 621 | version = "0.4.0" 622 | source = "registry+https://github.com/rust-lang/crates.io-index" 623 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 624 | 625 | [[package]] 626 | name = "winapi-x86_64-pc-windows-gnu" 627 | version = "0.4.0" 628 | source = "registry+https://github.com/rust-lang/crates.io-index" 629 | 630 | [[package]] 631 | name = "ws2_32-sys" 632 | version = "0.2.1" 633 | source = "registry+https://github.com/rust-lang/crates.io-index" 634 | dependencies = [ 635 | "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", 636 | "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", 637 | ] 638 | 639 | [metadata] 640 | "checksum arc-swap 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff" 641 | "checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" 642 | "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" 643 | "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" 644 | "checksum byteio 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1628aaab25b23803453d80873600ebffe6326577c1db81f5a3d123290e6287af" 645 | "checksum bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "10004c15deb332055f7a4a208190aed362cf9a7c2f6ab70a305fba50e1105f38" 646 | "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" 647 | "checksum chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e8493056968583b0193c1bb04d6f7684586f3726992d6c573261941a895dbd68" 648 | "checksum colored 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "433e7ac7d511768127ed85b0c4947f47a254131e37864b2dc13f52aa32cd37e5" 649 | "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" 650 | "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" 651 | "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" 652 | "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" 653 | "checksum futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6f16056ecbb57525ff698bb955162d0cd03bee84e6241c27ff75c08d8ca5987" 654 | "checksum futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fcae98ca17d102fd8a3603727b9259fcf7fa4239b603d2142926189bc8999b86" 655 | "checksum futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "79564c427afefab1dfb3298535b21eda083ef7935b4f0ecbfcb121f0aec10866" 656 | "checksum futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1e274736563f686a837a0568b478bdabfeaec2dca794b5649b04e2fe1627c231" 657 | "checksum futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e676577d229e70952ab25f3945795ba5b16d63ca794ca9d2c860e5595d20b5ff" 658 | "checksum futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52e7c56c15537adb4f76d0b7a76ad131cb4d2f4f32d3b0bcabcbe1c7c5e87764" 659 | "checksum futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "171be33efae63c2d59e6dbba34186fe0d6394fb378069a76dfd80fdcffd43c16" 660 | "checksum futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9" 661 | "checksum futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76" 662 | "checksum hermit-abi 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f629dc602392d3ec14bfc8a09b5e644d7ffd725102b48b81e59f90f2633621d7" 663 | "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" 664 | "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" 665 | "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" 666 | "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 667 | "checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" 668 | "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" 669 | "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" 670 | "checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" 671 | "checksum mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" 672 | "checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125" 673 | "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" 674 | "checksum miow 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "396aa0f2003d7df8395cb93e09871561ccc3e785f0acb369170e8cc74ddf9226" 675 | "checksum nano-leb128 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f778a7433cee0fc957551fe7e8bb7abb5caa7ea5d5ed52f9fabb63873415728e" 676 | "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" 677 | "checksum num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "b85e541ef8255f6cf42bbfe4ef361305c6c135d10919ecc26126c4e5ae94bc09" 678 | "checksum num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "443c53b3c3531dfcbfa499d8893944db78474ad7a1d87fa2d94d1a2231693ac6" 679 | "checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72" 680 | "checksum pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f0af6cbca0e6e3ce8692ee19fb8d734b641899e07b68eb73e9bbbd32f1703991" 681 | "checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" 682 | "checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" 683 | "checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" 684 | "checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" 685 | "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" 686 | "checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" 687 | "checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" 688 | "checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" 689 | "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" 690 | "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" 691 | "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" 692 | "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" 693 | "checksum serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "1217f97ab8e8904b57dd22eb61cde455fa7446a9c1cf43966066da047c1f3702" 694 | "checksum serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c6faef9a2e64b0064f48570289b4bf8823b7581f1d6157c1b52152306651d0" 695 | "checksum serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)" = "1a3351dcbc1f067e2c92ab7c3c1f288ad1a4cffc470b5aaddb4c2e0a3ae80043" 696 | "checksum signal-hook-registry 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41" 697 | "checksum simple_logger 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3a4756ecc75607ba957820ac0a2413a6c27e6c61191cda0c62c6dcea4da88870" 698 | "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" 699 | "checksum socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "e8b74de517221a2cb01a53349cf54182acdc31a074727d3079068448c0676d85" 700 | "checksum static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" 701 | "checksum syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f89693ae015201f8de93fd96bde2d065f8bfc3f97ce006d5bc9f900b97c0c7c0" 702 | "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" 703 | "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" 704 | "checksum tokio 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bcced6bb623d4bff3739c176c415f13c418f426395c169c9c3cd9a492c715b16" 705 | "checksum tokio-macros 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d5795a71419535c6dcecc9b6ca95bdd3c2d6142f7e8343d7beb9923f129aa87e" 706 | "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" 707 | "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" 708 | "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" 709 | "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" 710 | "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 711 | "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 712 | "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" 713 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 714 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tsdb" 3 | version = "0.1.0" 4 | authors = ["Matej "] 5 | edition = "2018" 6 | 7 | [dependencies] 8 | static_assertions = "1.1.0" 9 | nano-leb128 = "0.1.0" 10 | log = "0.4.13" 11 | simple_logger = "1.11.0" 12 | serde_json = "1.0.59" 13 | serde = { version = "1.0.125", features = ['derive'] } 14 | tokio = { version = "0.2.4", features = ['full'] } 15 | futures = "0.3.1" 16 | 17 | [dev-dependencies] 18 | tempdir = "0.3.7" 19 | 20 | [profile.release] 21 | debug = true -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | time-series database 2 | --------------------- 3 | 4 | Features: 5 | - [ ] ACID compliant 6 | - [ ] high performance 7 | - [ ] multi-threaded 8 | - [ ] query cache 9 | - [ ] B-Tree index 10 | - [x] sorted array index (bin-search) 11 | - [x] one scalar per data-point 12 | - [ ] aggregate functions 13 | - [ ] `WHERE` on timestamp 14 | - [ ] `WHERE` on value 15 | - [ ] `GROUP BY` support on timestamp 16 | - [x] textual command interface 17 | - [ ] binary command interface 18 | - [ ] multiple scalars (tuples) per data-point 19 | - [ ] old data downsampling 20 | - [ ] old data deletion 21 | - [ ] auth: name & password 22 | - [ ] ssl / tls 23 | 24 | ### ACID 25 | 26 | ### Data structure 27 | 28 | Data points are organized into individual **Series**. These series are independent 29 | of each other and can have different data-values with different timestamps. Each 30 | **Series** has a name and data schema associated with it. 31 | 32 | Schema of the **Series** dictates how are the data points encoded on the disk. 33 | 34 | Data points in one **Series** are split into multiple **Blocks**. Each block 35 | has a size of `4096` bytes. These block form a *log-structured merge-tree*. 36 | Multiple blocks (`2048`) are stored in one file. During the application execution 37 | some blocks are loaded in memory (cached). All blocks are stored on disk. 38 | 39 | The database also stores **Index** which is used to speed up queries on the data. Index 40 | object stores lowest and highest timestamp for each block. There is one **Index** for each 41 | **Series** object. Also the index is stored in one file and it is loaded in memory at all 42 | times. 43 | 44 | #### Schemas 45 | 46 | Currently these schemas are supported: `f32`. 47 | 48 | Planned schemas: `f32`, `i32`, `f64`, `i64`, `bool`. 49 | 50 | ##### f32 51 | 52 | - Timestamp encoding: delta-encoded LEB128 varints 53 | - Value encoding: low-endian without any compression 54 | 55 | ##### i32 56 | 57 | - Timestamp encoding: delta-encoded LEB128 varints 58 | - Value encoding: delta-encoded LEB128 varints 59 | 60 | ##### bool 61 | 62 | - Timestamp encoding: delta-encoded LEB128 varints 63 | - Value encoding: One boolean takes one bit. 64 | 65 | 66 | ### Operations on the data 67 | 68 | Database provides following operations on the data. 69 | 70 | #### Create series 71 | 72 | Creates a new **Series** object in the database. 73 | 74 | ``` 75 | CREATE SERIES {series} {schema} 76 | CREATE SERIES outdoor_sunlight f32 77 | ``` 78 | 79 | ```json 80 | { 81 | "CreateSeries": "default" 82 | } 83 | ``` 84 | 85 | #### List series 86 | 87 | Returns a list of all available series inside the database. 88 | 89 | ``` 90 | SHOW SERIES 91 | ``` 92 | 93 | #### Insert value 94 | 95 | Inserts a specified value to the specified time **Series** object. 96 | 97 | ``` 98 | INSERT INTO {series} {value} 99 | INSERT INTO outdoor_sunlight 4806.0 100 | ``` 101 | 102 | ```json 103 | { 104 | "Insert": { 105 | "to": "default", 106 | "value": 3.14 107 | } 108 | } 109 | ``` 110 | 111 | 112 | #### Query values 113 | 114 | Gets all data points inside the specified **Series** object in specified range of time. 115 | 116 | ``` 117 | SELECT {series} BETWEEN {start_timestamp} AND {end_timestamp} 118 | SELECT outdoor_sunlight BETWEEN 01-01-2019 AND 02-01-2019 119 | ``` 120 | 121 | ```json 122 | { 123 | "Select": { 124 | "from": "default", 125 | "between": { 126 | "min": null, 127 | "max": null 128 | } 129 | } 130 | } 131 | ``` 132 | 133 | It is also possible to query result of running an aggregate function on specified range data points in the specified **Series** object. 134 | 135 | ``` 136 | SELECT [AVG/SUM/MAX/MIN/COUNT] {series} BETWEEN {start_timestamp} AND {end_timestamp} GROUP BY [MINUTE/HOUR/DAY/WEEK/MONTH] 137 | SELECT AVG outdoor_sunlight BETWEEN 01-01-2019 AND 02-01-2019 GROUP BY HOUR 138 | ``` -------------------------------------------------------------------------------- /src/engine.rs: -------------------------------------------------------------------------------- 1 | use crate::engine::array_vec::ArrayVec; 2 | use std::ops::Sub; 3 | use serde::{Serialize, Deserialize}; 4 | 5 | /// Represents UNIX timestamp as number of seconds from the 6 | /// start of the epoch. 7 | /// 8 | /// Timestamps can be subtracted and converted into u64. 9 | #[derive(Eq, PartialEq, Debug, Copy, Clone, Ord, PartialOrd, Serialize, Deserialize)] 10 | pub struct Timestamp(u64); 11 | 12 | impl Into for u64 { 13 | fn into(self) -> Timestamp { 14 | Timestamp(self) 15 | } 16 | } 17 | 18 | impl Into for &Timestamp { 19 | fn into(self) -> u64 { 20 | self.0 21 | } 22 | } 23 | 24 | impl Sub for Timestamp { 25 | type Output = u64; 26 | 27 | fn sub(self, rhs: Timestamp) -> Self::Output { 28 | self.0 - rhs.0 29 | } 30 | } 31 | 32 | /// Represents a single data-point with associated Timestamp 33 | /// in Series object. 34 | #[derive(Eq, PartialEq, Debug, Copy, Clone)] 35 | pub struct Point { 36 | timestamp: Timestamp, 37 | value: V, 38 | } 39 | 40 | impl Point { 41 | #[inline] 42 | pub fn timestamp(&self) -> &Timestamp { 43 | &self.timestamp 44 | } 45 | 46 | #[inline] 47 | pub fn value(&self) -> &V { 48 | &self.value 49 | } 50 | } 51 | 52 | 53 | pub trait Decoder { 54 | fn new(min_timestamp: Timestamp) -> Self; 55 | } 56 | 57 | pub trait Schema { 58 | type EncState: Default; 59 | type DecState: Decoder; 60 | type Schema: Default; 61 | fn encode(state: &mut Self::EncState, point: Point) -> ArrayVec; 62 | fn decode(state: &mut Self::DecState, buff: &[u8]) -> (Point, usize); 63 | } 64 | 65 | pub mod array_vec; 66 | 67 | pub mod f32 { 68 | use crate::engine::{Schema, Point, Decoder, Timestamp}; 69 | use nano_leb128::ULEB128; 70 | use crate::engine::array_vec::ArrayVec; 71 | use serde::{Serialize, Deserialize}; 72 | 73 | pub struct F32; 74 | 75 | #[derive(Serialize, Deserialize, Copy, Clone)] 76 | pub struct F32Enc(Timestamp); 77 | 78 | impl Default for F32Enc { 79 | fn default() -> Self { 80 | F32Enc(0.into()) 81 | } 82 | } 83 | 84 | #[derive(Serialize, Deserialize, Copy, Clone)] 85 | pub struct F32Dec(u64); 86 | 87 | impl Schema for F32 { 88 | type EncState = F32Enc; 89 | type DecState = F32Dec; 90 | type Schema = (); 91 | 92 | fn encode(state: &mut Self::EncState, point: Point) -> ArrayVec { 93 | let mut array = ArrayVec::default(); 94 | let dt = point.timestamp - state.0; 95 | state.0 = point.timestamp; 96 | 97 | array.length += ULEB128::from(dt) 98 | .write_into(&mut array.data) 99 | .unwrap(); 100 | array.extend(&point.value.to_le_bytes()); 101 | 102 | array 103 | } 104 | 105 | fn decode(state: &mut Self::DecState, buff: &[u8]) -> (Point, usize) { 106 | let (dt, read) = ULEB128::read_from(buff).unwrap(); 107 | let value = f32::from_le_bytes([ 108 | buff[read + 0], 109 | buff[read + 1], 110 | buff[read + 2], 111 | buff[read + 3] 112 | ]); 113 | 114 | state.0 = state.0 + u64::from(dt); 115 | 116 | (Point { timestamp: state.0.into(), value }, read + 4) 117 | } 118 | } 119 | 120 | impl Decoder for F32Dec { 121 | fn new(min_timestamp: Timestamp) -> Self { 122 | F32Dec(min_timestamp.0) 123 | } 124 | } 125 | 126 | #[cfg(test)] 127 | mod tests { 128 | use crate::engine::{Point, Schema, Timestamp, Decoder}; 129 | use crate::engine::f32::{F32Enc, F32Dec, F32}; 130 | 131 | #[test] 132 | fn test_encode_decode() { 133 | let point1 = Point { timestamp: 40.into(), value: 3.14 }; 134 | let point2 = Point { timestamp: 120.into(), value: 10.0 }; 135 | 136 | let mut enc_state = F32Enc::default(); 137 | let mut dec_state = F32Dec::new(Timestamp(0)); 138 | 139 | let result1 = F32::encode(&mut enc_state, point1); 140 | let result2 = F32::encode(&mut enc_state, point2); 141 | 142 | let (decoded1, _) = F32::decode(&mut dec_state, &result1.data); 143 | let (decoded2, _) = F32::decode(&mut dec_state, &result2.data); 144 | 145 | assert_eq!(decoded1, point1); 146 | assert_eq!(decoded2, point2); 147 | } 148 | } 149 | } 150 | 151 | pub mod block { 152 | use static_assertions::{assert_eq_align, assert_eq_size}; 153 | 154 | /// Type representing metadata about the Block. 155 | pub struct BlockTail { 156 | free_bytes: u8, 157 | } 158 | 159 | pub const BLOCK_SIZE: usize = 4096; 160 | pub const BLOCK_TAIL_SIZE: usize = std::mem::size_of::(); 161 | 162 | /// Type owning data of a block. 163 | pub struct Block { 164 | pub data: [u8; BLOCK_SIZE - BLOCK_TAIL_SIZE], 165 | tail: BlockTail, 166 | } 167 | 168 | // These assertions ensure that we can safely interpret 169 | // any [u8; 4096] as Block and vice-versa. 170 | assert_eq_size!(Block, [u8; BLOCK_SIZE]); 171 | assert_eq_align!(Block, u8); 172 | 173 | impl Block { 174 | /// Stores specified value as free bytes inside the header of this 175 | /// block. 176 | pub fn set_free_bytes(&mut self, val: u8) { 177 | self.tail.free_bytes = val; 178 | } 179 | 180 | /// Returns the length of data in bytes. This function only returns 181 | /// valid result when the Block is closed. 182 | #[inline] 183 | pub fn data_len(&self) -> usize { 184 | (BLOCK_SIZE - BLOCK_TAIL_SIZE) - self.tail.free_bytes as usize 185 | } 186 | 187 | /// Returns the byte representation of this block including the header 188 | /// as immutable slice of bytes. 189 | pub fn as_slice(&self) -> &[u8] { 190 | // Safe: because Block has size of BLOCK_SIZE and alignment of 1. 191 | let ptr = self as *const Block as *const u8; 192 | unsafe { std::slice::from_raw_parts(ptr, BLOCK_SIZE) } 193 | } 194 | 195 | /// Returns the byte representation of this block including the header 196 | /// as mutable slice of bytes. 197 | pub fn as_slice_mut(&mut self) -> &mut [u8] { 198 | // Safe: because Block has size of BLOCK_SIZE and alignment of 1. 199 | let ptr = self as *mut Block as *mut u8; 200 | unsafe { std::slice::from_raw_parts_mut(ptr, BLOCK_SIZE) } 201 | } 202 | } 203 | 204 | impl Default for Block { 205 | fn default() -> Self { 206 | Block { 207 | tail: BlockTail { free_bytes: 0 }, 208 | data: [0; BLOCK_SIZE - BLOCK_TAIL_SIZE], 209 | } 210 | } 211 | } 212 | 213 | /// Type that represents a "handle" to potentially unloaded Block. 214 | #[derive(Hash, Eq, PartialEq, Copy, Clone, Debug)] 215 | pub struct BlockSpec { 216 | pub series_id: usize, 217 | pub block_id: usize, 218 | } 219 | } 220 | 221 | pub mod cache { 222 | use log::trace; 223 | use std::collections::HashMap; 224 | use std::hash::Hash; 225 | use std::fmt::Debug; 226 | 227 | /// An LRU cache backed by HashMap which owns the stored data. The 228 | /// least recently used algorithm is implemented using runtime 229 | /// allocated ring buffer as circular queue. 230 | pub struct Cache { 231 | items: HashMap, 232 | lru: Vec>, 233 | lru_idx: usize, 234 | } 235 | 236 | impl Cache where K: Hash + Eq + Copy + Debug { 237 | pub fn with_capacity(capacity: usize) -> Self { 238 | Cache { 239 | items: HashMap::with_capacity(capacity), 240 | lru: vec![None; capacity], 241 | lru_idx: 0, 242 | } 243 | } 244 | 245 | pub fn contains_key(&self, key: &K) -> bool { 246 | self.items.contains_key(key) 247 | } 248 | 249 | pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { 250 | self.items.get_mut(key) 251 | } 252 | 253 | fn evict_lru(&mut self) { 254 | if let Some(t) = self.lru[self.lru_idx] { 255 | self.remove(&t); 256 | self.lru[self.lru_idx] = None; 257 | } 258 | } 259 | 260 | pub fn insert(&mut self, key: K, value: V) { 261 | self.evict_lru(); 262 | 263 | self.items.insert(key, value); 264 | self.lru[self.lru_idx] = Some(key); 265 | self.lru_idx = (self.lru_idx + 1) % self.lru.len(); 266 | } 267 | 268 | /// Elements inserted with this method are not counted towards the 269 | /// cache capacity so it's possible to have more then `capacity` 270 | /// elements in the cache. 271 | pub fn insert_unevictable(&mut self, key: K, value: V) where K: Debug { 272 | trace!("Inserted unevictable entry: {:?}", &key); 273 | self.items.insert(key, value); 274 | } 275 | 276 | pub fn remove(&mut self, key: &K) where K: Debug { 277 | trace!("Removed entry: {:?}", key); 278 | self.items.remove(key); 279 | } 280 | 281 | pub fn len(&self) -> usize { 282 | self.items.len() 283 | } 284 | } 285 | 286 | #[cfg(test)] 287 | mod tests { 288 | use crate::engine::cache::Cache; 289 | 290 | #[test] 291 | fn test_get_mut() { 292 | let mut c = Cache::::with_capacity(8); 293 | c.insert(0, 65); 294 | c.insert(1, 60); 295 | 296 | let mut a = c.get_mut(&0); 297 | assert_eq!(*a.unwrap(), 65); 298 | } 299 | 300 | #[test] 301 | fn test_remove() { 302 | let mut c = Cache::::with_capacity(8); 303 | c.insert(0, 65); 304 | c.remove(&0); 305 | 306 | assert_eq!(c.len(), 0); 307 | } 308 | 309 | #[test] 310 | fn test_contains_without_eviction() { 311 | let mut c = Cache::::with_capacity(8); 312 | c.insert(0, 60); 313 | c.insert(1, 60); 314 | c.insert(2, 60); 315 | 316 | assert_eq!(c.len(), 3); 317 | assert!(c.contains_key(&0)); 318 | assert!(c.contains_key(&1)); 319 | assert!(c.contains_key(&2)); 320 | assert!(!c.contains_key(&3)); 321 | } 322 | 323 | #[test] 324 | fn test_eviction() { 325 | let mut c = Cache::::with_capacity(3); 326 | 327 | c.insert(0, 60); 328 | c.insert(1, 60); 329 | c.insert(2, 60); 330 | 331 | assert_eq!(c.len(), 3); 332 | assert!(c.contains_key(&0)); 333 | assert!(c.contains_key(&1)); 334 | assert!(c.contains_key(&2)); 335 | assert!(!c.contains_key(&3)); 336 | 337 | c.insert(3, 70); 338 | assert!(c.contains_key(&3)); 339 | assert!(!c.contains_key(&0)); 340 | 341 | c.insert_unevictable(5, 50); 342 | assert!(c.contains_key(&1)); 343 | assert!(c.contains_key(&2)); 344 | assert!(c.contains_key(&3)); 345 | assert!(c.contains_key(&5)); 346 | } 347 | } 348 | } 349 | 350 | pub mod io { 351 | use log::debug; 352 | use std::path::{Path, PathBuf}; 353 | use crate::engine::block::{Block, BlockSpec, BLOCK_SIZE}; 354 | use std::collections::HashMap; 355 | use std::fs::{File, OpenOptions}; 356 | use std::collections::hash_map::Entry; 357 | use std::io::{Write, Seek, SeekFrom, Read, Error, ErrorKind}; 358 | use serde::{Serialize, Deserialize}; 359 | 360 | #[derive(Copy, Clone, Debug)] 361 | #[derive(Serialize, Deserialize)] 362 | pub enum SyncPolicy { 363 | Immediate, 364 | Never, 365 | } 366 | 367 | struct StorageFile { 368 | file: File, 369 | length: usize, 370 | } 371 | 372 | impl StorageFile { 373 | const INITIAL_ALLOCATION_SIZE: usize = 2 * BLOCK_SIZE; 374 | 375 | /// Creates a StorageFile by creation a file on the disk and allocates 376 | /// the storage by filling the file with zeros. 377 | fn allocate_new(path: &Path) -> Result { 378 | let mut file = OpenOptions::new() 379 | .write(true) 380 | .read(true) 381 | .create_new(true) 382 | .open(path)?; 383 | 384 | // allocate space for some blocks 385 | file.write_all(&[0u8; StorageFile::INITIAL_ALLOCATION_SIZE])?; 386 | 387 | Ok(StorageFile { file, length: StorageFile::INITIAL_ALLOCATION_SIZE }) 388 | } 389 | 390 | /// Opens the file on the disk and returns a new StorageFile instance 391 | /// containing the opened file. 392 | fn open(path: &Path) -> Result { 393 | let file = OpenOptions::new() 394 | .write(true) 395 | .read(true) 396 | .open(path)?; 397 | 398 | let metadata = file.metadata()?; 399 | 400 | if (metadata.len() % BLOCK_SIZE as u64) != 0 { 401 | return Err(Error::new(ErrorKind::Other, "File length is not multiple of BLOCK_SIZE!")); 402 | } 403 | 404 | Ok(StorageFile { file, length: metadata.len() as usize }) 405 | } 406 | 407 | fn ensure_file_is_large_enough(&mut self, spec: &BlockSpec) -> Result<(), Error> { 408 | let start = spec.starting_pos_in_file(); 409 | 410 | // if the highest index in the file is smaller than 411 | // the starting position of this block, we must allocate the data 412 | if self.length - 1 < start { 413 | self.file.seek(SeekFrom::End(0))?; 414 | 415 | while self.length - 1 < start { 416 | self.file.write_all(&[0u8; BLOCK_SIZE])?; 417 | self.length += BLOCK_SIZE; 418 | } 419 | } 420 | 421 | Ok(()) 422 | } 423 | 424 | /// Loads data for block specified by BlockSpec into passed in 425 | /// mutable Block storage. 426 | /// 427 | /// It the file currently does not contain specified Block the space 428 | /// needed to store the block is allocated. 429 | fn read_into(&mut self, spec: &BlockSpec, block: &mut Block) -> Result<(), Error> { 430 | self.ensure_file_is_large_enough(spec)?; 431 | 432 | let seek = SeekFrom::Start(spec.starting_pos_in_file() as u64); 433 | self.file.seek(seek)?; 434 | self.file.read_exact(block.as_slice_mut()) 435 | } 436 | 437 | /// Writes part of block or whole block (if `dirty_from == 0`) to the 438 | /// file by seeking to `block_start + dirty_from` and writing the part 439 | /// of block that is dirty. 440 | fn write(&mut self, spec: &BlockSpec, block: &Block, dirty_from: usize) -> Result<(), Error> { 441 | let seek = SeekFrom::Start(spec.starting_pos_in_file() as u64 + dirty_from as u64); 442 | 443 | self.file.seek(seek)?; 444 | self.file.write_all(&block.as_slice()[dirty_from..]) 445 | } 446 | 447 | /// Syncs the file's contents (not necessarily metadata) to the disk 448 | /// to ensure all the data has been written. 449 | fn sync(&mut self) -> Result<(), Error> { 450 | self.file.sync_data() 451 | } 452 | } 453 | 454 | pub struct BlockIO { 455 | open_files: HashMap, 456 | sync_policy: SyncPolicy, 457 | storage: PathBuf, 458 | } 459 | 460 | const BLOCKS_PER_FILE: usize = 8192; 461 | 462 | impl BlockSpec { 463 | /// Returns file id this block resides in. The code in this function 464 | /// is equivalent to `self.block_id / BLOCKS_PER_FILE`. 465 | #[inline] 466 | pub fn file_id(&self) -> usize { 467 | self.block_id / BLOCKS_PER_FILE 468 | } 469 | 470 | /// Returns the offset in bytes inside the file the Block specified 471 | /// by this BlockSpec resides in. 472 | pub fn starting_pos_in_file(&self) -> usize { 473 | let file_local_block_id = self.block_id - (self.file_id() * BLOCKS_PER_FILE); 474 | file_local_block_id * BLOCK_SIZE 475 | } 476 | 477 | /// Returns the file path of the file the Block specified by this 478 | /// BlockSpec resides in. 479 | pub fn determine_file_path(&self, base_path: &Path) -> PathBuf { 480 | base_path.join(format!("{}-{}.dat", self.series_id, self.file_id())) 481 | } 482 | } 483 | 484 | impl BlockIO { 485 | pub fn new(storage: PathBuf, sync_policy: SyncPolicy) -> Self { 486 | BlockIO { 487 | open_files: Default::default(), 488 | sync_policy, 489 | storage, 490 | } 491 | } 492 | 493 | /// Creates or loads the file that contains block specified by `spec` 494 | /// parameter. This function does not load file's contents into memory. 495 | fn create_or_load_file(&mut self, spec: &BlockSpec) -> &mut StorageFile { 496 | // if the file is already open we just return in. in the 497 | // other case we need to either load the file or create it. 498 | match self.open_files.entry(*spec) { 499 | Entry::Occupied(t) => t.into_mut(), 500 | Entry::Vacant(t) => { 501 | let file_path = spec.determine_file_path(&self.storage); 502 | let storage_file = if file_path.exists() { 503 | debug!("Loading storage file: {}", file_path.to_string_lossy()); 504 | StorageFile::open(&file_path).expect("open block failed") 505 | } else { 506 | debug!("Creating storage file: {}", file_path.to_string_lossy()); 507 | StorageFile::allocate_new(&file_path).expect("create block failed") 508 | }; 509 | 510 | t.insert(storage_file) 511 | } 512 | } 513 | } 514 | 515 | /// This function either creates or loads the block specified by the 516 | /// BlockSpec to the memory and returns the Block instance owning 517 | /// the data. 518 | /// 519 | /// Each time this function is called a new instance of Block is 520 | /// created even if the BlockSpec is same. 521 | pub fn create_or_load_block(&mut self, spec: &BlockSpec) -> Block { 522 | let mut block = Block::default(); 523 | 524 | self.create_or_load_file(spec) 525 | .read_into(&spec, &mut block) 526 | .expect("load block failed"); 527 | 528 | block 529 | } 530 | 531 | /// Writes the data of whole Block into file specified by BlockSpec. 532 | /// 533 | /// It is preferred to call `write_block_partially` if only part of 534 | /// block's data has been changed. 535 | pub fn write_block_fully(&mut self, spec: &BlockSpec, block: &Block) { 536 | self.write_block_partially(spec, block, 0); 537 | } 538 | 539 | /// Writes the data of part of the Block into the file specified by 540 | /// BlockSpec. The written part of the Block is determined by `dirty_from` 541 | /// parameter which tells how many bytes in the Block are not dirty. 542 | /// 543 | /// This allows to write only the dirty part of the Block skipping the 544 | /// overwriting of the previously written data that is not changed. 545 | pub fn write_block_partially(&mut self, spec: &BlockSpec, block: &Block, dirty_from: usize) { 546 | let should_sync = match self.sync_policy { 547 | SyncPolicy::Immediate => true, 548 | SyncPolicy::Never => false, 549 | }; 550 | let file = self.create_or_load_file(spec); 551 | file.write(&spec, block, dirty_from) 552 | .expect("write block partially failed"); 553 | 554 | if should_sync { 555 | file.sync().expect("sync failed") 556 | } 557 | } 558 | } 559 | } 560 | 561 | pub mod index { 562 | use log::debug; 563 | use crate::engine::Timestamp; 564 | use std::fs::{File, OpenOptions}; 565 | use crate::engine::io::SyncPolicy; 566 | use std::path::Path; 567 | use std::io::{Error, Read, Seek, SeekFrom, Write}; 568 | 569 | #[derive(Debug)] 570 | struct MinMax { 571 | min: Timestamp, 572 | max: Timestamp, 573 | } 574 | 575 | pub struct TimestampIndex { 576 | file: File, 577 | sync_policy: SyncPolicy, 578 | data: Vec, 579 | dirty_from_element_idx: usize, 580 | } 581 | 582 | impl TimestampIndex { 583 | /// Loads file specified by path as timestamp index and 584 | /// returns TimestampIndex containing the data from file. 585 | fn load(path: &Path, sync_policy: SyncPolicy) -> Result { 586 | let mut file = OpenOptions::new() 587 | .write(true) 588 | .read(true) 589 | .open(path)?; 590 | 591 | let metadata = file.metadata()?; 592 | let elements = metadata.len() as usize / std::mem::size_of::(); 593 | let mut data = Vec::with_capacity(elements); 594 | 595 | debug!("Loading index from: {} ({} elements)", path.to_string_lossy(), elements); 596 | // todo: check if unsafe is safe? 597 | // safe: we take Vec's allocation, create a slice of u8 from it, read 598 | // the data into the slice. 599 | let ptr = data.as_mut_ptr() as *mut u8; 600 | let bytes = unsafe { 601 | std::slice::from_raw_parts_mut(ptr, elements * std::mem::size_of::()) 602 | }; 603 | 604 | file.read_exact(bytes)?; 605 | unsafe { data.set_len(elements); } 606 | 607 | Ok(TimestampIndex { 608 | file, 609 | sync_policy, 610 | dirty_from_element_idx: data.len(), 611 | data, 612 | }) 613 | } 614 | 615 | /// Creates a new empty index file and returns TimestampIndex 616 | /// associated with it. 617 | fn create(path: &Path, sync_policy: SyncPolicy) -> Result { 618 | debug!("Creating new index at: {}", path.to_string_lossy()); 619 | let file = OpenOptions::new() 620 | .write(true) 621 | .read(true) 622 | .create_new(true) 623 | .open(path)?; 624 | 625 | Ok(TimestampIndex { file, sync_policy, data: vec![], dirty_from_element_idx: 0 }) 626 | } 627 | 628 | /// Creates or loads the index file from specified path. 629 | pub fn create_or_load(path: &Path, sync_policy: SyncPolicy) -> Result { 630 | if !path.exists() { 631 | Self::create(path, sync_policy) 632 | } else { 633 | Self::load(path, sync_policy) 634 | } 635 | } 636 | 637 | /// Invalidates and removes all data in this index. The changes are not 638 | /// written to disk and the internal data structure is empty as if the 639 | /// index was newly created. 640 | pub fn invalidate(&mut self) { 641 | std::mem::replace(&mut self.data, vec![]); 642 | } 643 | 644 | #[inline] 645 | pub fn len(&self) -> usize { 646 | return self.data.len() 647 | } 648 | 649 | /// Ensures that the internal Vec storage is large enough to store 650 | /// data for block specified by `for_block_id`. If the internal Vec 651 | /// is not large enough the capacity of Vec is increased by 8 additional 652 | /// elements. 653 | /// 654 | /// This function is safe to call only with `for_block_id` less or equal 655 | /// to previous `for_block_id + 1`. 656 | /// 657 | /// # Panics 658 | /// This functions panic when `for_block_id` is larger then last 659 | /// `for_block_id + 1`. 660 | fn ensure_vec_has_enough_elements(&mut self, for_block_id: usize) { 661 | assert!(self.data.len() >= for_block_id); 662 | 663 | if self.data.len() < for_block_id + 1 { 664 | // when we push to full Vec the backing allocation size is 665 | // doubled. 666 | // 667 | // to prevent frequent reallocations and to prevent 668 | // exponential growth of the index (because it grows 669 | // linearly) we manually reserve 8 elements each time. 670 | if self.data.capacity() == self.data.len() { 671 | self.data.reserve(8); 672 | } 673 | self.data.push(MinMax { min: Timestamp(0), max: Timestamp(0) }); 674 | } 675 | } 676 | 677 | /// Sets the minimum timestamp for specified block. 678 | /// 679 | /// This function is safe to call only with `block_id` less or equal 680 | /// to previous `block_id + 1`. 681 | /// 682 | /// # Panics 683 | /// This functions panic when `block_id` is larger then last 684 | /// `block_id + 1`. 685 | pub fn set_min(&mut self, block_id: usize, timestamp: Timestamp) { 686 | self.ensure_vec_has_enough_elements(block_id); 687 | if let Some(t) = self.data.get_mut(block_id) { 688 | t.min = timestamp 689 | }; 690 | self.dirty_from_element_idx = self.dirty_from_element_idx.min(block_id); 691 | } 692 | 693 | /// Sets the maximum timestamp for specified block. 694 | /// 695 | /// This function is safe to call only with `block_id` less or equal 696 | /// to previous `block_id + 1`. 697 | /// 698 | /// # Panics 699 | /// This functions panic when `block_id` is larger then last 700 | /// `block_id + 1`. 701 | pub fn set_max(&mut self, block_id: usize, timestamp: Timestamp) { 702 | self.ensure_vec_has_enough_elements(block_id); 703 | if let Some(t) = self.data.get_mut(block_id) { 704 | t.max = timestamp 705 | }; 706 | self.dirty_from_element_idx = self.dirty_from_element_idx.min(block_id); 707 | } 708 | 709 | pub fn get_min(&self, block_id: usize) -> &Timestamp { 710 | &self.data.get(block_id).unwrap().min 711 | } 712 | 713 | pub fn get_max(&self, block_id: usize) -> &Timestamp { 714 | &self.data.get(block_id).unwrap().max 715 | } 716 | 717 | pub fn find_block(&self, timestamp: &Timestamp) -> usize { 718 | match self.data.binary_search_by(|x| x.max.0.cmp(×tamp.0)) { 719 | Ok(mut t) => { 720 | if t == 0 { 721 | return t; 722 | } 723 | let t_orig_max = self.get_max(t); 724 | 725 | while t > 0 && self.get_max(t - 1) == t_orig_max { 726 | t -= 1; 727 | } 728 | t 729 | } 730 | Err(t) => t 731 | } 732 | } 733 | 734 | pub fn write_dirty_part(&mut self) { 735 | let dirty_from_bytes = self.dirty_from_element_idx * std::mem::size_of::(); 736 | 737 | self.file.seek(SeekFrom::Start(dirty_from_bytes as u64)) 738 | .expect("cannot seek in index file"); 739 | 740 | // TODO: check if unsafe is safe 741 | // safe: we take Vec's allocation, create a slice of u8 from it 742 | // write the bytes of the slice to file 743 | let elements = self.data.len(); 744 | let ptr = self.data.as_ptr() as *const u8; 745 | let bytes = unsafe { 746 | std::slice::from_raw_parts(ptr, elements * std::mem::size_of::()) 747 | }; 748 | 749 | self.file.write_all(&bytes[dirty_from_bytes..]) 750 | .expect("cannot write to index file"); 751 | 752 | if let SyncPolicy::Immediate = &self.sync_policy { 753 | self.file.sync_data().expect("sync failed") 754 | } 755 | } 756 | } 757 | 758 | #[cfg(test)] 759 | mod tests { 760 | use crate::engine::index::TimestampIndex; 761 | use std::fs::File; 762 | use tempdir::TempDir; 763 | use crate::engine::io::SyncPolicy; 764 | use crate::engine::Timestamp; 765 | 766 | #[test] 767 | fn test_get_set_max() { 768 | let dir = TempDir::new("tsdb").unwrap(); 769 | let mut idx = TimestampIndex { 770 | file: File::create(dir.path().join("test_get_set_max")).unwrap(), 771 | sync_policy: SyncPolicy::Never, 772 | data: vec![], 773 | dirty_from_element_idx: 0, 774 | }; 775 | 776 | idx.set_min(0, Timestamp(5)); 777 | idx.set_min(1, Timestamp(10)); 778 | idx.set_min(2, Timestamp(16)); 779 | idx.set_min(3, Timestamp(20)); 780 | idx.set_min(4, Timestamp(21)); 781 | idx.set_min(5, Timestamp(50)); 782 | 783 | assert_eq!(*idx.get_min(0), Timestamp(5)); 784 | assert_eq!(*idx.get_min(4), Timestamp(21)); 785 | } 786 | 787 | #[test] 788 | fn test_binary_search() { 789 | let dir = TempDir::new("tsdb").unwrap(); 790 | let mut idx = TimestampIndex { 791 | file: File::create(dir.path().join("test_binary_search")).unwrap(), 792 | sync_policy: SyncPolicy::Never, 793 | data: vec![], 794 | dirty_from_element_idx: 0, 795 | }; 796 | 797 | // 5 | 10 | 16 | 20 | 20 | 21 | 50 798 | 799 | idx.set_max(0, Timestamp(5)); 800 | idx.set_max(1, Timestamp(10)); 801 | idx.set_max(2, Timestamp(16)); 802 | idx.set_max(3, Timestamp(20)); 803 | idx.set_max(4, Timestamp(20)); 804 | idx.set_max(5, Timestamp(21)); 805 | idx.set_max(6, Timestamp(50)); 806 | 807 | assert_eq!(idx.find_block(&Timestamp(0)), (0)); 808 | assert_eq!(idx.find_block(&Timestamp(5)), (0)); 809 | assert_eq!(idx.find_block(&Timestamp(6)), (1)); 810 | assert_eq!(idx.find_block(&Timestamp(18)), (3)); 811 | assert_eq!(idx.find_block(&Timestamp(19)), (3)); 812 | assert_eq!(idx.find_block(&Timestamp(20)), (3)); 813 | assert_eq!(idx.find_block(&Timestamp(21)), (5)); 814 | assert_eq!(idx.find_block(&Timestamp(25)), (6)); 815 | assert_eq!(idx.find_block(&Timestamp(1000)), (7)); 816 | } 817 | 818 | #[test] 819 | fn test_binary_search_same() { 820 | let dir = TempDir::new("tsdb").unwrap(); 821 | let mut idx = TimestampIndex { 822 | file: File::create(dir.path().join("test_binary_search_same")).unwrap(), 823 | sync_policy: SyncPolicy::Never, 824 | data: vec![], 825 | dirty_from_element_idx: 0, 826 | }; 827 | 828 | idx.set_max(0, Timestamp(5)); 829 | idx.set_max(1, Timestamp(5)); 830 | idx.set_max(2, Timestamp(5)); 831 | idx.set_max(3, Timestamp(10)); 832 | idx.set_max(4, Timestamp(10)); 833 | 834 | assert_eq!(idx.find_block(&Timestamp(0)), (0)); 835 | assert_eq!(idx.find_block(&Timestamp(5)), (0)); 836 | assert_eq!(idx.find_block(&Timestamp(6)), (3)); 837 | assert_eq!(idx.find_block(&Timestamp(7)), (3)); 838 | assert_eq!(idx.find_block(&Timestamp(8)), (3)); 839 | assert_eq!(idx.find_block(&Timestamp(9)), (3)); 840 | assert_eq!(idx.find_block(&Timestamp(10)), (3)); 841 | assert_eq!(idx.find_block(&Timestamp(11)), (5)); 842 | } 843 | 844 | #[test] 845 | fn test_save_load_index() { 846 | let dir = TempDir::new("tsdb").unwrap(); 847 | 848 | { 849 | let mut idx = TimestampIndex::create( 850 | &dir.path().join("test_save_load_index"), 851 | SyncPolicy::Never 852 | ).expect("cannot create idx"); 853 | 854 | idx.set_max(0, Timestamp(10)); 855 | idx.set_min(0, Timestamp(11)); 856 | idx.set_max(1, Timestamp(20)); 857 | idx.set_min(1, Timestamp(21)); 858 | 859 | assert_eq!(idx.len(), 2); 860 | 861 | idx.write_dirty_part(); 862 | } 863 | 864 | let mut idx = TimestampIndex::load( 865 | &dir.path().join("test_save_load_index"), 866 | SyncPolicy::Never 867 | ).expect("cannot load index"); 868 | 869 | assert_eq!(idx.len(), 2); 870 | 871 | assert_eq!(*idx.get_max(0), Timestamp(10)); 872 | assert_eq!(*idx.get_min(0), Timestamp(11)); 873 | assert_eq!(*idx.get_max(1), Timestamp(20)); 874 | assert_eq!(*idx.get_min(1), Timestamp(21)); 875 | } 876 | } 877 | } 878 | 879 | pub mod clock { 880 | use std::time::{Duration, SystemTime}; 881 | use crate::engine::Timestamp; 882 | 883 | /// Source of time that provides timestamps that do not go backwards in time. 884 | pub struct Clock { 885 | last_timestamp: Duration, 886 | } 887 | 888 | impl Clock { 889 | pub fn now(&mut self) -> Timestamp { 890 | self.last_timestamp = SystemTime::now() 891 | .duration_since(SystemTime::UNIX_EPOCH) 892 | .unwrap_or(self.last_timestamp); 893 | Timestamp(self.last_timestamp.as_secs()) 894 | } 895 | } 896 | 897 | impl Default for Clock { 898 | fn default() -> Self { 899 | Clock { 900 | last_timestamp: SystemTime::now() 901 | .duration_since(SystemTime::UNIX_EPOCH) 902 | .expect("cannot initialize initial_timestamp") 903 | } 904 | } 905 | } 906 | } 907 | 908 | pub mod server { 909 | use log::debug; 910 | use crate::engine::{Schema, Timestamp, Point}; 911 | use crate::engine::index::TimestampIndex; 912 | use crate::engine::block::{BlockSpec, BLOCK_TAIL_SIZE, BLOCK_SIZE, Block}; 913 | use crate::engine::cache::Cache; 914 | use crate::engine::io::{BlockIO, SyncPolicy}; 915 | use crate::engine::clock::Clock; 916 | use crate::engine::Decoder; 917 | use std::collections::HashMap; 918 | use std::path::PathBuf; 919 | use std::fmt::Debug; 920 | 921 | /// Type that contains metadata about series. 922 | pub struct Series where S: Schema { 923 | pub(crate) id: usize, 924 | pub(crate) enc_state: S::EncState, 925 | pub(crate) timestamp_index: TimestampIndex, 926 | pub(crate) blocks: usize, 927 | pub(crate) last_block_used_bytes: usize, 928 | pub(crate) last_timestamp: Timestamp, 929 | } 930 | 931 | impl Series where S: Schema { 932 | #[inline] 933 | pub fn create_block_spec(&self, block_id: usize) -> BlockSpec { 934 | BlockSpec { series_id: self.id, block_id } 935 | } 936 | 937 | /// Returns new BlockSpec referencing current last Block 938 | /// in this Series structure. 939 | pub fn last_block_spec(&self) -> BlockSpec { 940 | self.create_block_spec(self.blocks - 1) 941 | } 942 | 943 | #[inline] 944 | pub fn last_block_free_bytes(&self) -> usize { 945 | (BLOCK_SIZE - BLOCK_TAIL_SIZE) - self.last_block_used_bytes 946 | } 947 | } 948 | 949 | pub struct BlockLoader { 950 | cache: Cache, 951 | io: BlockIO, 952 | } 953 | 954 | impl BlockLoader { 955 | /// Creates new instance of Block loader with specified 956 | /// storage path, block cache capacity and sync policy for writing. 957 | pub fn new(storage: PathBuf, cache_capacity: usize, sync_policy: SyncPolicy) -> Self { 958 | if !storage.exists() { 959 | debug!("Storage dir does not exists - creating... {}", storage.to_string_lossy()); 960 | std::fs::create_dir_all(&storage) 961 | .expect("cannot create storage directory"); 962 | } 963 | 964 | debug!("Creating BlockLoader storage={} cache_capacity={} sync_policy={:?}", 965 | storage.to_string_lossy(), cache_capacity, sync_policy); 966 | BlockLoader { 967 | cache: Cache::with_capacity(cache_capacity), 968 | io: BlockIO::new(storage, sync_policy), 969 | } 970 | } 971 | 972 | /// Acquires block either from cache or by loading from disk and 973 | /// returns it as immutable reference. 974 | pub fn acquire_block(&mut self, spec: BlockSpec) -> &Block { 975 | if !self.cache.contains_key(&spec) { 976 | self.cache.insert(spec, self.io.create_or_load_block(&spec)); 977 | } 978 | 979 | self.cache.get_mut(&spec) 980 | .expect("block is not in cache after we loaded it!") 981 | } 982 | 983 | /// Removes the Block specified by `spec` parameter from the cache 984 | /// if it is present. Calling this method will remove Block from 985 | /// cache **even if it is unevictable**. 986 | pub fn remove(&mut self, spec: &BlockSpec) { 987 | self.cache.remove(spec); 988 | } 989 | 990 | /// Acquires mutable reference to Block specified by `spec` parameter, 991 | /// calls the specified closure with it and then writes the Block 992 | /// to disk partially using `dirty_from` parameter. 993 | pub fn acquire_then_write ())>(&mut self, 994 | spec: BlockSpec, 995 | dirty_from: usize, 996 | f: F) { 997 | if !self.cache.contains_key(&spec) { 998 | self.cache.insert(spec, self.io.create_or_load_block(&spec)); 999 | } 1000 | 1001 | let b = self.cache 1002 | .get_mut(&spec) 1003 | .expect("block is not in cache after we loaded it!"); 1004 | f(b); 1005 | self.io.write_block_partially(&spec, b, dirty_from); 1006 | } 1007 | 1008 | /// Acquires mutable reference to Block specified by `spec` parameter, 1009 | /// calls the specified closure with it and then writes the Block 1010 | /// to disk partially using `dirty_from` parameter. 1011 | /// 1012 | /// This method has same effect as calling `acquire_then_write` however 1013 | /// the block is loaded as unevictable meaning that it will not be 1014 | /// evicted when another Block is loaded into the cache. 1015 | pub fn acquire_unevictable_then_write ())>(&mut self, 1016 | spec: BlockSpec, 1017 | dirty_from: usize, 1018 | f: F) { 1019 | if !self.cache.contains_key(&spec) { 1020 | self.cache.insert_unevictable(spec, self.io.create_or_load_block(&spec)); 1021 | } 1022 | 1023 | let b = self.cache 1024 | .get_mut(&spec) 1025 | .expect("block is not in cache after we loaded it!"); 1026 | f(b); 1027 | self.io.write_block_partially(&spec, b, dirty_from); 1028 | } 1029 | 1030 | /// Acquires mutable reference to Block specified by `spec` parameter, 1031 | /// calls the specified closure with it. 1032 | pub fn acquire_unevictable ())>(&mut self, 1033 | spec: BlockSpec, 1034 | f: F) { 1035 | if !self.cache.contains_key(&spec) { 1036 | self.cache.insert_unevictable(spec, self.io.create_or_load_block(&spec)); 1037 | } 1038 | 1039 | let b = self.cache 1040 | .get_mut(&spec) 1041 | .expect("block is not in cache after we loaded it!"); 1042 | f(b); 1043 | } 1044 | } 1045 | 1046 | /// Type that represents fully-functional storage system for 1047 | /// time-series data. 1048 | pub struct Engine where S: Schema { 1049 | pub(crate) clock: Clock, 1050 | pub(crate) series: HashMap>, 1051 | pub(crate) last_series_id: usize, 1052 | pub(crate) storage: PathBuf, 1053 | pub(crate) sync_policy: SyncPolicy, 1054 | pub(crate) blocks: BlockLoader, 1055 | } 1056 | 1057 | impl Engine where S: Schema { 1058 | pub fn new(storage: PathBuf, cache_capacity: usize, sync_policy: SyncPolicy) -> Self { 1059 | Engine { 1060 | clock: Default::default(), 1061 | series: Default::default(), 1062 | last_series_id: 0, 1063 | storage: storage.clone(), 1064 | sync_policy, 1065 | blocks: BlockLoader::new(storage, cache_capacity, sync_policy), 1066 | } 1067 | } 1068 | 1069 | fn insert_point_at(&mut self, series_name: &str, value: V, timestamp: Timestamp) { 1070 | let mut series = self.series 1071 | .get_mut(series_name) 1072 | .unwrap(); 1073 | 1074 | let point = Point { timestamp, value }; 1075 | let encoded = S::encode(&mut series.enc_state, point); 1076 | { 1077 | let last_block_free_bytes = series.last_block_free_bytes(); 1078 | 1079 | if last_block_free_bytes < encoded.length { 1080 | let last_block_spec = series.last_block_spec(); 1081 | 1082 | // load the block if for some reason is not loaded and write the free bytes 1083 | self.blocks.acquire_then_write(last_block_spec, series.last_block_used_bytes, |b| { 1084 | debug!("Closing block {:?} with {} free bytes.", last_block_spec, last_block_free_bytes); 1085 | b.set_free_bytes(last_block_free_bytes as u8); 1086 | }); 1087 | 1088 | // as it might be loaded as unevictable we have to manually remove it from cache 1089 | self.blocks.remove(&last_block_spec); 1090 | 1091 | series.timestamp_index.set_max(last_block_spec.block_id, series.last_timestamp); 1092 | series.timestamp_index.set_min(last_block_spec.block_id + 1, timestamp); 1093 | series.timestamp_index.write_dirty_part(); 1094 | series.blocks += 1; 1095 | series.last_block_used_bytes = 0; 1096 | } 1097 | } 1098 | 1099 | // closure that writes the encoded data into block 1100 | let fn_write_encoded = |b: &mut Block| { 1101 | for (i, x) in encoded.data[0..encoded.length].iter().enumerate() { 1102 | b.data[series.last_block_used_bytes + i] = *x; 1103 | } 1104 | }; 1105 | 1106 | let block_spec = series.last_block_spec(); 1107 | self.blocks.acquire_unevictable_then_write( 1108 | block_spec, 1109 | series.last_block_used_bytes, 1110 | fn_write_encoded, 1111 | ); 1112 | series.timestamp_index.set_max(block_spec.block_id, timestamp); 1113 | series.timestamp_index.write_dirty_part(); 1114 | series.last_timestamp = timestamp; 1115 | series.last_block_used_bytes += encoded.length; 1116 | } 1117 | } 1118 | 1119 | pub enum CreateSeriesError { 1120 | SeriesAlreadyExists 1121 | } 1122 | 1123 | pub enum InsertPointError { 1124 | SeriesNotExists 1125 | } 1126 | 1127 | pub enum RetrievePointsError { 1128 | SeriesNotExists 1129 | } 1130 | 1131 | pub trait SimpleServer { 1132 | fn create_series(&mut self, name: &str) -> Result<(), CreateSeriesError>; 1133 | fn insert_point(&mut self, series_name: &str, value: V) -> Result<(), InsertPointError>; 1134 | fn retrieve_points(&mut self, series_name: &str, from: Option, to: Option) -> Result>, RetrievePointsError>; 1135 | } 1136 | 1137 | impl SimpleServer for Engine where S: Schema, V: Debug { 1138 | /// Creates a new series object. 1139 | fn create_series(&mut self, name: &str) -> Result<(), CreateSeriesError> { 1140 | if self.series.contains_key(name) { 1141 | return Err(CreateSeriesError::SeriesAlreadyExists); 1142 | } 1143 | 1144 | let series_id = self.last_series_id; 1145 | let index_file = self.storage.join(format!("{}.idx", series_id)); 1146 | let index = TimestampIndex::create_or_load(&index_file, self.sync_policy) 1147 | .unwrap(); 1148 | 1149 | debug!("Creating new series ID {} ('{}')", series_id, name); 1150 | self.series.insert(name.to_string(), Series { 1151 | id: series_id, 1152 | enc_state: Default::default(), 1153 | timestamp_index: index, 1154 | blocks: 1, 1155 | last_block_used_bytes: 0, 1156 | last_timestamp: Timestamp(0), 1157 | }); 1158 | self.last_series_id += 1; 1159 | Ok(()) 1160 | } 1161 | 1162 | /// Inserts specified point into the specified Series object. 1163 | fn insert_point(&mut self, series_name: &str, value: V) -> Result<(), InsertPointError> { 1164 | if !self.series.contains_key(series_name) { 1165 | return Err(InsertPointError::SeriesNotExists); 1166 | } 1167 | 1168 | let timestamp = self.clock.now(); 1169 | Ok(self.insert_point_at(series_name, value, timestamp)) 1170 | } 1171 | 1172 | /// Retrieves all data points that happened between `from` and `to` 1173 | /// parameters. If the parameters are empty (`None`) the range is 1174 | /// considered open from one or both sides. 1175 | fn retrieve_points(&mut self, 1176 | series_name: &str, 1177 | from: Option, 1178 | to: Option) -> Result>, RetrievePointsError> { 1179 | if !self.series.contains_key(series_name) { 1180 | return Err(RetrievePointsError::SeriesNotExists); 1181 | } 1182 | 1183 | let mut points = vec![]; 1184 | let series = self.series 1185 | .get_mut(series_name) 1186 | .unwrap(); 1187 | 1188 | let start_block = from 1189 | .or_else(|| Some(Timestamp(0))) 1190 | .map(|x| series.timestamp_index.find_block(&x)) 1191 | .unwrap(); 1192 | 1193 | // we start at block that does not exists yet -> empty result 1194 | if start_block > series.blocks - 1 { 1195 | return Ok(vec![]); 1196 | } 1197 | 1198 | let end_block = to 1199 | .map(|x| series.timestamp_index.find_block(&x)) 1200 | .map(|x| x.min(series.blocks - 1)) 1201 | .unwrap_or(series.blocks - 1); 1202 | 1203 | // first timestamp in block 0 is absolute and not relative 1204 | let min_timestamp = if start_block == 0 { Timestamp(0) } else { 1205 | *series.timestamp_index.get_min(start_block) 1206 | }; 1207 | 1208 | let mut dec_state = S::DecState::new(min_timestamp); 1209 | 1210 | for block_id in start_block..=end_block { 1211 | let spec = series.create_block_spec(block_id); 1212 | let block = self.blocks.acquire_block(spec); 1213 | 1214 | let mut read_bytes = 0; 1215 | let written_bytes = if block_id == end_block { series.last_block_used_bytes } else { block.data_len() }; 1216 | 1217 | while read_bytes < written_bytes { 1218 | let offset_buff = &block.data[read_bytes..]; 1219 | let (point, rb) = S::decode(&mut dec_state, offset_buff); 1220 | 1221 | if to.is_some() && to.unwrap().0 < point.timestamp.0 { 1222 | break; 1223 | } 1224 | 1225 | if from.is_none() || from.unwrap().0 <= point.timestamp.0 { 1226 | points.push(point); 1227 | } 1228 | 1229 | read_bytes += rb 1230 | } 1231 | } 1232 | 1233 | Ok(points) 1234 | } 1235 | } 1236 | 1237 | #[cfg(test)] 1238 | mod tests { 1239 | use crate::engine::server::{Engine, SimpleServer}; 1240 | use crate::engine::f32::F32; 1241 | use crate::engine::io::SyncPolicy; 1242 | use tempdir::TempDir; 1243 | use crate::engine::Timestamp; 1244 | 1245 | #[test] 1246 | fn test_simple_commands() { 1247 | let dir = TempDir::new("test_simple_commands").unwrap(); 1248 | let mut s: Engine = Engine::new( 1249 | dir.path().join("storage/"), 1250 | 1024, 1251 | SyncPolicy::Never, 1252 | ); 1253 | 1254 | s.create_series("default"); 1255 | s.insert_point_at("default", 1.0, Timestamp(1)); 1256 | s.insert_point_at("default", 2.0, Timestamp(1)); 1257 | s.insert_point_at("default", 3.0, Timestamp(2)); 1258 | s.insert_point_at("default", 4.0, Timestamp(3)); 1259 | s.insert_point_at("default", 5.0, Timestamp(4)); 1260 | s.insert_point_at("default", 6.0, Timestamp(5)); 1261 | s.insert_point_at("default", 7.0, Timestamp(5)); 1262 | s.insert_point_at("default", 8.0, Timestamp(5)); 1263 | s.insert_point_at("default", 9.0, Timestamp(10)); 1264 | s.insert_point_at("default", 10.0, Timestamp(11)); 1265 | 1266 | assert_eq!(s.retrieve_points("default", None, None).ok().unwrap().len(), 10); 1267 | assert_eq!(s.retrieve_points("default", Some(Timestamp(500)), None).ok().unwrap().len(), 0); 1268 | assert_eq!(s.retrieve_points("default", Some(Timestamp(1)), Some(Timestamp(11))).ok().unwrap().len(), 10); 1269 | assert_eq!(s.retrieve_points("default", Some(Timestamp(1)), Some(Timestamp(500))).ok().unwrap().len(), 10); 1270 | assert_eq!(s.retrieve_points("default", Some(Timestamp(1)), Some(Timestamp(10))).ok().unwrap().len(), 9); 1271 | assert_eq!(s.retrieve_points("default", Some(Timestamp(3)), Some(Timestamp(10))).ok().unwrap().len(), 6); 1272 | assert_eq!(s.retrieve_points("default", Some(Timestamp(5)), Some(Timestamp(5))).ok().unwrap().len(), 3); 1273 | assert_eq!(s.retrieve_points("default", Some(Timestamp(10)), Some(Timestamp(1))).ok().unwrap().len(), 0); 1274 | assert_eq!(s.retrieve_points("default", Some(Timestamp(11)), Some(Timestamp(100))).ok().unwrap().len(), 1); 1275 | } 1276 | 1277 | #[test] 1278 | fn test_block_boundary() { 1279 | let dir = TempDir::new("test_block_boundary").unwrap(); 1280 | let mut s: Engine = Engine::new( 1281 | dir.path().join("storage/"), 1282 | 1024, 1283 | SyncPolicy::Never, 1284 | ); 1285 | 1286 | s.create_series("default"); 1287 | for i in 0..3000 { 1288 | s.insert_point("default", i as f32); 1289 | } 1290 | 1291 | let results = s.retrieve_points("default", None, None).ok().unwrap(); 1292 | assert_eq!(results.len(), 3000) 1293 | } 1294 | } 1295 | } 1296 | -------------------------------------------------------------------------------- /src/engine/array_vec.rs: -------------------------------------------------------------------------------- 1 | const MAX_ARRAY_SIZE: usize = 16; 2 | 3 | #[derive(Default)] 4 | pub struct ArrayVec { 5 | pub data: [T; MAX_ARRAY_SIZE], 6 | pub length: usize, 7 | } 8 | 9 | impl<'a, A: 'a> Extend<&'a A> for ArrayVec where A: Copy { 10 | fn extend>(&mut self, iter: T) { 11 | for x in iter.into_iter() { 12 | self.data[self.length] = *x; 13 | self.length += 1; 14 | } 15 | } 16 | } 17 | 18 | #[cfg(test)] 19 | mod tests { 20 | use crate::engine::array_vec::ArrayVec; 21 | 22 | #[test] 23 | fn test_extend() { 24 | let mut av = ArrayVec::default(); 25 | av.extend(&[1, 2, 3, 4]); 26 | assert_eq!(av.length, 4); 27 | assert_eq!(av.data[0..5], [1, 2, 3, 4, 0]); 28 | } 29 | } -------------------------------------------------------------------------------- /src/engine/btree.rs: -------------------------------------------------------------------------------- 1 | struct BTreeNode { 2 | id: usize, 3 | elements: [(K, V); 511], 4 | pointers: [usize; 512] 5 | } -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod engine; 2 | pub mod server; 3 | //pub mod query_engine; -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use log::{debug, error, info, warn}; 2 | use std::io; 3 | use std::path::PathBuf; 4 | use tokio::net::{TcpListener, TcpStream}; 5 | use tokio::io::AsyncRead; 6 | use tsdb::engine::f32::F32; 7 | use tsdb::engine::io::SyncPolicy; 8 | use tsdb::server::{Server, Settings}; 9 | use futures::future::poll_fn; 10 | 11 | #[tokio::main] 12 | async fn main() -> io::Result<()> { 13 | simple_logger::init_with_level(log::Level::Debug).unwrap(); 14 | 15 | let settings = Settings { 16 | storage: PathBuf::from("./storage/"), 17 | block_cache_capacity: 1024 * 8, // 32 mb 18 | block_sync_policy: SyncPolicy::Never, 19 | index_sync_policy: SyncPolicy::Never, 20 | socket_read_timeout: 2000, 21 | socket_write_timeout: 5000, 22 | listen: "0.0.0.0:9087".to_string(), 23 | }; 24 | let mut listener = TcpListener::bind(&settings.listen).await?; 25 | info!("Listening on {}", &settings.listen); 26 | let db = Server::::new(settings); 27 | 28 | loop { 29 | let (socket, addr) = listener.accept().await?; 30 | info!("New client from {:?}", addr); 31 | 32 | handle_client(socket); 33 | } 34 | } 35 | 36 | async fn handle_client(socket: TcpStream) { 37 | let mut buf = [0; 10]; 38 | let result = poll_fn(|cx| { 39 | socket.poll_peek(cx, &mut buf) 40 | }).await; 41 | 42 | let mut de = serde_json::from_str(result.unwrap()); 43 | } 44 | -------------------------------------------------------------------------------- /src/query_engine.rs: -------------------------------------------------------------------------------- 1 | use crate::engine::{Timestamp, Schema, Point}; 2 | use crate::engine::server::{Engine, SimpleServer}; 3 | use std::fmt::Display; 4 | use static_assertions::_core::fmt::{Formatter, Error}; 5 | use std::ops::{AddAssign, Sub, Div}; 6 | use std::borrow::BorrowMut; 7 | use crate::query_engine::aggregates::{Count, Average, Aggregate}; 8 | 9 | /// Contains all aggregate functions with their 10 | /// structs (data types) that contain their internal state. 11 | /// 12 | /// All aggregate functions implement one trait called "Aggregate" 13 | /// which ensure that all of them have the same interface. 14 | pub mod aggregates { 15 | use std::ops::{AddAssign, Sub, Div}; 16 | 17 | pub trait One { 18 | const ONE: Self; 19 | } 20 | 21 | pub trait Zero { 22 | const ZERO: Self; 23 | } 24 | 25 | macro_rules! impl_one_zero { 26 | ($typ:ty) => { 27 | impl One for $typ { 28 | const ONE: $typ = 1 as $typ; 29 | } 30 | 31 | impl Zero for $typ { 32 | const ZERO: $typ = 0 as $typ; 33 | } 34 | }; 35 | ($typ:ty, $($y:ty),+) => ( 36 | impl_one_zero!($typ); 37 | impl_one_zero!($($y),+); 38 | ) 39 | } 40 | 41 | impl_one_zero!(u8, u16, u32, u64, u128, i8, i16, i32, i64, i128, f32, f64); 42 | 43 | /// Basic trait used for all aggregate functions. 44 | pub trait Aggregate { 45 | type Output; 46 | fn item(&mut self, item: T); 47 | fn result(&mut self) -> Self::Output; 48 | } 49 | 50 | /// Aggregate function that computes arithmetic mean of values. 51 | pub struct Average { 52 | avg: A, 53 | count: C, 54 | } 55 | 56 | macro_rules! impl_average { 57 | ($typ:ty) => { 58 | impl Aggregate for $typ 59 | where A: Copy + AddAssign + Sub + Div, 60 | C: One + Copy + AddAssign 61 | { 62 | type Output = A; 63 | 64 | fn item(&mut self, item: A) { 65 | self.avg += (item - self.avg) / self.count; 66 | self.count += C::ONE; 67 | } 68 | 69 | fn result(&mut self) -> Self::Output { 70 | self.avg 71 | } 72 | } 73 | }; 74 | } 75 | 76 | impl_average!(Average); 77 | impl_average!(&mut Average); 78 | 79 | 80 | impl Default for Average where A: Zero, C: One { 81 | fn default() -> Self { 82 | Average { avg: A::ZERO, count: C::ONE } 83 | } 84 | } 85 | 86 | /// Aggregate function that counts number of items. 87 | #[derive(Default)] 88 | pub struct Count(C); 89 | 90 | impl Aggregate for &mut Count 91 | where C: One + AddAssign + Copy 92 | { 93 | type Output = C; 94 | 95 | fn item(&mut self, _: T) { 96 | self.0 += C::ONE; 97 | } 98 | 99 | fn result(&mut self) -> Self::Output { 100 | self.0 101 | } 102 | } 103 | } 104 | 105 | pub enum AggFn { 106 | Count(Count), 107 | Average(Average), 108 | } 109 | 110 | impl AggFn where V: Copy + AddAssign + Sub + Div { 111 | fn next(&mut self, val: V) { 112 | match self { 113 | AggFn::Count(a) => a.borrow_mut().item(val), 114 | AggFn::Average(a) => a.borrow_mut().item(val), 115 | } 116 | } 117 | } 118 | 119 | 120 | impl Display for AggFn { 121 | fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { 122 | match self { 123 | AggFn::Count(_) => f.write_str("Count"), 124 | AggFn::Average(_) => f.write_str("Average"), 125 | } 126 | } 127 | } 128 | 129 | pub enum Selectable { 130 | Timestamp, 131 | Value, 132 | AggFn(AggFn), 133 | } 134 | 135 | pub enum From { 136 | Series(String), 137 | Subquery(Box>), 138 | } 139 | 140 | pub enum GroupBy { 141 | Minute, 142 | Hour, 143 | Day, 144 | Week, 145 | Month, 146 | Year, 147 | } 148 | 149 | #[derive(Default)] 150 | pub struct Between { 151 | min_timestamp: Option, 152 | max_timestamp: Option, 153 | } 154 | 155 | pub struct Query { 156 | from: From, 157 | select: Vec>, 158 | between: Between, 159 | group_by: Option, 160 | } 161 | 162 | struct Col<'a, A> { name: &'a str, data: Vec } 163 | 164 | pub trait Column { 165 | fn push(&mut self, val: &Point); 166 | } 167 | 168 | impl<'a, V> Column for Col<'a, V> where V: Copy { 169 | fn push(&mut self, val: &Point) { 170 | self.data.push(val.value); 171 | } 172 | } 173 | 174 | pub type QueryResult = Vec>>; 175 | 176 | pub trait QueryEngine { 177 | fn handle_query(&mut self, query: Query) -> QueryResult; 178 | } 179 | 180 | impl Engine where S: Schema {} 181 | 182 | impl QueryEngine for Engine 183 | where S: Schema, 184 | V: Copy + AddAssign + Sub + Div 185 | { 186 | fn handle_query(&mut self, query: Query) -> QueryResult { 187 | let mut res = QueryResult::new(); 188 | 189 | // prepare result vectors / agg fns 190 | query.select.iter().for_each(|it| { 191 | match it { 192 | Selectable::Timestamp => res.push(Box::new(Col:: { name: "timestamp", data: vec![] })), 193 | Selectable::Value => res.push(Box::new(Col:: { name: "value", data: vec![] })), 194 | _ => {} 195 | } 196 | }); 197 | 198 | if let From::Series(t) = query.from { 199 | let mut selectables = query.select; 200 | self.retrieve_points(&t, 201 | query.between.min_timestamp, 202 | query.between.max_timestamp) 203 | .iter() 204 | .for_each(|p| { 205 | selectables.iter_mut() 206 | .enumerate() 207 | .for_each(|(idx, s)| { 208 | match s { 209 | Selectable::Timestamp | Selectable::Value => res[idx].push(p), 210 | Selectable::AggFn(t) => t.next(p.value), 211 | } 212 | }); 213 | }); 214 | } 215 | 216 | res 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /src/server.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | use crate::engine::io::SyncPolicy; 3 | use crate::engine::server::{Engine, SimpleServer, BlockLoader, Series}; 4 | use log::{info, debug, error}; 5 | use crate::server::protocol::{Command, Error, Response, Insert, Select, Between}; 6 | use serde::{Deserialize, Serialize}; 7 | use crate::engine::{Schema, Timestamp}; 8 | use crate::engine::index::TimestampIndex; 9 | use std::time::Instant; 10 | use std::fmt::Debug; 11 | use crate::engine::Decoder; 12 | 13 | pub mod protocol { 14 | use serde::{Serialize, Deserialize}; 15 | 16 | #[derive(Serialize, Deserialize)] 17 | pub struct Between { 18 | pub min: Option, 19 | pub max: Option, 20 | } 21 | 22 | #[derive(Serialize, Deserialize)] 23 | pub struct Select { 24 | pub from: String, 25 | pub between: Between, 26 | } 27 | 28 | #[derive(Serialize, Deserialize)] 29 | pub struct Insert { 30 | pub to: String, 31 | pub value: V, 32 | } 33 | 34 | #[derive(Serialize, Deserialize)] 35 | pub enum Command { 36 | Select(Select), 37 | Insert(Insert), 38 | CreateSeries(String), 39 | } 40 | 41 | #[derive(Serialize, Deserialize)] 42 | pub enum Error { 43 | AuthError, 44 | InvalidQuery, 45 | TableNotFound, 46 | TableExists, 47 | } 48 | 49 | #[derive(Serialize, Deserialize)] 50 | pub enum Response { 51 | Created, 52 | Inserted, 53 | Data(Vec<(u64, V)>), 54 | } 55 | } 56 | 57 | #[derive(Serialize, Deserialize)] 58 | pub struct Settings { 59 | pub storage: PathBuf, 60 | pub block_cache_capacity: usize, 61 | pub block_sync_policy: SyncPolicy, 62 | pub index_sync_policy: SyncPolicy, 63 | 64 | /// Socket read timeout in milliseconds. 65 | pub socket_read_timeout: u64, 66 | 67 | /// Socket write timeout in milliseconds. 68 | pub socket_write_timeout: u64, 69 | 70 | /// Listen string in 'hostname:port' format. 71 | pub listen: String, 72 | } 73 | 74 | #[derive(Serialize, Deserialize)] 75 | pub struct SeriesData { 76 | id: usize, 77 | name: String, 78 | enc_state: EncState, 79 | blocks: usize, 80 | last_block_used_bytes: usize, 81 | last_timestamp: u64, 82 | } 83 | 84 | #[derive(Serialize, Deserialize)] 85 | pub struct ServerData { 86 | series: Vec>, 87 | last_series_id: usize, 88 | } 89 | 90 | pub struct Server where S: Schema { 91 | settings: Settings, 92 | engine: Engine, 93 | } 94 | 95 | impl Server 96 | where S: Schema, 97 | V: Debug, 98 | for<'a> V: Copy + Serialize + Deserialize<'a>, 99 | for<'a> EncState: Serialize + Deserialize<'a> + Default + Copy 100 | { 101 | pub fn new(settings: Settings) -> Self { 102 | let path = settings.storage.join("server.json"); 103 | 104 | debug!("socket read timeout={}", settings.socket_read_timeout); 105 | debug!("socket write timeout={}", settings.socket_write_timeout); 106 | 107 | let server_data: ServerData = if path.exists() { 108 | let reader = std::fs::File::open(path).unwrap(); 109 | serde_json::from_reader(reader).unwrap() 110 | } else { 111 | ServerData { 112 | series: vec![], 113 | last_series_id: 0, 114 | } 115 | }; 116 | 117 | let series = server_data.series.into_iter() 118 | .map(|s| { 119 | let index_file = settings.storage.join(format!("{}.idx", s.id)); 120 | let index = TimestampIndex::create_or_load(&index_file, settings.index_sync_policy) 121 | .unwrap(); 122 | 123 | debug!("Loaded series: {} (ID {}) with {} blocks.", s.name, s.id, s.blocks); 124 | 125 | (s.name.to_owned(), Series { 126 | id: s.id, 127 | enc_state: s.enc_state, 128 | timestamp_index: index, 129 | blocks: s.blocks, 130 | last_block_used_bytes: s.last_block_used_bytes, 131 | last_timestamp: s.last_timestamp.into(), 132 | }) 133 | }) 134 | .collect(); 135 | 136 | let blocks = BlockLoader::new( 137 | settings.storage.clone(), 138 | settings.block_cache_capacity, 139 | settings.block_sync_policy, 140 | ); 141 | 142 | let mut engine = Engine { 143 | clock: Default::default(), 144 | storage: settings.storage.clone(), 145 | sync_policy: settings.block_sync_policy, 146 | last_series_id: server_data.last_series_id, 147 | series, 148 | blocks, 149 | }; 150 | 151 | // verify all indices is correct if not, rebuild it. 152 | for (k, v) in engine.series.iter_mut() { 153 | if v.blocks != v.timestamp_index.len() { 154 | error!("Loaded index for series {} has incorrect length. Rebuilding...", k); 155 | let start = Instant::now(); 156 | 157 | v.timestamp_index.invalidate(); 158 | 159 | let mut dec_state = S::DecState::new(0.into()); 160 | 161 | for block_id in 0..v.blocks { 162 | let spec = v.create_block_spec(block_id); 163 | let block = engine.blocks.acquire_block(spec); 164 | 165 | let mut read_bytes = 0; 166 | let written_bytes = if block_id == v.blocks - 1 { v.last_block_used_bytes } else { block.data_len() }; 167 | 168 | let mut min: Timestamp = std::u64::MAX.into(); 169 | let mut max: Timestamp = std::u64::MIN.into(); 170 | 171 | while read_bytes < written_bytes { 172 | let offset_buff = &block.data[read_bytes..]; 173 | let (point, rb) = S::decode(&mut dec_state, offset_buff); 174 | 175 | min = min.min(*point.timestamp()); 176 | max = max.max(*point.timestamp()); 177 | 178 | read_bytes += rb 179 | } 180 | 181 | v.timestamp_index.set_max(block_id, max); 182 | v.timestamp_index.set_min(block_id, min); 183 | } 184 | 185 | v.timestamp_index.write_dirty_part(); 186 | info!("Rebuilt index in {} secs...", start.elapsed().as_secs_f32()) 187 | } 188 | } 189 | 190 | Server { 191 | engine, 192 | settings 193 | } 194 | } 195 | 196 | /// Persists metadata (information about series) to the disk. 197 | fn persist_metadata(&self) { 198 | let path = self.settings.storage.join("server.json"); 199 | let data = ServerData { 200 | series: self.engine.series.iter().map(|(name, series)| { 201 | SeriesData { 202 | id: series.id, 203 | name: name.clone(), 204 | enc_state: series.enc_state, 205 | blocks: series.blocks, 206 | last_block_used_bytes: series.last_block_used_bytes, 207 | last_timestamp: (&series.last_timestamp).into(), 208 | } 209 | }).collect(), 210 | last_series_id: self.engine.last_series_id, 211 | }; 212 | 213 | if let Err(e) = std::fs::write(path, serde_json::to_string(&data).unwrap()) { 214 | error!("Cannot save server metadata! {}", e) 215 | } 216 | } 217 | 218 | /// Handles the command on the server and returns the appropriate 219 | /// result. The calling method will handle the result (usually 220 | /// writes the result into a socket). 221 | pub fn handle_command(&mut self, cmd: Command) -> Result, Error> { 222 | match cmd { 223 | Command::Select(Select { from, between }) => { 224 | let Between { min, max } = between; 225 | let data = self.engine 226 | .retrieve_points(&from, 227 | min.map(|x| x.into()), 228 | max.map(|x| x.into()), 229 | ) 230 | .map_err(|_| Error::TableNotFound)? 231 | .iter() 232 | .map(|p| (p.timestamp().into(), *p.value())) 233 | .collect(); 234 | self.persist_metadata(); 235 | Ok(Response::Data(data)) 236 | } 237 | Command::Insert(Insert { to, value }) => { 238 | self.engine 239 | .insert_point(&to, value) 240 | .map_err(|_| Error::TableNotFound)?; 241 | self.persist_metadata(); 242 | Ok(Response::Inserted) 243 | } 244 | Command::CreateSeries(name) => { 245 | self.engine 246 | .create_series(&name) 247 | .map_err(|_| Error::TableExists)?; 248 | self.persist_metadata(); 249 | Ok(Response::Created) 250 | } 251 | } 252 | } 253 | } 254 | 255 | #[cfg(test)] 256 | mod tests {} --------------------------------------------------------------------------------