├── .gitignore ├── Cargo.toml ├── LICENSE.md ├── Readme.md ├── src ├── btree.rs ├── iforest.rs ├── index.rs ├── lib.rs └── trace.rs └── ui ├── Cargo.toml └── src └── main.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gigatrace" 3 | version = "0.1.0" 4 | authors = ["Tristan Hume "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | fastrand = "1.3.5" 11 | 12 | [workspace] 13 | members = ["ui"] 14 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2021 Jane Street Group, LLC (A personal project of Tristan Hume, licensing owned and approved by Jane Street only due to similarity to my job) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Proof-of-concept for a gigabyte-scale trace viewer 2 | 3 | This repo includes: 4 | - A memory-efficient representation for event traces 5 | - An unusually simple and memory-efficient range aggregation index data structure (`IForestIndex`) for zooming traces of billions of events at 60fps 6 | - A proof-of-concept Druid UI to demo efficient trace zooming, that isn't remotely useable as a real trace viewer. 7 | 8 | It's a tech demo for the data structure described in [this blog post](https://thume.ca/2021/03/14/iforests/) 9 | -------------------------------------------------------------------------------- /src/btree.rs: -------------------------------------------------------------------------------- 1 | // I got part-way into implementing this then stopped because it was 2 | // getting way more complicated than I wanted. 3 | 4 | #[derive(Copy, Clone)] 5 | pub struct Ns([u8; 6]); 6 | 7 | impl Ns { 8 | pub const fn new(ts: u64) -> Self { 9 | let b = ts.to_le_bytes(); 10 | // assert!(b[6] == 0 && b[7] == 0); 11 | Ns([b[0], b[1], b[2], b[3], b[4], b[5]]) 12 | } 13 | 14 | #[inline] 15 | pub fn to_u64(self) -> u64 { 16 | let b = self.0; 17 | u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], 0, 0]) 18 | } 19 | } 20 | 21 | #[derive(Copy, Clone)] 22 | pub struct TraceEvent { 23 | kind: u16, 24 | ts: Ns, 25 | dur: Ns, 26 | } 27 | 28 | const NULL_EVENT: TraceEvent = TraceEvent { 29 | kind: 0, 30 | ts: Ns::new(0), 31 | dur: Ns::new(0), 32 | }; 33 | 34 | pub type NodeIndex = u32; 35 | 36 | pub struct NodePool { 37 | root_index: NodeIndex, 38 | nodes: Vec, 39 | } 40 | 41 | type LevelIndex = u8; 42 | 43 | pub struct Node { 44 | level: LevelIndex, 45 | allocated: u8, 46 | range: (Ns, Ns), 47 | body: NodeBody, 48 | } 49 | 50 | const EVENTS_PER_LEAF: usize = 16; 51 | const SUBNODES_PER_NODE: usize = 16; 52 | enum NodeBody { 53 | Leaf([TraceEvent; EVENTS_PER_LEAF]), 54 | Inner([NodeIndex; SUBNODES_PER_NODE]), 55 | } 56 | 57 | #[derive(Copy, Clone)] 58 | enum NodeType { 59 | Leaf, 60 | Inner { level: LevelIndex }, 61 | } 62 | 63 | #[derive(Debug)] 64 | struct Overflow(NodeIndex); 65 | 66 | impl NodePool { 67 | pub fn new() -> Self { 68 | let root = Node::new(NodeType::Leaf); 69 | NodePool { 70 | nodes: vec![root], 71 | root_index: 0, 72 | } 73 | } 74 | 75 | pub fn push(&mut self, ev: TraceEvent) { 76 | match self.push_into(self.root_index, ev) { 77 | Ok(()) => (), 78 | Err(Overflow(over_i)) => { 79 | // Allocate new root parent 80 | let old_level = self.nodes[self.root_index as usize].level; 81 | let new_root_i = self.alloc(NodeType::Inner { 82 | level: old_level + 1, 83 | }); 84 | let new_root = &mut self.nodes[new_root_i as usize]; 85 | match new_root.body { 86 | NodeBody::Leaf(_) => unreachable!(), 87 | NodeBody::Inner(ref mut nodes) => { 88 | nodes[0] = self.root_index; 89 | nodes[1] = over_i; 90 | } 91 | } 92 | new_root.allocated = 2; 93 | self.root_index = new_root_i; 94 | } 95 | } 96 | } 97 | 98 | fn alloc(&mut self, t: NodeType) -> NodeIndex { 99 | self.nodes.push(Node::new(t)); 100 | (self.nodes.len() - 1) as u32 101 | } 102 | 103 | fn push_into(&mut self, node_i: NodeIndex, ev: TraceEvent) -> Result<(), Overflow> { 104 | let node = &self.nodes[node_i as usize]; 105 | match node.body { 106 | NodeBody::Inner(nodes) => { 107 | let allocated = node.allocated; 108 | let level = node.level; 109 | assert!(allocated > 0); 110 | let last_child_i = nodes[(allocated-1) as usize]; 111 | match self.push_into(last_child_i, ev) { 112 | Ok(()) => Ok(()), 113 | Err(Overflow(over_i)) if allocated as usize == SUBNODES_PER_NODE => { 114 | // Allocate new sibling 115 | let new_sibling_i = self.alloc(NodeType::Inner(level)); 116 | 117 | } 118 | Err(Overflow(over_i)) => { 119 | // Allocate new child 120 | Ok(()) 121 | } 122 | } 123 | } 124 | NodeBody::Leaf(events) => { 125 | Ok(()) 126 | } 127 | } 128 | } 129 | } 130 | 131 | impl Node { 132 | fn new(t: NodeType) -> Self { 133 | match t { 134 | NodeType::Leaf => Node { 135 | level: 0, 136 | allocated: 0, 137 | range: (Ns::new(0), Ns::new(0)), 138 | body: NodeBody::Leaf([NULL_EVENT; EVENTS_PER_LEAF]), 139 | }, 140 | NodeType::Inner { level } => Node { 141 | level, 142 | allocated: 0, 143 | range: (Ns::new(0), Ns::new(0)), 144 | body: NodeBody::Inner([u32::MAX; SUBNODES_PER_NODE]), 145 | }, 146 | } 147 | } 148 | } 149 | 150 | #[cfg(test)] 151 | mod tests { 152 | #[test] 153 | fn it_works() { 154 | assert_eq!(2 + 2, 4); 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /src/iforest.rs: -------------------------------------------------------------------------------- 1 | use crate::trace::{TraceBlock, BlockPool, Track}; 2 | use crate::index::{Aggregate, TrackIndex}; 3 | use std::ops::Range; 4 | 5 | pub struct IForestIndex { 6 | pub vals: Vec, 7 | } 8 | 9 | // # 10 | // _______________| 11 | // _______|_______| # 12 | // ___|___|___|___|___| 13 | // 0|1|2|3|4|5|6|7|8|9| 14 | impl IForestIndex { 15 | pub fn new() -> Self { 16 | IForestIndex { vals: vec![] } 17 | } 18 | 19 | pub fn push(&mut self, block: &TraceBlock) { 20 | self.vals.push(A::from_block(block)); 21 | 22 | let len = self.vals.len(); 23 | // We want to index the first level every 2 nodes, 2nd level every 4 nodes... 24 | // This happens to correspond to the number of trailing ones in the index 25 | let levels_to_index = len.trailing_ones()-1; 26 | 27 | // Complete unfinished aggregation nodes which are now ready 28 | let mut cur = len-1; // The leaf we just pushed 29 | for level in 0..levels_to_index { 30 | let prev_higher_level = cur-(1 << level); // nodes at a level reach 2^level 31 | let combined = A::combine(&self.vals[prev_higher_level], &self.vals[cur]); 32 | self.vals[prev_higher_level] = combined; 33 | cur = prev_higher_level; 34 | } 35 | 36 | // Push new aggregation node going back one level further than we aggregated 37 | self.vals.push(self.vals[len-(1 << levels_to_index)].clone()); 38 | } 39 | 40 | /// See [havelessbemore's explanation] for more on why these bit tricks 41 | /// work. Thanks to him for the enhanced bit tricks for fewer branches. 42 | /// 43 | /// See the [old version] for a potentially easier to understand older 44 | /// version of this function that was a bit more complex and probably 45 | /// slower. 46 | /// 47 | /// [havelessbemore's explanation]: https://github.com/havelessbemore/dastal/blob/cd6a1d03872aa437f9272ce3fd42e2e2c006b2cc/src/segmentTree/inOrderSegmentTree.ts 48 | /// [old version]: https://github.com/trishume/gigatrace/blob/9e2fbb3c111529335f4f76a86ca788689dafd81c/src/iforest.rs 49 | pub fn range_query(&self, r: Range) -> A { 50 | /// offset past largest tree with left index x 51 | fn lsp(x: usize) -> usize { 52 | x & x.wrapping_neg() // leave the least significant bit 53 | } 54 | /// offset past largest tree up to x long 55 | fn msp(x: usize) -> usize { 56 | 1usize.reverse_bits() >> x.leading_zeros() // leave the most significant bit 57 | } 58 | fn largest_prefix_inside_skip(min: usize, max: usize) -> usize { 59 | lsp(min|msp(max-min)) // = usize::min(lsp(min),msp(max-min)) 60 | } 61 | fn agg_node(i: usize, offset: usize) -> usize { 62 | i + (offset >> 1) - 1 // 63 | } 64 | 65 | let mut ri = (r.start*2)..(r.end*2); // translate underlying to interior indices 66 | let len = self.vals.len(); 67 | assert!(ri.start <= len && ri.end <= len, "range {:?} not inside 0..{}", r, len/2); 68 | 69 | let mut combined = A::empty(); 70 | while ri.start < ri.end { 71 | let skip = largest_prefix_inside_skip(ri.start, ri.end); 72 | combined = A::combine(&combined, &self.vals[agg_node(ri.start, skip)]); 73 | ri.start += skip 74 | } 75 | combined 76 | } 77 | } 78 | 79 | impl TrackIndex for IForestIndex { 80 | fn build(track: &Track, pool: &BlockPool) -> IForestIndex { 81 | let mut forest = IForestIndex::new(); 82 | for i in &track.block_locs { 83 | forest.push(&pool.blocks[*i as usize]); 84 | } 85 | // TODO in parallel 86 | forest 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/index.rs: -------------------------------------------------------------------------------- 1 | use crate::trace::{TraceBlock, TraceEvent, Track, BlockPool}; 2 | 3 | pub trait Aggregate: Clone { 4 | fn empty() -> Self; 5 | fn from_event(ev: &TraceEvent) -> Self; 6 | fn combine(&self, other: &Self) -> Self; 7 | 8 | fn from_block(block: &TraceBlock) -> Self { 9 | let mut c = Self::empty(); 10 | for ev in block.events() { 11 | c = Self::combine(&c, &Self::from_event(ev)); 12 | } 13 | c 14 | } 15 | } 16 | 17 | pub trait TrackIndex { 18 | fn build(track: &Track, pool: &BlockPool) -> Self; 19 | } 20 | 21 | // === Concrete aggregations 22 | 23 | #[derive(Clone)] 24 | pub struct LongestEvent(pub Option); 25 | 26 | impl Aggregate for LongestEvent { 27 | fn empty() -> Self { 28 | LongestEvent(None) 29 | } 30 | 31 | fn from_event(ev: &TraceEvent) -> Self { 32 | LongestEvent(Some(ev.clone())) 33 | } 34 | 35 | fn combine(&self, other: &Self) -> Self { 36 | LongestEvent([self.0, other.0].iter() 37 | .filter_map(|x| x.as_ref()) 38 | .max_by_key(|ev| ev.dur.unpack()) 39 | .map(|x| x.clone())) 40 | } 41 | 42 | fn from_block(block: &TraceBlock) -> Self { 43 | LongestEvent(block.events().iter().max_by_key(|ev| ev.dur.unpack()).map(|x| x.clone())) 44 | } 45 | } 46 | 47 | // #[derive(Clone)] 48 | // pub struct LongestEventLoc { 49 | // dur: Ns, 50 | // index: usize, 51 | // } 52 | 53 | // impl Aggregate for LongestEventLoc { 54 | // fn empty() -> Self { 55 | // LongestEventLoc { dur: 0, index: usize::MAX } 56 | // } 57 | 58 | // fn from_event(ev: &TraceEvent) -> Self { 59 | // LongestEventLoc(Some(ev.clone())) 60 | // } 61 | 62 | // fn combine(&self, other: &Self) -> Self { 63 | // LongestEventLoc([self.0, other.0].iter() 64 | // .filter_map(|x| x.as_ref()) 65 | // .max_by_key(|ev| ev.dur.unpack()) 66 | // .map(|x| x.clone())) 67 | // } 68 | 69 | // fn from_block(block: &TraceBlock) -> Self { 70 | // LongestEventLoc(block.events().iter().max_by_key(|ev| ev.dur.unpack()).map(|x| x.clone())) 71 | // } 72 | // } 73 | 74 | /// For debugging 75 | #[derive(Clone)] 76 | pub struct EventCount(pub usize); 77 | 78 | impl Aggregate for EventCount { 79 | fn empty() -> Self { 80 | EventCount(0) 81 | } 82 | 83 | fn from_event(_ev: &TraceEvent) -> Self { 84 | EventCount(1) 85 | } 86 | 87 | fn combine(&self, other: &Self) -> Self { 88 | EventCount(self.0 + other.0) 89 | } 90 | } 91 | 92 | #[derive(Clone, PartialEq, Eq, Debug)] 93 | pub struct TsSum(pub u64); 94 | 95 | impl Aggregate for TsSum { 96 | fn empty() -> Self { 97 | Self(0) 98 | } 99 | 100 | fn from_event(ev: &TraceEvent) -> Self { 101 | Self(ev.ts.unpack()) 102 | } 103 | 104 | fn combine(&self, other: &Self) -> Self { 105 | Self(self.0 + other.0) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod iforest; 2 | pub mod index; 3 | pub mod trace; 4 | 5 | use crate::iforest::IForestIndex; 6 | use crate::index::{Aggregate, LongestEvent, TrackIndex}; 7 | use crate::trace::{BlockPool, Ns, BlockIndex, Track}; 8 | use std::ops::Range; 9 | use std::mem; 10 | use fastrand::Rng; 11 | 12 | 13 | pub fn aggregate_by_steps( 14 | pool: &BlockPool, 15 | block_locs: &[BlockIndex], 16 | index: &IForestIndex, 17 | time_span: Range, 18 | time_step: u64, 19 | ) -> Vec { 20 | let mut out = vec![]; 21 | 22 | let mut block_i = 0; 23 | let mut target_time = time_span.start; 24 | let mut combined = A::empty(); 25 | 'outer: loop { 26 | if block_i >= block_locs.len() { 27 | break; 28 | } 29 | 30 | // == Skip to last block with a start_time before target_time 31 | let bsearch_res = block_locs[block_i..] 32 | .binary_search_by_key(&target_time, |i| pool.blocks[*i as usize].start_time()) 33 | .unwrap_or_else(|i| i); 34 | if bsearch_res > 1 { 35 | let skip = bsearch_res - 1; 36 | // == aggregate range using the index 37 | combined = A::combine(&combined, &index.range_query(block_i..(block_i+skip))); 38 | block_i += skip; 39 | } 40 | 41 | let block = &pool.blocks[block_locs[block_i] as usize]; 42 | for ev in block.events() { 43 | let ev_ts = ev.ts.unpack(); 44 | while ev_ts >= target_time { 45 | // TODO add trait bool fn to allow skipping adding empty stuff to lists 46 | out.push(mem::replace(&mut combined, A::empty())); 47 | if target_time >= time_span.end { 48 | break 'outer; 49 | } 50 | target_time = target_time + time_step; 51 | } 52 | combined = A::combine(&combined, &A::from_event(ev)); 53 | } 54 | 55 | block_i += 1; 56 | } 57 | 58 | out 59 | } 60 | 61 | pub fn aggregate_by_steps_unindexed( 62 | pool: &BlockPool, 63 | block_locs: &[BlockIndex], 64 | time_span: Range, 65 | time_step: u64, 66 | ) -> Vec { 67 | let mut out = vec![]; 68 | 69 | let mut target_time = time_span.start; 70 | let mut combined = A::empty(); 71 | 'outer: for block_i in block_locs { 72 | let block = &pool.blocks[*block_i as usize]; 73 | for ev in block.events() { 74 | let ev_ts = ev.ts.unpack(); 75 | while ev_ts >= target_time { 76 | // TODO add trait bool fn to allow skipping adding empty stuff to lists 77 | out.push(mem::replace(&mut combined, A::empty())); 78 | if target_time >= time_span.end { 79 | break 'outer; 80 | } 81 | target_time = target_time + time_step; 82 | } 83 | combined = A::combine(&combined, &A::from_event(ev)); 84 | } 85 | } 86 | out 87 | } 88 | 89 | pub struct TrackInfo { 90 | pub track: Track, 91 | pub zoom_index: IForestIndex, 92 | } 93 | 94 | pub struct Trace { 95 | pub pool: BlockPool, 96 | pub tracks: Vec, 97 | } 98 | 99 | impl Trace { 100 | pub fn new() -> Self { 101 | Trace { 102 | pool: BlockPool::new(), 103 | tracks: vec![], 104 | } 105 | } 106 | 107 | pub fn demo_trace(tracks: usize, events_per_track: usize) -> Self { 108 | let mut trace = Self::new(); 109 | let rng = Rng::new(); 110 | for _ in 0..tracks { 111 | let mut track = Track::new(); 112 | track.add_dummy_events(&mut trace.pool, &rng, events_per_track); 113 | trace.tracks.push(TrackInfo { 114 | zoom_index: IForestIndex::build(&track, &trace.pool), 115 | track, 116 | }); 117 | } 118 | trace 119 | } 120 | 121 | pub fn time_bounds(&self) -> Option> { 122 | let start = self.tracks.iter().filter_map(|t| t.track.start_time(&self.pool)).min(); 123 | let end = self.tracks.iter().filter_map(|t| t.track.after_last_time(&self.pool)).max(); 124 | match (start, end) { 125 | (Some(s), Some(e)) => Some(s..e), 126 | (_, _) => None 127 | } 128 | } 129 | } 130 | 131 | #[cfg(test)] 132 | mod tests { 133 | use crate::iforest::*; 134 | use crate::index::*; 135 | use crate::trace::*; 136 | use fastrand::Rng; 137 | 138 | #[test] 139 | fn it_works() { 140 | assert_eq!(2 + 2, 4); 141 | } 142 | 143 | #[test] 144 | fn dummy_trace() { 145 | let mut pool = BlockPool::new(); 146 | let mut track = Track::new(); 147 | let rng = Rng::new(); 148 | track.add_dummy_events(&mut pool, &rng, 300); 149 | 150 | let mut maxes = vec![]; 151 | for i in &track.block_locs { 152 | maxes.push( 153 | LongestEvent::from_block(&pool.blocks[*i as usize]) 154 | .0 155 | .unwrap() 156 | .dur 157 | .unpack(), 158 | ); 159 | } 160 | println!("maxes: {:?}", maxes); 161 | 162 | let index = IForestIndex::::build(&track, &pool); 163 | let index_vals = index 164 | .vals 165 | .iter() 166 | .map(|x| x.0.unwrap().dur.unpack()) 167 | .collect::>(); 168 | println!("index: {:?}", index_vals); 169 | 170 | // assert!(false); 171 | // TODO some test 172 | } 173 | 174 | #[test] 175 | fn block_count() { 176 | let mut pool = BlockPool::new(); 177 | let mut track = Track::new(); 178 | let rng = Rng::new(); 179 | track.add_dummy_events(&mut pool, &rng, 325); 180 | 181 | let index = IForestIndex::::build(&track, &pool); 182 | let index_vals = index.vals.iter().map(|x| x.0).collect::>(); 183 | println!("index: {:?}", index_vals); 184 | 185 | // assert!(false); 186 | // TODO some test 187 | } 188 | 189 | #[test] 190 | fn aggregate_by_steps_unindexed() { 191 | let mut pool = BlockPool::new(); 192 | let mut track = Track::new(); 193 | let ev_ts = &[10, 15, 20, 100, 101, 150, 170]; 194 | for t in ev_ts { 195 | track.push(&mut pool, TraceEvent { 196 | kind: 0, 197 | ts: PackedNs::new(*t), 198 | dur: PackedNs::new(0), 199 | }); 200 | } 201 | 202 | let span = 13..150; 203 | let res = crate::aggregate_by_steps_unindexed::(&pool, &track.block_locs, span, 10); 204 | let res_ts = res.iter().map(|x| x.0).collect::>(); 205 | assert_eq!(&res_ts[..], &[10, 35, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 150]); 206 | } 207 | 208 | #[test] 209 | fn prop_test_range_query() { 210 | let mut pool = BlockPool::new(); 211 | let mut track = Track::new(); 212 | let rng = Rng::new(); 213 | track.add_dummy_events(&mut pool, &rng, 325); 214 | 215 | let index = IForestIndex::::build(&track, &pool); 216 | // let index_vals = index.vals.iter().map(|x| x.0).collect::>(); 217 | // println!("index: {:?}", index_vals); 218 | 219 | for _ in 0..100_000 { 220 | let start = rng.usize(..=track.block_locs.len()); 221 | let end = rng.usize(start..=track.block_locs.len()); 222 | let EventCount(count) = index.range_query(start..end); 223 | let correct: usize = track.block_locs[start..end].iter() 224 | .map(|i| pool.blocks[*i as usize].len as usize).sum(); 225 | assert_eq!(count, correct, "failed for {}..{}", start, end); 226 | } 227 | } 228 | 229 | 230 | #[test] 231 | fn prop_test_aggregate_by_steps() { 232 | let mut pool = BlockPool::new(); 233 | let mut track = Track::new(); 234 | let rng = Rng::new(); 235 | track.add_dummy_events(&mut pool, &rng, 325); 236 | 237 | let index = IForestIndex::::build(&track, &pool); 238 | // let index_vals = index.vals.iter().map(|x| x.0).collect::>(); 239 | // println!("index: {:?}", index_vals); 240 | 241 | let time_bounds = 0..=(track.end_time(&pool).unwrap()+100_000); 242 | for _ in 0..100_000 { 243 | let t1 = rng.u64(time_bounds.clone()); 244 | let t2 = rng.u64(time_bounds.clone()); 245 | let t_range = if t2 > t1 { t1..t2 } else { t2..t1 }; 246 | let range_size = t_range.end - t_range.start; 247 | let step = (range_size / rng.u64(1..10)) + rng.u64(0..100); 248 | let res1 = crate::aggregate_by_steps::(&pool, &track.block_locs, &index, t_range.clone(), step); 249 | let res2 = crate::aggregate_by_steps_unindexed::(&pool, &track.block_locs, t_range.clone(), step); 250 | assert_eq!(res1, res2, "failed for {:?} - {}", t_range, step); 251 | } 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /src/trace.rs: -------------------------------------------------------------------------------- 1 | use fastrand::Rng; 2 | use std::ops::Range; 3 | 4 | pub type Ns = u64; 5 | #[derive(Copy, Clone)] 6 | pub struct PackedNs([u8; 6]); 7 | 8 | impl PackedNs { 9 | pub const fn new(ts: Ns) -> Self { 10 | let b = ts.to_le_bytes(); 11 | // assert!(b[6] == 0 && b[7] == 0); 12 | PackedNs([b[0], b[1], b[2], b[3], b[4], b[5]]) 13 | } 14 | 15 | #[inline] 16 | pub fn unpack(self) -> Ns { 17 | let b = self.0; 18 | u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], 0, 0]) 19 | } 20 | } 21 | 22 | #[derive(Copy, Clone)] 23 | pub struct TraceEvent { 24 | pub kind: u16, 25 | pub ts: PackedNs, 26 | pub dur: PackedNs, 27 | } 28 | 29 | pub const NULL_EVENT: TraceEvent = TraceEvent { 30 | kind: 0, 31 | ts: PackedNs::new(0), 32 | dur: PackedNs::new(0), 33 | }; 34 | 35 | pub type BlockIndex = u32; 36 | 37 | const EVENTS_PER_BLOCK: usize = 16; 38 | pub struct TraceBlock { 39 | pub len: u16, 40 | events: [TraceEvent; EVENTS_PER_BLOCK], 41 | } 42 | 43 | impl TraceBlock { 44 | pub fn new() -> Self { 45 | Self { 46 | len: 0, 47 | events: [NULL_EVENT; EVENTS_PER_BLOCK], 48 | } 49 | } 50 | 51 | pub fn is_full(&self) -> bool { 52 | self.len as usize == EVENTS_PER_BLOCK 53 | } 54 | 55 | pub fn push(&mut self, ev: TraceEvent) { 56 | assert!(!self.is_full()); 57 | self.events[self.len as usize] = ev; 58 | self.len += 1; 59 | } 60 | 61 | #[inline] 62 | pub fn events(&self) -> &[TraceEvent] { 63 | &self.events[..self.len as usize] 64 | } 65 | 66 | /// Returns 0 if block is empty, `Track` has a useful invariant that 67 | /// blocks are never empty. 68 | #[inline] 69 | pub fn start_time(&self) -> Ns { 70 | self.events[0].ts.unpack() 71 | } 72 | } 73 | 74 | pub struct BlockPool { 75 | pub blocks: Vec, 76 | } 77 | 78 | impl BlockPool { 79 | pub fn new() -> Self { 80 | BlockPool { 81 | blocks: vec![], 82 | } 83 | } 84 | 85 | pub fn alloc(&mut self) -> BlockIndex { 86 | let i = self.blocks.len(); 87 | self.blocks.push(TraceBlock::new()); 88 | i as BlockIndex 89 | } 90 | } 91 | 92 | pub struct Track { 93 | pub block_locs: Vec, 94 | } 95 | 96 | impl Track { 97 | pub fn new() -> Self { 98 | Self { 99 | block_locs: vec![] 100 | } 101 | } 102 | 103 | fn new_block(&mut self, pool: &mut BlockPool) -> BlockIndex { 104 | let i = pool.alloc(); 105 | self.block_locs.push(i); 106 | i 107 | } 108 | 109 | pub fn push(&mut self, pool: &mut BlockPool, ev: TraceEvent) { 110 | let last = match self.block_locs.last() { 111 | None => self.new_block(pool), 112 | Some(&i) if pool.blocks[i as usize].is_full() => self.new_block(pool), 113 | Some(&i) => i 114 | }; 115 | pool.blocks[last as usize].push(ev) 116 | } 117 | 118 | pub fn add_dummy_events(&mut self, pool: &mut BlockPool, rng: &Rng, n: usize) { 119 | let mut ts = 0; 120 | ts += rng.u64(..100_000); 121 | for _ in 0..n { 122 | ts += rng.u64(..10_000); 123 | let dur = rng.u64(..20_000); 124 | self.push(pool, TraceEvent { 125 | kind: rng.u16(4..250), 126 | ts: PackedNs::new(ts), 127 | dur: PackedNs::new(dur), 128 | }); 129 | ts += dur; 130 | } 131 | } 132 | 133 | pub fn start_time(&self, pool: &BlockPool) -> Option { 134 | self.block_locs.get(0).map(|i| pool.blocks[*i as usize].start_time()) 135 | } 136 | 137 | pub fn end_time(&self, pool: &BlockPool) -> Option { 138 | self.block_locs.last().and_then(|i| pool.blocks[*i as usize].events().last()).map(|x| x.ts.unpack()) 139 | } 140 | 141 | pub fn after_last_time(&self, pool: &BlockPool) -> Option { 142 | self.block_locs.last().and_then(|i| pool.blocks[*i as usize].events().last()).map(|x| x.ts.unpack() + x.dur.unpack()) 143 | } 144 | 145 | pub fn events<'a>(&'a self, pool: &'a BlockPool) -> impl Iterator + 'a { 146 | self.block_locs.iter().flat_map(move |i| pool.blocks[*i as usize].events()) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /ui/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gigatrace_ui" 3 | version = "0.1.0" 4 | authors = ["Tristan Hume "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | druid = { git = "https://github.com/linebender/druid" } 11 | gigatrace = { path = "../" } 12 | -------------------------------------------------------------------------------- /ui/src/main.rs: -------------------------------------------------------------------------------- 1 | use druid::kurbo::{Size}; 2 | use druid::piet::{FontFamily, ImageFormat, InterpolationMode}; 3 | use druid::widget::prelude::*; 4 | use druid::{ 5 | Affine, AppLauncher, Color, FontDescriptor, LocalizedString, Point, Rect, TextLayout, 6 | WindowDesc, 7 | }; 8 | use std::sync::Arc; 9 | use std::ops::{Deref, Range}; 10 | use std::u64; 11 | 12 | use gigatrace::trace::Ns; 13 | use gigatrace::index::LongestEvent; 14 | use gigatrace::{Trace, TrackInfo, self}; 15 | 16 | struct ViewMap { 17 | start: f64, 18 | scale: f64, 19 | } 20 | 21 | impl ViewMap { 22 | pub fn new(r: &Range, width: f64) -> Self { 23 | Self { start: r.start as f64, scale: width/((r.end - r.start) as f64) } 24 | } 25 | 26 | pub fn to_x(&self, t: Ns) -> f64 { 27 | ((t as f64) - self.start) * self.scale 28 | } 29 | 30 | pub fn to_ns(&self, x: f64) -> f64 { 31 | self.start + (x / self.scale) 32 | } 33 | } 34 | 35 | struct ViewQuant { 36 | pub time_step: Ns, 37 | } 38 | 39 | impl ViewQuant { 40 | pub fn new(r: &Range, width: f64) -> Self { 41 | let ns_per_px = (r.end-r.start)/(width as u64); 42 | let min_event_px = 2; 43 | let step = ns_per_px * min_event_px; 44 | let step = 1 << (64-step.leading_zeros()); 45 | Self { time_step: u64::max(1, step) } 46 | } 47 | 48 | pub fn round_down(&self, x: Ns) -> Ns { 49 | x - (x % self.time_step) 50 | } 51 | 52 | pub fn quantize(&self, r: &Range) -> Range { 53 | self.round_down(r.start)..(self.round_down(r.end)+self.time_step) 54 | } 55 | } 56 | 57 | struct TimelineWidget { 58 | view_range: Range, 59 | } 60 | 61 | impl TimelineWidget { 62 | fn paint_track(&self, ctx: &mut PaintCtx, trace: &Trace, env: &Env, track: &TrackInfo, size: Size) { 63 | // let rect = Rect::from_origin_size(Point::ORIGIN, size); 64 | // let fill_color = Color::rgb8(0x77, 0x00, 0x00); 65 | // ctx.fill(rect, &fill_color); 66 | 67 | let view = ViewMap::new(&self.view_range, size.width); 68 | let quant = ViewQuant::new(&self.view_range, size.width); 69 | let visible_events = gigatrace::aggregate_by_steps(&trace.pool, &track.track.block_locs, &track.zoom_index, quant.quantize(&self.view_range), quant.time_step); 70 | // for ev in track.track.events(&trace.pool) { 71 | for ev in visible_events.iter().filter_map(|x| x.0) { 72 | let ts = ev.ts.unpack(); 73 | let dur = ev.dur.unpack(); 74 | let (start, end) = if dur > quant.time_step { 75 | (ts, ts+dur) 76 | } else { 77 | let start = quant.round_down(ts); 78 | (start, start+quant.time_step) 79 | }; 80 | // println!("{:?}: {} - {} -> {:.1} - {:.1} / {}", self.view_range, start, end, view.to_x(start), view.to_x(end), size.width); 81 | let rect = Rect::new(view.to_x(start),0.0,view.to_x(end),size.height); 82 | let fill_color = Color::rgb8(0x00, 0x00, (ev.kind % 250) as u8); 83 | ctx.fill(rect, &fill_color); 84 | } 85 | } 86 | 87 | fn zoom(&mut self, zoom_factor: f64, at_x: f64, size: Size) -> bool { 88 | let delta_time = self.view_range.end - self.view_range.start; 89 | let new_delta_time = (delta_time as f64) * zoom_factor; // TODO max zoom 90 | let view = ViewMap::new(&self.view_range, size.width); 91 | let zoom_time = view.to_ns(at_x); 92 | let r = at_x / size.width; 93 | let new_start = zoom_time - new_delta_time * r; 94 | let new_end = new_start + new_delta_time; 95 | self.view_range = (new_start as u64)..(new_end as u64); 96 | true 97 | } 98 | 99 | fn zoom_ratio(delta: f64) -> f64 { 100 | let wheel_zoom_speed = -0.02; 101 | let sign = if delta.is_sign_positive() { 1.0 } else { -1.0 }; 102 | let delta_log = sign * (delta.abs()+1.0).log2(); 103 | 1.0-(delta_log*wheel_zoom_speed) 104 | } 105 | } 106 | 107 | impl Widget> for TimelineWidget { 108 | fn event(&mut self, ctx: &mut EventCtx, event: &Event, _data: &mut Arc, _env: &Env) { 109 | if !ctx.is_handled() { 110 | if let Event::Wheel(mouse) = event { 111 | let factor = Self::zoom_ratio(mouse.wheel_delta.y); 112 | if self.zoom(factor, mouse.pos.x, ctx.size()) { 113 | ctx.request_paint(); 114 | ctx.set_handled(); 115 | } 116 | } 117 | } 118 | } 119 | 120 | fn lifecycle( 121 | &mut self, 122 | _ctx: &mut LifeCycleCtx, 123 | _event: &LifeCycle, 124 | _data: &Arc, 125 | _env: &Env, 126 | ) { 127 | } 128 | 129 | fn update(&mut self, _ctx: &mut UpdateCtx, _old_data: &Arc, _data: &Arc, _env: &Env) {} 130 | 131 | fn layout( 132 | &mut self, 133 | _layout_ctx: &mut LayoutCtx, 134 | bc: &BoxConstraints, 135 | _data: &Arc, 136 | _env: &Env, 137 | ) -> Size { 138 | bc.max() 139 | } 140 | 141 | fn paint(&mut self, ctx: &mut PaintCtx, data: &Arc, env: &Env) { 142 | // Clear background 143 | let size = ctx.size(); 144 | let rect = Rect::from_origin_size(Point::ORIGIN, size); 145 | ctx.fill(rect, &Color::WHITE); 146 | 147 | let trace = data.deref(); 148 | let track_height = 30.0; 149 | ctx.with_save(|ctx| { 150 | for track in &data.tracks { 151 | self.paint_track(ctx, trace, env, track, Size::new(size.width, track_height)); 152 | ctx.transform(Affine::translate((0.0, track_height))); 153 | } 154 | }); 155 | 156 | // Text is easy; in real use TextLayout should be stored in the widget 157 | // and reused. 158 | let text_color = Color::rgb8(0xFF, 0x00, 0x00); 159 | let mut layout = TextLayout::new("Gigatrace!"); 160 | layout.set_font(FontDescriptor::new(FontFamily::SYSTEM_UI).with_size(12.0)); 161 | layout.set_text_color(text_color); 162 | layout.rebuild_if_needed(ctx.text(), env); 163 | // layout.draw(ctx, (10.0, 10.0)); 164 | } 165 | } 166 | 167 | pub fn main() { 168 | // let trace = Trace::demo_trace(5, 200_000_000); 169 | let trace = Trace::demo_trace(5, 2_000_000); 170 | let timeline = TimelineWidget { 171 | view_range: trace.time_bounds().unwrap_or(0..1000) 172 | }; 173 | 174 | let window = WindowDesc::new(move || timeline).title( 175 | LocalizedString::new("gigatrace-window-title").with_placeholder("Gigatrace"), 176 | ); 177 | AppLauncher::with_window(window) 178 | .use_simple_logger() 179 | .launch(Arc::new(trace)) 180 | .expect("launch failed"); 181 | } 182 | --------------------------------------------------------------------------------