├── LICENSE
├── README.org
├── atomic_list.cpp
├── include
    ├── atomic_list.hpp
    ├── color.hpp
    ├── fixed_list_manager.hpp
    ├── gc.hpp
    ├── impl_details.hpp
    ├── large_block_list.hpp
    ├── marker.hpp
    ├── mutator.hpp
    ├── node_pool.hpp
    ├── phase.hpp
    ├── stub_list.hpp
    └── write_barrier.hpp
└── mutator.cpp


/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2016 Mark Thom
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.org:
--------------------------------------------------------------------------------
 1 | #+TITLE:         On-the-fly Concurrent Mark Sweep GC
 2 | #+AUTHOR:        Mark Thom
 3 | #+EMAIL:         markjordanthom@gmail.com
 4 | 
 5 | On-the-fly GC is a concurrent mark-sweep garbage collector designed
 6 | for accurate garbage collection of language runtimes written in C++
 7 | 1x.  It implements the sliding views technique described in the paper
 8 | "An on-the-fly mark and sweep garbage collector based on sliding
 9 | views", which is available at
10 | 
11 | http://grothoff.org/christian/teaching/2007/4705/ms-sliding-views.pdf
12 | 
13 | A goal of on-the-fly is to deliver extremely low mutator pause times,
14 | on the order of tens of microseconds, independently of heap size.
15 | 
16 | For an example of on-the-fly in action, see
17 | 
18 | https://github.com/mthom/managed-ctrie
19 | 


--------------------------------------------------------------------------------
/atomic_list.cpp:
--------------------------------------------------------------------------------
 1 | #include "atomic_list.hpp"
 2 | #include "large_block_list.hpp"
 3 | #include "node_pool.hpp"
 4 | #include "stub_list.hpp"
 5 | 
 6 | namespace otf_gc
 7 | {
 8 |   template <typename T>
 9 |   node_pool<atomic_list<T>>& atomic_list_pool()
10 |   {
11 |     static thread_local node_pool<atomic_list<T>> pool;
12 |     return pool;
13 |   }
14 |   
15 |   template <typename T>
16 |   node_pool<list<T>>& list_pool()
17 |   {
18 |     static thread_local node_pool<list<T>> pool;
19 |     return pool;
20 |   }
21 | 
22 |   template <typename T>
23 |   void* list<T>::node_type::operator new(std::size_t)
24 |   {
25 |     return list_pool<T>().get();
26 |   }
27 | 
28 |   template <typename T>
29 |   void list<T>::node_type::operator delete(void* ptr, std::size_t)
30 |   {
31 |     list_pool<T>().put(ptr);
32 |   }
33 |   
34 |   template <typename T>
35 |   void* atomic_list<T>::node_type::operator new(std::size_t)
36 |   {
37 |     return atomic_list_pool<T>().get();
38 |   }
39 | 
40 |   template <typename T>
41 |   void atomic_list<T>::node_type::operator delete(void* ptr, std::size_t)
42 |   {
43 |     atomic_list_pool<T>().put(ptr);
44 |   }
45 |   
46 |   template class list<void*>;
47 |   
48 |   template class atomic_list<list<void*>>;
49 |   template class atomic_list<stub_list>;
50 | 
51 |   template node_pool<list<void*>>& list_pool<void*>();
52 |   template node_pool<list<list<void*>>>& list_pool<list<void*>>();
53 | 
54 |   template node_pool<atomic_list<list<void*>>>& atomic_list_pool<list<void*>>();
55 |   template node_pool<atomic_list<stub_list>>& atomic_list_pool<stub_list>();
56 | }
57 | 


--------------------------------------------------------------------------------
/include/atomic_list.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef ATOMIC_LIST_HPP_INCLUDED
  2 | #define ATOMIC_LIST_HPP_INCLUDED
  3 | 
  4 | #include <atomic>
  5 | #include <cassert>
  6 | #include <initializer_list>
  7 | #include <utility>
  8 | 
  9 | namespace otf_gc
 10 | {
 11 |   template <typename T>
 12 |   class list
 13 |   {
 14 |   private:
 15 |     struct list_node
 16 |     {
 17 |       T data;
 18 |       list_node* next;
 19 | 
 20 |       static void* operator new(std::size_t);
 21 |       static void operator delete(void*, std::size_t);
 22 |     };
 23 |   
 24 |     struct list_node_iterator
 25 |     {
 26 |       list_node* focus;
 27 | 
 28 |       list_node_iterator operator++(int) const
 29 |       {
 30 | 	return list_node_iterator { focus->next };
 31 |       }
 32 |       
 33 |       list_node_iterator& operator++()
 34 |       {
 35 | 	focus = focus->next;
 36 | 	return *this;
 37 |       }
 38 | 
 39 |       T operator*()
 40 |       {
 41 | 	return focus->data;
 42 |       }
 43 |     
 44 |       list_node_iterator& operator=(const list_node_iterator& it)
 45 |       {
 46 | 	focus = it.focus;
 47 | 	return *this;
 48 |       }
 49 | 
 50 |       inline bool operator==(const list_node_iterator& it)
 51 |       {
 52 | 	return focus == it.focus;
 53 |       }
 54 | 
 55 |       inline bool operator!=(const list_node_iterator& it)
 56 |       {
 57 | 	return focus != it.focus;
 58 |       }      
 59 |     };
 60 |       
 61 |     list_node* head;
 62 |     list_node* tail;    
 63 |   public:
 64 |     using node_type = list_node;
 65 |     
 66 |     list() noexcept : head(nullptr), tail(nullptr) {}
 67 |     list(list_node* head_) noexcept : head(head_), tail(head_) {}     
 68 |     list(list_node* head_, list_node* tail_) noexcept : head(head_), tail(tail_) {}
 69 |     list(std::initializer_list<T> lst) : head(nullptr), tail(nullptr)
 70 |     {
 71 |       for(const T& t : lst)
 72 | 	push_back(t);
 73 |     }
 74 |     
 75 |     inline void* head_ptr() {
 76 |       return reinterpret_cast<void*>(head);
 77 |     }
 78 |     
 79 |     inline list_node_iterator begin() const
 80 |     {
 81 |       return { head };
 82 |     }
 83 |     
 84 |     inline list_node_iterator end() const
 85 |     {
 86 |       return { nullptr };
 87 |     }
 88 | 
 89 |     inline void reset()
 90 |     {
 91 |       head = tail = nullptr;
 92 |     }    
 93 |     
 94 |     void pop_front()
 95 |     {
 96 |       list_node* old_head = head;
 97 |       head = head->next;
 98 | 
 99 |       if(head == nullptr)
100 | 	tail = nullptr;
101 |       
102 |       delete old_head;
103 |     }
104 | 
105 |     list_node* node_pop_front()
106 |     {
107 |       assert(head);
108 | 
109 |       list_node* old_head = head;      
110 |       head = head->next;
111 | 
112 |       if(head == nullptr)
113 | 	tail = nullptr;
114 | 
115 |       return old_head;
116 |     }
117 |     
118 |     T& front()
119 |     {
120 |       return head->data;
121 |     }
122 |     
123 |     list_node* front_ptr()
124 |     {
125 |       return head;
126 |     }
127 | 
128 |     bool empty() const
129 |     {
130 |       return head == nullptr;
131 |     }
132 | 
133 |     void push_front(list_node* node)
134 |     {
135 |       assert(node);
136 | 
137 |       node->next = head;
138 | 
139 |       if(head == nullptr) {
140 | 	head = tail = node;
141 |       } else {
142 | 	head = node;
143 |       }
144 |     }
145 |     
146 |     void push_front(const T& data)
147 |     {
148 |       head = new list_node{data, head};
149 |       
150 |       if(tail == nullptr)
151 | 	tail = head;
152 |     }
153 | 
154 |     template <class Q = T>
155 |     std::enable_if_t<std::is_same<Q, void*>::value, void>
156 |     node_push_front(list_node* chunk)
157 |     {
158 |       chunk->data = reinterpret_cast<void*>(chunk);
159 |       chunk->next = head;
160 | 
161 |       if(tail == nullptr)
162 | 	tail = chunk;
163 | 
164 |       head = chunk;
165 |     }
166 | 
167 |     void push_back(const T& data)
168 |     {
169 |       if(head == nullptr) {
170 | 	head = new list_node{data, head};
171 | 	tail = head;
172 |       } else {
173 | 	tail->next = new list_node{data, nullptr};
174 | 	tail = tail->next;
175 |       }
176 |     }
177 | 
178 |     void clear()
179 |     {
180 |       auto next = head;
181 |       
182 |       while(next) {
183 | 	auto node = next->next;
184 | 	delete next;
185 | 	next = node;
186 |       }
187 | 
188 |       head = tail = nullptr;
189 |     }
190 | 
191 |     list<T> append(list<T>&& lst)
192 |     {
193 |       if(head) {
194 | 	tail->next = lst.head;
195 | 	if(lst.tail)
196 | 	  tail = lst.tail;
197 |       } else {
198 | 	head = lst.head;
199 | 	tail = lst.tail;
200 |       }
201 | 
202 |       lst.head = lst.tail = nullptr;
203 |       
204 |       return *this;
205 |     }
206 |     
207 |     void atomic_vacate_and_append(std::atomic<list<T>>& lst)
208 |     {
209 |       if(head == nullptr)
210 | 	return;
211 |     
212 |       list<T> copy_lst(head, tail);
213 |       head = tail = nullptr;
214 |       list<T> atomic_lst = lst.exchange(nullptr, std::memory_order_relaxed);
215 |     
216 |       while(true)
217 |       {
218 | 	if(!atomic_lst.empty()) {
219 | 	  copy_lst.append(std::move(atomic_lst));
220 | 	}
221 |       
222 | 	copy_lst = lst.exchange(copy_lst, std::memory_order_relaxed);
223 |       
224 | 	if(!copy_lst.empty()) {
225 | 	  atomic_lst = lst.exchange(nullptr, std::memory_order_relaxed);
226 | 	} else {
227 | 	  break;
228 | 	}
229 |       }
230 |     }
231 |   };
232 | 
233 |   template <typename T>
234 |   class atomic_list
235 |   {
236 |   private:
237 |     struct atomic_list_node;
238 | 
239 |     struct counted_node_ptr
240 |     {
241 |       int external_count;
242 |       atomic_list_node* ptr;
243 |     };
244 | 
245 |     struct atomic_list_node
246 |     {
247 |       static_assert(std::is_trivially_copyable<T>::value, "needs trivially copyable type.");
248 | 
249 |       T data;
250 |       std::atomic<int> internal_count;
251 |       counted_node_ptr next;
252 | 
253 |       atomic_list_node(T const& data_)
254 | 	: data(data_)
255 | 	, internal_count(0)
256 |       {}
257 | 
258 |       static void* operator new(std::size_t);
259 |       static void operator delete(void*, std::size_t);
260 |     };
261 | 
262 |     std::atomic<counted_node_ptr> head;
263 | 
264 |     void increase_head_count(counted_node_ptr& old_counter)
265 |     {
266 |       counted_node_ptr new_counter;
267 | 
268 |       do
269 | 	{
270 | 	  new_counter = old_counter;
271 | 	  ++new_counter.external_count;
272 | 	} while(!head.compare_exchange_strong(old_counter,
273 | 					      new_counter,
274 | 					      std::memory_order_acquire,
275 | 					      std::memory_order_relaxed));
276 | 
277 |       old_counter.external_count = new_counter.external_count;
278 |     }
279 | 
280 |   public:
281 |     using node_type = atomic_list_node;
282 | 
283 |     atomic_list() : head(counted_node_ptr{1, nullptr}) {}
284 |   
285 |     void push_front(atomic_list_node* node)
286 |     {
287 |       counted_node_ptr new_node;
288 |     
289 |       new_node.ptr = node;
290 |       new_node.external_count = 1;
291 |       new_node.ptr->next = head.load(std::memory_order_relaxed);
292 | 
293 |       while(!head.compare_exchange_weak(new_node.ptr->next,
294 | 					new_node,
295 | 					std::memory_order_release,
296 | 					std::memory_order_relaxed));
297 |     }
298 |   
299 |     void push_front(const T& data)
300 |     {
301 |       counted_node_ptr new_node;
302 |     
303 |       new_node.ptr = new atomic_list_node(data);
304 |       new_node.external_count = 1;
305 |       new_node.ptr->next = head.load(std::memory_order_relaxed);
306 | 
307 |       while(!head.compare_exchange_weak(new_node.ptr->next,
308 | 					new_node,
309 | 					std::memory_order_release,
310 | 					std::memory_order_relaxed));
311 |     }
312 | 
313 |     T pop_front()
314 |     {
315 |       counted_node_ptr old_head = head.load(std::memory_order_relaxed);
316 | 
317 |       while(true)
318 | 	{
319 | 	  increase_head_count(old_head);
320 | 	  atomic_list_node* const ptr = old_head.ptr;
321 | 
322 | 	  if(!ptr)
323 | 	    return T();
324 | 
325 | 	  if(head.compare_exchange_strong(old_head,
326 | 					  ptr->next,
327 | 					  std::memory_order_relaxed))
328 | 	    {
329 | 	      T res(ptr->data);
330 | 	
331 | 	      int const count_increase = old_head.external_count-2;
332 | 
333 | 	      if(ptr->internal_count.fetch_add(count_increase, std::memory_order_release) == count_increase)
334 | 		{
335 | 		  delete ptr;
336 | 		}
337 | 
338 | 	      return res;
339 | 	    } else if(ptr->internal_count.fetch_add(-1, std::memory_order_relaxed) == 1) {
340 | 	    ptr->internal_count.load(std::memory_order_acquire);
341 | 	    delete ptr;
342 | 	  }	
343 | 	}
344 |     }
345 | 
346 |     atomic_list_node* node_pop_front()
347 |     {
348 |       counted_node_ptr old_head = head.load(std::memory_order_relaxed);
349 | 
350 |       while(true)
351 | 	{            
352 | 	  atomic_list_node* const ptr = old_head.ptr;
353 |       
354 | 	  if(!ptr)
355 | 	    return nullptr;
356 |       
357 | 	  if(head.compare_exchange_strong(old_head, ptr->next, std::memory_order_relaxed))      	
358 | 	    return ptr;
359 | 	}
360 |     }
361 |   
362 |     bool empty() const {
363 |       return !head.load(std::memory_order_relaxed).ptr;
364 |     }
365 |   };
366 | 
367 |   template <typename>
368 |   class node_pool;
369 | 
370 |   template <typename T>
371 |   node_pool<list<T>>& list_pool();
372 | 
373 |   template <typename T>
374 |   node_pool<atomic_list<T>>& atomic_list_pool();
375 | }
376 | #endif
377 | 


--------------------------------------------------------------------------------
/include/color.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef COLOR_HPP_INCLUDED
 2 | #define COLOR_HPP_INCLUDED
 3 | 
 4 | namespace otf_gc
 5 | {
 6 |   struct color
 7 |   {
 8 |     enum class color_t : int8_t
 9 |     {
10 |       Blue = 0x00,
11 |       Black = 0x01,
12 |       White = 0x02
13 |     };
14 | 
15 |     color_t c;
16 | 
17 |     color(int8_t h) noexcept : c(static_cast<color_t>(h))
18 |     {
19 |       assert(0 <= h && h <= 2);
20 |     }
21 |     
22 |     color(color_t c_ = color_t::Black) noexcept
23 |       : c(c_) {}
24 | 
25 |     inline color flip() const {
26 |       if(c == color_t::Black)
27 | 	return color_t::White;
28 |       else
29 | 	return color_t::Black;
30 |     }
31 | 
32 |     inline bool operator!=(const color& co) const
33 |     {
34 |       return c != co.c;
35 |     }
36 |     
37 |     inline bool operator==(const color& co) const
38 |     {
39 |       return c == co.c;
40 |     }
41 |     
42 |     operator color_t() {
43 |       return c;
44 |     }
45 |   };
46 | }
47 | 
48 | #endif
49 | 


--------------------------------------------------------------------------------
/include/fixed_list_manager.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef FIXED_LIST_MANAGER_HPP_INCLUDED
  2 | #define FIXED_LIST_MANAGER_HPP_INCLUDED
  3 | 
  4 | #include <atomic>
  5 | #include <cstdlib>
  6 | #include <memory>
  7 | 
  8 | #include "impl_details.hpp"
  9 | #include "stub_list.hpp"
 10 | 
 11 | namespace otf_gc
 12 | {
 13 |   class fixed_list_manager
 14 |   {
 15 |   private:
 16 |     stub* alloc;
 17 |     size_t offset;
 18 |     const size_t obj_size;
 19 |     size_t log_multiplier;
 20 | 
 21 |     stub_list free_list, used_list;
 22 |   public:
 23 |     fixed_list_manager(size_t size_)
 24 |       : alloc{nullptr}
 25 |       , offset{0}
 26 |       , obj_size{size_}
 27 |       , log_multiplier{3}
 28 |     {}
 29 | 
 30 |     inline stub_list release_free_list() {
 31 |       auto result = free_list;
 32 |       free_list.reset();
 33 |       return result;
 34 |     }
 35 | 
 36 |     inline stub_list release_used_list() {
 37 |       auto result = used_list;
 38 |       used_list.reset();
 39 |       return result;
 40 |     }
 41 | 
 42 |     inline void push_front(void* blk, size_t sz)
 43 |     {
 44 |       assert(blk != nullptr);
 45 |       stub* st = new stub(blk, sz);
 46 | 
 47 |       if(alloc)
 48 | 	free_list.push_back(st);
 49 |       else
 50 | 	alloc = st;
 51 |     }
 52 | 
 53 |     inline void append(stub_list&& sl)
 54 |     {
 55 |       free_list.append(std::move(sl));
 56 | 
 57 |       if(!alloc) {
 58 | 	alloc = free_list.front();
 59 | 	free_list.pop_front();
 60 |       }	
 61 |     }
 62 | 
 63 |     inline void* get_block()
 64 |     {
 65 |       if(!alloc)
 66 | 	return nullptr;
 67 | 
 68 |       assert(alloc->start != nullptr);
 69 | 
 70 |       void* ptr = nullptr;
 71 |       std::ptrdiff_t st = reinterpret_cast<std::ptrdiff_t>(alloc->start);
 72 | 
 73 |       if(offset < alloc->size) {
 74 | 	ptr = reinterpret_cast<void*>(st + offset);
 75 | 	offset += (1ULL << obj_size);
 76 |       } else {
 77 | 	assert(offset == alloc->size);
 78 | 	alloc = free_list.front();
 79 | 
 80 | 	if(alloc) {
 81 | 	  free_list.pop_front();
 82 | 	  ptr = alloc->start;
 83 | 	  offset = (1ULL << obj_size);
 84 | 	}
 85 |       }
 86 | 
 87 |       if(ptr)
 88 | 	used_list.push_back(new stub(ptr, 1ULL << obj_size));
 89 | 
 90 |       return ptr;
 91 |     }
 92 | 
 93 |     inline void* get_new_block()
 94 |     {
 95 |       assert(!alloc);
 96 |       void* blk = aligned_alloc(alignof(impl_details::header_t), 1ULL << (obj_size + log_multiplier));
 97 |       
 98 |       push_front(blk, 1ULL << (obj_size + log_multiplier));
 99 | 
100 |       if(obj_size + log_multiplier < impl_details::small_block_size_limit + obj_size)
101 | 	++log_multiplier;
102 | 
103 |       offset = 1ULL << obj_size;
104 | 
105 |       if(alloc->start)
106 | 	used_list.push_back(new stub(alloc->start, 1ULL << obj_size));
107 | 
108 |       assert(alloc->start == blk);
109 |       
110 |       return blk;
111 |     }
112 |   };
113 | }
114 | #endif
115 | 


--------------------------------------------------------------------------------
/include/gc.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef GC_HPP_INCLUDED
  2 | #define GC_HPP_INCLUDED
  3 | 
  4 | #include <atomic>
  5 | #include <cassert>
  6 | #include <cstdlib>
  7 | #include <memory>
  8 | #include <mutex>
  9 | #include <thread>
 10 | 
 11 | #include "atomic_list.hpp"
 12 | #include "color.hpp"
 13 | #include "impl_details.hpp"
 14 | #include "large_block_list.hpp"
 15 | #include "marker.hpp"
 16 | #include "mutator.hpp"
 17 | #include "phase.hpp"
 18 | #include "stub_list.hpp"
 19 | 
 20 | namespace otf_gc
 21 | {
 22 |   class gc
 23 |   {
 24 |   private:
 25 |     std::atomic<list<void*>> allocation_dump;
 26 | 
 27 |     std::atomic<stub_list> small_used_lists[impl_details::small_size_classes];
 28 |     std::atomic<large_block_list> large_used_list;
 29 | 
 30 |     atomic_list<stub_list> small_free_lists[impl_details::small_size_classes];
 31 | 
 32 |     std::atomic<bool> running;
 33 |     std::atomic<color> alloc_color;
 34 |     std::atomic<phase> gc_phase;
 35 |     std::atomic<unsigned> active, shook;
 36 | 
 37 |     std::mutex reg_mut;
 38 | 
 39 |     friend class mutator;
 40 |   public:
 41 |     static std::unique_ptr<gc> collector;
 42 |   private:
 43 |     std::atomic<list<void*>> root_set;
 44 |     atomic_list<list<void*>> buffer_set;
 45 | 
 46 |     bool try_advance()
 47 |     {
 48 |       if(shook.load(std::memory_order_relaxed) == active.load(std::memory_order_relaxed))
 49 |       {
 50 | 	std::lock_guard<std::mutex> lk(reg_mut);
 51 | 
 52 | 	if(shook.load(std::memory_order_relaxed) == active.load(std::memory_order_relaxed))
 53 | 	{
 54 | 	  shook.store(0, std::memory_order_relaxed);
 55 | 	  phase p(gc_phase.load(std::memory_order_relaxed));
 56 | 
 57 | 	  if(p == phase(phase::phase_t::Second_h))
 58 | 	  {
 59 | 	    color prev_color(alloc_color.load(std::memory_order_relaxed));
 60 | 	    alloc_color.store(prev_color.flip(), std::memory_order_relaxed);
 61 | 	  }
 62 | 
 63 | 	  gc_phase.store(p.advance(), std::memory_order_relaxed);
 64 | 
 65 | 	  return true;
 66 | 	}
 67 |       }
 68 | 
 69 |       return false;
 70 |     }
 71 | 
 72 |     void dump_thread_local_allocations()
 73 |     {
 74 |       list<void*> result = list_pool<void*>().reset_allocation_dump();
 75 | 
 76 |       result.append(list_pool<list<void*>>().reset_allocation_dump());
 77 |       
 78 |       result.append(atomic_list_pool<list<void*>>().reset_allocation_dump());
 79 |       result.append(atomic_list_pool<stub_list>().reset_allocation_dump());
 80 |       
 81 |       result.append(stub_list_pool().reset_allocation_dump());
 82 | 
 83 |       result.atomic_vacate_and_append(allocation_dump);
 84 |     }    
 85 |   public:
 86 |     class registered_mutator : public mutator
 87 |     {
 88 |     private:
 89 |       friend class gc;
 90 |       
 91 |       bool inactive, snoop, trace_on;
 92 |       std::function<list<void*>()> root_callback;
 93 |       phase current_phase;
 94 |       list<void*> buffer, snooped;
 95 |     public:
 96 |       registered_mutator()
 97 | 	: mutator(collector->alloc_color.load(std::memory_order_relaxed))
 98 | 	, inactive(false)
 99 | 	, snoop(collector->gc_phase.load(std::memory_order_relaxed).snooping())
100 | 	, trace_on(collector->gc_phase.load(std::memory_order_relaxed).tracing())
101 | 	, root_callback([]() { return nullptr; })
102 | 	, current_phase(collector->gc_phase.load(std::memory_order_relaxed))
103 |       {
104 | 	collector->active.fetch_add(1, std::memory_order_relaxed);
105 | 	collector->shook.fetch_add(1, std::memory_order_relaxed);
106 |       }
107 | 
108 |       inline bool tracing() const {
109 | 	return trace_on;
110 |       }
111 | 
112 |       inline bool snooping() const {
113 | 	return snoop;
114 |       }
115 | 
116 |       inline phase mut_phase() {
117 | 	return current_phase;
118 |       }
119 | 
120 |       inline void append_front_buffer(list<void*>&& buf) {
121 | 	buffer = buf.append(std::move(buffer));
122 |       }
123 | 
124 |       inline void push_front_buffer(void* p) {
125 | 	buffer.push_front(p);
126 |       }
127 | 
128 |       inline void push_front_snooping(void* p) {
129 | 	snooped.push_front(p);
130 |       }
131 | 
132 |       inline void* buffer_ptr() {
133 | 	if(buffer.empty())
134 | 	  return nullptr;
135 | 	else
136 | 	  return buffer.head_ptr();
137 |       }
138 | 
139 |       inline color mut_color() const {
140 | 	return alloc_color;
141 |       }
142 | 
143 |       inline void set_root_callback(std::function<list<void*>()> root_callback_)
144 |       {
145 | 	root_callback = root_callback_;
146 |       }
147 | 
148 |       inline void poll_for_sync()
149 |       {
150 | 	assert(!inactive);
151 | 	phase gc_phase = collector->gc_phase.load(std::memory_order_relaxed);
152 | 
153 | 	if(current_phase != gc_phase)
154 | 	{
155 | 	  current_phase = gc_phase;
156 | 
157 | 	  if(current_phase == phase(phase::phase_t::Third_h)) {
158 | 	    list<void*> roots = root_callback();
159 | 
160 | 	    roots.append(std::move(snooped));
161 | 	    roots.atomic_vacate_and_append(collector->root_set);
162 | 
163 | 	    for(size_t i = 0; i < impl_details::small_size_classes; ++i)
164 | 	      if(auto small_ul = vacate_small_used_list(i))
165 | 		small_ul.atomic_vacate_and_append(collector->small_used_lists[i]);
166 | 
167 | 	    if(auto large_ul = vacate_large_used_list())
168 | 	      large_ul.atomic_vacate_and_append(collector->large_used_list);
169 | 
170 | 	    alloc_color = collector->alloc_color.load(std::memory_order_relaxed);
171 | 	  } else if(current_phase == phase(phase::phase_t::Fourth_h)) {
172 | 	    collector->buffer_set.push_front(buffer);
173 | 	    buffer.reset();
174 | 	  }
175 | 
176 | 	  snoop = current_phase.snooping();
177 | 	  trace_on = current_phase.tracing();
178 | 
179 | 	  collector->shook.fetch_add(1, std::memory_order_relaxed);
180 | 	}
181 |       }
182 | 
183 |       ~registered_mutator()
184 |       {
185 | 	collector->buffer_set.push_front(buffer);
186 | 
187 | 	for(size_t i = 0; i < impl_details::small_size_classes; ++i) {
188 | 	  if(auto fl = fixed_managers[i].release_free_list())
189 | 	    collector->small_free_lists[i].push_front(fl);
190 | 
191 | 	  if(auto ul = fixed_managers[i].release_used_list())
192 | 	    ul.atomic_vacate_and_append(collector->small_used_lists[i]);
193 | 	}
194 | 
195 | 	collector->dump_thread_local_allocations();
196 | 	
197 | 	large_used_list.atomic_vacate_and_append(collector->large_used_list);
198 | 
199 | 	std::lock_guard<std::mutex> lk(collector->reg_mut);
200 | 
201 | 	collector->active.fetch_sub(!inactive, std::memory_order_relaxed);
202 | 
203 | 	if(!inactive && current_phase == collector->gc_phase.load(std::memory_order_relaxed))
204 | 	  collector->shook.fetch_sub(1, std::memory_order_relaxed);
205 |       }
206 |     };
207 |   private:
208 |     gc()
209 |       : running(false)
210 |       , active(0)
211 |       , shook(0)
212 |     {}
213 | 
214 |     impl_details::underlying_header_t header(void* p)
215 |     {
216 |       impl_details::header_t& h = *reinterpret_cast<impl_details::header_t*>(p);
217 |       return h.load(std::memory_order_relaxed);
218 |     }
219 | 
220 |     template <class Policy>
221 |     void destroy_objects()
222 |     {
223 |       using namespace impl_details;
224 | 
225 |       stub_list remaining_used;
226 | 
227 |       for(size_t i = 0; i < impl_details::small_size_classes; ++i)
228 |       {
229 |       	remaining_used = small_used_lists[i].exchange(nullptr, std::memory_order_relaxed);
230 | 
231 | 	while(remaining_used) {
232 | 	  stub* st = remaining_used.front();
233 | 	  remaining_used.pop_front();
234 | 
235 | 	  for(auto p = reinterpret_cast<std::uint64_t>(st->start);
236 | 	      p < reinterpret_cast<std::uint64_t>(st->start) + st->size;
237 | 	      p += (1ULL << (i+3)))
238 | 	  {
239 | 	    underlying_header_t h = header(reinterpret_cast<void*>(p + log_ptr_size));
240 | 
241 | 	    Policy::destroy(h, reinterpret_cast<header_t*>(p + log_ptr_size));
242 | 
243 | 	    reinterpret_cast<log_ptr_t*>(p)->~log_ptr_t();
244 | 	    reinterpret_cast<header_t*>(p + log_ptr_size)->~header_t();
245 | 	  }
246 | 	}
247 |       }
248 | 
249 |       large_block_list processed_large_used;
250 |       large_block_list remaining_large_used =
251 | 	large_used_list.exchange(nullptr, std::memory_order_relaxed);      
252 | 
253 |       while(remaining_large_used) {
254 | 	void* fr = remaining_large_used.front();
255 | 	
256 | 	remaining_large_used.pop_front();
257 | 	processed_large_used.push_back(fr);
258 | 
259 | 	block_cursor blk_c(fr);
260 | 	blk_c.recalculate();
261 | 
262 | 	underlying_header_t h = blk_c.header()->load(std::memory_order_relaxed);
263 | 
264 | 	Policy::destroy(h, blk_c.header());
265 |       }
266 | 
267 |       processed_large_used.atomic_vacate_and_append(large_used_list);
268 |     }
269 | 
270 |   public:
271 |     template <class Policy>
272 |     void destroy()
273 |     {
274 |       while(active.load(std::memory_order_relaxed) > 0);
275 | 
276 |       destroy_objects<Policy>();
277 | 
278 |       list<void*> records =
279 | 	allocation_dump.exchange(nullptr, std::memory_order_relaxed);
280 | 
281 |       while(!records.empty()) {
282 | 	void* record = records.front();
283 | 	records.pop_front();
284 | 	free(record);
285 |       }
286 | 
287 |       large_block_list used_large_records =
288 | 	large_used_list.exchange(nullptr, std::memory_order_relaxed);
289 |       
290 |       while(!used_large_records.empty()) {
291 | 	void* record = used_large_records.front();
292 | 	used_large_records.pop_front();
293 | 	free(record);
294 |       }
295 |     }
296 | 
297 |     inline static void initialize()
298 |     {
299 |       if(collector == nullptr)
300 | 	collector = std::unique_ptr<gc>(new gc());
301 |     }    
302 |     
303 |     inline static std::unique_ptr<registered_mutator> create_mutator()
304 |     {
305 |       std::lock_guard<std::mutex> lk(collector->reg_mut);
306 |       return std::make_unique<registered_mutator>();
307 |     }
308 |     
309 |     inline void stop()
310 |     {
311 |       running.store(false, std::memory_order_relaxed);
312 |     }
313 | 
314 |     template <class Tracer>
315 |     inline void clear_buffers()
316 |     {
317 |       using namespace impl_details;
318 | 
319 |       while(!buffer_set.empty())
320 |       {
321 | 	list<void*> buf(buffer_set.pop_front());
322 | 
323 | 	while(!buf.empty()) {
324 | 	  void* root = buf.front();
325 | 	  buf.pop_front();
326 | 
327 | 	  if(!root) continue;
328 | 
329 | 	  auto rp = reinterpret_cast<std::ptrdiff_t>(root);
330 | 
331 | 	  if(rp & 1ULL)
332 | 	  {
333 | 	    rp = rp ^ 1ULL;
334 | 
335 | 	    header_t* header_w = reinterpret_cast<header_t*>(rp - header_size);
336 | 	    underlying_header_t header_c = header_w->load(std::memory_order_relaxed);
337 | 
338 | 	    size_t num_log_ptrs = Tracer::num_log_ptrs(header_c);
339 | 
340 | 	    for(std::size_t p = rp - header_size - num_log_ptrs * log_ptr_size;
341 | 		p < rp - header_size;
342 | 		p += log_ptr_size)
343 | 	      reinterpret_cast<log_ptr_t*>(p)->store(nullptr);
344 | 	  }
345 | 	}
346 |       }
347 |     }
348 | 
349 |     template <class Policy>
350 |     void sweep(color free_color)
351 |     {
352 |       using namespace impl_details;
353 | 
354 |       static size_t ticks = 0;
355 | 
356 |       stub_list remaining_free, remaining_used, processed_used;      
357 | 
358 |       for(size_t i = 0; i < impl_details::small_size_classes; ++i)
359 |       {
360 |       	remaining_used = small_used_lists[i].exchange(nullptr, std::memory_order_relaxed);
361 | 
362 |       	while(remaining_used)
363 |       	{
364 |       	  stub* st = remaining_used.front();
365 |       	  remaining_used.pop_front();
366 | 
367 |       	  while(remaining_used)
368 |       	  {
369 |       	    stub* new_st = remaining_used.front();
370 |       	    void* offset = reinterpret_cast<void*>(reinterpret_cast<std::ptrdiff_t>(st->start) + st->size);
371 | 
372 |       	    if(offset == new_st->start) {
373 |       	      st->size += new_st->size;
374 |       	      remaining_used.pop_front();
375 |       	      delete new_st;
376 |       	    } else {
377 |       	      break;
378 |       	    }
379 |       	  }
380 | 
381 |       	  for(auto p = reinterpret_cast<std::uint64_t>(st->start);
382 |       	      p < reinterpret_cast<std::uint64_t>(st->start) + st->size;
383 |       	      p += (1ULL << (i+3)))
384 |       	  {
385 |       	    underlying_header_t h = header(reinterpret_cast<void*>(p + log_ptr_size));
386 |       	    bool free_status = color(h & header_color_mask) == free_color;
387 | 
388 | 	    if(ticks % tick_frequency == 0 && !running.load(std::memory_order_relaxed)) {
389 | 	      processed_used.push_front(new stub(reinterpret_cast<void*>(p),
390 | 						 reinterpret_cast<std::uint64_t>(st->start) + st->size - p));
391 | 
392 | 	      processed_used.atomic_vacate_and_append(small_used_lists[i]);
393 | 	      remaining_used.atomic_vacate_and_append(small_used_lists[i]);
394 | 
395 | 	      return;
396 | 	    }
397 | 
398 |       	    if(free_status) {
399 |       	      Policy::destroy(h, reinterpret_cast<header_t*>(p + log_ptr_size));
400 | 
401 |       	      reinterpret_cast<log_ptr_t*>(p)->~log_ptr_t();
402 |       	      reinterpret_cast<header_t*>(p + log_ptr_size)->~header_t();
403 |       	    }
404 | 
405 |       	    size_t coalesced = 1ULL << (i+3);
406 | 
407 |       	    if(free_status) {
408 |       	      for(p += (1ULL << (i+3));
409 |       		  p < reinterpret_cast<std::uint64_t>(st->start) + st->size;
410 |       		  p += (1ULL << (i+3)))
411 |       	      {
412 |       		if(++ticks % tick_frequency == 0) {
413 | 		  small_free_lists[i].push_front(remaining_free);
414 | 		  remaining_free.reset();
415 |       		}
416 | 
417 |       		h = header(reinterpret_cast<void*>(p + log_ptr_size));
418 | 
419 |       		if(color(h & header_color_mask) == free_color)
420 |       		{
421 |       		  Policy::destroy(h, reinterpret_cast<header_t*>(p + log_ptr_size));
422 | 
423 |       		  reinterpret_cast<log_ptr_t*>(p)->~log_ptr_t();
424 |       		  reinterpret_cast<header_t*>(p + log_ptr_size)->~header_t();
425 | 
426 |       		  coalesced += 1ULL << (i+3);
427 |       		} else {
428 |       		  break;
429 |       		}
430 |       	      }
431 |       	    } else {
432 |       	      for(p += (1ULL << (i+3));
433 |       		  p < reinterpret_cast<std::uint64_t>(st->start) + st->size;
434 |       		  p += (1ULL << (i+3)))
435 |       	      {
436 |       		if(++ticks % tick_frequency == 0) {
437 | 		  small_free_lists[i].push_front(remaining_free);
438 | 		  remaining_free.reset();
439 |       		}
440 | 
441 |       		h = header(reinterpret_cast<void*>(p + log_ptr_size));
442 | 
443 |       		if(color(h & header_color_mask) != free_color) {
444 |       		  coalesced += 1ULL << (i+3);
445 |       		} else {
446 |       		  break;
447 |       		}
448 |       	      }
449 |       	    }
450 | 
451 |       	    if(coalesced < st->size)
452 |       	    {
453 |       	      stub* new_stub = new stub(reinterpret_cast<void*>(p), st->size - coalesced);
454 |       	      st->size = coalesced;
455 | 
456 |       	      free_status ? remaining_free.push_front(st) : processed_used.push_front(st);
457 |       	      remaining_used.push_front(new_stub);
458 |       	      break;
459 |       	    } else {
460 |       	      assert(p == reinterpret_cast<underlying_header_t>(st->start) + st->size - (1ULL << (i+3)));
461 |       	      free_status ? remaining_free.push_front(st) : processed_used.push_front(st);
462 |       	    }
463 |       	  }
464 |       	}
465 | 
466 |       	processed_used.atomic_vacate_and_append(small_used_lists[i]);
467 | 
468 | 	small_free_lists[i].push_front(remaining_free);
469 | 	remaining_free.reset();
470 |       }
471 | 
472 |       large_block_list remaining_large_used =
473 | 	large_used_list.exchange(nullptr, std::memory_order_relaxed);
474 |       large_block_list processed_large_used;
475 | 
476 |       while(remaining_large_used)
477 |       {
478 | 	using namespace impl_details;
479 | 
480 | 	void* fr = remaining_large_used.front();
481 | 	remaining_large_used.pop_front();
482 | 
483 | 	block_cursor blk_c(fr);
484 | 	blk_c.recalculate();
485 | 
486 | 	underlying_header_t h = blk_c.header()->load(std::memory_order_relaxed);
487 | 	bool free_status = color(h & header_color_mask) == free_color;
488 | 
489 | 	if(ticks % tick_frequency == 0 && !running.load(std::memory_order_relaxed)) {
490 | 	  remaining_large_used.push_front(blk_c);
491 | 
492 | 	  remaining_large_used.atomic_vacate_and_append(large_used_list);
493 | 	  processed_large_used.atomic_vacate_and_append(large_used_list);
494 | 
495 | 	  return;
496 | 	}
497 | 
498 | 	if(free_status)
499 | 	{
500 | 	  Policy::destroy(h, blk_c.header());
501 | 	  free(reinterpret_cast<void*>(blk_c.start()));
502 | 	} else {
503 | 	  processed_large_used.push_front(blk_c);
504 | 	}
505 |       }
506 |       
507 |       processed_large_used.atomic_vacate_and_append(large_used_list);
508 |     }
509 | 
510 |     template <class Policy, class Tracer>
511 |     inline void run()
512 |     {
513 |       running.store(true, std::memory_order_relaxed);
514 | 
515 |       while(running.load(std::memory_order_relaxed))
516 |       {
517 | 	assert(shook.load(std::memory_order_relaxed) <= active.load(std::memory_order_relaxed));
518 | 
519 | 	if(try_advance())
520 | 	{
521 | 	  phase::phase_t p = gc_phase.load(std::memory_order_relaxed);
522 | 
523 | 	  switch(p)
524 | 	  {
525 | 	  case phase::phase_t::Tracing:
526 | 	    {
527 | 	      list<void*> r = root_set.exchange(nullptr, std::memory_order_relaxed);
528 | 
529 | 	      marker<Tracer> m(std::move(r), running);
530 | 	      m.mark(alloc_color.load(std::memory_order_relaxed));
531 | 
532 | 	      break;
533 | 	    }
534 | 	  case phase::phase_t::Sweep:
535 | 	    sweep<Policy>(alloc_color.load(std::memory_order_relaxed).flip());
536 | 	    clear_buffers<Tracer>();
537 | 	    break;
538 | 
539 | 	  default:
540 | 	    break;
541 | 	  }
542 | 	}
543 |       }
544 | 
545 |       dump_thread_local_allocations();            
546 |     }
547 |   };
548 | }
549 | #endif
550 | 


--------------------------------------------------------------------------------
/include/impl_details.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef IMPL_DETAILS_HPP_INCLUDED
 2 | #define IMPL_DETAILS_HPP_INCLUDED
 3 | 
 4 | #include <atomic>
 5 | 
 6 | namespace otf_gc
 7 | {
 8 |   namespace impl_details
 9 |   {
10 |     using underlying_header_t = uint64_t;
11 |     using header_t = std::atomic<underlying_header_t>;    
12 | 
13 |     using underlying_log_ptr_t = void*;
14 |     using log_ptr_t = std::atomic<underlying_log_ptr_t>;
15 |     
16 |     static constexpr underlying_header_t zeroed_header = 0;
17 |     static constexpr underlying_log_ptr_t zeroed_log_ptr = 0;
18 | 
19 |     static constexpr uint64_t color_bits = 2;
20 |     static constexpr uint64_t tag_bits = 8;
21 | 
22 |     static constexpr uint64_t header_tag_mask = ((1 << tag_bits) - 1) << color_bits;
23 |     static constexpr uint64_t header_color_mask = 0x3;
24 |     static constexpr std::size_t header_size = sizeof(header_t);
25 |     static constexpr std::size_t log_ptr_size = sizeof(log_ptr_t);
26 |     static constexpr std::size_t log_ptr_offset = 2*sizeof(std::size_t) + 2*sizeof(void*);
27 |     static constexpr std::size_t search_depth = 32;
28 |     static constexpr std::size_t segment_size = 64; // size of a segment in bytes.
29 |     static constexpr uint64_t small_block_metadata_size = header_size + log_ptr_size;
30 |     static constexpr uint64_t small_block_size_limit    = 6;
31 |     static constexpr uint64_t split_bits                = 32;
32 |     static constexpr uint64_t split_mask                = (1ULL << split_bits) - 1;
33 |     static constexpr uint64_t split_switch_bits         = 32;
34 |     static constexpr uint64_t split_switch_mask         = ((1ULL << split_switch_bits) - 1) << split_bits;    
35 |     static constexpr uint64_t large_block_metadata_size = 2*sizeof(void*) + header_size + sizeof(std::size_t);
36 |     static constexpr uint64_t large_obj_min_bits  = 10;
37 |     static constexpr uint64_t large_obj_threshold = 1 << (large_obj_min_bits - 1);
38 |     static constexpr std::size_t mark_tick_frequency = 64;
39 |     static constexpr std::size_t pool_chunk_size = 64;
40 |     static constexpr std::size_t small_size_classes = 7;
41 |     static constexpr std::size_t tick_frequency = 32;
42 |   }
43 | }
44 | 
45 | #endif
46 | 


--------------------------------------------------------------------------------
/include/large_block_list.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef LARGE_BLOCK_LIST_HPP_INCLUDED
  2 | #define LARGE_BLOCK_LIST_HPP_INCLUDED
  3 | 
  4 | #include <atomic>
  5 | #include <cassert>
  6 | #include <new>
  7 | 
  8 | #include "impl_details.hpp"
  9 | 
 10 | namespace otf_gc
 11 | {
 12 |   using namespace impl_details;
 13 | 
 14 |   struct block_cursor
 15 |   {
 16 |     std::ptrdiff_t prev_d, next_d, log_ptr_d, header_d;
 17 | 
 18 |     block_cursor(void* blk, std::size_t num_log_ptrs = 0)
 19 |       : prev_d(reinterpret_cast<std::ptrdiff_t>(blk))
 20 |       , next_d(prev_d + sizeof(void*))
 21 |       , log_ptr_d(next_d + sizeof(void*))
 22 |       , header_d(log_ptr_d + sizeof(std::size_t) + log_ptr_size * num_log_ptrs)
 23 |     {}
 24 | 
 25 |     block_cursor(std::ptrdiff_t blk, std::size_t num_log_ptrs = 0)
 26 |       : prev_d(blk)
 27 |       , next_d(prev_d + sizeof(void*))
 28 |       , log_ptr_d(next_d + sizeof(void*))
 29 |       , header_d(log_ptr_d + sizeof(std::size_t) + log_ptr_size * num_log_ptrs)
 30 |     {}
 31 | 
 32 |     inline std::ptrdiff_t start() const
 33 |     {
 34 |       return prev_d;
 35 |     }
 36 | 
 37 |     inline void** prev()
 38 |     {
 39 |       return reinterpret_cast<void**>(prev_d);
 40 |     }
 41 | 
 42 |     inline void** next()
 43 |     {
 44 |       return reinterpret_cast<void**>(next_d);
 45 |     }
 46 | 
 47 |     inline std::ptrdiff_t data()
 48 |     {
 49 |       return start() + large_block_metadata_size + log_ptr_size * num_log_ptrs();
 50 |     }
 51 | 
 52 |     inline header_t* header()
 53 |     {
 54 |       return reinterpret_cast<header_t*>(header_d);
 55 |     }
 56 | 
 57 |     inline std::size_t& num_log_ptrs()
 58 |     {
 59 |       return *reinterpret_cast<std::size_t*>(log_ptr_d);
 60 |     }
 61 | 
 62 |     inline void recalculate()
 63 |     {
 64 |       header_d = num_log_ptrs() * log_ptr_size + log_ptr_d + sizeof(std::size_t);
 65 |     }
 66 | 
 67 |     inline log_ptr_t* log_ptr(std::size_t i)
 68 |     {
 69 |       return reinterpret_cast<log_ptr_t*>(log_ptr_d + sizeof(std::size_t) + i * log_ptr_size);
 70 |     }
 71 | 
 72 |     inline bool null_block() const
 73 |     {
 74 |       return prev_d == 0;
 75 |     }
 76 | 
 77 |     inline void write(void* prev_, void* next_)
 78 |     {
 79 |       *next() = next_;
 80 |       *prev() = prev_;
 81 |     }
 82 | 
 83 |     block_cursor& operator=(std::ptrdiff_t) = delete;
 84 | 
 85 |     inline block_cursor& operator=(void* blk)
 86 |     {
 87 |       prev_d = reinterpret_cast<std::ptrdiff_t>(blk);
 88 |       next_d = prev_d + sizeof(void*);
 89 |       log_ptr_d = next_d + sizeof(void*);
 90 | 
 91 |       if(blk)
 92 | 	recalculate();
 93 | 
 94 |       return *this;
 95 |     }
 96 | 
 97 |     inline void unlink(void*& head, void*& tail)
 98 |     {
 99 |       block_cursor prev_c(*prev()), next_c(*next());
100 | 
101 |       if(tail == reinterpret_cast<void*>(start()))
102 | 	tail = *prev();
103 | 
104 |       if(*prev()) {
105 | 	*prev_c.next() = reinterpret_cast<void*>(next_c.start());
106 | 	*prev() = nullptr;
107 |       }
108 | 
109 |       if(head == reinterpret_cast<void*>(start()))
110 | 	head = *next();
111 | 
112 |       if(*next()) {
113 | 	*next_c.prev() = reinterpret_cast<void*>(prev_c.start());
114 | 	*next() = nullptr;
115 |       }
116 |     }
117 |   };
118 | 
119 |   class large_block_list
120 |   {
121 |   private:
122 |     void* head;
123 |     void* tail;
124 | 
125 |   public:
126 |     large_block_list(void* head_ = nullptr, void* tail_ = nullptr) noexcept
127 |       : head(head_)
128 |       , tail(tail_)
129 |     {}
130 | 
131 |     inline operator bool() const {
132 |       return !empty();
133 |     }
134 | 
135 |     inline bool empty() const
136 |     {
137 |       return head == nullptr;
138 |     }
139 | 
140 |     inline void*& front()
141 |     {
142 |       return head;
143 |     }
144 | 
145 |     inline void reset()
146 |     {
147 |       head = tail = nullptr;
148 |     }
149 |     
150 |     inline void pop_front()
151 |     {
152 |       if(head) {
153 | 	block_cursor blk_c(head);
154 | 	head = *blk_c.next();
155 | 
156 | 	*blk_c.next() = nullptr;
157 | 	blk_c = head;
158 | 
159 | 	if(head)
160 | 	  *blk_c.prev() = nullptr;
161 | 	else
162 | 	  tail = nullptr;
163 |       }
164 |     }
165 | 
166 |     inline void*& back()
167 |     {
168 |       return tail;
169 |     }
170 | 
171 |     inline void pop_back()
172 |     {
173 |       if(tail)
174 |       {
175 | 	block_cursor blk_c(tail);
176 | 	tail = *blk_c.prev();
177 | 
178 | 	*blk_c.prev() = nullptr;
179 | 	blk_c = tail;
180 | 
181 | 	if(tail)
182 | 	  *blk_c.next() = nullptr;
183 | 	else
184 | 	  head = nullptr;
185 |       }
186 |     }
187 | 
188 |     inline void push_front(block_cursor blk)
189 |     {
190 |       push_front(reinterpret_cast<void*>(blk.start()));
191 |     }
192 | 
193 |     inline void push_back(void* blk)
194 |     {
195 |       assert(blk != nullptr);
196 | 
197 |       block_cursor blk_c(blk), tail_c(tail);
198 |       blk_c.write(tail, nullptr);
199 | 
200 |       if(tail)
201 | 	*tail_c.next() = blk;
202 |       else {
203 | 	assert(!head);
204 | 	head = blk;
205 |       }
206 | 
207 |       tail = blk;
208 |     }
209 | 
210 |     inline void push_front(void* blk)
211 |     {
212 |       assert(blk != nullptr);
213 | 
214 |       block_cursor blk_c(blk), head_c(head);
215 |       blk_c.write(nullptr, head);
216 | 
217 |       if(head)
218 | 	*head_c.prev() = blk;
219 |       else {
220 | 	assert(!tail);
221 | 	tail = blk;
222 |       }
223 | 
224 |       head = blk;
225 |     }
226 | 
227 |     inline void append(large_block_list&& blk)
228 |     {
229 |       if(head) {
230 | 	block_cursor blk_c(blk.head);
231 | 	block_cursor tail_c(tail);
232 | 
233 | 	if(blk.head) {
234 | 	  *tail_c.next() = blk.head;
235 | 	  *blk_c.prev()  = tail;
236 | 	}
237 |       } else {
238 | 	head = blk.head;
239 |       }
240 | 
241 |       tail = blk.tail;
242 |       blk.head = blk.tail = nullptr;
243 |     }
244 | 
245 |     inline void atomic_vacate_and_append(std::atomic<large_block_list>& lbl)
246 |     {
247 |       if(head == nullptr)
248 | 	return;
249 | 
250 |       large_block_list copy_lbl(head, tail);
251 |       head = tail = nullptr;
252 |       large_block_list atomic_lbl = lbl.exchange(nullptr, std::memory_order_relaxed);
253 | 
254 |       while(true)
255 |       {
256 | 	if(!atomic_lbl.empty())
257 | 	  copy_lbl.append(std::move(atomic_lbl));
258 | 
259 | 	copy_lbl = lbl.exchange(copy_lbl, std::memory_order_relaxed);
260 | 
261 | 	if(!copy_lbl.empty()) {
262 | 	  atomic_lbl = lbl.exchange(nullptr, std::memory_order_relaxed);
263 | 	} else {
264 | 	  break;
265 | 	}
266 |       }
267 |     }
268 |   };
269 | }
270 | #endif
271 | 


--------------------------------------------------------------------------------
/include/marker.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef MARKER_HPP_INCLUDED
  2 | #define MARKER_HPP_INCLUDED
  3 | 
  4 | #include <atomic>
  5 | #include <cstring>
  6 | 
  7 | #include "atomic_list.hpp"
  8 | #include "impl_details.hpp"
  9 | #include "large_block_list.hpp"
 10 | 
 11 | namespace otf_gc
 12 | {
 13 |   template <class Tracer>
 14 |   class marker
 15 |   {
 16 |   private:
 17 |     list<void*> roots;
 18 |     std::atomic<bool>& running;
 19 |     
 20 |     inline uint64_t set_color(impl_details::underlying_header_t header, color c)
 21 |     {
 22 |       using namespace impl_details;
 23 |       
 24 |       return ((header >> color_bits) << color_bits) | static_cast<underlying_header_t>(c.c);
 25 |     }
 26 | 
 27 |     inline typename impl_details::header_t& header(void* root)
 28 |     {
 29 |       std::ptrdiff_t pr = reinterpret_cast<std::ptrdiff_t>(root) - sizeof(impl_details::header_size);
 30 |       return *reinterpret_cast<impl_details::header_t*>(pr);
 31 |     }
 32 | 
 33 |     inline void clear_copy(void* buf)
 34 |     {
 35 |       if(!buf) return;
 36 |       
 37 |       header_t* hp = reinterpret_cast<header_t*>(reinterpret_cast<std::ptrdiff_t>(buf) - header_size);
 38 |       hp->~header_t();
 39 |       
 40 |       free(reinterpret_cast<void*>(hp));      
 41 |     }
 42 |     
 43 |     inline void mark_indiv(void* root, const color& c)
 44 |     {
 45 |       using namespace impl_details;
 46 |       
 47 |       header_t& header_w = header(root);
 48 |       underlying_header_t header_c = header_w.load(std::memory_order_relaxed);
 49 | 
 50 |       if(color(header_c & header_color_mask) != c)
 51 |       {
 52 | 	size_t num_log_ptrs = Tracer::num_log_ptrs(header_c);
 53 | 
 54 | 	auto rp = reinterpret_cast<std::size_t>(root);
 55 | 
 56 | 	if(num_log_ptrs == 0) {
 57 | 	  void* buf = Tracer::copy_obj(header_c, root);
 58 | 
 59 | 	  if(buf)
 60 | 	    roots.append(Tracer::get_derived_ptrs(header_c, buf));
 61 | 
 62 | 	  clear_copy(buf);
 63 | 	} else {		  	
 64 | 	  std::size_t rp_start = rp - header_size - num_log_ptrs * log_ptr_size;	
 65 | 	
 66 | 	  for(std::size_t p = rp_start; p < rp - header_size; p += log_ptr_size)
 67 | 	  {
 68 | 	    bool dirtied = false;
 69 | 	    log_ptr_t* lp = reinterpret_cast<log_ptr_t*>(p);	  
 70 | 	  
 71 | 	    if(lp->load() == nullptr) {
 72 | 	      std::size_t obj_seg = (p - rp_start) / log_ptr_size;	    
 73 | 	      void* buf = Tracer::copy_obj_segment(header_c, root, obj_seg);
 74 | 	      
 75 | 	      if(lp->load() == nullptr && buf)
 76 | 		roots.append(Tracer::derived_ptrs_of_obj_segment(header_c, buf, obj_seg));
 77 | 	      else
 78 | 		dirtied = true;	    
 79 | 
 80 | 	      clear_copy(buf);
 81 | 	    } else {
 82 | 	      dirtied = true;
 83 | 	    }
 84 | 	  
 85 | 	    if(dirtied) {
 86 | 	      auto lpp = lp->load();
 87 | 	    
 88 | 	      if(lpp) {
 89 | 		list<void*> buffer_list(reinterpret_cast<typename list<void*>::node_type*>(lpp));	      	      
 90 | 		auto it = buffer_list.begin();
 91 | 		
 92 | 		if(it != buffer_list.end() && *it)
 93 | 		{
 94 | 		  assert((reinterpret_cast<std::ptrdiff_t>(*it) & 1ULL) != 0ULL);
 95 | 		    
 96 | 		  ++it;
 97 | 		
 98 | 		  for(; it != buffer_list.end(); ++it) {
 99 | 		    if(!(*it)) continue;
100 | 		    
101 | 		    if((reinterpret_cast<std::ptrdiff_t>(*it) & 1ULL) != 0ULL)
102 | 		      break;
103 | 		    else 
104 | 		      roots.push_front(*it);
105 | 		  }
106 | 		}
107 | 	      }
108 | 	    }
109 | 	  }
110 | 	}
111 | 
112 | 	header_w.store(set_color(header_c, c), std::memory_order_relaxed);
113 |       } else {
114 | 	assert(color(header_c & header_color_mask) == c);
115 |       }            
116 |     }  
117 |   public:
118 |     marker(list<void*>&& roots_, std::atomic<bool>& running_)
119 |       : roots(std::move(roots_)), running(running_)
120 |     {}
121 |     
122 |     inline void mark(const color& ep)
123 |     {
124 |       size_t ticks = 0;
125 |       
126 |       while(!roots.empty()) {	  	  
127 | 	void* root = roots.front();
128 | 	roots.pop_front();
129 | 	
130 | 	if(root) mark_indiv(root, ep);
131 | 	if(++ticks % impl_details::mark_tick_frequency == 0 && !running.load(std::memory_order_relaxed))
132 | 	  break;
133 |       }      
134 |     }
135 |   };
136 | }
137 | #endif
138 | 


--------------------------------------------------------------------------------
/include/mutator.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef MUTATOR_HPP_INCLUDED
 2 | #define MUTATOR_HPP_INCLUDED
 3 | 
 4 | #include <array>
 5 | 
 6 | #include "atomic_list.hpp"
 7 | #include "color.hpp"
 8 | #include "fixed_list_manager.hpp"
 9 | #include "large_block_list.hpp"
10 | 
11 | namespace otf_gc
12 | {
13 |   class mutator
14 |   {
15 |   protected:
16 |     std::array<fixed_list_manager, impl_details::small_size_classes> fixed_managers;
17 |     large_block_list large_used_list;
18 |     list<void*> allocation_dump;
19 |     color alloc_color;
20 | 
21 |     inline impl_details::underlying_header_t create_header(impl_details::underlying_header_t);    
22 |     inline bool transfer_small_blocks_from_collector(size_t);
23 |     
24 |     void* allocate_small(size_t, impl_details::underlying_header_t);
25 |     void* allocate_large(size_t, impl_details::underlying_header_t, size_t);
26 | 
27 |     mutator(color c)
28 |       : fixed_managers{{fixed_list_manager(3)
29 | 	  , fixed_list_manager(4)
30 | 	  , fixed_list_manager(5)
31 | 	  , fixed_list_manager(6)
32 | 	  , fixed_list_manager(7)
33 | 	  , fixed_list_manager(8)
34 | 	  , fixed_list_manager(9)}}
35 |       , alloc_color(c)
36 |     {}
37 |   public:
38 |     virtual ~mutator() {}
39 | 
40 |     inline static size_t binary_log(int);
41 | 
42 |     void* allocate(int, impl_details::underlying_header_t, size_t);
43 | 
44 |     stub_list vacate_small_used_list(size_t);
45 |     large_block_list vacate_large_used_list();
46 |   };
47 | }
48 | #endif
49 | 


--------------------------------------------------------------------------------
/include/node_pool.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef NODE_POOL_HPP_INCLUDED
 2 | #define NODE_POOL_HPP_INCLUDED
 3 | 
 4 | #include <cstdlib>
 5 | 
 6 | #include "atomic_list.hpp"
 7 | #include "impl_details.hpp"
 8 | 
 9 | namespace otf_gc
10 | {
11 |   template <class List>
12 |   class node_pool
13 |   {
14 |   private:
15 |     void* chunk;
16 |     std::size_t offset;
17 | 
18 |     list<void*> allocation_dump;
19 |     
20 |     List pool;
21 |   public:
22 |     node_pool()
23 |       : chunk(nullptr)
24 |       , offset(0)
25 |     {}
26 |     
27 |     void* get()
28 |     {
29 |       static constexpr std::size_t sz =
30 | 	impl_details::pool_chunk_size * sizeof(typename List::node_type) + sizeof(typename list<void*>::node_type);
31 |       
32 |       if(pool.empty())
33 |       {		
34 | 	if(!chunk || offset == sz)	                       
35 | 	{
36 | 	  chunk = aligned_alloc(alignof(typename list<void*>::node_type), sz);
37 | 	  offset = sizeof(typename list<void*>::node_type);
38 | 
39 | 	  allocation_dump.node_push_front(reinterpret_cast<typename list<void*>::node_type*>(chunk));
40 | 	}
41 | 
42 | 	std::size_t node = reinterpret_cast<std::size_t>(chunk) + offset;
43 | 	offset += sizeof(typename List::node_type);
44 | 	return reinterpret_cast<void*>(node);
45 |       } else {
46 | 	typename List::node_type* node = pool.node_pop_front();
47 | 	return reinterpret_cast<void*>(node);
48 |       }
49 |     }
50 | 
51 |     inline void put(void* node)
52 |     {
53 |       pool.push_front(reinterpret_cast<typename List::node_type*>(node));
54 |     }
55 | 
56 |     inline list<void*> reset_allocation_dump()
57 |     {
58 |       list<void*> result(allocation_dump);
59 |       allocation_dump.reset();
60 |       
61 |       return result;
62 |     }
63 |   };
64 | }
65 | #endif
66 | 


--------------------------------------------------------------------------------
/include/phase.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef PHASE_HPP_INCLUDED
 2 | #define PHASE_HPP_INCLUDED
 3 | 
 4 | namespace otf_gc
 5 | {
 6 |   struct phase
 7 |   {
 8 |     enum class phase_t : int8_t
 9 |     {
10 |       First_h  = 0x00,
11 |       Second_h,
12 |       Third_h,
13 |       Tracing,
14 |       Fourth_h,
15 |       Sweep      
16 |     };
17 | 
18 |     phase_t p;
19 | 
20 |     phase() noexcept : p(phase_t::First_h) {}
21 | 
22 |     phase(phase_t p_) noexcept : p(p_) {}
23 |   
24 |     inline phase_t& advance() {
25 |       return p = phase_t((static_cast<int8_t>(p) + 1) % 6);
26 |     }
27 | 
28 |     inline phase_t prev() const {
29 |       return phase_t((static_cast<int8_t>(p) - 1) % 6);
30 |     }
31 |     
32 |     inline bool snooping() {
33 |       return static_cast<int8_t>(p) <= 1;      
34 |     }
35 |       
36 |     inline bool tracing() {      
37 |       auto val = static_cast<int8_t>(p);      
38 |       return (val >= 1) && (val <= 3);
39 |     }
40 | 
41 |     operator phase_t() {
42 |       return p;
43 |     }
44 |   
45 |     inline bool operator==(const phase& ph) const {
46 |       return p == ph.p;
47 |     }
48 |   };
49 | }
50 | #endif
51 | 


--------------------------------------------------------------------------------
/include/stub_list.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef STUB_LIST_HPP_INCLUDED
  2 | #define STUB_LIST_HPP_INCLUDED
  3 | 
  4 | #include <atomic>
  5 | #include <cassert>
  6 | 
  7 | #include "node_pool.hpp"
  8 | 
  9 | namespace otf_gc
 10 | {
 11 |   struct stub {
 12 |     void* start;
 13 |     size_t size;
 14 | 
 15 |     stub* next;
 16 |     stub* prev;
 17 | 
 18 |     stub(void* start_, size_t size_)
 19 |       : start(start_), size(size_), next(nullptr), prev(nullptr)
 20 |     {}
 21 | 
 22 |     inline static void* operator new(size_t);
 23 |     inline static void operator delete(void*, size_t);
 24 |   };
 25 | 
 26 |   class stub_list
 27 |   {
 28 |   private:
 29 |     stub* head;
 30 |     stub* tail;
 31 |   public:
 32 |     using node_type = stub;
 33 | 
 34 |     stub_list(stub* head_ = nullptr, stub* tail_ = nullptr) noexcept
 35 |       : head(head_)
 36 |       , tail(tail_)
 37 |     {}
 38 | 
 39 |     inline void append(stub_list&& sl)
 40 |     {
 41 |       if(tail) {
 42 | 	tail->next = sl.head;
 43 | 
 44 | 	if(sl.head)
 45 | 	  sl.head->prev = tail;
 46 |       } else {
 47 | 	head = sl.head;
 48 |       }
 49 | 
 50 |       tail = sl.tail;
 51 | 
 52 |       sl.head = sl.tail = nullptr;
 53 |     }
 54 | 
 55 |     inline void reset()
 56 |     {
 57 |       head = tail = nullptr;
 58 |     }
 59 | 
 60 |     inline operator bool() const {
 61 |       return !empty();
 62 |     }
 63 |     
 64 |     inline void atomic_vacate_and_append(std::atomic<stub_list>& sl)
 65 |     {
 66 |       if(head == nullptr)
 67 | 	return;
 68 | 
 69 |       stub_list copy_sl(head, tail);
 70 |       head = tail = nullptr;
 71 |       stub_list atomic_sl = sl.exchange(nullptr, std::memory_order_relaxed);
 72 | 
 73 |       while(true)
 74 |       {
 75 | 	if(atomic_sl)
 76 | 	  copy_sl.append(std::move(atomic_sl));
 77 | 
 78 | 	copy_sl = sl.exchange(copy_sl, std::memory_order_relaxed);
 79 | 
 80 | 	if(copy_sl) {
 81 | 	  atomic_sl = sl.exchange(nullptr, std::memory_order_relaxed);
 82 | 	} else {
 83 | 	  break;
 84 | 	}
 85 |       }
 86 |     }
 87 | 
 88 |     inline stub* erase(stub* st)
 89 |     {
 90 |       assert(st != nullptr);
 91 |       if(st->prev)
 92 | 	st->prev->next = st->next;
 93 |       if(st->next)
 94 | 	st->next->prev = st->prev;
 95 |       st->next = st->prev = nullptr;
 96 |       return st;
 97 |     }
 98 | 
 99 |     inline void push_front(stub* st)
100 |     {
101 |       assert(st != nullptr);
102 |       st->next = head;
103 |       if(head)
104 | 	head->prev = st;
105 |       else
106 | 	tail = st;
107 | 
108 |       head = st;
109 |       st->prev = nullptr;
110 |     }
111 |     
112 |     inline void push_back(stub* st)
113 |     {
114 |       assert(st != nullptr);
115 |       st->prev = tail;
116 |       if(tail)
117 | 	tail->next = st;
118 |       else
119 | 	head = st;
120 |       tail = st;
121 |       st->next = nullptr;
122 |     }
123 | 
124 |     inline stub* front() {
125 |       return head;
126 |     }
127 | 
128 |     inline stub* front_ptr() {
129 |       return head;
130 |     }
131 |     
132 |     inline stub* back() {
133 |       return tail;
134 |     }    
135 |     
136 |     inline void pop_front()
137 |     {
138 |       assert(head != nullptr);
139 |       assert(head->prev == nullptr);
140 | 
141 |       stub* st = head;
142 | 
143 |       if(head->next)
144 | 	head->next->prev = nullptr;
145 |       else
146 | 	tail = nullptr;
147 | 
148 |       head = head->next;
149 |       st->next = nullptr;      
150 |     }
151 | 
152 |     inline stub* node_pop_front()
153 |     {
154 |       stub* fr = front();
155 |       pop_front();
156 |       return fr;
157 |     }
158 |     
159 |     inline void pop_back()
160 |     {
161 |       assert(tail != nullptr);
162 |       assert(tail->next == nullptr);
163 | 
164 |       stub* st = tail;
165 | 
166 |       if(tail->prev)
167 | 	tail->prev->next = nullptr;
168 |       else
169 | 	head = nullptr;
170 | 
171 |       tail = tail->prev;
172 |       st->prev = nullptr;
173 |     }
174 | 
175 |     inline bool empty() const
176 |     {
177 |       return head == nullptr;
178 |     }
179 |   };
180 | 
181 |   static node_pool<stub_list>& stub_list_pool()
182 |   {
183 |     static thread_local node_pool<stub_list> pool;
184 |     return pool;
185 |   }
186 | 
187 |   void* stub::operator new(size_t)
188 |   {
189 |     return stub_list_pool().get();
190 |   }
191 | 
192 |   void stub::operator delete(void* ptr, size_t)
193 |   {
194 |     stub_list_pool().put(ptr);
195 |   }
196 | }
197 | 
198 | #endif
199 | 


--------------------------------------------------------------------------------
/include/write_barrier.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef WRITE_BARRIER_HPP_INCLUDED
  2 | #define WRITE_BARRIER_HPP_INCLUDED
  3 | 
  4 | #include <atomic>
  5 | #include <memory>
  6 | 
  7 | #include "color.hpp"
  8 | #include "impl_details.hpp"
  9 | #include "gc.hpp"
 10 | 
 11 | namespace otf_gc
 12 | {
 13 |   template <std::unique_ptr<gc::registered_mutator>&(*Alloc)(), class Tracer, class T>
 14 |   class otf_write_barrier_impl
 15 |   {
 16 |   protected:
 17 |     void prelude(void* parent, T data)
 18 |     {
 19 |       using namespace impl_details;
 20 |       
 21 |       if(parent && Alloc()->tracing()) {
 22 | 	auto hp = reinterpret_cast<std::ptrdiff_t>(parent) - header_size;
 23 | 	auto h  = reinterpret_cast<header_t*>(hp)->load(std::memory_order_relaxed);
 24 | 	
 25 | 	if(color(h & header_color_mask) != Alloc()->mut_color())
 26 | 	{
 27 | 	  size_t seg_num =
 28 | 	    (reinterpret_cast<std::ptrdiff_t>(&data) - reinterpret_cast<std::ptrdiff_t>(parent)) / segment_size;
 29 | 	  
 30 | 	  log_ptr_t* lp = Tracer::log_ptr(h, parent, seg_num);
 31 | 	  assert(lp != nullptr);
 32 | 	  
 33 | 	  if(!lp->load()) {
 34 | 	    list<void*> temp_buf = Tracer::derived_ptrs_of_obj_segment(h, parent, seg_num);
 35 | 	    	    
 36 | 	    if(!temp_buf.empty() && !lp->load()) {
 37 | 		Alloc()->append_front_buffer(std::move(temp_buf));
 38 | 		assert((reinterpret_cast<std::ptrdiff_t>(parent) & 1ULL) == 0ULL);
 39 | 		Alloc()->push_front_buffer(reinterpret_cast<void*>(reinterpret_cast<std::ptrdiff_t>(parent) | 1ULL));
 40 | 
 41 | 		lp->store(Alloc()->buffer_ptr());
 42 | 	    }
 43 | 	  }
 44 | 	}
 45 |       }
 46 |     }
 47 |   };
 48 |   
 49 |   template <std::unique_ptr<gc::registered_mutator>&(*)(), class, class>
 50 |   class otf_write_barrier {};
 51 |   
 52 |   template <std::unique_ptr<gc::registered_mutator>&(*Alloc)(), class Tracer, class T>
 53 |   class otf_write_barrier<Alloc, Tracer, T*> : private otf_write_barrier_impl<Alloc, Tracer, T*>
 54 |   {
 55 |   private:
 56 |     T* data;
 57 | 
 58 |     using otf_write_barrier_impl<Alloc, Tracer, T*>::prelude;    
 59 |   public:
 60 |     template <typename... Ts>
 61 |     otf_write_barrier(Ts&&... items) : data(std::forward<Ts>(items)...)
 62 |     {}
 63 | 
 64 |     otf_write_barrier(T* data_ = nullptr) : data(data_) {}
 65 |     
 66 |     inline operator T*() {
 67 |       return data;
 68 |     }
 69 |     
 70 |     inline operator void*() {
 71 |       return reinterpret_cast<void*>(data);
 72 |     }
 73 | 
 74 |     otf_write_barrier<Alloc, Tracer, T*>& operator=(T*) = delete;
 75 |   
 76 |     inline void write(void* parent, T* data_)
 77 |     {
 78 |       prelude(parent, data);
 79 |       
 80 |       data = data_;
 81 |       if(Alloc()->snooping() && data)
 82 | 	Alloc()->push_front_snooping(data->derived_ptr());
 83 |     }
 84 |   
 85 |     inline T* get() {
 86 |       return data;
 87 |     }
 88 |   
 89 |     inline T* const operator->() const
 90 |     {
 91 |       return data;
 92 |     }
 93 | 
 94 |     inline bool operator==(const otf_write_barrier<Alloc, Tracer, T*>& it) const
 95 |     {
 96 |       return data == it.data;
 97 |     }
 98 | 
 99 |     inline bool operator!=(const otf_write_barrier<Alloc, Tracer, T*>& it) const
100 |     {
101 |       return data != it.data;
102 |     }
103 |   };
104 | 
105 |   template <std::unique_ptr<gc::registered_mutator>&(*Alloc)(), class Tracer, class T>
106 |   class otf_write_barrier<Alloc, Tracer, std::atomic<T*>> : private otf_write_barrier_impl<Alloc, Tracer, std::atomic<T*>&>
107 |   {
108 |   private:
109 |     std::atomic<T*> data;
110 | 
111 |     using otf_write_barrier_impl<Alloc, Tracer, std::atomic<T*>&>::prelude;
112 |   public:
113 |     otf_write_barrier(T* data_) : data(data_) {}
114 | 
115 |     inline bool operator==(T* const t) const
116 |     {
117 |       return data == t;
118 |     }
119 | 
120 |     inline T* load(std::memory_order mem) const
121 |     {
122 |       return data.load(mem);
123 |     }
124 | 
125 |     inline void store(void* parent, T* val, std::memory_order mem)
126 |     {
127 |       prelude(parent, data);
128 | 
129 |       data.store(val, mem);
130 |       if(Alloc()->snooping() && val) 
131 | 	Alloc()->push_front_snooping(val->derived_ptr());      
132 |     }
133 | 
134 |     inline bool compare_exchange_strong(void* parent,
135 | 					T*& expected,
136 | 					T* desired,
137 | 					std::memory_order success,
138 | 					std::memory_order failure)
139 |     {
140 |       prelude(parent, data);
141 |       
142 |       bool result = data.compare_exchange_strong(expected, desired, success, failure);
143 |       
144 |       if(result && desired && Alloc()->snooping())
145 | 	Alloc()->push_front_snooping(desired->derived_ptr());
146 |     
147 |       return result;
148 |     }
149 |   };
150 | }
151 | #endif
152 | 


--------------------------------------------------------------------------------
/mutator.cpp:
--------------------------------------------------------------------------------
  1 | #include <atomic>
  2 | #include <cstdlib>
  3 | #include <memory>
  4 | 
  5 | #include "atomic_list.hpp"
  6 | #include "impl_details.hpp"
  7 | #include "gc.hpp"
  8 | #include "mutator.hpp"
  9 | 
 10 | using namespace std;
 11 | 
 12 | namespace otf_gc
 13 | {
 14 |   inline size_t mutator::binary_log(int sz)
 15 |   {
 16 |     assert(sz >= 8);
 17 | 
 18 |     sz--;
 19 |     sz |= sz >> 1;
 20 |     sz |= sz >> 2;
 21 |     sz |= sz >> 4;
 22 |     sz |= sz >> 8;
 23 |     sz |= sz >> 16;
 24 | 
 25 |     sz -= (sz >> 1) & 0x5555555555555555ULL;
 26 |     sz = (sz & 0x3333333333333333ULL) + ((sz >> 2) & 0x3333333333333333ULL);
 27 |     sz = (sz + (sz >> 4)) & 0x0f0f0f0f0f0f0f0fULL;
 28 | 
 29 |     return (sz * 0x0101010101010101ULL) >> 56;
 30 |   }
 31 | 
 32 |   inline impl_details::underlying_header_t
 33 |   mutator::create_header(impl_details::underlying_header_t desc)
 34 |   {
 35 |     using namespace impl_details;
 36 |     return static_cast<underlying_header_t>(alloc_color.c) | (desc << color_bits);
 37 |   }
 38 | 
 39 |   inline bool mutator::transfer_small_blocks_from_collector(size_t power)
 40 |   {
 41 |     stub_list stubs = gc::collector->small_free_lists[power-3].pop_front();
 42 | 
 43 |     if(stubs) {
 44 |       fixed_managers[power-3].append(std::move(stubs));
 45 |       return true;
 46 |     }
 47 | 
 48 |     return false;
 49 |   }
 50 | 
 51 |   void* mutator::allocate_small(size_t power, impl_details::underlying_header_t desc)
 52 |   {
 53 |     void* ptr = fixed_managers[power-3].get_block();
 54 | 
 55 |     if(!ptr) {
 56 |       if(transfer_small_blocks_from_collector(power))
 57 | 	ptr = fixed_managers[power-3].get_block();
 58 | 
 59 |       if(!ptr) {
 60 | 	void* blk = fixed_managers[power-3].get_new_block();
 61 | 	allocation_dump.push_front(blk);
 62 | 	ptr = blk;
 63 |       }
 64 |     }
 65 | 
 66 |     new(ptr) impl_details::log_ptr_t(nullptr);
 67 |     new(reinterpret_cast<header_t*>(reinterpret_cast<std::ptrdiff_t>(ptr) + log_ptr_size))
 68 |       impl_details::header_t(create_header(desc));
 69 | 
 70 |     return ptr;
 71 |   }
 72 | 
 73 |   void* mutator::allocate_large(size_t sz,
 74 | 				impl_details::underlying_header_t desc,
 75 | 				size_t num_log_ptrs)
 76 |   {
 77 |     void* blk = aligned_alloc(alignof(impl_details::header_t), sz);
 78 |     large_used_list.push_front(blk);
 79 | 
 80 |     block_cursor blk_c(blk);
 81 |     
 82 |     blk_c.num_log_ptrs() = num_log_ptrs;
 83 |     blk_c.recalculate();
 84 | 
 85 |     for(size_t i = 0; i < num_log_ptrs; ++i)
 86 |       new(blk_c.log_ptr(i)) impl_details::log_ptr_t(nullptr);
 87 | 
 88 |     new(blk_c.header()) impl_details::header_t(create_header(desc));
 89 | 
 90 |     return blk;
 91 |   }
 92 | 
 93 |   stub_list mutator::vacate_small_used_list(size_t i)
 94 |   {    
 95 |     return fixed_managers[i].release_used_list();
 96 |   }
 97 | 
 98 |   large_block_list mutator::vacate_large_used_list()
 99 |   {
100 |     auto result = large_used_list;
101 |     large_used_list.reset();
102 |     return result;
103 |   }
104 | 
105 |   void* mutator::allocate(int raw_sz,
106 | 			  impl_details::underlying_header_t desc,
107 | 			  size_t num_log_ptrs)
108 |   {
109 |     using namespace impl_details;
110 | 
111 |     if(raw_sz + small_block_metadata_size <= large_obj_threshold) {
112 |       void* p = allocate_small(mutator::binary_log(raw_sz + small_block_metadata_size), desc);
113 | 
114 |       return reinterpret_cast<void*>(reinterpret_cast<std::ptrdiff_t>(p) + small_block_metadata_size);
115 |     } else {
116 |       size_t preamble_sz = large_block_metadata_size + num_log_ptrs * log_ptr_size;
117 |       void *p = allocate_large(preamble_sz + raw_sz, desc, num_log_ptrs);
118 | 
119 |       return reinterpret_cast<void*>(reinterpret_cast<std::ptrdiff_t>(p) + preamble_sz);
120 |     }
121 |   }
122 | }
123 | 


--------------------------------------------------------------------------------