├── Job.cpp ├── Job.h ├── JobQueue.cpp ├── JobQueue.h ├── JobSystem.cpp ├── JobSystem.h ├── LICENSE ├── README.md ├── Worker.cpp ├── Worker.h ├── docs ├── initial.png ├── pop.png ├── push.png ├── push_2.png └── steal.png └── main.cpp /Job.cpp: -------------------------------------------------------------------------------- 1 | #include "Job.h" 2 | 3 | Job::Job( JobFunction function, Job *parent, void *data ) 4 | : function( function ), parent ( parent ), data ( data ), pendingJobs { 1 } 5 | { 6 | if ( parent != nullptr ) 7 | { 8 | parent->pendingJobs++; 9 | } 10 | } 11 | 12 | bool Job::IsFinished() const 13 | { 14 | return ( pendingJobs == 0 ); 15 | } 16 | 17 | void Job::Execute() 18 | { 19 | if ( function != nullptr ) 20 | { 21 | function( data ); 22 | } 23 | Finish(); 24 | } 25 | 26 | void Job::Finish() 27 | { 28 | pendingJobs--; 29 | 30 | if ( IsFinished() && parent != nullptr ) 31 | { 32 | parent->Finish(); 33 | } 34 | } -------------------------------------------------------------------------------- /Job.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | using JobFunction = void(*)( void *data ); 6 | 7 | class Job 8 | { 9 | public: 10 | Job( JobFunction function, Job *parent, void *data ); 11 | 12 | void Execute(); 13 | void Finish(); 14 | bool IsFinished() const; 15 | 16 | private: 17 | JobFunction function; 18 | Job *parent; 19 | std::atomic_size_t pendingJobs; 20 | void *data; 21 | }; -------------------------------------------------------------------------------- /JobQueue.cpp: -------------------------------------------------------------------------------- 1 | #include "JobQueue.h" 2 | 3 | #include 4 | 5 | JobQueue::JobQueue( size_t maxJobs ) 6 | : maxJobs ( maxJobs ) 7 | { 8 | bottomIndex = 0; 9 | topIndex = 0; 10 | queue.resize( maxJobs ); 11 | } 12 | 13 | void JobQueue::Push( Job *job ) 14 | { 15 | int bottom = bottomIndex.load( std::memory_order_seq_cst ); 16 | queue[ bottom ] = job; 17 | bottomIndex.store( bottom + 1, std::memory_order_seq_cst ); 18 | } 19 | 20 | Job* JobQueue::Pop() 21 | { 22 | int bottom = bottomIndex.load( std::memory_order_seq_cst ); 23 | bottom = std::max ( 0, bottom - 1 ); 24 | bottomIndex.store( bottom, std::memory_order_seq_cst ); 25 | int top = topIndex.load( std::memory_order_seq_cst); 26 | 27 | if ( top <= bottom ) 28 | { 29 | Job *job = queue[ bottom ]; 30 | 31 | // There are several jobs in the queue, we don't need to worry about Steal() 32 | if ( top != bottom ) 33 | { 34 | return job; 35 | } 36 | 37 | // This is the last item in the queue, we need to check if a Steal() has increased top 38 | int stolenTop = top + 1; 39 | if( topIndex.compare_exchange_strong( stolenTop, top + 1, std::memory_order_seq_cst ) ) 40 | { 41 | // An Steal() call has stolen our job (https://www.youtube.com/watch?v=DEiWU1MbBfk) 42 | bottomIndex.store( stolenTop, std::memory_order_release ); 43 | return nullptr; 44 | } 45 | return job; 46 | } 47 | else 48 | { 49 | bottomIndex.store( top, std::memory_order_seq_cst ); 50 | return nullptr; 51 | } 52 | } 53 | 54 | Job* JobQueue::Steal() 55 | { 56 | int top = topIndex.load( std::memory_order_seq_cst ); 57 | int bottom = bottomIndex.load( std::memory_order_seq_cst ); 58 | 59 | if ( topIndex < bottomIndex ) 60 | { 61 | Job *job = queue[ top ]; 62 | 63 | // Check if a Pop() or another Steal() operation has stolen our job 64 | if( topIndex.compare_exchange_strong( top, top + 1, std::memory_order_seq_cst ) ) 65 | { 66 | return job; 67 | } 68 | 69 | return nullptr; 70 | } 71 | else 72 | { 73 | // nothing left to steal 74 | return nullptr; 75 | } 76 | } 77 | 78 | size_t JobQueue::Size() const 79 | { 80 | return bottomIndex - topIndex; 81 | } 82 | 83 | void JobQueue::Clear() 84 | { 85 | bottomIndex = 0; 86 | topIndex = 0; 87 | } -------------------------------------------------------------------------------- /JobQueue.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "Job.h" 7 | 8 | class JobQueue { 9 | public: 10 | JobQueue( size_t maxJobs ); 11 | 12 | void Push( Job *job ); 13 | Job* Pop(); 14 | Job* Steal(); 15 | 16 | size_t Size() const; 17 | void Clear(); 18 | 19 | private: 20 | int maxJobs; 21 | 22 | std::atomic_int bottomIndex; 23 | std::atomic_int topIndex; 24 | std::vector< Job * > queue; 25 | }; 26 | -------------------------------------------------------------------------------- /JobSystem.cpp: -------------------------------------------------------------------------------- 1 | #include "JobSystem.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "JobQueue.h" 7 | #include "Worker.h" 8 | 9 | JobSystem::JobSystem( size_t workersCount, size_t jobsPerWorker) : workersCount( workersCount ) 10 | { 11 | queues.reserve( workersCount ); 12 | workers.reserve( workersCount ); 13 | 14 | // Add a main thread worker 15 | JobQueue *queue = new JobQueue( jobsPerWorker ); 16 | queues.push_back( queue ); 17 | Worker *mainThreadWorker = new Worker( this, queue ); 18 | workers.push_back( mainThreadWorker ); 19 | 20 | // Add the rest of the workers 21 | for ( size_t i = 0; i < workersCount; ++i ) 22 | { 23 | JobQueue *queue = new JobQueue( jobsPerWorker ); 24 | queues.push_back( queue ); 25 | Worker *worker = new Worker( this, queue ); 26 | workers.push_back( worker ); 27 | } 28 | 29 | // Start background thread for all the workers except the main thread one (first on the vector) 30 | for ( size_t i = 1; i <= workersCount; ++i ) 31 | { 32 | workers[i]->StartBackgroundThread(); 33 | } 34 | } 35 | 36 | JobSystem::~JobSystem() 37 | { 38 | for ( Worker *worker : workers) 39 | { 40 | delete worker; 41 | } 42 | workers.clear(); 43 | 44 | for ( JobQueue *queue : queues ) 45 | { 46 | delete queue; 47 | } 48 | queues.clear(); 49 | } 50 | 51 | Job* JobSystem::CreateEmptyJob() const 52 | { 53 | return CreateJob( nullptr, nullptr ); 54 | } 55 | 56 | Job* JobSystem::CreateJob( JobFunction function ) const 57 | { 58 | return CreateJob( function, nullptr ); 59 | } 60 | 61 | Job* JobSystem::CreateJob( JobFunction function, void *data ) const 62 | { 63 | // TODO (jonathan): Change this for a pool of jobs 64 | return new Job( function, nullptr, data ); 65 | } 66 | 67 | Job* JobSystem::CreateJobAsChild( JobFunction function, Job *parent ) const 68 | { 69 | return CreateJobAsChild( function, parent, nullptr ); 70 | } 71 | 72 | Job* JobSystem::CreateJobAsChild( JobFunction function, Job *parent, void *data ) const 73 | { 74 | // TODO (jonathan): Change this for a pool of jobs 75 | return new Job( function, parent, data ); 76 | } 77 | 78 | void JobSystem::Run( Job *job ) 79 | { 80 | Worker * worker = FindWorkerWithThreadID( std::this_thread::get_id() ); 81 | if ( worker != nullptr ) 82 | { 83 | worker->Submit( job ); 84 | } 85 | } 86 | 87 | void JobSystem::Wait( Job *job ) 88 | { 89 | Worker * worker = FindWorkerWithThreadID( std::this_thread::get_id() ); 90 | if ( worker != nullptr ) 91 | { 92 | worker->Wait( job ); 93 | } 94 | } 95 | 96 | JobQueue* JobSystem::GetRandomJobQueue() 97 | { 98 | static std::random_device rd; 99 | static std::mt19937 gen(rd()); 100 | static std::uniform_int_distribution<> distribution( 0, workersCount ); 101 | 102 | size_t index = static_cast ( std::round( distribution( gen ) ) ); 103 | return queues[ index ]; 104 | } 105 | 106 | Worker * JobSystem::FindWorkerWithThreadID( const std::thread::id &id ) const 107 | { 108 | for ( Worker *worker : workers ) 109 | { 110 | if ( id == worker->GetThreadId() ) 111 | { 112 | return worker; 113 | } 114 | } 115 | return nullptr; 116 | } 117 | 118 | void JobSystem::ClearJobQueues() 119 | { 120 | for ( JobQueue *queue : queues ) 121 | { 122 | queue->Clear(); 123 | } 124 | } -------------------------------------------------------------------------------- /JobSystem.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "Job.h" 7 | 8 | class JobQueue; 9 | class Worker; 10 | 11 | class JobSystem 12 | { 13 | public: 14 | JobSystem( size_t workersCount, size_t jobsPerWorker ); 15 | ~JobSystem(); 16 | 17 | Job* CreateEmptyJob() const; 18 | Job* CreateJob( JobFunction function ) const; 19 | Job* CreateJob( JobFunction function, void * data ) const; 20 | Job* CreateJobAsChild( JobFunction function, Job *parent ) const; 21 | Job* CreateJobAsChild( JobFunction function, Job *parent, void *data ) const; 22 | 23 | void Run( Job *job ); 24 | void Wait( Job *job ); 25 | 26 | void ClearJobQueues(); 27 | JobQueue* GetRandomJobQueue(); 28 | 29 | private: 30 | size_t workersCount; 31 | 32 | std::vector< Worker * > workers; 33 | std::vector< JobQueue * > queues; 34 | 35 | Worker * FindWorkerWithThreadID( const std::thread::id &id ) const; 36 | }; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Jonathan Maldonado Contreras 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # TinyJob 3 | TinyJob is a simple job system implemented in C++11. It is based on the execution of several workers in parallel that are fed a job via a lock-free stealing queue. The system also supports hierarchy so we can wait in the main thread until all the jobs that are pending from a root one get executed before continuing. 4 | 5 | ## Workers 6 | The system contains N workers and a job queue per worker. Workers are in charge of executing jobs or wait until a job has been completed. Jobs can only be added to the worker running in the current thread. There will always be a worker on the main thread. 7 | 8 | Workers get the jobs that they must execute via a method called GetJob(). This method will try first to Pop() a job from the workers' own queue and if there is no job left to be executed will try to Steal() a Job from the queue of another worker. By doing this, workers are always running and tasks are distributed via Steal() operations. 9 | 10 | ## Free lock job stealing queue 11 | The key of the whole system is the implementation of a lock-free stealing queue. By lock-free we mean that there is no lock system, like mutexes or spin locks, in place to coordinate different threads. In order to achieve this we must use atomic variables and compare and swap operations. 12 | 13 | The implementation of the JobQueue is based on two atomic integers: top and bottom. Each queue provides three main functionalities: Push(), Pop() and Steal(). The first two operations: Push() and Pop() must always be executed by the same thread and never concurrently, the last one on the other hand must only be executed concurrently. 14 | 15 |

16 | initial state 17 |

18 | 19 | Each worker will Push() elements to its own queue and Pop() elements from its own queue in a LIFO way. Other workers will request a Steal() operation using a FIFO strategy. This means that the queue behaves like a FIFO and a LIFO at the same time depending of the operation that we are calling. 20 | 21 | ### Push 22 | This operation adds a new element into the queue. It only increments the bottom index and doesn't need to perform any CAS operation. 23 | 24 |

25 | push one element 26 |

27 | 28 |

29 | push several elements 30 |

31 | 32 | ### Pop 33 | This operation extracts an element from the queue. It only decrements the bottom index. In case there is only an element left in the queue it must execute a CAS operation with the top index value to check if a Steal() operation has already taken away the last job before we could complete the Pop(). 34 |

35 | pop 36 |

37 | 38 | ### Steal 39 | Other concurrent workers may request via GetJob() a Steal() operation. This operation extracts an element from the queue. It only increments the top index. In case there is only an element left in the queue it must execute a CAS operation with the bottom index value to check if a Pop() operation has already taken away the last job before we could complete the Steal(). 40 |

41 | steal 42 |

43 | 44 | 45 | ### Synchronizing without locks 46 | If we would synchronize our queues with mutexes most of our background workers will be constantly put to sleep and awaken. This operation is costly and in performance sensitive systems we can't afford that. We could also try to use spin locks but they are still more expensive that implementing a queue that is lock free. 47 | 48 | The main point of the queue is that Push() and Pop() only touch the bottom index and Steal() only touches the top index. By implementing the queue like this we only need to execute a CAS operation when there is only one element left in the queue to ensure that Steal() hasn't stolen a job while we were doing a Pop() and viceversa, that a Pop() hasn't happened before we completed a Steal() and we don't need to synchronize any other case. 49 | 50 | It is also crucial that in the Push() and Pop() operations the variable bottom index always gets read before top index and in the Steal() operation the variable top gets read before the bottom one. In order to assure that no compiler optimization may change the order of the read or write sentences we must add compiler barriers. 51 | 52 | ```c++ 53 | void JobQueue::Push( Job *job ) 54 | { 55 | int bottom = bottomIndex.load( std::memory_order_acquire ); 56 | queue[ bottom ] = job; 57 | bottomIndex.store( bottom + 1, std::memory_order_release ); 58 | } 59 | ``` 60 | 61 | Compiler barriers are implemented via the API offered by the atomic library in C++11: http://en.cppreference.com/w/cpp/atomic/atomic 62 | 63 | ## Adding jobs to the system 64 | 65 | Jobs are created and added via the JobSystem class. In order to add a job to the system we must simply call Run(). At that moment the thread worker will submit that job to its queue. 66 | 67 | ```c++ 68 | Job *job = jobSystem.CreateJob( function , data ); 69 | jobSystem.Run( job ); 70 | ``` 71 | 72 | If we want the thread worker to wait until a certain job has finished its execution we must call Wait(). Our parent job only acts as a fence, and it may be executed before its children. 73 | 74 | ```c++ 75 | Job *parent = jobSystem.CreateEmptyJob(); 76 | 77 | for ( int i = 0; i < 10; ++i ) 78 | { 79 | Job *job = jobSystem.CreateJobAsChild( another_function, parent ); 80 | jobSystem.Run( job ); 81 | } 82 | jobSystem.Run( parent ); 83 | jobSystem.Wait( parent ); 84 | ``` 85 | 86 | 87 | ## Use cases 88 | The system is design with the idea that the main thread will create a high level job and wait until it finishes. This high level job will maybe create more jobs to complete its execution and it will add them to the system via the worker in which is being executed. 89 | 90 | You can for example create a high level job for creating a save file that will then create several jobs, one per sub systems, each of them saving the information of their own parts. You can as well create a job that will trigger a request and put that background worker to wait until the request is completed. 91 | 92 | ## Documentation 93 | There are a few great articles regarding job systems and lock-free programming. TinyJob is based on the articles from molecular matters: https://blog.molecular-matters.com that talk about their job system and you may find a nice introduction of lock-free programing in the Preshing on Programming blog: http://preshing.com/20120612/an-introduction-to-lock-free-programming/. 94 | -------------------------------------------------------------------------------- /Worker.cpp: -------------------------------------------------------------------------------- 1 | #include "Worker.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "JobQueue.h" 7 | #include "JobSystem.h" 8 | #include "Job.h" 9 | 10 | Worker::Worker( JobSystem *system, JobQueue *queue ) 11 | : system( system ), queue( queue ), thread( nullptr ), threadId(std::this_thread::get_id()) {} 12 | 13 | Worker::~Worker() 14 | { 15 | Stop(); 16 | if ( thread != nullptr ) 17 | { 18 | thread->join(); 19 | delete thread; 20 | } 21 | } 22 | 23 | void Worker::StartBackgroundThread() 24 | { 25 | state = State::RUNNING; 26 | thread = new std::thread( &Worker::Loop, this ); 27 | threadId = thread->get_id(); 28 | } 29 | 30 | void Worker::Stop() 31 | { 32 | state = State::IDLE; 33 | } 34 | 35 | void Worker::Submit( Job *job ) 36 | { 37 | queue->Push( job ); 38 | } 39 | 40 | void Worker::Wait( Job *sentinel ) 41 | { 42 | while ( !sentinel->IsFinished() ) 43 | { 44 | Job *job = GetJob(); 45 | if ( job != nullptr ) 46 | { 47 | job->Execute(); 48 | if ( job->IsFinished() ) 49 | { 50 | delete job; 51 | } 52 | } 53 | } 54 | } 55 | 56 | void Worker::Loop() 57 | { 58 | while ( IsRunning() ) 59 | { 60 | Job *job = GetJob(); 61 | if ( job != nullptr ) { 62 | job->Execute(); 63 | if ( job->IsFinished() ) 64 | { 65 | delete job; 66 | } 67 | } 68 | } 69 | } 70 | 71 | Job* Worker::GetJob() 72 | { 73 | Job *job = queue->Pop(); 74 | 75 | if ( job == nullptr ) 76 | { 77 | JobQueue *randomQueue = system->GetRandomJobQueue(); 78 | if ( randomQueue == nullptr ) 79 | { 80 | std::this_thread::yield(); 81 | return nullptr; 82 | } 83 | 84 | if ( queue == randomQueue ) 85 | { 86 | std::this_thread::yield(); 87 | return nullptr; 88 | } 89 | 90 | job = randomQueue->Steal(); 91 | if ( job == nullptr ) 92 | { 93 | std::this_thread::yield(); 94 | return nullptr; 95 | } 96 | } 97 | return job; 98 | } 99 | 100 | bool Worker::IsRunning() 101 | { 102 | return ( state == State::RUNNING ); 103 | } 104 | 105 | const std::thread::id& Worker::GetThreadId() const 106 | { 107 | return threadId; 108 | } 109 | -------------------------------------------------------------------------------- /Worker.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | class JobSystem; 6 | class JobQueue; 7 | class Job; 8 | 9 | class Worker 10 | { 11 | public: 12 | enum class State : unsigned int 13 | { 14 | RUNNING = 0, 15 | IDLE 16 | }; 17 | 18 | Worker( JobSystem *, JobQueue * ); 19 | Worker( const Worker & ) = delete; 20 | ~Worker(); 21 | 22 | void StartBackgroundThread(); 23 | void Stop(); 24 | void Submit( Job *job ); 25 | void Wait( Job *sentinel ); 26 | bool IsRunning(); 27 | 28 | const std::thread::id& GetThreadId() const; 29 | 30 | private: 31 | State state; 32 | JobQueue *queue; 33 | JobSystem *system; 34 | std::thread *thread; 35 | std::thread::id threadId; 36 | 37 | Job* GetJob(); 38 | void Loop(); 39 | }; -------------------------------------------------------------------------------- /docs/initial.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jonazan2/TinyJob/29bf967e3ae5c5becae4d2aecdb04ac93dd483fb/docs/initial.png -------------------------------------------------------------------------------- /docs/pop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jonazan2/TinyJob/29bf967e3ae5c5becae4d2aecdb04ac93dd483fb/docs/pop.png -------------------------------------------------------------------------------- /docs/push.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jonazan2/TinyJob/29bf967e3ae5c5becae4d2aecdb04ac93dd483fb/docs/push.png -------------------------------------------------------------------------------- /docs/push_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jonazan2/TinyJob/29bf967e3ae5c5becae4d2aecdb04ac93dd483fb/docs/push_2.png -------------------------------------------------------------------------------- /docs/steal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jonazan2/TinyJob/29bf967e3ae5c5becae4d2aecdb04ac93dd483fb/docs/steal.png -------------------------------------------------------------------------------- /main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include "JobSystem.h" 5 | 6 | void PrintSomethingNice( void *data ) 7 | { 8 | std::cout << (char*)data; 9 | } 10 | 11 | void PrintSomething( void *data ) 12 | { 13 | std::cout << "You killed my father!\n"; 14 | } 15 | 16 | int main(int argc, char **argv) 17 | { 18 | JobSystem jobSystem( 7, 65536 ); 19 | 20 | /* Example of how to use a job as a parent to create a fence */ 21 | std::string data = "No Job, I'm your father\n"; 22 | Job* parent = jobSystem.CreateJob(PrintSomethingNice, (void*) data.c_str() ); 23 | 24 | for ( int i = 0; i < 10000; ++i ) 25 | { 26 | Job *job = jobSystem.CreateJobAsChild( PrintSomething, parent ); 27 | jobSystem.Run( job ); 28 | } 29 | jobSystem.Run( parent ); 30 | jobSystem.Wait( parent ); 31 | } --------------------------------------------------------------------------------