├── .clang-format ├── .github └── workflows │ └── ci.yml ├── CMakeLists.txt ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── cmake └── TargetArch.cmake ├── docs └── SchedulerStrategies.md ├── examples ├── CMakeLists.txt ├── condition-variable++.cpp ├── connect++.cpp ├── echo++.cpp ├── echo-timeout++.cpp ├── echo-timeout.c ├── echo.c ├── fibonacci++.cpp └── key-value++.cpp ├── include └── fev │ ├── config.h.in │ ├── fev++.hpp │ └── fev.h ├── libfev.pc.in ├── src ├── fev_alloc.c ├── fev_alloc.h ├── fev_arch.h ├── fev_arch_x86.h ├── fev_assert.h ├── fev_bounded_mpmc_queue.h ├── fev_bounded_spmc_queue.h ├── fev_compiler.h ├── fev_cond.c ├── fev_cond_impl.h ├── fev_cond_intf.h ├── fev_context.h ├── fev_context_i386.S ├── fev_context_x86_64_posix.S ├── fev_context_x86_family.c ├── fev_fiber.c ├── fev_fiber.h ├── fev_fiber_attr.c ├── fev_fiber_attr.h ├── fev_ilock.c ├── fev_ilock_impl.h ├── fev_ilock_intf.h ├── fev_mutex.c ├── fev_mutex_impl.h ├── fev_mutex_intf.h ├── fev_os.h ├── fev_os_nix.c ├── fev_os_win.c ├── fev_poller.h ├── fev_poller_epoll.c ├── fev_poller_epoll.h ├── fev_poller_io_uring.c ├── fev_poller_io_uring.h ├── fev_poller_kqueue.c ├── fev_poller_kqueue.h ├── fev_poller_reactor.c ├── fev_poller_reactor.h ├── fev_qsbr.h ├── fev_qsbr_queue.h ├── fev_sched_attr.c ├── fev_sched_attr.h ├── fev_sched_common.c ├── fev_sched_impl.h ├── fev_sched_intf.h ├── fev_sched_shr_bounded_mpmc.c ├── fev_sched_shr_bounded_mpmc_impl.h ├── fev_sched_shr_bounded_mpmc_intf.h ├── fev_sched_shr_locking.c ├── fev_sched_shr_locking_impl.h ├── fev_sched_shr_locking_intf.h ├── fev_sched_shr_simple_mpmc.c ├── fev_sched_shr_simple_mpmc_impl.h ├── fev_sched_shr_simple_mpmc_intf.h ├── fev_sched_steal_bounded_mpmc.c ├── fev_sched_steal_bounded_mpmc_impl.h ├── fev_sched_steal_bounded_mpmc_intf.h ├── fev_sched_steal_bounded_spmc.c ├── fev_sched_steal_bounded_spmc_impl.h ├── fev_sched_steal_bounded_spmc_intf.h ├── fev_sched_steal_locking.c ├── fev_sched_steal_locking_impl.h ├── fev_sched_steal_locking_intf.h ├── fev_sem.c ├── fev_sem.h ├── fev_simple_mpmc_pool.h ├── fev_simple_mpmc_queue.h ├── fev_simple_mpmc_stack.h ├── fev_socket.h ├── fev_socket_io_uring.c ├── fev_socket_io_uring.h ├── fev_socket_reactor.c ├── fev_socket_reactor.h ├── fev_spinlock_impl.h ├── fev_spinlock_intf.h ├── fev_stack.h ├── fev_stack_posix.c ├── fev_thr.h ├── fev_thr_mutex.h ├── fev_thr_mutex_linux.c ├── fev_thr_mutex_linux.h ├── fev_thr_mutex_posix.h ├── fev_thr_mutex_win.h ├── fev_thr_posix.h ├── fev_thr_sem.h ├── fev_thr_sem_macos.c ├── fev_thr_sem_macos.h ├── fev_thr_sem_posix.h ├── fev_thr_sem_win.h ├── fev_thr_win.h ├── fev_time.h ├── fev_timers.c ├── fev_timers.h ├── fev_timers_binheap.c ├── fev_timers_binheap.h ├── fev_timers_rbtree.c ├── fev_timers_rbtree.h ├── fev_util.h ├── fev_waiter_impl.h ├── fev_waiter_intf.h ├── fev_waiters_queue_impl.h └── fev_waiters_queue_intf.h ├── tests ├── CMakeLists.txt ├── sleep.c ├── stress_cond.c ├── stress_cond_with_timeout.c ├── stress_ilock.c ├── stress_mpmc_queue.c ├── stress_mutex.c ├── stress_mutex_with_timeout.c ├── stress_qsbr_queue.c ├── stress_sem.c ├── stress_sem_with_timeout.c ├── stress_thr_mutex.c ├── timers_bucket.c └── util.h └── third_party ├── queue.h └── tree.h /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: LLVM 2 | BreakBeforeBraces: Custom 3 | BraceWrapping: 4 | AfterFunction: true 5 | ColumnLimit: 100 6 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | name: Build 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: 12 | - macos-10.15 13 | - ubuntu-20.04 14 | sched: 15 | - work-sharing-locking 16 | - work-sharing-bounded-mpmc 17 | - work-sharing-simple-mpmc 18 | - work-stealing-locking 19 | - work-stealing-bounded-mpmc 20 | - work-stealing-bounded-spmc 21 | timers: 22 | - binheap 23 | - rbtree 24 | steps: 25 | - name: Checkout repository 26 | uses: actions/checkout@v2 27 | - name: Configure 28 | run: | 29 | cmake -B build \ 30 | -DCMAKE_BUILD_TYPE=Release \ 31 | -DFEV_BUILD_EXAMPLES=On \ 32 | -DFEV_BUILD_TESTS=On \ 33 | -DFEV_SCHED=${{ matrix.sched }} \ 34 | -DFEV_TIMERS=${{ matrix.timers }} 35 | - name: Build 36 | run: cmake --build build --config Release --parallel 37 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy 2 | of this software and associated documentation files (the "Software"), to deal 3 | in the Software without restriction, including without limitation the rights 4 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 5 | copies of the Software, and to permit persons to whom the Software is 6 | furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all 9 | copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 17 | SOFTWARE. 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # libfev 2 | 3 | A library for events and fibers (a.k.a. green threads/goroutines/stackful coroutines). 4 | 5 | [![CI](https://github.com/patrykstefanski/libfev/workflows/CI/badge.svg)](https://github.com/patrykstefanski/libfev/actions) 6 | 7 | ## Overview 8 | 9 | libfev is an abstraction over event-driven, non-blocking I/O for writing programs in C and C++ in a 10 | simple blocking style. It provides: 11 | 12 | * Few multithreaded schedulers 13 | * Backends for epoll and kqueue (and experimental io\_uring backend) 14 | * Timers 15 | * Synchronization primitives (mutex, condition variable and semaphore) 16 | 17 | ## Performance 18 | In a throughput benchmark libfev can handle up to 172% more requests per second than 19 | [Boost.Asio](https://www.boost.org/doc/libs/1_74_0/doc/html/boost_asio.html), up to 77% more than 20 | [Tokio](https://tokio.rs/), up to 40% more than [async-std](https://async.rs/) and up to 16% more 21 | than [Go](https://golang.org/). See [async-bench](https://github.com/patrykstefanski/async-bench) 22 | for more data, there is also a comparison of the available schedulers. 23 | 24 | ## Support 25 | 26 | Following platforms are currently supported: 27 | 28 | * x86 (both 32- and 64-bit) 29 | * FreeBSD, Linux, macOS 30 | * DragonFlyBSD, NetBSD, OpenBSD should work too, but I haven't tested them yet 31 | * Clang >= 9 or GCC >= 8 32 | 33 | ## Example 34 | 35 | ```cpp 36 | void echo(fev::socket &&socket) try { 37 | char buffer[1024]; 38 | for (;;) { 39 | std::size_t num_read = socket.read(buffer, sizeof(buffer)); 40 | if (num_read == 0) 41 | break; 42 | 43 | socket.write(buffer, num_read); 44 | } 45 | } catch (const std::system_error &e) { 46 | std::cerr << "[echo] " << e.what() << '\n'; 47 | } 48 | 49 | void acceptor() { 50 | fev::socket socket; 51 | socket.open(AF_INET, SOCK_STREAM, 0); 52 | socket.set_reuse_addr(); 53 | socket.bind(reinterpret_cast(&server_addr), sizeof(server_addr)); 54 | socket.listen(1024); 55 | for (;;) { 56 | auto new_socket = socket.accept(); 57 | fev::fiber::spawn(&echo, std::move(new_socket)); 58 | } 59 | } 60 | ``` 61 | 62 | See also [examples](examples). 63 | 64 | ## Documentation 65 | 66 | * [Scheduler strategies](docs/SchedulerStrategies.md) 67 | 68 | ## License 69 | 70 | Licensed under either of 71 | 72 | * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or 73 | http://www.apache.org/licenses/LICENSE-2.0) 74 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) 75 | 76 | at your option. 77 | 78 | ### Third party 79 | 80 | This library includes some code written by third parties. Check [third\_party](third_party) for 81 | their licenses. 82 | 83 | ### Contribution 84 | 85 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the 86 | work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any 87 | additional terms or conditions. 88 | -------------------------------------------------------------------------------- /docs/SchedulerStrategies.md: -------------------------------------------------------------------------------- 1 | # Scheduler strategies 2 | 3 | The main role of libfev's schedulers is to schedule runnable fibers. libfev implements few 4 | strategies to achieve that. They can be split into 2 categories: work sharing and work stealing. 5 | 6 | ## Work sharing 7 | 8 | Work sharing schedulers use a single, global queue for runnable fibers, the queue is shared by all 9 | workers. 10 | 11 | The main advantage of these strategies is that fibers are scheduled fairly (FIFO). However, since 12 | the queue is shared, a high contention on the queue is possible. 13 | 14 | Note that libfev implements cooperative multitasking, so it is possible that some fibers will get 15 | much more CPU time that others. 16 | 17 | ### work-sharing-bounded-mpmc 18 | 19 | A [bounded MPMC queue](http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue) 20 | is used for the queue implementation. Since the queue is bounded, there may be not enough space for 21 | all runnable fibers. In that case, an additional fallback queue is used, which is protected by a 22 | global mutex. The fallback queue is based on an intrusive list. 23 | 24 | ### work-sharing-locking 25 | 26 | This strategy uses an intrusive linked list for its global queue implementation. The list is 27 | protected by a global mutex. 28 | 29 | ### work-sharing-simple-mpmc 30 | 31 | This strategy implements a simple lock-free queue based on 32 | [Michael Scott's queue](https://www.cs.rochester.edu/u/scott/papers/1996_PODC_queues.pdf). 33 | 34 | It needs to allocate memory for each node. A memory pool is used for the allocation. The nodes are 35 | never released to the underlying operating system. 36 | 37 | If an allocation fails, the whole process is aborted. This is the only strategy that can fail due to 38 | failed allocation in user space. 39 | 40 | ## Work stealing 41 | 42 | These strategies use a queue per worker for runnable fibers. The scheduling is in most cases 43 | independent. A fiber that becomes runnable (for example a newly spawned fiber) is pushed to the 44 | current worker's queue. If a worker is out of work, it tries to steal some fibers from other workers 45 | before going to sleep. 46 | 47 | The main advantage is that a contention of the queues is minimized, and thus these strategies should 48 | scale better. However, it is possible that some fibers will be scheduled more often than others, and 49 | thus the scheduling is not fair. 50 | 51 | ### work-stealing-bounded-mpmc 52 | 53 | A [bounded MPMC queue](http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue) 54 | per worker is created. If a queue is full, the fiber is pushed to a global fallback queue based on 55 | an intrusive list. The fallback queue is protected by a global mutex. 56 | 57 | ### work-stealing-bounded-spmc 58 | 59 | A bounded SPMC queue per worker is created. Similarly to work-stealing-bounded-mpmc, if a queue is 60 | full, the fiber is pushed to a global fallback queue, which is protected by a global mutex. 61 | 62 | ### work-stealing-locking 63 | 64 | An intrusive list per worker is created. Each list is protected by its own lock. 65 | 66 | The queue's lock strategy can be controlled by **FEV_SCHED_STEAL_LOCKING_LOCK** option. Currently, 67 | a mutex or a spinlock can be used as the queue's lock. 68 | 69 | ## Further reading 70 | 71 | * https://tokio.rs/blog/2019-10-scheduler 72 | * http://www.cs.columbia.edu/~aho/cs6998/reports/12-12-11_DeshpandeSponslerWeiss_GO.pdf 73 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | foreach(target echo echo-timeout) 2 | add_executable(${target} ${target}.c) 3 | target_link_libraries(${target} PRIVATE fev) 4 | set_property(TARGET ${target} PROPERTY C_STANDARD 11) 5 | fev_set_compile_options(${target}) 6 | endforeach() 7 | 8 | foreach(target condition-variable++ connect++ echo++ echo-timeout++ fibonacci++ key-value++) 9 | add_executable(${target} ${target}.cpp) 10 | target_link_libraries(${target} PRIVATE fev) 11 | set_property(TARGET ${target} PROPERTY CXX_STANDARD 17) 12 | fev_set_compile_options(${target}) 13 | endforeach() 14 | -------------------------------------------------------------------------------- /examples/condition-variable++.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | // Based on: 8 | // https://en.cppreference.com/w/cpp/thread/condition_variable 9 | 10 | namespace { 11 | 12 | fev::mutex m{}; 13 | fev::condition_variable cv{}; 14 | std::string data{}; 15 | bool ready = false; 16 | bool processed = false; 17 | 18 | void worker() 19 | { 20 | std::unique_lock lock{m}; 21 | cv.wait(lock, [] { return ready; }); 22 | 23 | std::cout << "worker is processing data\n"; 24 | data += " after processing"; 25 | 26 | processed = true; 27 | std::cout << "worker signals data processing completed\n"; 28 | 29 | lock.unlock(); 30 | cv.notify_one(); 31 | } 32 | 33 | void manager() 34 | { 35 | data = "example"; 36 | 37 | { 38 | std::lock_guard lock{m}; 39 | ready = true; 40 | std::cout << "manager signals data ready for processing\n"; 41 | } 42 | cv.notify_one(); 43 | 44 | { 45 | std::unique_lock lock{m}; 46 | cv.wait(lock, [] { return processed; }); 47 | } 48 | std::cout << "manager received processed data: " << data << "\n"; 49 | } 50 | 51 | } // namespace 52 | 53 | int main() 54 | { 55 | fev::sched sched{}; 56 | fev::fiber::spawn(sched, &manager); 57 | fev::fiber::spawn(sched, &worker); 58 | sched.run(); 59 | } 60 | -------------------------------------------------------------------------------- /examples/connect++.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | namespace { 14 | 15 | struct sockaddr_in server_addr; 16 | 17 | std::string_view message; 18 | 19 | void client() 20 | try { 21 | fev::socket socket; 22 | socket.open(AF_INET, SOCK_STREAM, 0); 23 | socket.connect(reinterpret_cast(&server_addr), sizeof(server_addr)); 24 | 25 | // Send hello world message. 26 | socket.write(message.data(), message.length()); 27 | 28 | // Receive response. 29 | char buffer[1024]; 30 | std::size_t num_read = socket.read(buffer, sizeof(buffer)); 31 | 32 | std::string_view response{buffer, num_read}; 33 | std::cout << "Response: " << response << '\n'; 34 | } catch (const std::system_error &e) { 35 | std::cerr << "[client] " << e.what() << '\n'; 36 | } 37 | 38 | } // namespace 39 | 40 | int main(int argc, char **argv) 41 | { 42 | // Parse arguments. 43 | 44 | if (argc != 4) { 45 | std::cerr << "Usage: " << argv[0] << " \n"; 46 | return 1; 47 | } 48 | 49 | auto host = argv[1]; 50 | 51 | std::uint16_t port; 52 | if (auto [_, ec] = std::from_chars(argv[2], argv[2] + std::strlen(argv[2]), port); 53 | ec != std::errc{}) { 54 | std::cerr << "Failed to parse port\n"; 55 | return 1; 56 | } 57 | 58 | message = argv[3]; 59 | 60 | // Initialize server address. 61 | 62 | server_addr.sin_family = AF_INET; 63 | server_addr.sin_port = htons(port); 64 | if (inet_aton(host, &server_addr.sin_addr) != 1) { 65 | std::cerr << "Converting host IPv4 '" << host << "' failed\n"; 66 | return 1; 67 | } 68 | 69 | // Run. 70 | 71 | fev::sched sched{}; 72 | 73 | // Spawn a fiber in `sched`. 74 | fev::fiber::spawn(sched, &client); 75 | 76 | sched.run(); 77 | 78 | return 0; 79 | } 80 | -------------------------------------------------------------------------------- /examples/echo++.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | namespace { 14 | 15 | struct sockaddr_in server_addr; 16 | 17 | void echo(fev::socket &&socket) 18 | try { 19 | char buffer[1024]; 20 | for (;;) { 21 | std::size_t num_read = socket.read(buffer, sizeof(buffer)); 22 | if (num_read == 0) 23 | break; 24 | 25 | socket.write(buffer, num_read); 26 | } 27 | } catch (const std::system_error &e) { 28 | std::cerr << "[echo] " << e.what() << '\n'; 29 | } 30 | 31 | void acceptor() 32 | { 33 | fev::socket socket; 34 | socket.open(AF_INET, SOCK_STREAM, 0); 35 | socket.set_reuse_addr(); 36 | socket.bind(reinterpret_cast(&server_addr), sizeof(server_addr)); 37 | socket.listen(1024); 38 | for (;;) { 39 | auto new_socket = socket.accept(); 40 | 41 | // If we don't specify any sched, the fiber will be spawned in the current scheduler. 42 | fev::fiber::spawn(&echo, std::move(new_socket)); 43 | } 44 | } 45 | 46 | } // namespace 47 | 48 | int main(int argc, char **argv) 49 | { 50 | // Parse arguments. 51 | 52 | if (argc != 3) { 53 | std::cerr << "Usage: " << argv[0] << " \n"; 54 | return 1; 55 | } 56 | 57 | auto host = argv[1]; 58 | 59 | std::uint16_t port; 60 | if (auto [_, ec] = std::from_chars(argv[2], argv[2] + std::strlen(argv[2]), port); 61 | ec != std::errc{}) { 62 | std::cerr << "Failed to parse port\n"; 63 | return 1; 64 | } 65 | 66 | // Initialize server address. 67 | 68 | server_addr.sin_family = AF_INET; 69 | server_addr.sin_port = htons(port); 70 | if (inet_aton(host, &server_addr.sin_addr) != 1) { 71 | std::cerr << "Converting host IPv4 '" << host << "' failed\n"; 72 | return 1; 73 | } 74 | 75 | // Run. 76 | 77 | fev::sched sched{}; 78 | 79 | // Spawn a fiber in `sched`. 80 | fev::fiber::spawn(sched, &acceptor); 81 | 82 | sched.run(); 83 | 84 | return 0; 85 | } 86 | -------------------------------------------------------------------------------- /examples/echo-timeout++.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | 14 | namespace { 15 | 16 | struct sockaddr_in server_addr; 17 | 18 | void echo(fev::socket &&socket) 19 | try { 20 | char buffer[1024]; 21 | 22 | for (;;) { 23 | using namespace std::chrono_literals; 24 | std::size_t num_read = socket.try_read_for(buffer, sizeof(buffer), 3s); 25 | if (num_read == 0) 26 | break; 27 | 28 | socket.write(buffer, num_read); 29 | } 30 | } catch (const std::system_error &e) { 31 | std::cerr << "[echo] " << e.what() << '\n'; 32 | } 33 | 34 | void acceptor() 35 | { 36 | fev::socket socket; 37 | socket.open(AF_INET, SOCK_STREAM, 0); 38 | socket.set_reuse_addr(); 39 | socket.bind(reinterpret_cast(&server_addr), sizeof(server_addr)); 40 | socket.listen(1024); 41 | for (;;) { 42 | auto new_socket = socket.accept(); 43 | 44 | // If we don't specify any sched, the fiber will be spawned in the current scheduler. 45 | fev::fiber::spawn(&echo, std::move(new_socket)); 46 | } 47 | } 48 | 49 | } // namespace 50 | 51 | int main(int argc, char **argv) 52 | { 53 | // Parse arguments. 54 | 55 | if (argc != 3) { 56 | std::cerr << "Usage: " << argv[0] << " \n"; 57 | return 1; 58 | } 59 | 60 | auto host = argv[1]; 61 | 62 | std::uint16_t port; 63 | if (auto [_, ec] = std::from_chars(argv[2], argv[2] + std::strlen(argv[2]), port); 64 | ec != std::errc{}) { 65 | std::cerr << "Failed to parse port\n"; 66 | return 1; 67 | } 68 | 69 | // Initialize server address. 70 | 71 | server_addr.sin_family = AF_INET; 72 | server_addr.sin_port = htons(port); 73 | if (inet_aton(host, &server_addr.sin_addr) != 1) { 74 | std::cerr << "Converting host IPv4 '" << host << "' failed\n"; 75 | return 1; 76 | } 77 | 78 | // Run. 79 | 80 | fev::sched sched{}; 81 | 82 | // Spawn a fiber in `sched`. 83 | fev::fiber::spawn(sched, &acceptor); 84 | 85 | sched.run(); 86 | 87 | return 0; 88 | } 89 | -------------------------------------------------------------------------------- /examples/echo.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | static struct sockaddr_in server_addr; 12 | 13 | static void *echo(void *arg) 14 | { 15 | char buffer[1024]; 16 | struct fev_socket *socket = arg; 17 | 18 | for (;;) { 19 | ssize_t num_read = fev_socket_read(socket, buffer, sizeof(buffer)); 20 | if (num_read <= 0) { 21 | if (num_read < 0) { 22 | int err = (int)(-num_read); 23 | fprintf(stderr, "Reading from socket failed: %s\n", strerror(err)); 24 | } 25 | break; 26 | } 27 | 28 | ssize_t num_written = fev_socket_write(socket, buffer, (size_t)num_read); 29 | if (num_written != num_read) { 30 | if (num_written < 0) { 31 | int err = (int)(-num_written); 32 | fprintf(stderr, "Writing to socket failed: %s\n", strerror(err)); 33 | } 34 | break; 35 | } 36 | } 37 | 38 | fev_socket_close(socket); 39 | fev_socket_destroy(socket); 40 | 41 | return NULL; 42 | } 43 | 44 | static void *acceptor(void *arg) 45 | { 46 | struct fev_socket *socket; 47 | int ret; 48 | 49 | (void)arg; 50 | 51 | ret = fev_socket_create(&socket); 52 | if (ret != 0) { 53 | fprintf(stderr, "Creating socket failed: %s\n", strerror(-ret)); 54 | goto out; 55 | } 56 | 57 | ret = fev_socket_open(socket, AF_INET, SOCK_STREAM, 0); 58 | if (ret != 0) { 59 | fprintf(stderr, "Opening socket failed: %s\n", strerror(-ret)); 60 | goto out_destroy; 61 | } 62 | 63 | ret = fev_socket_set_opt(socket, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int)); 64 | if (ret != 0) { 65 | fprintf(stderr, "Setting SO_REUSEADDR failed: %s\n", strerror(-ret)); 66 | goto out_close; 67 | } 68 | 69 | ret = fev_socket_bind(socket, (struct sockaddr *)&server_addr, sizeof(server_addr)); 70 | if (ret != 0) { 71 | fprintf(stderr, "Binding socket failed: %s\n", strerror(-ret)); 72 | goto out_close; 73 | } 74 | 75 | ret = fev_socket_listen(socket, /*backlog=*/1024); 76 | if (ret != 0) { 77 | fprintf(stderr, "Listening on socket failed: %s\n", strerror(-ret)); 78 | goto out_close; 79 | } 80 | 81 | for (;;) { 82 | struct fev_socket *new_socket; 83 | 84 | ret = fev_socket_create(&new_socket); 85 | if (ret != 0) { 86 | fprintf(stderr, "Creating new socket failed: %s\n", strerror(-ret)); 87 | goto out_close; 88 | } 89 | 90 | ret = fev_socket_accept(socket, new_socket, /*address=*/NULL, /*address_len=*/NULL); 91 | if (ret != 0) { 92 | fprintf(stderr, "Accepting socket failed: %s\n", strerror(-ret)); 93 | fev_socket_destroy(new_socket); 94 | goto out_close; 95 | } 96 | 97 | /* We can pass NULL as sched, the fiber will be spawned in the current scheduler. */ 98 | ret = fev_fiber_spawn(/*sched=*/NULL, &echo, new_socket); 99 | if (ret != 0) { 100 | fprintf(stderr, "Spawning echo fiber failed: %s\n", strerror(-ret)); 101 | fev_socket_destroy(new_socket); 102 | goto out_close; 103 | } 104 | } 105 | 106 | out_close: 107 | fev_socket_close(socket); 108 | 109 | out_destroy: 110 | fev_socket_destroy(socket); 111 | 112 | out: 113 | return NULL; 114 | } 115 | 116 | int main(int argc, char **argv) 117 | { 118 | struct fev_sched *sched; 119 | const char *host; 120 | uint16_t port; 121 | int err, ret = 1; 122 | 123 | /* Parse arguments. */ 124 | 125 | if (argc != 3) { 126 | fprintf(stderr, "Usage: %s \n", argv[0]); 127 | return 1; 128 | } 129 | 130 | host = argv[1]; 131 | 132 | if (sscanf(argv[2], "%" SCNu16, &port) != 1) { 133 | fputs("Parsing port failed\n", stderr); 134 | return 1; 135 | } 136 | 137 | /* Initialize server address. */ 138 | 139 | server_addr.sin_family = AF_INET; 140 | server_addr.sin_port = htons(port); 141 | if (inet_aton(host, &server_addr.sin_addr) != 1) { 142 | fprintf(stderr, "Converting host IPv4 '%s' failed\n", host); 143 | return 1; 144 | } 145 | 146 | /* Run. */ 147 | 148 | err = fev_sched_create(&sched, /*attr=*/NULL); 149 | if (err != 0) { 150 | fprintf(stderr, "Creating scheduler failed: %s\n", strerror(-err)); 151 | return 1; 152 | } 153 | 154 | /* Spawn a fiber in `sched`. */ 155 | err = fev_fiber_spawn(sched, &acceptor, NULL); 156 | if (err != 0) { 157 | fprintf(stderr, "Spawning fiber failed: %s\n", strerror(-err)); 158 | goto out_sched; 159 | } 160 | 161 | err = fev_sched_run(sched); 162 | if (err != 0) { 163 | fprintf(stderr, "Running scheduler failed: %s\n", strerror(-err)); 164 | goto out_sched; 165 | } 166 | 167 | ret = 0; 168 | 169 | out_sched: 170 | fev_sched_destroy(sched); 171 | 172 | return ret; 173 | } 174 | -------------------------------------------------------------------------------- /examples/fibonacci++.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | // Note this library is not the right one for tasks like this. If you are looking for parallel 7 | // tasks with dependencies, you should check taskflow library. 8 | 9 | namespace { 10 | 11 | constexpr int N = 20; 12 | 13 | void fibonacci(int n, int &result) 14 | { 15 | if (n <= 1) { 16 | result = n; 17 | return; 18 | } 19 | 20 | int result1, result2; 21 | fev::fiber fiber1{&fibonacci, n - 1, std::ref(result1)}; 22 | fev::fiber fiber2{&fibonacci, n - 2, std::ref(result2)}; 23 | fiber1.join(); 24 | fiber2.join(); 25 | result = result1 + result2; 26 | } 27 | 28 | } // namespace 29 | 30 | int main() 31 | { 32 | fev::sched sched{}; 33 | 34 | int result; 35 | fev::fiber::spawn(sched, &fibonacci, N, std::ref(result)); 36 | 37 | sched.run(); 38 | 39 | std::cout << "fibonacci(" << N << ") = " << result << '\n'; 40 | } 41 | -------------------------------------------------------------------------------- /examples/key-value++.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | namespace { 19 | 20 | struct sockaddr_in server_addr; 21 | 22 | // A mutex between fibers. 23 | fev::mutex mutex{}; 24 | 25 | std::unordered_map data{}; 26 | 27 | void client(fev::socket &&socket) 28 | try { 29 | char buffer[1024]; 30 | for (;;) { 31 | std::size_t num_read = socket.read(buffer, sizeof(buffer)); 32 | if (num_read == 0) { 33 | // EOF 34 | break; 35 | } 36 | 37 | std::string_view msg{buffer, num_read}; 38 | 39 | // Trim 40 | while (msg.size() > 0 && std::isspace(msg[0])) 41 | msg.remove_prefix(1); 42 | while (msg.size() > 0 && std::isspace(msg[msg.size() - 1])) 43 | msg.remove_suffix(1); 44 | 45 | std::string_view cmd{}, key{}; 46 | if (auto cmd_end = msg.find(' '); cmd_end != std::string_view::npos) { 47 | cmd = msg.substr(0, cmd_end); 48 | key = msg.substr(cmd_end + 1); 49 | } 50 | 51 | if (key.empty()) { 52 | std::string_view resp{"Parsing failed\n"}; 53 | socket.write(resp.data(), resp.size()); 54 | continue; 55 | } 56 | 57 | if (cmd == "get") { 58 | auto lock = std::unique_lock{mutex}; 59 | auto search = data.find(std::string{key}); 60 | if (search != data.end()) { 61 | auto value = search->second; 62 | lock.unlock(); 63 | value += '\n'; 64 | socket.write(value.data(), value.size()); 65 | } else { 66 | lock.unlock(); 67 | std::string_view resp{"Not found\n"}; 68 | socket.write(resp.data(), resp.size()); 69 | } 70 | } else if (cmd == "set") { 71 | std::string k{}, value{}; 72 | if (auto key_end = key.find(' '); key_end != std::string_view::npos) { 73 | k = key.substr(0, key_end); 74 | value = key.substr(key_end + 1); 75 | auto lock = std::unique_lock{mutex}; 76 | data[k] = value; 77 | lock.unlock(); 78 | std::string_view resp{"OK\n"}; 79 | socket.write(resp.data(), resp.size()); 80 | } else { 81 | std::string_view resp{"Parsing failed\n"}; 82 | socket.write(resp.data(), resp.size()); 83 | } 84 | } else { 85 | std::string_view resp{"Unknown command\n"}; 86 | socket.write(resp.data(), resp.size()); 87 | } 88 | } 89 | } catch (const std::system_error &e) { 90 | std::cerr << "[client] " << e.what() << '\n'; 91 | } 92 | 93 | void acceptor() 94 | { 95 | fev::socket socket; 96 | socket.open(AF_INET, SOCK_STREAM, 0); 97 | socket.set_reuse_addr(); 98 | socket.bind(reinterpret_cast(&server_addr), sizeof(server_addr)); 99 | socket.listen(1024); 100 | for (;;) { 101 | auto new_socket = socket.accept(); 102 | fev::fiber::spawn(&client, std::move(new_socket)); 103 | } 104 | } 105 | 106 | } // namespace 107 | 108 | int main(int argc, char **argv) 109 | { 110 | // Parse arguments. 111 | 112 | if (argc != 3) { 113 | std::cerr << "Usage: " << argv[0] << " \n"; 114 | return 1; 115 | } 116 | 117 | auto host = argv[1]; 118 | 119 | std::uint16_t port; 120 | if (auto [_, ec] = std::from_chars(argv[2], argv[2] + std::strlen(argv[2]), port); 121 | ec != std::errc{}) { 122 | std::cerr << "Failed to parse port\n"; 123 | return 1; 124 | } 125 | 126 | // Initialize server address. 127 | 128 | server_addr.sin_family = AF_INET; 129 | server_addr.sin_port = htons(port); 130 | if (inet_aton(host, &server_addr.sin_addr) != 1) { 131 | std::cerr << "Converting host IPv4 '" << host << "' failed\n"; 132 | return 1; 133 | } 134 | 135 | // Run. 136 | 137 | fev::sched sched{}; 138 | fev::fiber::spawn(sched, &acceptor); 139 | sched.run(); 140 | 141 | return 0; 142 | } 143 | -------------------------------------------------------------------------------- /include/fev/config.h.in: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_CONFIG_H 11 | #define FEV_CONFIG_H 12 | 13 | /* Version */ 14 | 15 | #define FEV_VERSION_MAJOR @FEV_VERSION_MAJOR@ 16 | #define FEV_VERSION_MINOR @FEV_VERSION_MINOR@ 17 | #define FEV_VERSION_PATCH @FEV_VERSION_PATCH@ 18 | 19 | /* Architecture */ 20 | 21 | #cmakedefine FEV_ARCH_I386 22 | #cmakedefine FEV_ARCH_X86_64 23 | 24 | /* Compiler */ 25 | 26 | #cmakedefine FEV_COMPILER_CLANG 27 | #cmakedefine FEV_COMPILER_GCC 28 | #cmakedefine FEV_COMPILER_MSVC 29 | 30 | /* OS */ 31 | 32 | #cmakedefine FEV_OS_MACOS 33 | #cmakedefine FEV_OS_DARWIN 34 | 35 | #cmakedefine FEV_OS_LINUX 36 | 37 | #cmakedefine FEV_OS_DRAGONFLYBSD 38 | #cmakedefine FEV_OS_FREEBSD 39 | #cmakedefine FEV_OS_NETBSD 40 | #cmakedefine FEV_OS_OPENBSD 41 | 42 | #cmakedefine FEV_OS_WIN 43 | 44 | #cmakedefine FEV_OS_BSD 45 | #cmakedefine FEV_OS_POSIX 46 | 47 | /* Functions */ 48 | 49 | #cmakedefine FEV_HAVE_ACCEPT4 50 | 51 | /* Page and cache sizes */ 52 | 53 | #define FEV_PAGE_SIZE @FEV_PAGE_SIZE@ 54 | #define FEV_DCACHE_LINE_SIZE @FEV_DCACHE_LINE_SIZE@ 55 | #define FEV_ICACHE_LINE_SIZE @FEV_ICACHE_LINE_SIZE@ 56 | 57 | /* Fiber's stack */ 58 | 59 | #define FEV_DEFAULT_STACK_SIZE @FEV_DEFAULT_STACK_SIZE@ 60 | #define FEV_DEFAULT_GUARD_SIZE @FEV_DEFAULT_GUARD_SIZE@ 61 | 62 | /* Scheduler */ 63 | 64 | #cmakedefine FEV_SCHED_WORK_SHARING_LOCKING 65 | #cmakedefine FEV_SCHED_WORK_SHARING_BOUNDED_MPMC 66 | #cmakedefine FEV_SCHED_WORK_SHARING_SIMPLE_MPMC 67 | #cmakedefine FEV_SCHED_WORK_STEALING_LOCKING 68 | #cmakedefine FEV_SCHED_WORK_STEALING_BOUNDED_MPMC 69 | #cmakedefine FEV_SCHED_WORK_STEALING_BOUNDED_SPMC 70 | 71 | #define FEV_SCHED_SHR_BOUNDED_MPMC_QUEUE_CAPACITY @FEV_SCHED_SHR_BOUNDED_MPMC_QUEUE_CAPACITY@ 72 | 73 | #define FEV_SCHED_SHR_SIMPLE_MPMC_LOCAL_POOL_SIZE @FEV_SCHED_SHR_SIMPLE_MPMC_LOCAL_POOL_SIZE@ 74 | 75 | #cmakedefine FEV_SCHED_STEAL_LOCKING_LOCK_MUTEX 76 | #cmakedefine FEV_SCHED_STEAL_LOCKING_LOCK_SPINLOCK 77 | #define FEV_SCHED_STEAL_LOCKING_STEAL_COUNT @FEV_SCHED_STEAL_LOCKING_STEAL_COUNT@ 78 | 79 | #define FEV_SCHED_STEAL_BOUNDED_MPMC_QUEUE_CAPACITY @FEV_SCHED_STEAL_BOUNDED_MPMC_QUEUE_CAPACITY@ 80 | #define FEV_SCHED_STEAL_BOUNDED_MPMC_STEAL_COUNT @FEV_SCHED_STEAL_BOUNDED_MPMC_STEAL_COUNT@ 81 | 82 | #define FEV_SCHED_STEAL_BOUNDED_SPMC_QUEUE_CAPACITY @FEV_SCHED_STEAL_BOUNDED_SPMC_QUEUE_CAPACITY@ 83 | #define FEV_SCHED_STEAL_BOUNDED_SPMC_STEAL_COUNT @FEV_SCHED_STEAL_BOUNDED_SPMC_STEAL_COUNT@ 84 | 85 | /* Poller */ 86 | 87 | #cmakedefine FEV_POLLER_EPOLL 88 | #cmakedefine FEV_POLLER_IO_URING 89 | #cmakedefine FEV_POLLER_KQUEUE 90 | 91 | #define FEV_IO_URING_ENTRIES_PER_WORKER @FEV_IO_URING_ENTRIES_PER_WORKER@ 92 | 93 | /* Timers */ 94 | 95 | #cmakedefine FEV_TIMERS_BINHEAP 96 | #cmakedefine FEV_TIMERS_RBTREE 97 | 98 | #cmakedefine FEV_TIMERS_MIN_LOCK_MUTEX 99 | #cmakedefine FEV_TIMERS_MIN_LOCK_SPINLOCK 100 | 101 | /* Internal lock */ 102 | 103 | #cmakedefine FEV_ILOCK_LOCK_MUTEX 104 | #cmakedefine FEV_ILOCK_LOCK_SPINLOCK 105 | 106 | /* Threads */ 107 | 108 | #cmakedefine FEV_THR_POSIX 109 | #cmakedefine FEV_THR_WIN 110 | 111 | /* Thread mutex */ 112 | 113 | #cmakedefine FEV_THR_MUTEX_LINUX 114 | #cmakedefine FEV_THR_MUTEX_POSIX 115 | #cmakedefine FEV_THR_MUTEX_WIN 116 | 117 | /* Thread semaphore */ 118 | 119 | #cmakedefine FEV_THR_SEM_MACOS 120 | #cmakedefine FEV_THR_SEM_POSIX 121 | #cmakedefine FEV_THR_SEM_WIN 122 | 123 | /* Other */ 124 | 125 | #cmakedefine FEV_ENABLE_DEBUG_ASSERT 126 | #cmakedefine FEV_ENABLE_ASAN 127 | #cmakedefine FEV_ASSUME_MALLOC_NEVER_FAILS 128 | 129 | #endif /* !FEV_CONFIG_H */ 130 | -------------------------------------------------------------------------------- /libfev.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@CMAKE_INSTALL_PREFIX@ 2 | exec_prefix=${prefix} 3 | libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ 4 | includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ 5 | 6 | Name: @PROJECT_NAME@ 7 | Description: @PROJECT_DESCRIPTION@ 8 | Version: @FEV_VERSION_MAJOR@.@FEV_VERSION_MINOR@.@FEV_VERSION_PATCH@ 9 | URL: @PROJECT_HOMEPAGE_URL@ 10 | 11 | Requires: 12 | Libs: -L${libdir} -lfev 13 | Cflags: -I${includedir} 14 | -------------------------------------------------------------------------------- /src/fev_alloc.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_alloc.h" 11 | 12 | #include 13 | 14 | #include "fev_compiler.h" 15 | 16 | fev_realloc_t fev_realloc_ptr = &realloc; 17 | 18 | fev_realloc_t fev_get_realloc(void) { return fev_realloc_ptr; } 19 | 20 | FEV_NONNULL(1) void fev_set_realloc(fev_realloc_t realloc_ptr) { fev_realloc_ptr = realloc_ptr; } 21 | -------------------------------------------------------------------------------- /src/fev_alloc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_ALLOC_H 11 | #define FEV_ALLOC_H 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include "fev_assert.h" 19 | #include "fev_compiler.h" 20 | 21 | #ifdef FEV_ASSUME_MALLOC_NEVER_FAILS 22 | #define FEV_ALLOC_RETURN FEV_RETURNS_NONNULL 23 | #else 24 | #define FEV_ALLOC_RETURN 25 | #endif 26 | 27 | extern fev_realloc_t fev_realloc_ptr; 28 | 29 | FEV_ALLOC_RETURN FEV_ALLOC_SIZE(1) FEV_MALLOC static inline void *fev_malloc(size_t size) 30 | { 31 | void *ptr; 32 | 33 | ptr = fev_realloc_ptr(NULL, size); 34 | 35 | /* The returned pointer should be at least aligned to sizeof(void *). */ 36 | FEV_ASSERT((uintptr_t)ptr % sizeof(void *) == 0); 37 | 38 | return ptr; 39 | } 40 | 41 | FEV_ALLOC_RETURN FEV_ALLOC_SIZE(2) static inline void *fev_realloc(void *ptr, size_t size) 42 | { 43 | return fev_realloc_ptr(ptr, size); 44 | } 45 | 46 | static inline void fev_free(void *ptr) 47 | { 48 | if (ptr != NULL) 49 | fev_realloc_ptr(ptr, 0); 50 | } 51 | 52 | FEV_ALLOC_RETURN FEV_ALLOC_ALIGN(1) FEV_ALLOC_SIZE(2) FEV_MALLOC 53 | static inline void *fev_aligned_alloc(size_t alignment, size_t size) 54 | { 55 | void *ptr, *ret; 56 | 57 | FEV_ASSERT(alignment != 0); 58 | FEV_ASSERT(alignment % sizeof(void *) == 0); 59 | FEV_ASSERT((alignment & (alignment - 1)) == 0); 60 | FEV_ASSERT(size <= SIZE_MAX - alignment); 61 | 62 | ptr = fev_malloc(size + alignment); 63 | 64 | #ifndef FEV_ASSUME_MALLOC_NEVER_FAILS 65 | if (FEV_UNLIKELY(ptr == NULL)) 66 | return NULL; 67 | #endif 68 | 69 | ret = (void *)(((uintptr_t)ptr + alignment) & ~(alignment - 1)); 70 | 71 | /* 72 | * Check if [ret, ret+size) is valid (or the difference below is no more than 73 | * `alignment` since we allocated `size` + `alignment` bytes. 74 | */ 75 | FEV_ASSERT((uintptr_t)ret - (uintptr_t)ptr <= alignment); 76 | 77 | FEV_ASSERT((uintptr_t)ptr <= (uintptr_t)(&((void **)ret)[-1])); 78 | ((void **)ret)[-1] = ptr; 79 | 80 | return ret; 81 | } 82 | 83 | FEV_NONNULL(1) static inline void fev_aligned_free(void *ptr) { fev_free(((void **)ptr)[-1]); } 84 | 85 | #endif /* !FEV_ALLOC_H */ 86 | -------------------------------------------------------------------------------- /src/fev_arch.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_ARCH_H 11 | #define FEV_ARCH_H 12 | 13 | #include 14 | 15 | #if defined(FEV_ARCH_I386) || defined(FEV_ARCH_X86_64) 16 | #include "fev_arch_x86.h" 17 | #else 18 | #error Your architecture is unsupported, currently only x86 is supported. 19 | #endif 20 | 21 | #endif /* !FEV_ARCH_H */ 22 | -------------------------------------------------------------------------------- /src/fev_arch_x86.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_ARCH_X86_H 11 | #define FEV_ARCH_X86_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_assert.h" 18 | #include "fev_compiler.h" 19 | 20 | /* pause */ 21 | 22 | static inline void fev_pause(void) { asm volatile("pause"); } 23 | 24 | /* cmpxchg8b/cmpxchg16b */ 25 | 26 | #if defined(FEV_ARCH_X86_64) 27 | #define FEV_CMPXCHG2_OPCODE "cmpxchg16b" 28 | #else 29 | #define FEV_CMPXCHG2_OPCODE "cmpxchg8b" 30 | #endif 31 | 32 | #define fev_cmpxchg2(ptr, expected0, expected1, desired0, desired1) \ 33 | ({ \ 34 | bool fev_exchanged; \ 35 | FEV_ASSERT((uintptr_t)(ptr) % (2 * sizeof(void *)) == 0); \ 36 | static_assert(sizeof(*(expected0)) == sizeof(void *), "wrong size"); \ 37 | static_assert(sizeof(*(expected1)) == sizeof(void *), "wrong size"); \ 38 | static_assert(sizeof(desired0) == sizeof(void *), "wrong size"); \ 39 | static_assert(sizeof(desired1) == sizeof(void *), "wrong size"); \ 40 | asm volatile("lock; " FEV_CMPXCHG2_OPCODE " %1" \ 41 | : "=@cce"(fev_exchanged), "+m"(*(uintptr_t *)(ptr)), "+a"(*(expected0)), \ 42 | "+d"(*(expected1)) \ 43 | : "b"(desired0), "c"(desired1) \ 44 | : "memory", "cc"); \ 45 | fev_exchanged; \ 46 | }) 47 | 48 | #endif /* !FEV_ARCH_X86_H */ 49 | -------------------------------------------------------------------------------- /src/fev_assert.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_ASSERT_H 11 | #define FEV_ASSERT_H 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include "fev_compiler.h" 19 | 20 | #ifdef FEV_ENABLE_DEBUG_ASSERT 21 | 22 | #define FEV_ASSERT(expr) \ 23 | do { \ 24 | if (FEV_UNLIKELY(!(expr))) { \ 25 | fprintf(stderr, "Asseration \"%s\" failed in %s (%s:%u)\n", #expr, __func__, __FILE__, \ 26 | __LINE__); \ 27 | abort(); \ 28 | } \ 29 | } while (0) 30 | 31 | #else 32 | 33 | #define FEV_ASSERT(expr) ((void)0) 34 | 35 | #endif 36 | 37 | #endif /* !FEV_ASSERT_H */ 38 | -------------------------------------------------------------------------------- /src/fev_compiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_COMPILER_H 11 | #define FEV_COMPILER_H 12 | 13 | #include 14 | 15 | /* Attributes */ 16 | 17 | /* FEV_NONNULL, FEV_NORETURN and FEV_PURE are defined in fev/fev.h. */ 18 | 19 | #if FEV_HAS_ATTRIBUTE(__always_inline__) || FEV_GCC_VERSION 20 | #define FEV_ALWAYS_INLINE __attribute__((__always_inline__)) 21 | #else 22 | #define FEV_ALWAYS_INLINE /* __attribute__((__always_inline__)) */ 23 | #endif 24 | 25 | #if FEV_HAS_ATTRIBUTE(__aligned__) || FEV_GCC_VERSION 26 | #define FEV_ALIGNED(alignment) __attribute__((__aligned__(alignment))) 27 | #else 28 | #define FEV_ALIGNED(alignment) /* __attribute__((__aligned__(alignment))) */ 29 | #endif 30 | 31 | #if FEV_HAS_ATTRIBUTE(__const__) || FEV_GCC_VERSION 32 | #define FEV_CONST __attribute__((__const__)) 33 | #else 34 | #define FEV_CONST /* __attribute__((__const__)) */ 35 | #endif 36 | 37 | #if FEV_HAS_ATTRIBUTE(__malloc__) || FEV_GCC_VERSION 38 | #define FEV_MALLOC __attribute__((__malloc__)) 39 | #else 40 | #define FEV_MALLOC /* __attribute__((__malloc__)) */ 41 | #endif 42 | 43 | #if FEV_HAS_ATTRIBUTE(__noinline__) || FEV_GCC_VERSION 44 | #define FEV_NOINLINE __attribute__((__noinline__)) 45 | #else 46 | #define FEV_NOINLINE /* __attribute__((__noinline__)) */ 47 | #endif 48 | 49 | #if FEV_HAS_ATTRIBUTE(__warn_unused_result__) || FEV_GCC_VERSION 50 | #define FEV_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) 51 | #else 52 | #define FEV_WARN_UNUSED_RESULT /* __attribute__((__warn_unused_result__)) */ 53 | #endif 54 | 55 | #if FEV_HAS_ATTRIBUTE(__alloc_size__) || FEV_GCC_VERSION >= 40300 56 | #define FEV_ALLOC_SIZE(...) __attribute__((__alloc_size__(__VA_ARGS__))) 57 | #else 58 | #define FEV_ALLOC_SIZE(...) /* __attribute__((__alloc_size__(__VA_ARGS__))) */ 59 | #endif 60 | 61 | #if FEV_HAS_ATTRIBUTE(__cold__) || FEV_GCC_VERSION >= 40300 62 | #define FEV_COLD __attribute__((__cold__)) 63 | #else 64 | #define FEV_COLD /* __attribute__((__cold__)) */ 65 | #endif 66 | 67 | #if FEV_HAS_ATTRIBUTE(__hot__) || FEV_GCC_VERSION >= 40300 68 | #define FEV_HOT __attribute__((__hot__)) 69 | #else 70 | #define FEV_HOT /* __attribute__((__hot__)) */ 71 | #endif 72 | 73 | #if FEV_HAS_ATTRIBUTE(__alloc_align__) || FEV_GCC_VERSION >= 40900 74 | #define FEV_ALLOC_ALIGN(n) __attribute__((__alloc_align__(n))) 75 | #else 76 | #define FEV_ALLOC_ALIGN(n) /* __attribute__((__alloc_align__(n))) */ 77 | #endif 78 | 79 | #if FEV_HAS_ATTRIBUTE(__returns_nonnull__) || FEV_GCC_VERSION >= 40900 80 | #define FEV_RETURNS_NONNULL __attribute__((__returns_nonnull__)) 81 | #else 82 | #define FEV_RETURNS_NONNULL /* __attribute__((__returns_nonnull__)) */ 83 | #endif 84 | 85 | /* Builtins */ 86 | 87 | #ifdef __has_builtin 88 | #define FEV_HAS_BUILTIN(builtin) __has_builtin(builtin) 89 | #else 90 | #define FEV_HAS_BUILTIN(builtin) 0 91 | #endif 92 | 93 | #if FEV_HAS_BUILTIN(__builtin_expect) || FEV_GCC_VERSION 94 | #define FEV_LIKELY(exp) __builtin_expect((exp), 1) 95 | #define FEV_UNLIKELY(exp) __builtin_expect((exp), 0) 96 | #else 97 | #define FEV_LIKELY(exp) (exp) 98 | #define FEV_UNLIKELY(exp) (exp) 99 | #endif 100 | 101 | #if FEV_HAS_BUILTIN(__builtin_unreachable) || FEV_GCC_VERSION >= 40500 102 | #define FEV_UNREACHABLE() __builtin_unreachable() 103 | #else 104 | #define FEV_UNREACHABLE() /* __builtin_unreachable() */ 105 | #endif 106 | 107 | #endif /* !FEV_COMPILER_H */ 108 | -------------------------------------------------------------------------------- /src/fev_cond.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_cond_impl.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_alloc.h" 18 | #include "fev_assert.h" 19 | #include "fev_compiler.h" 20 | #include "fev_mutex_intf.h" 21 | #include "fev_time.h" 22 | #include "fev_waiters_queue_impl.h" 23 | 24 | FEV_NONNULL(1) int fev_cond_create(struct fev_cond **cond_ptr) 25 | { 26 | struct fev_cond *cond; 27 | int ret; 28 | 29 | cond = fev_malloc(sizeof(*cond)); 30 | if (FEV_UNLIKELY(cond == NULL)) 31 | return -ENOMEM; 32 | 33 | ret = fev_cond_init(cond); 34 | if (FEV_UNLIKELY(ret != 0)) { 35 | fev_free(cond); 36 | return ret; 37 | } 38 | 39 | *cond_ptr = cond; 40 | return 0; 41 | } 42 | 43 | FEV_NONNULL(1) void fev_cond_destroy(struct fev_cond *cond) 44 | { 45 | fev_cond_fini(cond); 46 | fev_free(cond); 47 | } 48 | 49 | static bool fev_cond_wait_recheck(void *arg) 50 | { 51 | struct fev_mutex *mutex = arg; 52 | 53 | fev_mutex_unlock(mutex); 54 | return true; 55 | } 56 | 57 | FEV_NONNULL(1, 2) void fev_cond_wait(struct fev_cond *cond, struct fev_mutex *mutex) 58 | { 59 | int res = fev_waiters_queue_wait(&cond->wq, /*abs_time=*/NULL, &fev_cond_wait_recheck, mutex); 60 | (void)res; 61 | FEV_ASSERT(res == 0); 62 | 63 | fev_mutex_lock(mutex); 64 | } 65 | 66 | FEV_NONNULL(1, 2, 3) 67 | int fev_cond_wait_until(struct fev_cond *cond, struct fev_mutex *mutex, 68 | const struct timespec *abs_time) 69 | { 70 | int res = fev_waiters_queue_wait(&cond->wq, abs_time, &fev_cond_wait_recheck, mutex); 71 | 72 | if (res == -ENOMEM || res == -ETIMEDOUT) 73 | return res; 74 | 75 | /* 76 | * We were woken up by fev_cond_notify_one()/all() or spuriously, in both cases we need to 77 | * reacquire the mutex. 78 | */ 79 | FEV_ASSERT(res == 0 || res == -EAGAIN); 80 | return fev_mutex_try_lock_until(mutex, abs_time); 81 | } 82 | 83 | FEV_NONNULL(1, 2, 3) 84 | int fev_cond_wait_for(struct fev_cond *cond, struct fev_mutex *mutex, 85 | const struct timespec *rel_time) 86 | { 87 | struct timespec abs_time; 88 | 89 | fev_get_abs_time_since_now(&abs_time, rel_time); 90 | return fev_cond_wait_until(cond, mutex, &abs_time); 91 | } 92 | 93 | FEV_NONNULL(1) void fev_cond_notify_one(struct fev_cond *cond) 94 | { 95 | fev_waiters_queue_wake(&cond->wq, /*max_waiters=*/1, /*callback=*/NULL, /*callback_arg=*/NULL); 96 | } 97 | 98 | FEV_NONNULL(1) void fev_cond_notify_all(struct fev_cond *cond) 99 | { 100 | fev_waiters_queue_wake(&cond->wq, /*max_waiters=*/UINT32_MAX, /*callback=*/NULL, 101 | /*callback_arg=*/NULL); 102 | } 103 | -------------------------------------------------------------------------------- /src/fev_cond_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_COND_IMPL_H 11 | #define FEV_COND_IMPL_H 12 | 13 | #include "fev_cond_intf.h" 14 | 15 | #include "fev_compiler.h" 16 | #include "fev_waiters_queue_impl.h" 17 | 18 | FEV_NONNULL(1) FEV_WARN_UNUSED_RESULT static inline int fev_cond_init(struct fev_cond *cond) 19 | { 20 | return fev_waiters_queue_init(&cond->wq); 21 | } 22 | 23 | FEV_NONNULL(1) static inline void fev_cond_fini(struct fev_cond *cond) 24 | { 25 | fev_waiters_queue_fini(&cond->wq); 26 | } 27 | 28 | #endif /* !FEV_COND_IMPL_H */ 29 | -------------------------------------------------------------------------------- /src/fev_cond_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_COND_INTF_H 11 | #define FEV_COND_INTF_H 12 | 13 | #include 14 | 15 | #include "fev_waiters_queue_intf.h" 16 | 17 | struct fev_cond { 18 | struct fev_waiters_queue wq; 19 | }; 20 | 21 | #endif /* !FEV_COND_INTF_H */ 22 | -------------------------------------------------------------------------------- /src/fev_context.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_CONTEXT_H 11 | #define FEV_CONTEXT_H 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include "fev_compiler.h" 19 | 20 | #if defined(FEV_ARCH_X86_64) 21 | 22 | struct fev_context { 23 | uint32_t mxcsr; 24 | uint16_t fpucw; 25 | uint16_t _pad; 26 | uint64_t rsp; 27 | uint64_t rbp; 28 | uint64_t rbx; 29 | uint64_t r12; 30 | uint64_t r13; 31 | uint64_t r14; 32 | uint64_t r15; 33 | 34 | #ifdef FEV_ENABLE_ASAN 35 | void *fake_stack; 36 | const void *stack_bottom; 37 | size_t stack_size; 38 | #endif 39 | }; 40 | 41 | #elif defined(FEV_ARCH_I386) 42 | 43 | /* TODO: Add support for ASAN. */ 44 | struct fev_context { 45 | uint32_t mxcsr; 46 | uint16_t fpucw; 47 | uint16_t _pad; 48 | uint32_t esp; 49 | uint32_t ebp; 50 | uint32_t ebx; 51 | uint32_t edi; 52 | uint32_t esi; 53 | }; 54 | 55 | #else /* !FEV_ARCH_X86_64 && !FEV_ARCH_I386 */ 56 | #error The architecture is unsupported 57 | #endif /* FEV_ARCH_X86_64 */ 58 | 59 | FEV_NONNULL(1, 2, 4) 60 | void fev_context_init(struct fev_context *restrict context, uint8_t *restrict stack_bottom, 61 | size_t stack_size, const void *restrict start_addr); 62 | 63 | FEV_NONNULL(1, 2) 64 | void fev_context_switch(struct fev_context *restrict from, struct fev_context *restrict to); 65 | 66 | FEV_NONNULL(2, 3, 4) 67 | void fev_context_switch_and_call(void *restrict post_arg, void *restrict post_routine, 68 | struct fev_context *restrict from, 69 | struct fev_context *restrict to); 70 | 71 | #endif /* !FEV_CONTEXT_H */ 72 | -------------------------------------------------------------------------------- /src/fev_context_i386.S: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | .text 11 | 12 | 13 | .p2align 4,,15 14 | 15 | #ifdef __APPLE__ 16 | .globl _fev_context_switch 17 | #else 18 | .globl fev_context_switch 19 | #endif 20 | 21 | #ifdef __ELF__ 22 | .type fev_context_switch,@function 23 | #endif 24 | 25 | #ifdef __APPLE__ 26 | _fev_context_switch: 27 | #else 28 | fev_context_switch: 29 | #endif 30 | 31 | movl 4(%esp), %eax 32 | movl 8(%esp), %ecx 33 | 34 | stmxcsr (%eax) 35 | fnstcw 4(%eax) 36 | movl %esp, 8(%eax) 37 | movl %ebp, 12(%eax) 38 | movl %ebx, 16(%eax) 39 | movl %edi, 20(%eax) 40 | movl %esi, 24(%eax) 41 | 42 | ldmxcsr (%ecx) 43 | fldcw 4(%ecx) 44 | movl 8(%ecx), %esp 45 | movl 12(%ecx), %ebp 46 | movl 16(%ecx), %ebx 47 | movl 20(%ecx), %edi 48 | movl 24(%ecx), %esi 49 | 50 | ret 51 | 52 | #ifdef __ELF__ 53 | .size fev_context_switch,.-fev_context_switch 54 | #endif 55 | 56 | 57 | .p2align 4,,15 58 | 59 | #ifdef __APPLE__ 60 | .globl _fev_context_switch_and_call 61 | #else 62 | .globl fev_context_switch_and_call 63 | #endif 64 | 65 | #ifdef __ELF__ 66 | .type fev_context_switch_and_call,@function 67 | #endif 68 | 69 | #ifdef __APPLE__ 70 | _fev_context_switch_and_call: 71 | #else 72 | fev_context_switch_and_call: 73 | #endif 74 | 75 | movl 12(%esp), %eax 76 | movl 16(%esp), %ecx 77 | 78 | stmxcsr (%eax) 79 | fnstcw 4(%eax) 80 | movl %esp, 8(%eax) 81 | movl %ebp, 12(%eax) 82 | movl %ebx, 16(%eax) 83 | movl %edi, 20(%eax) 84 | movl %esi, 24(%eax) 85 | 86 | mov 4(%esp), %eax 87 | mov 8(%esp), %edx 88 | 89 | ldmxcsr (%ecx) 90 | fldcw 4(%ecx) 91 | movl 8(%ecx), %esp 92 | movl 12(%ecx), %ebp 93 | movl 16(%ecx), %ebx 94 | movl 20(%ecx), %edi 95 | movl 24(%ecx), %esi 96 | 97 | pushl %eax 98 | call *%edx 99 | popl %eax 100 | 101 | ret 102 | 103 | #ifdef __ELF__ 104 | .size fev_context_switch_and_call,.-fev_context_switch_and_call 105 | #endif 106 | 107 | 108 | #if defined(__linux__) && defined(__ELF__) 109 | .section .note.GNU-stack,"",%progbits 110 | #endif 111 | -------------------------------------------------------------------------------- /src/fev_context_x86_64_posix.S: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include 11 | 12 | .text 13 | 14 | 15 | .p2align 4,,15 16 | 17 | #ifdef __APPLE__ 18 | .globl _fev_context_switch 19 | #else 20 | .globl fev_context_switch 21 | #endif 22 | 23 | #ifdef __ELF__ 24 | .type fev_context_switch,@function 25 | #endif 26 | 27 | #ifdef __APPLE__ 28 | _fev_context_switch: 29 | #else 30 | fev_context_switch: 31 | #endif 32 | 33 | #ifdef FEV_ENABLE_ASAN 34 | pushq %rdi 35 | pushq %rsi 36 | leaq 64(%rdi), %rdi /* &from->fake_stack */ 37 | movq 80(%rsi), %rdx /* to->stack_size */ 38 | movq 72(%rsi), %rsi /* to->stack_bottom */ 39 | call __sanitizer_start_switch_fiber 40 | popq %rsi 41 | popq %rdi 42 | #endif 43 | 44 | stmxcsr (%rdi) 45 | fnstcw 4(%rdi) 46 | movq %rsp, 8(%rdi) 47 | movq %rbp, 16(%rdi) 48 | movq %rbx, 24(%rdi) 49 | movq %r12, 32(%rdi) 50 | movq %r13, 40(%rdi) 51 | movq %r14, 48(%rdi) 52 | movq %r15, 56(%rdi) 53 | 54 | ldmxcsr (%rsi) 55 | fldcw 4(%rsi) 56 | movq 8(%rsi), %rsp 57 | movq 16(%rsi), %rbp 58 | movq 24(%rsi), %rbx 59 | movq 32(%rsi), %r12 60 | movq 40(%rsi), %r13 61 | movq 48(%rsi), %r14 62 | movq 56(%rsi), %r15 63 | 64 | #ifdef FEV_ENABLE_ASAN 65 | movq %rdi, %rax 66 | movq 64(%rsi), %rdi /* to->fake_stack */ 67 | leaq 72(%rax), %rsi /* &from->stack_bottom */ 68 | leaq 80(%rax), %rdx /* &from->stack_size */ 69 | call __sanitizer_finish_switch_fiber 70 | #endif 71 | 72 | ret 73 | 74 | #ifdef __ELF__ 75 | .size fev_context_switch,.-fev_context_switch 76 | #endif 77 | 78 | 79 | .p2align 4,,15 80 | 81 | #ifdef __APPLE__ 82 | .globl _fev_context_switch_and_call 83 | #else 84 | .globl fev_context_switch_and_call 85 | #endif 86 | 87 | #ifdef __ELF__ 88 | .type fev_context_switch_and_call,@function 89 | #endif 90 | 91 | #ifdef __APPLE__ 92 | _fev_context_switch_and_call: 93 | #else 94 | fev_context_switch_and_call: 95 | #endif 96 | 97 | #ifdef FEV_ENABLE_ASAN 98 | pushq %rdi 99 | pushq %rsi 100 | pushq %rdx 101 | pushq %rcx 102 | leaq 64(%rdx), %rdi /* &from->fake_stack */ 103 | movq 72(%rcx), %rsi /* to->stack_bottom */ 104 | movq 80(%rcx), %rdx /* to->stack_size */ 105 | call __sanitizer_start_switch_fiber 106 | popq %rcx 107 | popq %rdx 108 | popq %rsi 109 | popq %rdi 110 | #endif 111 | 112 | stmxcsr (%rdx) 113 | fnstcw 4(%rdx) 114 | movq %rsp, 8(%rdx) 115 | movq %rbp, 16(%rdx) 116 | movq %rbx, 24(%rdx) 117 | movq %r12, 32(%rdx) 118 | movq %r13, 40(%rdx) 119 | movq %r14, 48(%rdx) 120 | movq %r15, 56(%rdx) 121 | 122 | ldmxcsr (%rcx) 123 | fldcw 4(%rcx) 124 | movq 8(%rcx), %rsp 125 | movq 16(%rcx), %rbp 126 | movq 24(%rcx), %rbx 127 | movq 32(%rcx), %r12 128 | movq 40(%rcx), %r13 129 | movq 48(%rcx), %r14 130 | movq 56(%rcx), %r15 131 | 132 | #ifdef FEV_ENABLE_ASAN 133 | pushq %rdi 134 | pushq %rsi 135 | movq 64(%rcx), %rdi /* to->fake_stack */ 136 | leaq 72(%rdx), %rsi /* &from->stack_bottom */ 137 | leaq 80(%rdx), %rdx /* &from->stack_size */ 138 | call __sanitizer_finish_switch_fiber 139 | popq %rsi 140 | popq %rdi 141 | #endif 142 | 143 | jmp *%rsi 144 | 145 | #ifdef __ELF__ 146 | .size fev_context_switch_and_call,.-fev_context_switch_and_call 147 | #endif 148 | 149 | 150 | #if defined(__linux__) && defined(__ELF__) 151 | .section .note.GNU-stack,"",%progbits 152 | #endif 153 | -------------------------------------------------------------------------------- /src/fev_context_x86_family.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_context.h" 11 | 12 | #include 13 | 14 | #include 15 | #include 16 | 17 | #include "fev_assert.h" 18 | #include "fev_compiler.h" 19 | 20 | FEV_NONNULL(1, 2, 4) 21 | void fev_context_init(struct fev_context *restrict context, uint8_t *restrict stack_bottom, 22 | size_t stack_size, const void *restrict start_addr) 23 | { 24 | uintptr_t *stack_ptr; 25 | 26 | FEV_ASSERT((uintptr_t)stack_bottom <= UINTPTR_MAX - stack_size); 27 | 28 | stack_ptr = (uintptr_t *)(stack_bottom + stack_size); 29 | FEV_ASSERT((uintptr_t)stack_ptr % sizeof(*stack_ptr) == 0); 30 | 31 | *--stack_ptr = 0xdeadbabe; 32 | *--stack_ptr = (uintptr_t)start_addr; 33 | 34 | memset(context, 0, sizeof(*context)); 35 | 36 | #if defined(FEV_ARCH_I386) 37 | context->esp = (uintptr_t)stack_ptr; 38 | #elif defined(FEV_ARCH_X86_64) 39 | context->rsp = (uintptr_t)stack_ptr; 40 | #endif 41 | 42 | #if defined(FEV_ARCH_X86_64) && defined(FEV_ENABLE_ASAN) 43 | context->stack_bottom = stack_bottom; 44 | context->stack_size = stack_size; 45 | #endif 46 | } 47 | -------------------------------------------------------------------------------- /src/fev_fiber.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_FIBER_H 11 | #define FEV_FIBER_H 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | #include "fev_cond_intf.h" 22 | #include "fev_context.h" 23 | #include "fev_mutex_intf.h" 24 | 25 | /* Fiber flags */ 26 | enum { 27 | /* The fiber has exited. */ 28 | FEV_FIBER_DEAD = 1 << 0, 29 | 30 | /* The fiber is joinable (not detached). */ 31 | FEV_FIBER_JOINABLE = 1 << 1, 32 | 33 | /* Another fiber is joining on the fiber. */ 34 | FEV_FIBER_JOINING = 1 << 2, 35 | }; 36 | 37 | struct fev_fiber { 38 | union { 39 | STAILQ_ENTRY(fev_fiber) stq_entry; 40 | TAILQ_ENTRY(fev_fiber) tq_entry; 41 | }; 42 | 43 | /* Fiber's arch-specific context (registers, PC etc.). */ 44 | struct fev_context context; 45 | 46 | /* Stack address and its total size (usable and guard size). */ 47 | void *stack_addr; 48 | size_t total_stack_size; 49 | 50 | /* 51 | * If true, the fiber is using the user supplied stack (in 'attr' parameter). If false, the stack 52 | * was allocated in fev_fiber_create(). 53 | */ 54 | bool user_stack; 55 | 56 | /* The start routine of the fiber and its argument and return value. */ 57 | void *(*start_routine)(void *); 58 | void *arg; 59 | void *return_value; 60 | 61 | /* Fiber flags (see above). */ 62 | int flags; 63 | 64 | /* Synchronization for joining the fiber. */ 65 | struct fev_cond cond; 66 | struct fev_mutex mutex; 67 | 68 | /* Number of refs, the fiber itself + joiner (if not detached). */ 69 | atomic_uint ref_count; 70 | }; 71 | 72 | typedef STAILQ_HEAD(fev_fiber_stq_head, fev_fiber) fev_fiber_stq_head_t; 73 | typedef TAILQ_HEAD(fev_fiber_tq_head, fev_fiber) fev_fiber_tq_head_t; 74 | 75 | #endif /* !FEV_FIBER_H */ 76 | -------------------------------------------------------------------------------- /src/fev_fiber_attr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_fiber_attr.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "fev_alloc.h" 20 | #include "fev_compiler.h" 21 | 22 | static_assert(FEV_DEFAULT_STACK_SIZE % FEV_PAGE_SIZE == 0, 23 | "FEV_DEFAULT_STACK_SIZE must be multiple of FEV_PAGE_SIZE"); 24 | static_assert(FEV_DEFAULT_GUARD_SIZE % FEV_PAGE_SIZE == 0, 25 | "FEV_DEFAULT_GUARD_SIZE must be multiple of FEV_PAGE_SIZE"); 26 | 27 | const struct fev_fiber_attr fev_fiber_create_default_attr = { 28 | .stack_addr = NULL, 29 | .stack_size = FEV_DEFAULT_STACK_SIZE, 30 | .guard_size = FEV_DEFAULT_GUARD_SIZE, 31 | .detached = false, 32 | }; 33 | 34 | const struct fev_fiber_attr fev_fiber_spawn_default_attr = { 35 | .stack_addr = NULL, 36 | .stack_size = FEV_DEFAULT_STACK_SIZE, 37 | .guard_size = FEV_DEFAULT_GUARD_SIZE, 38 | .detached = true, 39 | }; 40 | 41 | FEV_NONNULL(1) int fev_fiber_attr_create(struct fev_fiber_attr **attr_ptr) 42 | { 43 | struct fev_fiber_attr *attr; 44 | 45 | attr = fev_malloc(sizeof(*attr)); 46 | if (FEV_UNLIKELY(attr == NULL)) 47 | return -ENOMEM; 48 | 49 | /* 50 | * Zero-initialize the padding to not leak any information. In C11 no guarantee is given for the 51 | * bits of the padding of fev_fiber_create_default_attr, thus a memcpy is not enough. 52 | */ 53 | memset(attr, 0, sizeof(*attr)); 54 | attr->stack_addr = fev_fiber_create_default_attr.stack_addr; 55 | attr->stack_size = fev_fiber_create_default_attr.stack_size; 56 | attr->guard_size = fev_fiber_create_default_attr.guard_size; 57 | attr->detached = fev_fiber_create_default_attr.detached; 58 | 59 | *attr_ptr = attr; 60 | return 0; 61 | } 62 | 63 | FEV_NONNULL(1) void fev_fiber_attr_destroy(struct fev_fiber_attr *attr) { fev_free(attr); } 64 | 65 | FEV_NONNULL(1, 2, 3) 66 | void fev_fiber_attr_get_stack(const struct fev_fiber_attr *attr, void **stack_addr, 67 | size_t *stack_size) 68 | { 69 | *stack_addr = attr->stack_addr; 70 | *stack_size = attr->stack_size; 71 | } 72 | 73 | FEV_NONNULL(1, 2) 74 | int fev_fiber_attr_set_stack(struct fev_fiber_attr *attr, void *stack_addr, size_t stack_size) 75 | { 76 | if (FEV_UNLIKELY((uintptr_t)stack_addr % FEV_PAGE_SIZE != 0)) 77 | return -EINVAL; 78 | 79 | if (FEV_UNLIKELY(stack_size % FEV_PAGE_SIZE != 0)) 80 | return -EINVAL; 81 | 82 | attr->stack_addr = stack_addr; 83 | attr->stack_size = stack_size; 84 | return 0; 85 | } 86 | 87 | FEV_NONNULL(1) FEV_PURE size_t fev_fiber_attr_get_stack_size(const struct fev_fiber_attr *attr) 88 | { 89 | return attr->stack_size; 90 | } 91 | 92 | FEV_NONNULL(1) int fev_fiber_attr_set_stack_size(struct fev_fiber_attr *attr, size_t stack_size) 93 | { 94 | if (FEV_UNLIKELY(stack_size % FEV_PAGE_SIZE != 0)) 95 | return -EINVAL; 96 | 97 | attr->stack_size = stack_size; 98 | return 0; 99 | } 100 | 101 | FEV_NONNULL(1) FEV_PURE size_t fev_fiber_attr_get_guard_size(const struct fev_fiber_attr *attr) 102 | { 103 | return attr->guard_size; 104 | } 105 | 106 | FEV_NONNULL(1) int fev_fiber_attr_set_guard_size(struct fev_fiber_attr *attr, size_t guard_size) 107 | { 108 | if (FEV_UNLIKELY(guard_size % FEV_PAGE_SIZE != 0)) 109 | return -EINVAL; 110 | 111 | attr->guard_size = guard_size; 112 | return 0; 113 | } 114 | 115 | FEV_NONNULL(1) FEV_PURE bool fev_fiber_attr_get_detached(const struct fev_fiber_attr *attr) 116 | { 117 | return attr->detached; 118 | } 119 | 120 | FEV_NONNULL(1) void fev_fiber_attr_set_detached(struct fev_fiber_attr *attr, bool detached) 121 | { 122 | attr->detached = detached; 123 | } 124 | -------------------------------------------------------------------------------- /src/fev_fiber_attr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_FIBER_ATTR_H 11 | #define FEV_FIBER_ATTR_H 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | struct fev_fiber_attr { 19 | void *stack_addr; 20 | size_t stack_size; 21 | size_t guard_size; 22 | bool detached; 23 | }; 24 | 25 | extern const struct fev_fiber_attr fev_fiber_create_default_attr; 26 | extern const struct fev_fiber_attr fev_fiber_spawn_default_attr; 27 | 28 | #endif /* !FEV_FIBER_ATTR_H */ 29 | -------------------------------------------------------------------------------- /src/fev_ilock.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_ilock_impl.h" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | #include "fev_assert.h" 19 | #include "fev_compiler.h" 20 | #include "fev_context.h" 21 | #include "fev_fiber.h" 22 | #include "fev_sched_intf.h" 23 | 24 | FEV_NONNULL(1) static void fev_ilock_lock_post(fev_ilock_lock_t *lock) 25 | { 26 | struct fev_sched_worker *cur_worker; 27 | struct fev_sched *sched; 28 | 29 | cur_worker = fev_cur_sched_worker; 30 | FEV_ASSERT(cur_worker != NULL); 31 | 32 | sched = cur_worker->sched; 33 | FEV_ASSERT(sched != NULL); 34 | 35 | /* Decrease the ready counter, fev_wake_one/_stq will increase it. */ 36 | atomic_fetch_sub_explicit(&sched->num_run_fibers, 1, memory_order_relaxed); 37 | 38 | fev_ilock_lock_unlock(lock); 39 | } 40 | 41 | FEV_NONNULL(1) bool fev_ilock_lock_slow(struct fev_ilock *ilock) 42 | { 43 | struct fev_sched_worker *cur_worker; 44 | struct fev_fiber *cur_fiber; 45 | unsigned state; 46 | 47 | cur_worker = fev_cur_sched_worker; 48 | FEV_ASSERT(cur_worker != NULL); 49 | 50 | cur_fiber = cur_worker->cur_fiber; 51 | FEV_ASSERT(cur_fiber != NULL); 52 | 53 | fev_ilock_lock_lock(&ilock->lock); 54 | 55 | /* Update the state to 2 (locked, some waiters), as we are appending a waiter. */ 56 | state = atomic_exchange_explicit(&ilock->state, 2, memory_order_relaxed); 57 | if (state == 0) { 58 | /* 59 | * The lock was unlocked inbetween the two swaps, update the state to 1 (locked, no waiters) and 60 | * return. 61 | */ 62 | atomic_store_explicit(&ilock->state, 1, memory_order_relaxed); 63 | fev_ilock_lock_unlock(&ilock->lock); 64 | atomic_thread_fence(memory_order_acquire); 65 | return false; 66 | } 67 | 68 | /* Append the waiter. */ 69 | STAILQ_INSERT_TAIL(&ilock->waiters, cur_fiber, stq_entry); 70 | 71 | /* 72 | * The spinlock must be unlocked after the context switch. Otherwise, a worker may be switching to 73 | * this fiber (after the ilock was unlocked and the waiter was woken up) while we are switching to 74 | * the scheduler at the same time. 75 | */ 76 | fev_context_switch_and_call(&ilock->lock, &fev_ilock_lock_post, &cur_fiber->context, 77 | &cur_worker->context); 78 | 79 | /* At this point we own the ilock. */ 80 | FEV_ASSERT(atomic_load(&ilock->state) > 0); 81 | 82 | atomic_thread_fence(memory_order_acquire); 83 | return true; 84 | } 85 | 86 | FEV_NONNULL(1) 87 | FEV_RETURNS_NONNULL FEV_WARN_UNUSED_RESULT struct fev_fiber * 88 | fev_ilock_unlock_slow(struct fev_ilock *ilock) 89 | { 90 | struct fev_fiber *fiber; 91 | 92 | fev_ilock_lock_lock(&ilock->lock); 93 | 94 | /* 95 | * At this point the state must be 2 (locked, some waiters): 96 | * 1. The ilock must be locked, thus the state must be at least 1. 97 | * 2. We have failed to swap from 1 to 0 in the fast path. 98 | * 3. The lock function cannot update the state from 2 to 1. 99 | */ 100 | FEV_ASSERT(atomic_load(&ilock->state) == 2); 101 | 102 | /* Get the first waiting fiber. */ 103 | fiber = STAILQ_FIRST(&ilock->waiters); 104 | FEV_ASSERT(fiber != NULL); 105 | 106 | /* Remove the queue's head. */ 107 | if ((ilock->waiters.stqh_first = fiber->stq_entry.stqe_next) == NULL) { 108 | /* The queue is empty. */ 109 | ilock->waiters.stqh_last = &ilock->waiters.stqh_first; 110 | 111 | /* No waiters, update the state to 1 (locked, no waiters). */ 112 | atomic_store_explicit(&ilock->state, 1, memory_order_relaxed); 113 | } 114 | 115 | fev_ilock_lock_unlock(&ilock->lock); 116 | 117 | return fiber; 118 | } 119 | -------------------------------------------------------------------------------- /src/fev_ilock_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_ILOCK_IMPL_H 11 | #define FEV_ILOCK_IMPL_H 12 | 13 | #include "fev_ilock_intf.h" 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | #include "fev_compiler.h" 22 | #include "fev_fiber.h" 23 | #include "fev_sched_impl.h" 24 | #include "fev_spinlock_impl.h" 25 | #include "fev_thr_mutex.h" 26 | 27 | /* TODO: Maybe rename the functions... */ 28 | #if defined(FEV_ILOCK_LOCK_MUTEX) 29 | #define fev_ilock_lock_init fev_thr_mutex_init 30 | #define fev_ilock_lock_fini fev_thr_mutex_fini 31 | #define fev_ilock_lock_lock fev_thr_mutex_lock 32 | #define fev_ilock_lock_unlock fev_thr_mutex_unlock 33 | #elif defined(FEV_ILOCK_LOCK_SPINLOCK) 34 | #define fev_ilock_lock_init fev_spinlock_init 35 | #define fev_ilock_lock_fini fev_spinlock_fini 36 | #define fev_ilock_lock_lock fev_spinlock_lock 37 | #define fev_ilock_lock_unlock fev_spinlock_unlock 38 | #endif 39 | 40 | FEV_NONNULL(1) FEV_WARN_UNUSED_RESULT static inline int fev_ilock_init(struct fev_ilock *ilock) 41 | { 42 | int ret = fev_ilock_lock_init(&ilock->lock); 43 | if (FEV_UNLIKELY(ret != 0)) 44 | return ret; 45 | 46 | atomic_init(&ilock->state, 0); 47 | STAILQ_INIT(&ilock->waiters); 48 | return 0; 49 | } 50 | 51 | FEV_NONNULL(1) static inline void fev_ilock_fini(struct fev_ilock *ilock) 52 | { 53 | fev_ilock_lock_fini(&ilock->lock); 54 | } 55 | 56 | /* 57 | * Locks the internal lock. Returns true, if we switched back to the scheduler and were blocked for 58 | * some time. Returns false, if the internal lock was acquired without switching and blocking. After 59 | * returning, the internal lock is owned by the caller, there are no spurious wake ups. 60 | */ 61 | FEV_NONNULL(1) static inline bool fev_ilock_lock(struct fev_ilock *ilock) 62 | { 63 | unsigned expected = 0; 64 | bool success; 65 | 66 | success = atomic_compare_exchange_weak_explicit(&ilock->state, &expected, 1, memory_order_acquire, 67 | memory_order_relaxed); 68 | if (FEV_LIKELY(success)) 69 | return false; 70 | return fev_ilock_lock_slow(ilock); 71 | } 72 | 73 | /* 74 | * Unlocks the internal lock and returns the next fiber in the waiters queue, i.e. the fiber that 75 | * now owns the lock and should be woken up by the caller, or returns NULL if there is no such 76 | * fiber. 77 | */ 78 | FEV_NONNULL(1) 79 | FEV_WARN_UNUSED_RESULT static inline struct fev_fiber *fev_ilock_unlock(struct fev_ilock *ilock) 80 | { 81 | unsigned expected = 1; 82 | bool success; 83 | 84 | success = atomic_compare_exchange_strong_explicit(&ilock->state, &expected, 0, 85 | memory_order_release, memory_order_relaxed); 86 | if (FEV_LIKELY(success)) 87 | return NULL; 88 | return fev_ilock_unlock_slow(ilock); 89 | } 90 | 91 | /* Unlocks the internal lock and wakes up the next fiber in the waiters queue. */ 92 | FEV_NONNULL(1) static inline void fev_ilock_unlock_and_wake(struct fev_ilock *ilock) 93 | { 94 | struct fev_fiber *fiber; 95 | 96 | fiber = fev_ilock_unlock(ilock); 97 | if (fiber != NULL) 98 | fev_cur_wake_one(fiber); 99 | } 100 | 101 | #endif /* !FEV_ILOCK_IMPL_H */ 102 | -------------------------------------------------------------------------------- /src/fev_ilock_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_ILOCK_INTF_H 11 | #define FEV_ILOCK_INTF_H 12 | 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | #include "fev_compiler.h" 19 | #include "fev_spinlock_intf.h" 20 | #include "fev_thr_mutex.h" 21 | 22 | #if defined(FEV_ILOCK_LOCK_MUTEX) 23 | typedef struct fev_thr_mutex fev_ilock_lock_t; 24 | #elif defined(FEV_ILOCK_LOCK_SPINLOCK) 25 | typedef struct fev_spinlock fev_ilock_lock_t; 26 | #else 27 | #error Wrong lock strategy for ilock selected, define either FEV_ILOCK_LOCK_MUTEX or \ 28 | FEV_ILOCK_LOCK_SPINLOCK. 29 | #endif 30 | 31 | /* Internal lock. It is used to implement higher-level primitives. */ 32 | 33 | struct fev_fiber; 34 | 35 | struct fev_ilock { 36 | /* 37 | * State of the lock: 38 | * 0 - unlocked 39 | * 1 - locked, no waiters 40 | * 2 - locked, some waiters 41 | */ 42 | atomic_uint state; 43 | 44 | fev_ilock_lock_t lock; 45 | STAILQ_HEAD(, fev_fiber) waiters; 46 | }; 47 | 48 | /* Slow path variants, don't use directly. Use the functions defined in fev_ilock_impl.h. */ 49 | 50 | FEV_NONNULL(1) bool fev_ilock_lock_slow(struct fev_ilock *ilock); 51 | 52 | FEV_NONNULL(1) 53 | FEV_RETURNS_NONNULL FEV_WARN_UNUSED_RESULT struct fev_fiber * 54 | fev_ilock_unlock_slow(struct fev_ilock *ilock); 55 | 56 | #endif /* !FEV_ILOCK_INTF_H */ 57 | -------------------------------------------------------------------------------- /src/fev_mutex_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_MUTEX_IMPL_H 11 | #define FEV_MUTEX_IMPL_H 12 | 13 | #include "fev_mutex_intf.h" 14 | 15 | #include 16 | 17 | #include "fev_compiler.h" 18 | #include "fev_waiters_queue_impl.h" 19 | 20 | FEV_NONNULL(1) FEV_WARN_UNUSED_RESULT static inline int fev_mutex_init(struct fev_mutex *mutex) 21 | { 22 | int ret = fev_waiters_queue_init(&mutex->wq); 23 | if (FEV_UNLIKELY(ret != 0)) 24 | return ret; 25 | 26 | atomic_init(&mutex->state, 0); 27 | return 0; 28 | } 29 | 30 | FEV_NONNULL(1) static inline void fev_mutex_fini(struct fev_mutex *mutex) 31 | { 32 | fev_waiters_queue_fini(&mutex->wq); 33 | } 34 | 35 | #endif /* !FEV_MUTEX_IMPL_H */ 36 | -------------------------------------------------------------------------------- /src/fev_mutex_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_MUTEX_INTF_H 11 | #define FEV_MUTEX_INTF_H 12 | 13 | #include 14 | 15 | #include 16 | 17 | #include "fev_waiters_queue_intf.h" 18 | 19 | struct fev_mutex { 20 | /* 21 | * State of the lock: 22 | * 0 - unlocked 23 | * 1 - locked, no waiters 24 | * 2 - locked, some waiters 25 | */ 26 | atomic_uint state; 27 | 28 | struct fev_waiters_queue wq; 29 | }; 30 | 31 | #endif /* !FEV_MUTEX_INTF_H */ 32 | -------------------------------------------------------------------------------- /src/fev_os.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_OS_H 11 | #define FEV_OS_H 12 | 13 | #include 14 | 15 | uint32_t fev_get_num_processors(void); 16 | 17 | #endif /* !FEV_OS_H */ 18 | -------------------------------------------------------------------------------- /src/fev_os_nix.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_os.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "fev_compiler.h" 21 | 22 | uint32_t fev_get_num_processors(void) 23 | { 24 | long value; 25 | 26 | errno = 0; 27 | value = sysconf(_SC_NPROCESSORS_ONLN); 28 | if (FEV_UNLIKELY(value == -1)) { 29 | /* errno theoretically can be 0 here. */ 30 | fprintf(stderr, "Failed to get number of processors: %s\n", strerror(errno)); 31 | abort(); 32 | } 33 | 34 | if (FEV_UNLIKELY(value < 1)) { 35 | fprintf(stderr, "Got %li as number of processors, should be at least 1\n", value); 36 | abort(); 37 | } 38 | 39 | #if LONG_MAX > UINT32_MAX 40 | if (FEV_UNLIKELY(value >= UINT32_MAX)) 41 | value = UINT32_MAX; 42 | #endif 43 | 44 | return (uint32_t)value; 45 | } 46 | -------------------------------------------------------------------------------- /src/fev_os_win.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_os.h" 11 | 12 | #include 13 | #include 14 | 15 | uint32_t fev_get_num_processors(void) 16 | { 17 | SYSTEM_INFO info; 18 | GetSystemInfo(&info); 19 | return info.dwNumberOfProcessors; 20 | } 21 | -------------------------------------------------------------------------------- /src/fev_poller.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_POLLER_H 11 | #define FEV_POLLER_H 12 | 13 | #include 14 | 15 | #if defined(FEV_POLLER_EPOLL) 16 | #include "fev_poller_epoll.h" 17 | #include "fev_poller_reactor.h" 18 | #elif defined(FEV_POLLER_KQUEUE) 19 | #include "fev_poller_kqueue.h" 20 | #include "fev_poller_reactor.h" 21 | #elif defined(FEV_POLLER_IO_URING) 22 | #include "fev_poller_io_uring.h" 23 | #else 24 | #error Wrong poller 25 | #endif 26 | 27 | #endif /* !FEV_POLLER_H */ 28 | -------------------------------------------------------------------------------- /src/fev_poller_epoll.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_POLLER_EPOLL_H 11 | #define FEV_POLLER_EPOLL_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_compiler.h" 18 | #include "fev_qsbr.h" 19 | #include "fev_time.h" 20 | 21 | struct fev_sched_worker; 22 | struct fev_timers_bucket; 23 | struct fev_socket; 24 | 25 | /* The maxevents param when calling epoll_wait(). */ 26 | #ifndef FEV_POLLER_MAX_EVENTS 27 | #define FEV_POLLER_MAX_EVENTS 64u 28 | #endif 29 | 30 | enum fev_poller_flag { 31 | FEV_POLLER_IN = EPOLLIN, 32 | FEV_POLLER_OUT = EPOLLOUT, 33 | }; 34 | 35 | struct fev_poller { 36 | int epoll_fd; 37 | int event_fd; 38 | 39 | struct fev_qsbr_global sockets_global_qsbr; 40 | _Atomic uint32_t num_sockets_to_free; 41 | }; 42 | 43 | struct fev_worker_poller_data { 44 | int epoll_fd; 45 | int event_fd; 46 | struct fev_qsbr_local sockets_local_qsbr; 47 | }; 48 | 49 | struct fev_poller_timers_bucket_data { 50 | int timer_fd; 51 | }; 52 | 53 | FEV_COLD FEV_NONNULL(1) int fev_poller_init(struct fev_sched *sched); 54 | 55 | FEV_COLD FEV_NONNULL(1) void fev_poller_fini(struct fev_sched *sched); 56 | 57 | FEV_NONNULL(1, 2) 58 | int fev_poller_register(const struct fev_sched_worker *worker, struct fev_socket *socket, 59 | enum fev_poller_flag flag); 60 | 61 | FEV_NONNULL(1, 2) 62 | void fev_poller_set_timeout(const struct fev_timers_bucket *bucket, 63 | const struct timespec *abs_time); 64 | 65 | FEV_NONNULL(1) void fev_poller_interrupt(const struct fev_poller *poller); 66 | 67 | FEV_NONNULL(1) void fev_poller_process(struct fev_sched_worker *worker, int timeout); 68 | 69 | FEV_NONNULL(1) static inline void fev_poller_check(struct fev_sched_worker *worker) 70 | { 71 | /* epoll_wait() won't block. */ 72 | fev_poller_process(worker, /*timeout=*/0); 73 | } 74 | 75 | FEV_NONNULL(1) static inline void fev_poller_wait(struct fev_sched_worker *worker) 76 | { 77 | /* epoll_wait() will block indefinitely. */ 78 | fev_poller_process(worker, /*timeout=*/-1); 79 | } 80 | 81 | #endif /* !FEV_POLLER_EPOLL_H */ 82 | -------------------------------------------------------------------------------- /src/fev_poller_kqueue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_POLLER_KQUEUE_H 11 | #define FEV_POLLER_KQUEUE_H 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "fev_compiler.h" 19 | #include "fev_qsbr.h" 20 | #include "fev_time.h" 21 | 22 | struct fev_sched_worker; 23 | struct fev_timers_bucket; 24 | struct fev_socket; 25 | 26 | /* The nevents param when calling kqueue(). */ 27 | #ifndef FEV_POLLER_MAX_EVENTS 28 | #define FEV_POLLER_MAX_EVENTS 64u 29 | #endif 30 | 31 | enum fev_poller_flag { 32 | FEV_POLLER_IN = EVFILT_READ, 33 | FEV_POLLER_OUT = EVFILT_WRITE, 34 | }; 35 | 36 | struct fev_poller { 37 | int kqueue_fd; 38 | 39 | struct fev_qsbr_global sockets_global_qsbr; 40 | _Atomic uint32_t num_sockets_to_free; 41 | }; 42 | 43 | struct fev_worker_poller_data { 44 | int kqueue_fd; 45 | struct fev_qsbr_local sockets_local_qsbr; 46 | }; 47 | 48 | struct fev_poller_timers_bucket_data { 49 | int kqueue_fd; 50 | }; 51 | 52 | FEV_COLD FEV_NONNULL(1) int fev_poller_init(struct fev_sched *sched); 53 | 54 | FEV_COLD FEV_NONNULL(1) void fev_poller_fini(struct fev_sched *sched); 55 | 56 | FEV_NONNULL(1, 2) 57 | int fev_poller_register(const struct fev_sched_worker *worker, struct fev_socket *socket, 58 | enum fev_poller_flag flag); 59 | 60 | FEV_NONNULL(1, 2) 61 | void fev_poller_set_timeout(const struct fev_timers_bucket *bucket, 62 | const struct timespec *abs_time); 63 | 64 | FEV_NONNULL(1) void fev_poller_interrupt(const struct fev_poller *poller); 65 | 66 | FEV_NONNULL(1) 67 | void fev_poller_process(struct fev_sched_worker *worker, const struct timespec *timeout); 68 | 69 | FEV_NONNULL(1) static inline void fev_poller_check(struct fev_sched_worker *worker) 70 | { 71 | /* kevent() won't block. */ 72 | const struct timespec timeout = { 73 | .tv_sec = 0, 74 | .tv_nsec = 0, 75 | }; 76 | fev_poller_process(worker, &timeout); 77 | } 78 | 79 | FEV_NONNULL(1) static inline void fev_poller_wait(struct fev_sched_worker *worker) 80 | { 81 | /* kevent() will block indefinitely. */ 82 | fev_poller_process(worker, /*timeout=*/NULL); 83 | } 84 | 85 | #endif /* !FEV_POLLER_KQUEUE_H */ 86 | -------------------------------------------------------------------------------- /src/fev_poller_reactor.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_poller.h" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include "fev_alloc.h" 17 | #include "fev_compiler.h" 18 | #include "fev_qsbr.h" 19 | #include "fev_sched_intf.h" 20 | #include "fev_socket.h" 21 | #include "fev_util.h" 22 | 23 | #define FEV_POLLER_WAKE_ALL_THRESHOLD 64 24 | #define FEV_POLLER_WAKE_ALL_STEP 32 25 | 26 | FEV_NONNULL(1, 2) 27 | void fev_poller_free_socket(struct fev_sched_worker *worker, struct fev_socket *socket) 28 | { 29 | struct fev_sched *sched = worker->sched; 30 | 31 | /* 32 | * The implementation of QSBR does not work if the number of threads is 1. But in that case, we 33 | * know there are no references to the socket, thus it can be freed now. 34 | */ 35 | if (sched->num_workers == 1) { 36 | fev_free(socket); 37 | } else { 38 | struct fev_poller *poller = &sched->poller; 39 | struct fev_worker_poller_data *poller_data = &worker->poller_data; 40 | uint32_t num_sockets_to_free; 41 | 42 | fev_qsbr_free(&poller->sockets_global_qsbr, &poller_data->sockets_local_qsbr, 43 | &socket->qsbr_entry); 44 | 45 | /* 46 | * Wake sleeping workers from time to time so that they can go through quiescent phase, since 47 | * we can only free sockets if all workers have acknowledged these frees. 48 | */ 49 | num_sockets_to_free = 50 | atomic_fetch_add_explicit(&poller->num_sockets_to_free, 1, memory_order_relaxed); 51 | if (FEV_UNLIKELY(num_sockets_to_free >= FEV_POLLER_WAKE_ALL_THRESHOLD)) { 52 | if (num_sockets_to_free % FEV_POLLER_WAKE_ALL_STEP == 0) 53 | fev_sched_wake_all_workers(sched); 54 | } 55 | } 56 | } 57 | 58 | static inline uint32_t fev_poller_free_socket_list(struct fev_qsbr_entry *cur) 59 | { 60 | uint32_t n = 0; 61 | 62 | while (cur != NULL) { 63 | struct fev_qsbr_entry *next = atomic_load_explicit(&cur->next, memory_order_relaxed); 64 | struct fev_socket *socket = FEV_CONTAINER_OF(cur, struct fev_socket, qsbr_entry); 65 | fev_free(socket); 66 | cur = next; 67 | n++; 68 | } 69 | 70 | return n; 71 | } 72 | 73 | FEV_NONNULL(1) void fev_poller_quiescent(struct fev_sched_worker *worker) 74 | { 75 | struct fev_poller *poller = &worker->sched->poller; 76 | struct fev_worker_poller_data *poller_data = &worker->poller_data; 77 | struct fev_qsbr_entry *cur; 78 | uint32_t num_freed; 79 | 80 | /* 81 | * This works regardless of the number of threads. If the number of threads is 1, then the local 82 | * epoch will be always equal to the global epoch, and thus this will return NULL. 83 | */ 84 | cur = fev_qsbr_quiescent(&poller->sockets_global_qsbr, &poller_data->sockets_local_qsbr); 85 | num_freed = fev_poller_free_socket_list(cur); 86 | 87 | if (num_freed > 0) 88 | atomic_fetch_sub_explicit(&poller->num_sockets_to_free, num_freed, memory_order_relaxed); 89 | } 90 | 91 | FEV_COLD FEV_NONNULL(1) void fev_poller_free_remaining_sockets(struct fev_poller *poller) 92 | { 93 | struct fev_qsbr_entry *to_free1, *to_free2; 94 | 95 | fev_qsbr_fini_global(&poller->sockets_global_qsbr, &to_free1, &to_free2); 96 | fev_poller_free_socket_list(to_free1); 97 | fev_poller_free_socket_list(to_free2); 98 | 99 | /* We don't need to update `num_sockets_to_free`, as it won't be used anymore. */ 100 | } 101 | -------------------------------------------------------------------------------- /src/fev_poller_reactor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_POLLER_REACTOR_H 11 | #define FEV_POLLER_REACTOR_H 12 | 13 | #include "fev_compiler.h" 14 | 15 | struct fev_poller; 16 | struct fev_sched_worker; 17 | struct fev_worker_poller_data; 18 | struct fev_socket; 19 | 20 | /* 21 | * Free socket logically, physical free (via fev_free()) will be delayed to a point where nobody can 22 | * reference this socket. 23 | */ 24 | FEV_NONNULL(1, 2) 25 | void fev_poller_free_socket(struct fev_sched_worker *worker, struct fev_socket *socket); 26 | 27 | /* 28 | * Marks a state where the current worker cannot hold references to sockets. This can also 29 | * physically free (via fev_free()) some sockets that were previously logically freed (via 30 | * fev_poller_free_socket()). 31 | */ 32 | FEV_NONNULL(1) void fev_poller_quiescent(struct fev_sched_worker *worker); 33 | 34 | /* 35 | * Free all remaining sockets that were previously logically freed regardless if anyone has any 36 | * references to the sockets. 37 | */ 38 | FEV_COLD FEV_NONNULL(1) void fev_poller_free_remaining_sockets(struct fev_poller *poller); 39 | 40 | #endif /* !FEV_POLLER_REACTOR_H */ 41 | -------------------------------------------------------------------------------- /src/fev_qsbr_queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_QSBR_QUEUE_H 11 | #define FEV_QSBR_QUEUE_H 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "fev_compiler.h" 21 | #include "fev_qsbr.h" 22 | 23 | struct fev_qsbr_queue_node { 24 | struct fev_qsbr_entry qsbr_entry; 25 | void *value; 26 | _Atomic(struct fev_qsbr_queue_node *) next; 27 | }; 28 | 29 | struct fev_qsbr_queue { 30 | alignas(FEV_DCACHE_LINE_SIZE) _Atomic(struct fev_qsbr_queue_node *) head; 31 | alignas(FEV_DCACHE_LINE_SIZE) _Atomic(struct fev_qsbr_queue_node *) tail; 32 | }; 33 | 34 | FEV_NONNULL(1) 35 | static inline void fev_qsbr_queue_init(struct fev_qsbr_queue *queue, 36 | struct fev_qsbr_queue_node *init_node) 37 | { 38 | atomic_init(&init_node->next, NULL); 39 | atomic_init(&queue->head, init_node); 40 | atomic_init(&queue->tail, init_node); 41 | } 42 | 43 | FEV_NONNULL(1, 2) 44 | static inline void fev_qsbr_queue_fini(struct fev_qsbr_queue *queue, 45 | struct fev_qsbr_queue_node **released_node) 46 | { 47 | *released_node = atomic_load_explicit(&queue->head, memory_order_relaxed); 48 | } 49 | 50 | FEV_NONNULL(1, 2) 51 | static inline void fev_qsbr_queue_push(struct fev_qsbr_queue *queue, 52 | struct fev_qsbr_queue_node *node, void *value) 53 | { 54 | struct fev_qsbr_queue_node *tail, *next; 55 | 56 | node->value = value; 57 | atomic_store_explicit(&node->next, NULL, memory_order_relaxed); 58 | 59 | tail = atomic_load_explicit(&queue->tail, memory_order_relaxed); 60 | for (;;) { 61 | next = NULL; 62 | if (atomic_compare_exchange_weak_explicit(&tail->next, &next, node, memory_order_release, 63 | memory_order_relaxed)) 64 | break; 65 | atomic_compare_exchange_weak_explicit(&queue->tail, &tail, next, memory_order_relaxed, 66 | memory_order_relaxed); 67 | } 68 | atomic_compare_exchange_strong(&queue->tail, &tail, node); 69 | } 70 | 71 | FEV_NONNULL(1, 2, 3) 72 | static inline bool fev_qsbr_queue_pop(struct fev_qsbr_queue *queue, 73 | struct fev_qsbr_queue_node **released_node, void **value_ptr) 74 | { 75 | struct fev_qsbr_queue_node *head, *next; 76 | 77 | head = atomic_load_explicit(&queue->head, memory_order_consume); 78 | for (;;) { 79 | next = atomic_load_explicit(&head->next, memory_order_acquire); 80 | if (next == NULL) 81 | return false; 82 | if (atomic_compare_exchange_weak_explicit(&queue->head, &head, next, memory_order_acquire, 83 | memory_order_consume)) 84 | break; 85 | } 86 | 87 | *value_ptr = next->value; 88 | *released_node = head; 89 | return true; 90 | } 91 | 92 | #endif /* !FEV_QSBR_QUEUE_H */ 93 | -------------------------------------------------------------------------------- /src/fev_sched_attr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_sched_attr.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_alloc.h" 18 | #include "fev_compiler.h" 19 | 20 | const struct fev_sched_attr fev_sched_default_attr = { 21 | .num_workers = 0u, 22 | }; 23 | 24 | FEV_NONNULL(1) int fev_sched_attr_create(struct fev_sched_attr **attr_ptr) 25 | { 26 | struct fev_sched_attr *attr; 27 | 28 | attr = fev_malloc(sizeof(*attr)); 29 | if (FEV_UNLIKELY(attr == NULL)) 30 | return -ENOMEM; 31 | 32 | memset(attr, 0, sizeof(*attr)); 33 | attr->num_workers = fev_sched_default_attr.num_workers; 34 | 35 | *attr_ptr = attr; 36 | return 0; 37 | } 38 | 39 | FEV_NONNULL(1) void fev_sched_attr_destroy(struct fev_sched_attr *attr) { fev_free(attr); } 40 | 41 | FEV_NONNULL(1) FEV_PURE uint32_t fev_sched_attr_get_num_workers(const struct fev_sched_attr *attr) 42 | { 43 | return attr->num_workers; 44 | } 45 | 46 | FEV_NONNULL(1) 47 | void fev_sched_attr_set_num_workers(struct fev_sched_attr *attr, uint32_t num_workers) 48 | { 49 | attr->num_workers = num_workers; 50 | } 51 | -------------------------------------------------------------------------------- /src/fev_sched_attr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_ATTR_H 11 | #define FEV_SCHED_ATTR_H 12 | 13 | #include 14 | 15 | #include 16 | 17 | struct fev_sched_attr { 18 | uint32_t num_workers; 19 | }; 20 | 21 | extern const struct fev_sched_attr fev_sched_default_attr; 22 | 23 | #endif /* !FEV_SCHED_ATTR_H */ 24 | -------------------------------------------------------------------------------- /src/fev_sched_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_sched_intf.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_alloc.h" 18 | #include "fev_assert.h" 19 | #include "fev_compiler.h" 20 | #include "fev_os.h" 21 | #include "fev_poller.h" 22 | #include "fev_sched_attr.h" 23 | #include "fev_thr.h" 24 | #include "fev_thr_sem.h" 25 | 26 | FEV_NONNULL(1) 27 | void fev_wake_workers_slow(struct fev_sched_worker *worker, uint32_t num_waiting, 28 | uint32_t num_fibers) 29 | { 30 | struct fev_sched *sched = worker->sched; 31 | uint32_t n = num_fibers; 32 | 33 | if (n > num_waiting) 34 | n = num_waiting; 35 | 36 | #ifdef FEV_POLLER_IO_URING 37 | while (n--) 38 | fev_poller_interrupt(&sched->poller); 39 | #else 40 | if (atomic_load(&sched->poller_waiting)) { 41 | fev_poller_interrupt(&sched->poller); 42 | n--; 43 | } 44 | 45 | while (n--) 46 | fev_thr_sem_post(&sched->sem); 47 | #endif 48 | } 49 | 50 | FEV_NONNULL(1) void fev_sched_wake_all_workers(struct fev_sched *sched) 51 | { 52 | #ifdef FEV_POLLER_IO_URING 53 | for (uint32_t i = 0; i < sched->num_workers; i++) 54 | fev_poller_interrupt(&sched->poller); 55 | #else 56 | fev_poller_interrupt(&sched->poller); 57 | for (uint32_t i = 0; i < sched->num_workers; i++) 58 | fev_thr_sem_post(&sched->sem); 59 | #endif 60 | } 61 | 62 | FEV_COLD FEV_NONNULL(1) static void fev_sched_work_start(struct fev_sched_worker *cur_worker) 63 | { 64 | fev_cur_sched_worker = cur_worker; 65 | fev_sched_work(cur_worker); 66 | } 67 | 68 | FEV_COLD FEV_NONNULL(1) static void *fev_sched_thread_proc(void *arg) 69 | { 70 | struct fev_sched_worker *cur_worker = arg; 71 | 72 | /* Do not try to do any work before all threads are successfully created. */ 73 | fev_thr_sem_wait(cur_worker->sched->start_sem); 74 | 75 | fev_sched_work_start(cur_worker); 76 | 77 | return NULL; 78 | } 79 | 80 | FEV_COLD FEV_NONNULL(1) int fev_sched_run(struct fev_sched *sched) 81 | { 82 | struct fev_thr_sem start_sem; 83 | struct fev_thr *thrs; 84 | uint32_t num_workers = sched->num_workers; 85 | uint32_t n; /* number of successfully created workers */ 86 | int ret; 87 | 88 | FEV_ASSERT(num_workers > 0); 89 | 90 | ret = fev_thr_sem_init(&start_sem, 0); 91 | if (FEV_UNLIKELY(ret != 0)) 92 | goto out; 93 | 94 | sched->start_sem = &start_sem; 95 | 96 | ret = -ENOMEM; 97 | thrs = fev_malloc(num_workers * sizeof(*thrs)); 98 | if (FEV_UNLIKELY(thrs == NULL)) 99 | goto out_sem; 100 | 101 | for (n = 1; n < num_workers; n++) { 102 | ret = fev_thr_create(&thrs[n], &fev_sched_thread_proc, &sched->workers[n]); 103 | if (FEV_UNLIKELY(ret != 0)) 104 | goto fail_thrs; 105 | } 106 | 107 | /* Allow other workers to start executing. */ 108 | for (uint32_t i = 1; i < num_workers; i++) 109 | fev_thr_sem_post(&start_sem); 110 | 111 | fev_sched_work_start(&sched->workers[0]); 112 | 113 | ret = 0; 114 | goto out_thrs; 115 | 116 | fail_thrs: 117 | for (uint32_t i = 1; i < n; i++) 118 | fev_thr_cancel(&thrs[i]); 119 | 120 | out_thrs: 121 | for (uint32_t i = 1; i < n; i++) 122 | fev_thr_join(&thrs[i], NULL); 123 | 124 | fev_free(thrs); 125 | 126 | out_sem: 127 | fev_thr_sem_fini(&start_sem); 128 | sched->start_sem = NULL; 129 | 130 | out: 131 | return ret; 132 | } 133 | 134 | FEV_COLD FEV_NONNULL(1) int fev_sched_create(struct fev_sched **sched_ptr, 135 | const struct fev_sched_attr *attr) 136 | { 137 | struct fev_sched *sched; 138 | uint32_t num_workers; 139 | int ret; 140 | 141 | if (attr == NULL) 142 | attr = &fev_sched_default_attr; 143 | 144 | sched = fev_aligned_alloc(FEV_DCACHE_LINE_SIZE, sizeof(*sched)); 145 | if (FEV_UNLIKELY(sched == NULL)) 146 | return -ENOMEM; 147 | 148 | num_workers = attr->num_workers; 149 | if (num_workers == 0) 150 | num_workers = fev_get_num_processors(); 151 | 152 | ret = fev_sched_init(sched, num_workers); 153 | if (FEV_UNLIKELY(ret != 0)) { 154 | fev_aligned_free(sched); 155 | return ret; 156 | } 157 | 158 | *sched_ptr = sched; 159 | return 0; 160 | } 161 | 162 | FEV_COLD FEV_NONNULL(1) void fev_sched_destroy(struct fev_sched *sched) 163 | { 164 | fev_sched_fini(sched); 165 | fev_aligned_free(sched); 166 | } 167 | -------------------------------------------------------------------------------- /src/fev_sched_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_IMPL_H 11 | #define FEV_SCHED_IMPL_H 12 | 13 | #include "fev_sched_intf.h" 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "fev_compiler.h" 21 | #include "fev_fiber.h" 22 | 23 | #if defined(FEV_SCHED_WORK_SHARING_LOCKING) 24 | #include "fev_sched_shr_locking_impl.h" 25 | #elif defined(FEV_SCHED_WORK_SHARING_SIMPLE_MPMC) 26 | #include "fev_sched_shr_simple_mpmc_impl.h" 27 | #elif defined(FEV_SCHED_WORK_SHARING_BOUNDED_MPMC) 28 | #include "fev_sched_shr_bounded_mpmc_impl.h" 29 | #elif defined(FEV_SCHED_WORK_STEALING_LOCKING) 30 | #include "fev_sched_steal_locking_impl.h" 31 | #elif defined(FEV_SCHED_WORK_STEALING_BOUNDED_MPMC) 32 | #include "fev_sched_steal_bounded_mpmc_impl.h" 33 | #elif defined(FEV_SCHED_WORK_STEALING_BOUNDED_SPMC) 34 | #include "fev_sched_steal_bounded_spmc_impl.h" 35 | #endif 36 | 37 | FEV_NONNULL(1) 38 | static inline void fev_wake_up_waiting_workers(struct fev_sched_worker *worker, uint32_t num_fibers) 39 | { 40 | struct fev_sched *sched = worker->sched; 41 | 42 | atomic_fetch_add(&sched->num_run_fibers, num_fibers); 43 | 44 | /* Get the number of waiting workers. */ 45 | uint32_t num_waiting = atomic_load(&sched->num_waiting); 46 | if (FEV_LIKELY(num_waiting == 0)) { 47 | /* No worker is waiting, we don't have to do anything. */ 48 | return; 49 | } 50 | 51 | fev_wake_workers_slow(worker, num_waiting, num_fibers); 52 | } 53 | 54 | FEV_NONNULL(1, 2) 55 | static inline void fev_wake_one(struct fev_sched_worker *worker, struct fev_fiber *fiber) 56 | { 57 | fev_push_one(worker, fiber); 58 | fev_wake_up_waiting_workers(worker, /*num_fibers=*/1); 59 | } 60 | 61 | FEV_NONNULL(1, 2) 62 | static inline void fev_wake_stq(struct fev_sched_worker *worker, fev_fiber_stq_head_t *fibers, 63 | uint32_t num_fibers) 64 | { 65 | fev_push_stq(worker, fibers, num_fibers); 66 | fev_wake_up_waiting_workers(worker, num_fibers); 67 | } 68 | 69 | FEV_NONNULL(1) static inline void fev_cur_wake_one(struct fev_fiber *fiber) 70 | { 71 | fev_wake_one(fev_cur_sched_worker, fiber); 72 | } 73 | 74 | FEV_NONNULL(1) 75 | static inline void fev_cur_wake_stq(fev_fiber_stq_head_t *fibers, uint32_t num_fibers) 76 | { 77 | fev_wake_stq(fev_cur_sched_worker, fibers, num_fibers); 78 | } 79 | 80 | FEV_NONNULL(1) static inline bool fev_sched_is_running(struct fev_sched *sched) 81 | { 82 | return sched->start_sem != NULL; 83 | } 84 | 85 | static inline struct fev_fiber *fev_cur_fiber(void) { return fev_cur_sched_worker->cur_fiber; } 86 | 87 | #endif /* !FEV_SCHED_IMPL_H */ 88 | -------------------------------------------------------------------------------- /src/fev_sched_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_INTF_H 11 | #define FEV_SCHED_INTF_H 12 | 13 | #include 14 | 15 | #include 16 | 17 | #include "fev_compiler.h" 18 | 19 | #if defined(FEV_SCHED_WORK_SHARING_LOCKING) 20 | #include "fev_sched_shr_locking_intf.h" 21 | #elif defined(FEV_SCHED_WORK_SHARING_BOUNDED_MPMC) 22 | #include "fev_sched_shr_bounded_mpmc_intf.h" 23 | #elif defined(FEV_SCHED_WORK_SHARING_SIMPLE_MPMC) 24 | #include "fev_sched_shr_simple_mpmc_intf.h" 25 | #elif defined(FEV_SCHED_WORK_STEALING_LOCKING) 26 | #include "fev_sched_steal_locking_intf.h" 27 | #elif defined(FEV_SCHED_WORK_STEALING_BOUNDED_MPMC) 28 | #include "fev_sched_steal_bounded_mpmc_intf.h" 29 | #elif defined(FEV_SCHED_WORK_STEALING_BOUNDED_SPMC) 30 | #include "fev_sched_steal_bounded_spmc_intf.h" 31 | #else 32 | #error Wrong scheduler 33 | #endif 34 | 35 | extern _Thread_local struct fev_sched_worker *fev_cur_sched_worker; 36 | 37 | FEV_NONNULL(1) 38 | void fev_wake_workers_slow(struct fev_sched_worker *worker, uint32_t num_waiting, 39 | uint32_t num_fibers); 40 | 41 | FEV_NONNULL(1) void fev_sched_wake_all_workers(struct fev_sched *sched); 42 | 43 | FEV_NONNULL(1, 2) int fev_sched_put(struct fev_sched *sched, struct fev_fiber *fiber); 44 | 45 | FEV_NONNULL(1) void fev_sched_work(struct fev_sched_worker *cur_worker); 46 | 47 | FEV_NONNULL(1) int fev_sched_init(struct fev_sched *sched, uint32_t num_workers); 48 | 49 | FEV_NONNULL(1) void fev_sched_fini(struct fev_sched *sched); 50 | 51 | #endif /* !FEV_SCHED_INTF_H */ 52 | -------------------------------------------------------------------------------- /src/fev_sched_shr_bounded_mpmc_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_SHR_BOUNDED_MPMC_IMPL_H 11 | #define FEV_SCHED_SHR_BOUNDED_MPMC_IMPL_H 12 | 13 | #include "fev_sched_intf.h" 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "fev_assert.h" 21 | #include "fev_bounded_mpmc_queue.h" 22 | #include "fev_compiler.h" 23 | #include "fev_fiber.h" 24 | 25 | FEV_NONNULL(1, 2) 26 | static inline void fev_push_one(struct fev_sched_worker *worker, struct fev_fiber *fiber) 27 | { 28 | struct fev_sched *sched = worker->sched; 29 | bool pushed = fev_bounded_mpmc_queue_push(&sched->run_queue, fiber); 30 | if (FEV_UNLIKELY(!pushed)) 31 | fev_push_one_fallback(sched, fiber); 32 | } 33 | 34 | FEV_NONNULL(1, 2) 35 | static inline void fev_push_stq(struct fev_sched_worker *worker, fev_fiber_stq_head_t *fibers, 36 | uint32_t num_fibers) 37 | { 38 | struct fev_sched *sched = worker->sched; 39 | uint32_t n = num_fibers; 40 | 41 | FEV_ASSERT(!STAILQ_EMPTY(fibers)); 42 | FEV_ASSERT(num_fibers > 0); 43 | 44 | fev_bounded_mpmc_queue_push_stq(&sched->run_queue, fibers, &n); 45 | if (FEV_UNLIKELY(n != num_fibers)) { 46 | FEV_ASSERT(n < num_fibers); 47 | fev_push_stq_fallback(sched, fibers, num_fibers - n); 48 | } 49 | } 50 | 51 | #endif /* !FEV_SCHED_SHR_BOUNDED_MPMC_IMPL_H */ 52 | -------------------------------------------------------------------------------- /src/fev_sched_shr_bounded_mpmc_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_SHR_BOUNDED_MPMC_INTF_H 11 | #define FEV_SCHED_SHR_BOUNDED_MPMC_INTF_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_bounded_mpmc_queue.h" 18 | #include "fev_compiler.h" 19 | #include "fev_context.h" 20 | #include "fev_fiber.h" 21 | #include "fev_poller.h" 22 | #include "fev_thr_mutex.h" 23 | #include "fev_thr_sem.h" 24 | #include "fev_timers.h" 25 | 26 | struct fev_sched_worker { 27 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_fiber *cur_fiber; 28 | struct fev_context context; 29 | struct fev_worker_poller_data poller_data; 30 | struct fev_sched *sched; 31 | }; 32 | 33 | struct fev_sched { 34 | _Atomic uint32_t poller_backoff; 35 | 36 | /* Number of waiting workers. */ 37 | _Atomic uint32_t num_waiting; 38 | 39 | /* Is any worker waiting on poller? */ 40 | atomic_bool poller_waiting; 41 | 42 | /* Number of runnable fibers. */ 43 | _Atomic uint32_t num_run_fibers; 44 | 45 | /* Total number of fibers (runnable & blocked). */ 46 | _Atomic uint32_t num_fibers; 47 | 48 | struct fev_thr_mutex fallback_queue_lock; 49 | fev_fiber_stq_head_t fallback_queue; 50 | _Atomic uint32_t fallback_queue_len; 51 | 52 | struct fev_poller poller; 53 | struct fev_timers timers; 54 | 55 | struct fev_thr_sem sem; 56 | 57 | struct fev_bounded_mpmc_queue run_queue; 58 | 59 | struct fev_sched_worker *workers; 60 | uint32_t num_workers; 61 | 62 | struct fev_thr_sem *start_sem; 63 | }; 64 | 65 | FEV_NONNULL(1, 2) 66 | void fev_push_one_fallback(struct fev_sched *sched, struct fev_fiber *fiber); 67 | 68 | FEV_NONNULL(1, 2) 69 | void fev_push_stq_fallback(struct fev_sched *sched, fev_fiber_stq_head_t *fibers, 70 | uint32_t num_fibers); 71 | 72 | #endif /* !FEV_SCHED_SHR_BOUNDED_MPMC_INTF_H */ 73 | -------------------------------------------------------------------------------- /src/fev_sched_shr_locking_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_SHR_LOCKING_IMPL_H 11 | #define FEV_SCHED_SHR_LOCKING_IMPL_H 12 | 13 | #include "fev_sched_intf.h" 14 | 15 | #include 16 | 17 | #include 18 | 19 | #include "fev_assert.h" 20 | #include "fev_compiler.h" 21 | #include "fev_fiber.h" 22 | #include "fev_thr_mutex.h" 23 | 24 | FEV_NONNULL(1, 2) 25 | static inline void fev_push_one(struct fev_sched_worker *worker, struct fev_fiber *fiber) 26 | { 27 | struct fev_sched *sched = worker->sched; 28 | 29 | fev_thr_mutex_lock(&sched->run_queue_lock); 30 | STAILQ_INSERT_TAIL(&sched->run_queue, fiber, stq_entry); 31 | fev_thr_mutex_unlock(&sched->run_queue_lock); 32 | } 33 | 34 | FEV_NONNULL(1, 2) 35 | static inline void fev_push_stq(struct fev_sched_worker *worker, fev_fiber_stq_head_t *fibers, 36 | uint32_t num_fibers) 37 | { 38 | struct fev_sched *sched = worker->sched; 39 | fev_fiber_stq_head_t *run_queue = &sched->run_queue; 40 | struct fev_fiber *first = fibers->stqh_first, **last = fibers->stqh_last; 41 | 42 | (void)num_fibers; 43 | 44 | FEV_ASSERT(!STAILQ_EMPTY(fibers)); 45 | FEV_ASSERT(num_fibers > 0); 46 | 47 | fev_thr_mutex_lock(&sched->run_queue_lock); 48 | *run_queue->stqh_last = first; 49 | run_queue->stqh_last = last; 50 | fev_thr_mutex_unlock(&sched->run_queue_lock); 51 | } 52 | 53 | #endif /* !FEV_SCHED_SHR_LOCKING_IMPL_H */ 54 | -------------------------------------------------------------------------------- /src/fev_sched_shr_locking_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_SHR_LOCKING_INTF_H 11 | #define FEV_SCHED_SHR_LOCKING_INTF_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_context.h" 18 | #include "fev_fiber.h" 19 | #include "fev_poller.h" 20 | #include "fev_thr_mutex.h" 21 | #include "fev_thr_sem.h" 22 | #include "fev_timers.h" 23 | 24 | struct fev_sched_worker { 25 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_fiber *cur_fiber; 26 | struct fev_context context; 27 | struct fev_worker_poller_data poller_data; 28 | struct fev_sched *sched; 29 | }; 30 | 31 | struct fev_sched { 32 | _Atomic uint32_t poller_backoff; 33 | 34 | /* Number of waiting workers. */ 35 | _Atomic uint32_t num_waiting; 36 | 37 | /* Is any worker waiting on poller? */ 38 | atomic_bool poller_waiting; 39 | 40 | /* Number of runnable fibers. */ 41 | _Atomic uint32_t num_run_fibers; 42 | 43 | /* Total number of fibers (runnable & blocked). */ 44 | _Atomic uint32_t num_fibers; 45 | 46 | struct fev_poller poller; 47 | struct fev_timers timers; 48 | 49 | struct fev_thr_sem sem; 50 | 51 | fev_fiber_stq_head_t run_queue; 52 | struct fev_thr_mutex run_queue_lock; 53 | 54 | struct fev_sched_worker *workers; 55 | uint32_t num_workers; 56 | 57 | struct fev_thr_sem *start_sem; 58 | }; 59 | 60 | #endif /* !FEV_SCHED_SHR_LOCKING_INTF_H */ 61 | -------------------------------------------------------------------------------- /src/fev_sched_shr_simple_mpmc_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_SHR_SIMPLE_MPMC_IMPL_H 11 | #define FEV_SCHED_SHR_SIMPLE_MPMC_IMPL_H 12 | 13 | #include "fev_sched_intf.h" 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "fev_assert.h" 21 | #include "fev_compiler.h" 22 | #include "fev_fiber.h" 23 | #include "fev_simple_mpmc_pool.h" 24 | #include "fev_simple_mpmc_queue.h" 25 | 26 | FEV_NONNULL(1, 2) 27 | static inline void fev_push_one(struct fev_sched_worker *worker, struct fev_fiber *fiber) 28 | { 29 | struct fev_simple_mpmc_queue_node *node; 30 | 31 | node = fev_simple_mpmc_pool_alloc_local(&worker->pool_local); 32 | if (FEV_UNLIKELY(node == NULL)) 33 | fev_sched_oom(); 34 | 35 | fev_simple_mpmc_queue_push(worker->run_queue, node, fiber); 36 | } 37 | 38 | FEV_NONNULL(1, 2) 39 | static inline void fev_push_stq(struct fev_sched_worker *worker, fev_fiber_stq_head_t *fibers, 40 | uint32_t num_fibers) 41 | { 42 | struct fev_simple_mpmc_queue *run_queue = worker->run_queue; 43 | struct fev_simple_mpmc_queue_node *node; 44 | struct fev_fiber *cur; 45 | 46 | (void)num_fibers; 47 | 48 | FEV_ASSERT(!STAILQ_EMPTY(fibers)); 49 | FEV_ASSERT(num_fibers > 0); 50 | 51 | cur = STAILQ_FIRST(fibers); 52 | while (cur != NULL) { 53 | struct fev_fiber *next = STAILQ_NEXT(cur, stq_entry); 54 | 55 | node = fev_simple_mpmc_pool_alloc_local(&worker->pool_local); 56 | if (FEV_UNLIKELY(node == NULL)) 57 | fev_sched_oom(); 58 | 59 | fev_simple_mpmc_queue_push(run_queue, node, cur); 60 | 61 | cur = next; 62 | } 63 | } 64 | 65 | #endif /* !FEV_SCHED_SHR_SIMPLE_MPMC_IMPL_H */ 66 | -------------------------------------------------------------------------------- /src/fev_sched_shr_simple_mpmc_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_SHR_SIMPLE_MPMC_INTF_H 11 | #define FEV_SCHED_SHR_SIMPLE_MPMC_INTF_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_compiler.h" 18 | #include "fev_context.h" 19 | #include "fev_fiber.h" 20 | #include "fev_poller.h" 21 | #include "fev_simple_mpmc_pool.h" 22 | #include "fev_simple_mpmc_queue.h" 23 | #include "fev_thr_sem.h" 24 | #include "fev_timers.h" 25 | 26 | struct fev_sched_worker { 27 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_fiber *cur_fiber; 28 | struct fev_context context; 29 | 30 | struct fev_simple_mpmc_queue *run_queue; 31 | struct fev_simple_mpmc_pool_local pool_local; 32 | 33 | struct fev_worker_poller_data poller_data; 34 | struct fev_sched *sched; 35 | }; 36 | 37 | struct fev_sched { 38 | _Atomic uint32_t poller_backoff; 39 | 40 | /* Number of waiting workers. */ 41 | _Atomic uint32_t num_waiting; 42 | 43 | /* Is any worker waiting on poller? */ 44 | atomic_bool poller_waiting; 45 | 46 | /* Number of runnable fibers. */ 47 | _Atomic uint32_t num_run_fibers; 48 | 49 | /* Total number of fibers (runnable & blocked). */ 50 | _Atomic uint32_t num_fibers; 51 | 52 | struct fev_poller poller; 53 | struct fev_timers timers; 54 | 55 | struct fev_thr_sem sem; 56 | 57 | struct fev_simple_mpmc_queue *run_queue; 58 | struct fev_simple_mpmc_pool_global pool_global; 59 | 60 | struct fev_sched_worker *workers; 61 | uint32_t num_workers; 62 | 63 | struct fev_thr_sem *start_sem; 64 | }; 65 | 66 | FEV_COLD FEV_NOINLINE FEV_NORETURN void fev_sched_oom(void); 67 | 68 | #endif /* !FEV_SCHED_SHR_SIMPLE_MPMC_INTF_H */ 69 | -------------------------------------------------------------------------------- /src/fev_sched_steal_bounded_mpmc_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_STEAL_BOUNDED_MPMC_IMPL_H 11 | #define FEV_SCHED_STEAL_BOUNDED_MPMC_IMPL_H 12 | 13 | #include "fev_sched_intf.h" 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "fev_assert.h" 21 | #include "fev_bounded_mpmc_queue.h" 22 | #include "fev_compiler.h" 23 | #include "fev_fiber.h" 24 | 25 | FEV_NONNULL(1, 2) 26 | static inline void fev_push_one(struct fev_sched_worker *worker, struct fev_fiber *fiber) 27 | { 28 | bool pushed = fev_bounded_mpmc_queue_push(&worker->run_queue, fiber); 29 | if (FEV_UNLIKELY(!pushed)) 30 | fev_push_one_fallback(worker, fiber); 31 | } 32 | 33 | FEV_NONNULL(1, 2) 34 | static inline void fev_push_stq(struct fev_sched_worker *worker, fev_fiber_stq_head_t *fibers, 35 | uint32_t num_fibers) 36 | { 37 | uint32_t n = num_fibers; 38 | 39 | FEV_ASSERT(!STAILQ_EMPTY(fibers)); 40 | FEV_ASSERT(num_fibers > 0); 41 | 42 | fev_bounded_mpmc_queue_push_stq(&worker->run_queue, fibers, &n); 43 | if (FEV_UNLIKELY(n != num_fibers)) { 44 | FEV_ASSERT(n < num_fibers); 45 | fev_push_stq_fallback(worker, fibers, num_fibers - n); 46 | } 47 | } 48 | 49 | #endif /* !FEV_SCHED_STEAL_BOUNDED_MPMC_IMPL_H */ 50 | -------------------------------------------------------------------------------- /src/fev_sched_steal_bounded_mpmc_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_STEAL_BOUNDED_MPMC_INTF_H 11 | #define FEV_SCHED_STEAL_BOUNDED_MPMC_INTF_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_bounded_mpmc_queue.h" 18 | #include "fev_compiler.h" 19 | #include "fev_context.h" 20 | #include "fev_fiber.h" 21 | #include "fev_poller.h" 22 | #include "fev_thr_mutex.h" 23 | #include "fev_thr_sem.h" 24 | #include "fev_timers.h" 25 | 26 | struct fev_sched_worker { 27 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_bounded_mpmc_queue run_queue; 28 | 29 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_fiber *cur_fiber; 30 | struct fev_context context; 31 | struct fev_worker_poller_data poller_data; 32 | struct fev_sched *sched; 33 | uint32_t rnd; 34 | }; 35 | 36 | struct fev_sched { 37 | /* Number of waiting workers. */ 38 | _Atomic uint32_t num_waiting; 39 | 40 | /* Is any worker waiting on poller? */ 41 | atomic_bool poller_waiting; 42 | 43 | /* Number of runnable fibers. */ 44 | _Atomic uint32_t num_run_fibers; 45 | 46 | /* Total number of fibers (runnable & blocked). */ 47 | _Atomic uint32_t num_fibers; 48 | 49 | struct fev_thr_mutex fallback_queue_lock; 50 | fev_fiber_stq_head_t fallback_queue; 51 | _Atomic uint32_t fallback_queue_len; 52 | 53 | struct fev_poller poller; 54 | struct fev_timers timers; 55 | 56 | struct fev_thr_sem sem; 57 | 58 | struct fev_sched_worker *workers; 59 | uint32_t num_workers; 60 | 61 | struct fev_thr_sem *start_sem; 62 | }; 63 | 64 | FEV_NONNULL(1, 2) 65 | void fev_push_one_fallback(struct fev_sched_worker *worker, struct fev_fiber *fiber); 66 | 67 | FEV_NONNULL(1, 2) 68 | void fev_push_stq_fallback(struct fev_sched_worker *worker, fev_fiber_stq_head_t *fibers, 69 | uint32_t num_fibers); 70 | 71 | #endif /* !FEV_SCHED_STEAL_BOUNDED_MPMC_INTF_H */ 72 | -------------------------------------------------------------------------------- /src/fev_sched_steal_bounded_spmc_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_STEAL_BOUNDED_SPMC_IMPL_H 11 | #define FEV_SCHED_STEAL_BOUNDED_SPMC_IMPL_H 12 | 13 | #include "fev_sched_intf.h" 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "fev_assert.h" 21 | #include "fev_bounded_spmc_queue.h" 22 | #include "fev_compiler.h" 23 | #include "fev_fiber.h" 24 | 25 | FEV_NONNULL(1, 2) 26 | static inline void fev_push_one(struct fev_sched_worker *worker, struct fev_fiber *fiber) 27 | { 28 | bool pushed = fev_bounded_spmc_queue_push(&worker->run_queue, fiber); 29 | if (FEV_UNLIKELY(!pushed)) 30 | fev_push_one_fallback(worker, fiber); 31 | } 32 | 33 | FEV_NONNULL(1, 2) 34 | static inline void fev_push_stq(struct fev_sched_worker *worker, fev_fiber_stq_head_t *fibers, 35 | uint32_t num_fibers) 36 | { 37 | uint32_t n = num_fibers; 38 | 39 | FEV_ASSERT(!STAILQ_EMPTY(fibers)); 40 | FEV_ASSERT(num_fibers > 0); 41 | 42 | fev_bounded_spmc_queue_push_stq(&worker->run_queue, fibers, &n); 43 | if (FEV_UNLIKELY(n != num_fibers)) { 44 | FEV_ASSERT(n < num_fibers); 45 | fev_push_stq_fallback(worker, fibers, num_fibers - n); 46 | } 47 | } 48 | #endif /* !FEV_SCHED_STEAL_BOUNDED_SPMC_IMPL_H */ 49 | -------------------------------------------------------------------------------- /src/fev_sched_steal_bounded_spmc_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_STEAL_BOUNDED_SPMC_INTF_H 11 | #define FEV_SCHED_STEAL_BOUNDED_SPMC_INTF_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_bounded_spmc_queue.h" 18 | #include "fev_compiler.h" 19 | #include "fev_context.h" 20 | #include "fev_fiber.h" 21 | #include "fev_poller.h" 22 | #include "fev_thr_mutex.h" 23 | #include "fev_thr_sem.h" 24 | #include "fev_timers.h" 25 | 26 | struct fev_sched_worker { 27 | /* 28 | * The worker's queue. It can be shared with other workers if they want to steal a fiber from us. 29 | * Thus, to avoid false sharing it is aligned to cache line size. 30 | */ 31 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_bounded_spmc_queue run_queue; 32 | 33 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_fiber *cur_fiber; 34 | struct fev_context context; 35 | struct fev_worker_poller_data poller_data; 36 | struct fev_sched *sched; 37 | uint32_t rnd; 38 | }; 39 | 40 | struct fev_sched { 41 | /* Number of waiting workers. */ 42 | _Atomic uint32_t num_waiting; 43 | 44 | /* Is any worker waiting on poller? */ 45 | atomic_bool poller_waiting; 46 | 47 | /* Number of runnable fibers. */ 48 | _Atomic uint32_t num_run_fibers; 49 | 50 | /* Total number of fibers (runnable & blocked). */ 51 | _Atomic uint32_t num_fibers; 52 | 53 | struct fev_thr_mutex fallback_queue_lock; 54 | fev_fiber_stq_head_t fallback_queue; 55 | _Atomic uint32_t fallback_queue_len; 56 | 57 | struct fev_poller poller; 58 | struct fev_timers timers; 59 | 60 | struct fev_thr_sem sem; 61 | 62 | struct fev_sched_worker *workers; 63 | uint32_t num_workers; 64 | 65 | struct fev_thr_sem *start_sem; 66 | }; 67 | 68 | FEV_NONNULL(1, 2) 69 | void fev_push_one_fallback(struct fev_sched_worker *worker, struct fev_fiber *fiber); 70 | 71 | FEV_NONNULL(1, 2) 72 | void fev_push_stq_fallback(struct fev_sched_worker *worker, fev_fiber_stq_head_t *fibers, 73 | uint32_t num_fibers); 74 | 75 | #endif /* !FEV_SCHED_STEAL_BOUNDED_SPMC_INTF_H */ 76 | -------------------------------------------------------------------------------- /src/fev_sched_steal_locking_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_STEAL_LOCKING_IMPL_H 11 | #define FEV_SCHED_STEAL_LOCKING_IMPL_H 12 | 13 | #include "fev_sched_intf.h" 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | #include "fev_assert.h" 21 | #include "fev_compiler.h" 22 | #include "fev_fiber.h" 23 | #include "fev_spinlock_impl.h" 24 | #include "fev_thr_mutex.h" 25 | 26 | #if defined(FEV_SCHED_STEAL_LOCKING_LOCK_MUTEX) 27 | #define fev_sched_run_queue_lock_init fev_thr_mutex_init 28 | #define fev_sched_run_queue_lock_fini fev_thr_mutex_fini 29 | #define fev_sched_run_queue_lock fev_thr_mutex_lock 30 | #define fev_sched_run_queue_try_lock fev_thr_mutex_try_lock 31 | #define fev_sched_run_queue_unlock fev_thr_mutex_unlock 32 | #elif defined(FEV_SCHED_STEAL_LOCKING_LOCK_SPINLOCK) 33 | #define fev_sched_run_queue_lock_init fev_spinlock_init 34 | #define fev_sched_run_queue_lock_fini fev_spinlock_fini 35 | #define fev_sched_run_queue_lock fev_spinlock_lock 36 | #define fev_sched_run_queue_try_lock fev_spinlock_try_lock 37 | #define fev_sched_run_queue_unlock fev_spinlock_unlock 38 | #endif 39 | 40 | FEV_NONNULL(1, 2) 41 | static inline void fev_push_one(struct fev_sched_worker *worker, struct fev_fiber *fiber) 42 | { 43 | struct fev_sched_run_queue *run_queue = &worker->run_queue; 44 | 45 | fev_sched_run_queue_lock(&run_queue->lock); 46 | 47 | STAILQ_INSERT_TAIL(&run_queue->head, fiber, stq_entry); 48 | atomic_fetch_add_explicit(&run_queue->size, 1, memory_order_relaxed); 49 | 50 | fev_sched_run_queue_unlock(&run_queue->lock); 51 | } 52 | 53 | FEV_NONNULL(1, 2) 54 | static inline void fev_push_stq(struct fev_sched_worker *worker, fev_fiber_stq_head_t *fibers, 55 | uint32_t num_fibers) 56 | { 57 | struct fev_sched_run_queue *run_queue = &worker->run_queue; 58 | struct fev_fiber *first = fibers->stqh_first, **last = fibers->stqh_last; 59 | 60 | FEV_ASSERT(!STAILQ_EMPTY(fibers)); 61 | FEV_ASSERT(num_fibers > 0); 62 | 63 | fev_sched_run_queue_lock(&run_queue->lock); 64 | 65 | *run_queue->head.stqh_last = first; 66 | run_queue->head.stqh_last = last; 67 | 68 | atomic_fetch_add_explicit(&run_queue->size, num_fibers, memory_order_relaxed); 69 | 70 | fev_sched_run_queue_unlock(&run_queue->lock); 71 | } 72 | 73 | #endif /* !FEV_SCHED_STEAL_LOCKING_IMPL_H */ 74 | -------------------------------------------------------------------------------- /src/fev_sched_steal_locking_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SCHED_STEAL_LOCKING_INTF_H 11 | #define FEV_SCHED_STEAL_LOCKING_INTF_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_context.h" 18 | #include "fev_fiber.h" 19 | #include "fev_poller.h" 20 | #include "fev_spinlock_intf.h" 21 | #include "fev_thr_mutex.h" 22 | #include "fev_thr_sem.h" 23 | #include "fev_timers.h" 24 | 25 | #if defined(FEV_SCHED_STEAL_LOCKING_LOCK_MUTEX) 26 | typedef struct fev_thr_mutex fev_sched_run_queue_lock_t; 27 | #elif defined(FEV_SCHED_STEAL_LOCKING_LOCK_SPINLOCK) 28 | typedef struct fev_spinlock fev_sched_run_queue_lock_t; 29 | #else 30 | #error Wrong lock strategy for sched stealing locking selected, define either \ 31 | FEV_SCHED_STEAL_LOCKING_LOCK_MUTEX or FEV_SCHED_STEAL_LOCKING_LOCK_SPINLOCK. 32 | #endif 33 | 34 | struct fev_sched_run_queue { 35 | fev_fiber_stq_head_t head; 36 | fev_sched_run_queue_lock_t lock; 37 | _Atomic uint32_t size; 38 | }; 39 | 40 | struct fev_sched_worker { 41 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_fiber *cur_fiber; 42 | struct fev_context context; 43 | struct fev_worker_poller_data poller_data; 44 | struct fev_sched *sched; 45 | uint32_t rnd; 46 | 47 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_sched_run_queue run_queue; 48 | }; 49 | 50 | struct fev_sched { 51 | /* Number of waiting workers. */ 52 | _Atomic uint32_t num_waiting; 53 | 54 | /* Is any worker waiting on poller? */ 55 | atomic_bool poller_waiting; 56 | 57 | /* Number of runnable fibers. */ 58 | _Atomic uint32_t num_run_fibers; 59 | 60 | /* Total number of fibers (runnable & blocked). */ 61 | _Atomic uint32_t num_fibers; 62 | 63 | struct fev_poller poller; 64 | struct fev_timers timers; 65 | 66 | struct fev_thr_sem sem; 67 | 68 | struct fev_sched_worker *workers; 69 | uint32_t num_workers; 70 | 71 | struct fev_thr_sem *start_sem; 72 | }; 73 | 74 | #endif /* !FEV_SCHED_STEAL_LOCKING_INTF_H */ 75 | -------------------------------------------------------------------------------- /src/fev_sem.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_sem.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_alloc.h" 18 | #include "fev_assert.h" 19 | #include "fev_compiler.h" 20 | #include "fev_time.h" 21 | #include "fev_waiters_queue_impl.h" 22 | 23 | FEV_NONNULL(1) FEV_WARN_UNUSED_RESULT static int fev_sem_init(struct fev_sem *sem, int32_t value) 24 | { 25 | int ret = fev_waiters_queue_init(&sem->wq); 26 | if (FEV_UNLIKELY(ret != 0)) 27 | return ret; 28 | 29 | sem->value = value; 30 | return 0; 31 | } 32 | 33 | FEV_NONNULL(1) static void fev_sem_fini(struct fev_sem *sem) { fev_waiters_queue_fini(&sem->wq); } 34 | 35 | FEV_NONNULL(1) int fev_sem_create(struct fev_sem **sem_ptr, int32_t value) 36 | { 37 | struct fev_sem *sem; 38 | int ret; 39 | 40 | sem = fev_malloc(sizeof(*sem)); 41 | if (FEV_UNLIKELY(sem == NULL)) 42 | return -ENOMEM; 43 | 44 | ret = fev_sem_init(sem, value); 45 | if (FEV_UNLIKELY(ret != 0)) { 46 | fev_free(sem); 47 | return ret; 48 | } 49 | 50 | *sem_ptr = sem; 51 | return 0; 52 | } 53 | 54 | FEV_NONNULL(1) void fev_sem_destroy(struct fev_sem *sem) 55 | { 56 | fev_sem_fini(sem); 57 | fev_free(sem); 58 | } 59 | 60 | static bool fev_sem_wait_recheck(void *arg) 61 | { 62 | struct fev_sem *sem = arg; 63 | 64 | if (sem->value > 0) { 65 | sem->value--; 66 | return false; 67 | } 68 | 69 | return true; 70 | } 71 | 72 | FEV_NONNULL(1) void fev_sem_wait(struct fev_sem *sem) 73 | { 74 | int res = fev_waiters_queue_wait(&sem->wq, /*abs_time=*/NULL, &fev_sem_wait_recheck, sem); 75 | (void)res; 76 | FEV_ASSERT(res == 0); 77 | } 78 | 79 | FEV_NONNULL(1, 2) int fev_sem_wait_until(struct fev_sem *sem, const struct timespec *abs_time) 80 | { 81 | int res; 82 | 83 | do { 84 | res = fev_waiters_queue_wait(&sem->wq, abs_time, &fev_sem_wait_recheck, sem); 85 | } while (res == -EAGAIN); 86 | 87 | return res; 88 | } 89 | 90 | FEV_NONNULL(1, 2) int fev_sem_wait_for(struct fev_sem *sem, const struct timespec *rel_time) 91 | { 92 | struct timespec abs_time; 93 | 94 | fev_get_abs_time_since_now(&abs_time, rel_time); 95 | return fev_sem_wait_until(sem, &abs_time); 96 | } 97 | 98 | static void fev_sem_post_callback(void *arg, uint32_t num_woken, bool is_empty) 99 | { 100 | struct fev_sem *sem = arg; 101 | 102 | (void)is_empty; 103 | 104 | FEV_ASSERT(num_woken <= 1); 105 | 106 | if (num_woken == 0) 107 | sem->value++; 108 | } 109 | 110 | FEV_NONNULL(1) void fev_sem_post(struct fev_sem *sem) 111 | { 112 | fev_waiters_queue_wake(&sem->wq, /*max_waiters=*/1, &fev_sem_post_callback, sem); 113 | } 114 | -------------------------------------------------------------------------------- /src/fev_sem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SEM_H 11 | #define FEV_SEM_H 12 | 13 | #include 14 | 15 | #include 16 | 17 | #include "fev_waiters_queue_intf.h" 18 | 19 | struct fev_sem { 20 | int32_t value; 21 | struct fev_waiters_queue wq; 22 | }; 23 | 24 | #endif /* !FEV_SEM_H */ 25 | -------------------------------------------------------------------------------- /src/fev_simple_mpmc_stack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SIMPLE_MPMC_STACK_H 11 | #define FEV_SIMPLE_MPMC_STACK_H 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "fev_arch.h" 19 | #include "fev_compiler.h" 20 | 21 | /* 22 | * A simple MPMC lock-free stack. 23 | * 24 | * Allocated nodes must stay in memory, otherwise an use-after-free is possible in pop() operation 25 | * when dereferencing top->next. 26 | */ 27 | 28 | struct fev_simple_mpmc_stack_node { 29 | _Atomic(struct fev_simple_mpmc_stack_node *) next; 30 | }; 31 | 32 | struct fev_simple_mpmc_stack { 33 | /* 16-byte alignment is necessary for fev_cmpxchg2(). */ 34 | alignas(16) _Atomic(struct fev_simple_mpmc_stack_node *) top; 35 | atomic_uintptr_t count; 36 | }; 37 | 38 | FEV_NONNULL(1) 39 | static inline void fev_simple_mpmc_stack_init(struct fev_simple_mpmc_stack *stack) 40 | { 41 | atomic_init(&stack->top, NULL); 42 | atomic_init(&stack->count, 0); 43 | } 44 | 45 | FEV_NONNULL(1, 2) 46 | static inline void fev_simple_mpmc_stack_push(struct fev_simple_mpmc_stack *stack, 47 | struct fev_simple_mpmc_stack_node *node) 48 | { 49 | struct fev_simple_mpmc_stack_node *top; 50 | uintptr_t count; 51 | 52 | /* Read count before the pointer, make sure that they are not reordered. */ 53 | count = atomic_load_explicit(&stack->count, memory_order_acquire); 54 | top = atomic_load_explicit(&stack->top, memory_order_relaxed); 55 | 56 | do { 57 | /* 58 | * node->next must be updated before top, but fev_cmpxchg2 is fully ordered, thus the store can 59 | * be relaxed. 60 | */ 61 | atomic_store_explicit(&node->next, top, memory_order_relaxed); 62 | } while (!fev_cmpxchg2(stack, &top, &count, node, count + 1)); 63 | } 64 | 65 | FEV_NONNULL(1) 66 | static inline struct fev_simple_mpmc_stack_node * 67 | fev_simple_mpmc_stack_pop(struct fev_simple_mpmc_stack *stack) 68 | { 69 | struct fev_simple_mpmc_stack_node *top, *next; 70 | uintptr_t count; 71 | 72 | /* Read count before the pointer, make sure that they are not reordered. */ 73 | count = atomic_load_explicit(&stack->count, memory_order_acquire); 74 | top = atomic_load_explicit(&stack->top, memory_order_relaxed); 75 | 76 | do { 77 | if (top == NULL) 78 | return NULL; 79 | next = atomic_load_explicit(&top->next, memory_order_relaxed); 80 | } while (!fev_cmpxchg2(stack, &top, &count, next, count + 1)); 81 | 82 | return top; 83 | } 84 | 85 | #endif /* !FEV_SIMPLE_MPMC_STACK_H */ 86 | -------------------------------------------------------------------------------- /src/fev_socket.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SOCKET_H 11 | #define FEV_SOCKET_H 12 | 13 | #include 14 | 15 | #ifdef FEV_POLLER_IO_URING 16 | #include "fev_socket_io_uring.h" 17 | #else 18 | #include "fev_socket_reactor.h" 19 | #endif 20 | 21 | #endif /* !FEV_SOCKET_H */ 22 | -------------------------------------------------------------------------------- /src/fev_socket_io_uring.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SOCKET_IO_URING_H 11 | #define FEV_SOCKET_IO_URING_H 12 | 13 | #include "fev_fiber.h" 14 | 15 | struct fev_socket_end { 16 | struct fev_fiber *fiber; 17 | 18 | /* Number of entries still in uring for this socket. */ 19 | unsigned num_entries; 20 | 21 | /* Result from uring. */ 22 | int res; 23 | }; 24 | 25 | struct fev_socket { 26 | int fd; 27 | struct fev_socket_end read_end; 28 | struct fev_socket_end write_end; 29 | }; 30 | 31 | #endif /* !FEV_SOCKET_IO_URING_H */ 32 | -------------------------------------------------------------------------------- /src/fev_socket_reactor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SOCKET_REACTOR_H 11 | #define FEV_SOCKET_REACTOR_H 12 | 13 | #include 14 | 15 | #include "fev_qsbr.h" 16 | #include "fev_waiter_intf.h" 17 | 18 | struct fev_socket_end { 19 | struct fev_waiter waiter; 20 | bool active; 21 | }; 22 | 23 | struct fev_socket { 24 | struct fev_socket_end read_end; 25 | struct fev_socket_end write_end; 26 | int fd; 27 | unsigned error; 28 | struct fev_qsbr_entry qsbr_entry; 29 | }; 30 | 31 | #endif /* !FEV_SOCKET_REACTOR_H */ 32 | -------------------------------------------------------------------------------- /src/fev_spinlock_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SPINLOCK_IMPL_H 11 | #define FEV_SPINLOCK_IMPL_H 12 | 13 | #include "fev_spinlock_intf.h" 14 | 15 | #include 16 | #include 17 | 18 | #include "fev_arch.h" 19 | #include "fev_compiler.h" 20 | 21 | FEV_NONNULL(1) static inline int fev_spinlock_init(struct fev_spinlock *spinlock) 22 | { 23 | atomic_init(&spinlock->state, 0); 24 | return 0; 25 | } 26 | 27 | FEV_NONNULL(1) static inline void fev_spinlock_fini(struct fev_spinlock *spinlock) 28 | { 29 | (void)spinlock; 30 | } 31 | 32 | FEV_NONNULL(1) static inline bool fev_spinlock_try_lock(struct fev_spinlock *spinlock) 33 | { 34 | if (atomic_load_explicit(&spinlock->state, memory_order_relaxed) != 0) 35 | return false; 36 | if (atomic_exchange_explicit(&spinlock->state, 1, memory_order_acquire) != 0) 37 | return false; 38 | return true; 39 | } 40 | 41 | FEV_NONNULL(1) static inline void fev_spinlock_lock(struct fev_spinlock *spinlock) 42 | { 43 | while (atomic_exchange_explicit(&spinlock->state, 1, memory_order_acquire) != 0) { 44 | while (atomic_load_explicit(&spinlock->state, memory_order_relaxed) != 0) 45 | fev_pause(); 46 | } 47 | } 48 | 49 | FEV_NONNULL(1) static inline void fev_spinlock_unlock(struct fev_spinlock *spinlock) 50 | { 51 | atomic_store_explicit(&spinlock->state, 0, memory_order_release); 52 | } 53 | 54 | #endif /* !FEV_SPINLOCK_IMPL_H */ 55 | -------------------------------------------------------------------------------- /src/fev_spinlock_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_SPINLOCK_INTF_H 11 | #define FEV_SPINLOCK_INTF_H 12 | 13 | #include 14 | 15 | struct fev_spinlock { 16 | atomic_uint state; 17 | }; 18 | 19 | #endif /* !FEV_SPINLOCK_INTF_H */ 20 | -------------------------------------------------------------------------------- /src/fev_stack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_STACK_H 11 | #define FEV_STACK_H 12 | 13 | #include 14 | 15 | int fev_stack_alloc(void **addr_ptr, size_t usable_size, size_t guard_size); 16 | void fev_stack_free(void *addr, size_t total_size); 17 | 18 | #endif /* !FEV_STACK_H */ 19 | -------------------------------------------------------------------------------- /src/fev_stack_posix.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_stack.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_assert.h" 18 | #include "fev_compiler.h" 19 | #include "fev_util.h" 20 | 21 | #if defined(FEV_OS_BSD) || defined(FEV_OS_MACOS) || defined(FEV_OS_DARWIN) 22 | #define FEV_MAP_ANON MAP_ANON 23 | #elif defined(FEV_OS_LINUX) 24 | #define FEV_MAP_ANON MAP_ANONYMOUS 25 | #else 26 | #error Not implemented 27 | #endif 28 | 29 | int fev_stack_alloc(void **addr_ptr, size_t usable_size, size_t guard_size) 30 | { 31 | size_t total_size; 32 | void *addr; 33 | int ret; 34 | 35 | FEV_ASSERT(usable_size <= SIZE_MAX - guard_size); 36 | total_size = usable_size + guard_size; 37 | 38 | addr = mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | FEV_MAP_ANON, -1, 0); 39 | if (FEV_UNLIKELY(addr == MAP_FAILED)) { 40 | ret = -errno; 41 | goto fail; 42 | } 43 | 44 | if (guard_size > 0) { 45 | ret = mprotect(addr, guard_size, PROT_NONE); 46 | if (FEV_UNLIKELY(ret != 0)) { 47 | ret = -errno; 48 | goto fail_free; 49 | } 50 | } 51 | 52 | *addr_ptr = addr; 53 | return 0; 54 | 55 | fail_free: 56 | fev_stack_free(addr, total_size); 57 | 58 | fail: 59 | return ret; 60 | } 61 | 62 | void fev_stack_free(void *addr, size_t total_stack_size) 63 | { 64 | int ret = munmap(addr, total_stack_size); 65 | (void)ret; 66 | FEV_ASSERT(ret == 0); 67 | } 68 | -------------------------------------------------------------------------------- /src/fev_thr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_H 11 | #define FEV_THR_H 12 | 13 | #include 14 | 15 | #if defined(FEV_THR_POSIX) 16 | #include "fev_thr_posix.h" 17 | #elif defined(FEV_THR_WIN) 18 | #include "fev_thr_win.h" 19 | #else 20 | #error Your platform is unsupported! 21 | #endif 22 | 23 | #endif /* !FEV_THR_H */ 24 | -------------------------------------------------------------------------------- /src/fev_thr_mutex.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_MUTEX_H 11 | #define FEV_THR_MUTEX_H 12 | 13 | #include 14 | 15 | #if defined(FEV_THR_MUTEX_LINUX) 16 | #include "fev_thr_mutex_linux.h" 17 | #elif defined(FEV_THR_MUTEX_POSIX) 18 | #include "fev_thr_mutex_posix.h" 19 | #elif defined(FEV_THR_MUTEX_WIN) 20 | #include "fev_thr_mutex_win.h" 21 | #else 22 | #error Your platform is unsupported! 23 | #endif 24 | 25 | #endif /* !FEV_THR_MUTEX_H */ 26 | -------------------------------------------------------------------------------- /src/fev_thr_mutex_linux.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_thr_mutex_linux.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "fev_assert.h" 19 | #include "fev_compiler.h" 20 | 21 | FEV_NONNULL(1) static void fev_futex_wait(int *addr, int val) 22 | { 23 | long ret; 24 | 25 | /* 26 | * The wait operation returns 0 if the caller was woken up or an error: 27 | * EAGAIN - The value at addr was not equal to val. This is fine. 28 | * EINTR - The operation was interrupted. This is fine, we don't care. 29 | * EINVAL - An invalid argument or inconsistency (check man pages). It's a bug 30 | * in libfev, thus only an assert. 31 | * ENOSYS - The futex syscall is not supported. libfev's requirements assume 32 | * Linux >= 2.6.26, thus only an assert. 33 | */ 34 | ret = syscall(SYS_futex, addr, FUTEX_WAIT_PRIVATE, val, /*timeout=*/NULL); 35 | (void)ret; 36 | FEV_ASSERT(ret == 0 || errno == EAGAIN || errno == EINTR); 37 | } 38 | 39 | FEV_NONNULL(1) static void fev_futex_wake(int *addr, int val) 40 | { 41 | long ret; 42 | 43 | /* 44 | * The wake operation returns the number of woken waiters or an error: 45 | * EFAULT - addr did not point to a valid user-space address. This is a bug 46 | * in libfev, thus only an assert. 47 | * EINVAL - An invalid argument or an inconsistency between the user-space at 48 | * addr and the kernel state, there is a waiter waiting in 49 | * FUTEX_LOCK_PI. In both cases this is a bug in libfev, thus only 50 | * an assert. 51 | * ENOSYS - The futex syscall is not supported. libfev's requirements assume 52 | * Linux >= 2.6.26, thus only an assert. 53 | */ 54 | ret = syscall(SYS_futex, addr, FUTEX_WAKE_PRIVATE, val); 55 | (void)ret; 56 | FEV_ASSERT(ret >= 0); 57 | } 58 | 59 | FEV_NONNULL(1) void fev_thr_mutex_lock_slow(struct fev_thr_mutex *mutex) 60 | { 61 | int state = atomic_load_explicit(&mutex->state, memory_order_acquire); 62 | if (state != 2) 63 | state = atomic_exchange_explicit(&mutex->state, 2, memory_order_acquire); 64 | while (state != 0) { 65 | fev_futex_wait((int *)&mutex->state, 2); 66 | state = atomic_exchange_explicit(&mutex->state, 2, memory_order_acquire); 67 | } 68 | } 69 | 70 | FEV_NONNULL(1) void fev_thr_mutex_unlock_slow(struct fev_thr_mutex *mutex) 71 | { 72 | fev_futex_wake((int *)&mutex->state, 1); 73 | } 74 | -------------------------------------------------------------------------------- /src/fev_thr_mutex_linux.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_MUTEX_LINUX_H 11 | #define FEV_THR_MUTEX_LINUX_H 12 | 13 | #include 14 | #include 15 | 16 | #include "fev_compiler.h" 17 | 18 | /* 19 | * glibc uses a similar implementation, however the atomic instructions cannot be inlined unless 20 | * linked statically. Here, these instructions will be inlined and this improves performance a bit. 21 | * 22 | * Based on 'Futexes Are Tricky' by Ulrich Drepper. 23 | */ 24 | 25 | struct fev_thr_mutex { 26 | atomic_int state; 27 | }; 28 | 29 | FEV_NONNULL(1) void fev_thr_mutex_lock_slow(struct fev_thr_mutex *mutex); 30 | 31 | FEV_NONNULL(1) void fev_thr_mutex_unlock_slow(struct fev_thr_mutex *mutex); 32 | 33 | FEV_NONNULL(1) static inline int fev_thr_mutex_init(struct fev_thr_mutex *mutex) 34 | { 35 | atomic_init(&mutex->state, 0); 36 | return 0; 37 | } 38 | 39 | FEV_NONNULL(1) static inline void fev_thr_mutex_fini(struct fev_thr_mutex *mutex) { (void)mutex; } 40 | 41 | FEV_NONNULL(1) static inline bool fev_thr_mutex_try_lock(struct fev_thr_mutex *mutex) 42 | { 43 | int expected = 0; 44 | return atomic_compare_exchange_weak_explicit(&mutex->state, &expected, 1, memory_order_acquire, 45 | memory_order_relaxed); 46 | } 47 | 48 | FEV_NONNULL(1) static inline void fev_thr_mutex_lock(struct fev_thr_mutex *mutex) 49 | { 50 | int expected = 0; 51 | bool success = atomic_compare_exchange_weak_explicit(&mutex->state, &expected, 1, 52 | memory_order_acquire, memory_order_relaxed); 53 | if (FEV_UNLIKELY(!success)) 54 | fev_thr_mutex_lock_slow(mutex); 55 | } 56 | 57 | FEV_NONNULL(1) static inline void fev_thr_mutex_unlock(struct fev_thr_mutex *mutex) 58 | { 59 | int state = atomic_exchange_explicit(&mutex->state, 0, memory_order_release); 60 | if (FEV_UNLIKELY(state == 2)) 61 | fev_thr_mutex_unlock_slow(mutex); 62 | } 63 | 64 | #endif /* !FEV_THR_MUTEX_LINUX_H */ 65 | -------------------------------------------------------------------------------- /src/fev_thr_mutex_posix.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_MUTEX_POSIX_H 11 | #define FEV_THR_MUTEX_POSIX_H 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "fev_assert.h" 21 | #include "fev_compiler.h" 22 | #include "fev_util.h" 23 | 24 | struct fev_thr_mutex { 25 | pthread_mutex_t handle; 26 | }; 27 | 28 | FEV_NONNULL(1) 29 | FEV_WARN_UNUSED_RESULT static inline int fev_thr_mutex_init(struct fev_thr_mutex *mutex) 30 | { 31 | int ret = pthread_mutex_init(&mutex->handle, NULL); 32 | FEV_ASSERT(ret != EBUSY); 33 | FEV_ASSERT(ret != EINVAL); 34 | return FEV_LIKELY(ret == 0) ? 0 : -ret; 35 | } 36 | 37 | /* 38 | * MacOS, Darwin, Linux, DragonflyBSD, NetBSD and OpenBSD won't fail unless we make a mistake, thus 39 | * an assertion should suffice. FreeBSD can fail with ENOTRECOVERABLE. 40 | */ 41 | #if !defined(FEV_OS_MACOS) && !defined(FEV_OS_DARWIN) && !defined(FEV_OS_LINUX) && \ 42 | !defined(FEV_OS_DRAGONFLYBSD) && !defined(FEV_OS_NETBSD) && !defined(FEV_OS_OPENBSD) 43 | #define FEV_THR_MUTEX_CAN_FAIL 44 | #endif 45 | 46 | #ifdef FEV_THR_MUTEX_CAN_FAIL 47 | #define FEV_THR_MUTEX_CHECK(e) \ 48 | do { \ 49 | if (!(e)) { \ 50 | fprintf(stderr, #e " failed in %s\n", __func__); \ 51 | abort(); \ 52 | } \ 53 | } while (0) 54 | #else 55 | #define FEV_THR_MUTEX_CHECK(e) FEV_ASSERT(e) 56 | #endif 57 | 58 | FEV_NONNULL(1) static inline void fev_thr_mutex_fini(struct fev_thr_mutex *mutex) 59 | { 60 | int ret = pthread_mutex_destroy(&mutex->handle); 61 | (void)ret; 62 | FEV_THR_MUTEX_CHECK(ret == 0); 63 | } 64 | 65 | FEV_NONNULL(1) static inline bool fev_thr_mutex_try_lock(struct fev_thr_mutex *mutex) 66 | { 67 | int ret = pthread_mutex_trylock(&mutex->handle); 68 | FEV_THR_MUTEX_CHECK(ret == 0 || ret == EBUSY); 69 | return !ret; 70 | } 71 | 72 | FEV_NONNULL(1) static inline void fev_thr_mutex_lock(struct fev_thr_mutex *mutex) 73 | { 74 | int ret = pthread_mutex_lock(&mutex->handle); 75 | (void)ret; 76 | FEV_THR_MUTEX_CHECK(ret == 0); 77 | } 78 | 79 | FEV_NONNULL(1) static inline void fev_thr_mutex_unlock(struct fev_thr_mutex *mutex) 80 | { 81 | int ret = pthread_mutex_unlock(&mutex->handle); 82 | (void)ret; 83 | FEV_THR_MUTEX_CHECK(ret == 0); 84 | } 85 | 86 | #endif /* !FEV_THR_MUTEX_POSIX_H */ 87 | -------------------------------------------------------------------------------- /src/fev_thr_mutex_win.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_MUTEX_WIN_H 11 | #define FEV_THR_MUTEX_WIN_H 12 | 13 | #include 14 | #include 15 | 16 | #include "fev_compiler.h" 17 | 18 | struct fev_thr_mutex { 19 | CRITICAL_SECTION critical_section; 20 | }; 21 | 22 | FEV_NONNULL(1) static inline int fev_thr_mutex_init(struct fev_thr_mutex *mutex) 23 | { 24 | InitializeCriticalSection(&mutex->critical_section); 25 | return 0; 26 | } 27 | 28 | FEV_NONNULL(1) static inline void fev_thr_mutex_fini(struct fev_thr_mutex *mutex) 29 | { 30 | DeleteCriticalSection(&mutex->critical_section); 31 | } 32 | 33 | FEV_NONNULL(1) static inline bool fev_thr_mutex_try_lock(struct fev_thr_mutex *mutex) 34 | { 35 | return TryEnterCriticalSection(&mutex->critical_section); 36 | } 37 | 38 | FEV_NONNULL(1) static inline void fev_thr_mutex_lock(struct fev_thr_mutex *mutex) 39 | { 40 | EnterCriticalSection(&mutex->critical_section); 41 | } 42 | 43 | FEV_NONNULL(1) static inline void fev_thr_mutex_unlock(struct fev_thr_mutex *mutex) 44 | { 45 | LeaveCriticalSection(&mutex->critical_section); 46 | } 47 | 48 | #endif /* !FEV_THR_MUTEX_WIN_H */ 49 | -------------------------------------------------------------------------------- /src/fev_thr_posix.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_POSIX_H 11 | #define FEV_THR_POSIX_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_assert.h" 18 | #include "fev_compiler.h" 19 | #include "fev_util.h" 20 | 21 | struct fev_thr { 22 | pthread_t handle; 23 | }; 24 | 25 | FEV_NONNULL(1, 2) 26 | FEV_WARN_UNUSED_RESULT 27 | static inline int fev_thr_create(struct fev_thr *thr, void *(*start_routine)(void *), void *arg) 28 | { 29 | int ret = pthread_create(&thr->handle, NULL, start_routine, arg); 30 | FEV_ASSERT(ret != EINVAL); 31 | return -ret; 32 | } 33 | 34 | FEV_NONNULL(1) static inline void fev_thr_join(struct fev_thr *thr, void **ret_val_ptr) 35 | { 36 | int ret = pthread_join(thr->handle, ret_val_ptr); 37 | (void)ret; 38 | FEV_ASSERT(ret == 0); 39 | } 40 | 41 | FEV_NONNULL(1) static inline int fev_thr_cancel(struct fev_thr *thr) 42 | { 43 | int ret = pthread_cancel(thr->handle); 44 | return -ret; 45 | } 46 | 47 | #endif /* !FEV_THR_POSIX_H */ 48 | -------------------------------------------------------------------------------- /src/fev_thr_sem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_SEM_H 11 | #define FEV_THR_SEM_H 12 | 13 | #include 14 | 15 | #if defined(FEV_THR_SEM_MACOS) 16 | #include "fev_thr_sem_macos.h" 17 | #elif defined(FEV_THR_SEM_POSIX) 18 | #include "fev_thr_sem_posix.h" 19 | #elif defined(FEV_THR_SEM_WIN) 20 | #include "fev_thr_sem_win.h" 21 | #else 22 | #error Your platform is unsupported! 23 | #endif 24 | 25 | #endif /* !FEV_THR_SEM_H */ 26 | -------------------------------------------------------------------------------- /src/fev_thr_sem_macos.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_thr_sem.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "fev_assert.h" 21 | #include "fev_compiler.h" 22 | 23 | FEV_NONNULL(1) FEV_WARN_UNUSED_RESULT int fev_thr_sem_init(struct fev_thr_sem *sem, unsigned value) 24 | { 25 | char name[64]; 26 | pid_t pid; 27 | sem_t *handle; 28 | 29 | FEV_ASSERT(value <= SEM_VALUE_MAX); 30 | 31 | /* TODO: This can leak some information, like ASLR offset etc. */ 32 | pid = getpid(); 33 | snprintf(name, sizeof(name), "fev_sem_%i_%016" PRIxPTR, pid, (uintptr_t)sem); 34 | handle = sem_open(name, O_CREAT | O_EXCL, 0644, value); 35 | if (FEV_UNLIKELY(handle == SEM_FAILED)) 36 | return -errno; 37 | 38 | sem->handle = handle; 39 | return 0; 40 | } 41 | 42 | FEV_NONNULL(1) void fev_thr_sem_fini(struct fev_thr_sem *sem) 43 | { 44 | int ret = sem_close(sem->handle); 45 | (void)ret; 46 | FEV_ASSERT(ret == 0); 47 | } 48 | -------------------------------------------------------------------------------- /src/fev_thr_sem_macos.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_SEM_MACOS_H 11 | #define FEV_THR_SEM_MACOS_H 12 | 13 | #include 14 | #include 15 | 16 | #include "fev_assert.h" 17 | #include "fev_compiler.h" 18 | 19 | struct fev_thr_sem { 20 | sem_t *handle; 21 | }; 22 | 23 | FEV_NONNULL(1) FEV_WARN_UNUSED_RESULT int fev_thr_sem_init(struct fev_thr_sem *sem, unsigned value); 24 | 25 | FEV_NONNULL(1) void fev_thr_sem_fini(struct fev_thr_sem *sem); 26 | 27 | FEV_NONNULL(1) static inline void fev_thr_sem_wait(struct fev_thr_sem *sem) 28 | { 29 | int ret; 30 | do { 31 | ret = sem_wait(sem->handle); 32 | FEV_ASSERT(ret == 0 || (ret == -1 && errno == EINTR)); 33 | } while (ret != 0); 34 | } 35 | 36 | FEV_NONNULL(1) static inline void fev_thr_sem_post(struct fev_thr_sem *sem) 37 | { 38 | int ret = sem_post(sem->handle); 39 | (void)ret; 40 | FEV_ASSERT(ret == 0); 41 | } 42 | 43 | #endif /* !FEV_THR_SEM_MACOS_H */ 44 | -------------------------------------------------------------------------------- /src/fev_thr_sem_posix.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_SEM_POSIX_H 11 | #define FEV_THR_SEM_POSIX_H 12 | 13 | #include 14 | #include 15 | 16 | #include "fev_assert.h" 17 | #include "fev_compiler.h" 18 | 19 | struct fev_thr_sem { 20 | sem_t handle; 21 | }; 22 | 23 | FEV_NONNULL(1) 24 | FEV_WARN_UNUSED_RESULT static inline int fev_thr_sem_init(struct fev_thr_sem *sem, unsigned value) 25 | { 26 | int ret = sem_init(&sem->handle, /*pshared=*/0, value); 27 | return FEV_LIKELY(ret == 0) ? 0 : -errno; 28 | } 29 | 30 | FEV_NONNULL(1) static inline void fev_thr_sem_fini(struct fev_thr_sem *sem) 31 | { 32 | int ret = sem_destroy(&sem->handle); 33 | (void)ret; 34 | FEV_ASSERT(ret == 0); 35 | } 36 | 37 | FEV_NONNULL(1) static inline void fev_thr_sem_wait(struct fev_thr_sem *sem) 38 | { 39 | int ret; 40 | do { 41 | ret = sem_wait(&sem->handle); 42 | FEV_ASSERT(ret == 0 || (ret == -1 && errno == EINTR)); 43 | } while (ret != 0); 44 | } 45 | 46 | FEV_NONNULL(1) static inline void fev_thr_sem_post(struct fev_thr_sem *sem) 47 | { 48 | int ret = sem_post(&sem->handle); 49 | (void)ret; 50 | FEV_ASSERT(ret == 0); 51 | } 52 | 53 | #endif /* !FEV_THR_SEM_POSIX_H */ 54 | -------------------------------------------------------------------------------- /src/fev_thr_sem_win.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_SEM_WIN_H 11 | #define FEV_THR_SEM_WIN_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_assert.h" 18 | #include "fev_compiler.h" 19 | #include "fev_util.h" 20 | 21 | struct fev_thr_sem { 22 | HANDLE handle; 23 | }; 24 | 25 | FEV_NONNULL(1) 26 | FEV_WARN_UNUSED_RESULT static int fev_thr_sem_init(struct fev_thr_sem *sem, unsigned value) 27 | { 28 | sem->handle = CreateSemaphore(/*lpSemaphoreAttributes=*/NULL, /*lInitialCount=*/(LONG)value, 29 | /*lMaximumCount=*/(LONG)UINT_MAX, /*lpName=*/NULL); 30 | if (FEV_UNLIKELY(handle == NULL)) 31 | return fev_translate_win_error(GetLastError()); 32 | return 0; 33 | } 34 | 35 | FEV_NONNULL(1) static void fev_thr_sem_fini(struct fev_thr_sem *sem) 36 | { 37 | BOOL ok = CloseHandle(sem->handle); 38 | (void)ok; 39 | FEV_ASSERT(ok); 40 | } 41 | 42 | FEV_NONNULL(1) static void fev_thr_sem_wait(struct fev_thr_sem *sem) 43 | { 44 | DWORD result = WaitForSingleObject(sem->handle, /*dwMilliseconds=*/INFINITE); 45 | (void)result; 46 | FEV_ASSERT(result == WAIT_OBJECT_0); 47 | } 48 | 49 | FEV_NONNULL(1) static void fev_thr_sem_post(struct fev_thr_sem *sem) 50 | { 51 | BOOL ok = ReleaseSemaphore(sem->handle, /*lReleaseCount=*/1, /*lpPreviousCount=*/NULL); 52 | (void)ok; 53 | FEV_ASSERT(ok); 54 | } 55 | 56 | #endif /* !FEV_THR_SEM_WIN_H */ 57 | -------------------------------------------------------------------------------- /src/fev_thr_win.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_THR_WIN_H 11 | #define FEV_THR_WIN_H 12 | 13 | #error Not implemented 14 | 15 | #endif /* !FEV_THR_WIN_H */ 16 | -------------------------------------------------------------------------------- /src/fev_time.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_TIME_H 11 | #define FEV_TIME_H 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include "fev_assert.h" 19 | #include "fev_compiler.h" 20 | 21 | /* TODO: Move it to config. */ 22 | #ifndef FEV_CLOCK_ID 23 | #define FEV_CLOCK_ID CLOCK_MONOTONIC 24 | #endif 25 | 26 | #define FEV_NSECS_PER_SEC (1000 * 1000 * 1000) 27 | 28 | FEV_NONNULL(1) static inline void fev_clock_get_time(struct timespec *ts) 29 | { 30 | int ret; 31 | 32 | ret = clock_gettime(FEV_CLOCK_ID, ts); 33 | (void)ret; 34 | FEV_ASSERT(ret == 0); 35 | } 36 | 37 | FEV_NONNULL(1) static inline void fev_timespec_assert_valid(const struct timespec *ts) 38 | { 39 | (void)ts; 40 | FEV_ASSERT(ts->tv_sec >= 0); 41 | FEV_ASSERT(ts->tv_nsec >= 0 && ts->tv_nsec < FEV_NSECS_PER_SEC); 42 | } 43 | 44 | FEV_NONNULL(1, 2) 45 | FEV_PURE static inline long fev_timespec_cmp(const struct timespec *lhs, const struct timespec *rhs) 46 | { 47 | time_t lhs_sec, rhs_sec; 48 | long sec_cmp; 49 | 50 | fev_timespec_assert_valid(lhs); 51 | fev_timespec_assert_valid(rhs); 52 | 53 | /* tv_sec is of type time_t, which is unspecified (it may be unsigned). */ 54 | lhs_sec = lhs->tv_sec; 55 | rhs_sec = rhs->tv_sec; 56 | sec_cmp = _Generic(lhs_sec, long 57 | : lhs_sec - rhs_sec, default 58 | : lhs_sec < rhs_sec ? -1 : (lhs_sec > rhs_sec ? 1 : 0)); 59 | if (sec_cmp != 0) 60 | return sec_cmp; 61 | 62 | return lhs->tv_nsec - rhs->tv_nsec; 63 | } 64 | 65 | FEV_NONNULL(1, 2) 66 | static inline void fev_timespec_abs_to_rel(struct timespec *rel_time, 67 | const struct timespec *abs_time) 68 | { 69 | struct timespec ts; 70 | 71 | fev_timespec_assert_valid(abs_time); 72 | 73 | fev_clock_get_time(&ts); 74 | 75 | if (fev_timespec_cmp(&ts, abs_time) >= 0) { 76 | rel_time->tv_sec = 0; 77 | rel_time->tv_nsec = 0; 78 | return; 79 | } 80 | 81 | ts.tv_sec = abs_time->tv_sec - ts.tv_sec; 82 | 83 | ts.tv_nsec = abs_time->tv_nsec - ts.tv_nsec; 84 | if (ts.tv_nsec < 0) { 85 | FEV_ASSERT(ts.tv_sec > 0); 86 | ts.tv_sec -= 1; 87 | ts.tv_nsec += FEV_NSECS_PER_SEC; 88 | } 89 | 90 | fev_timespec_assert_valid(&ts); 91 | 92 | rel_time->tv_sec = ts.tv_sec; 93 | rel_time->tv_nsec = ts.tv_nsec; 94 | } 95 | 96 | FEV_NONNULL(1, 2) 97 | static inline void fev_get_abs_time_since_now(struct timespec *restrict abs_time, 98 | const struct timespec *restrict rel_time) 99 | { 100 | fev_timespec_assert_valid(rel_time); 101 | 102 | fev_clock_get_time(abs_time); 103 | 104 | abs_time->tv_sec += 105 | rel_time->tv_sec + (abs_time->tv_nsec + rel_time->tv_nsec) / FEV_NSECS_PER_SEC; 106 | abs_time->tv_nsec = (abs_time->tv_nsec + rel_time->tv_nsec) % FEV_NSECS_PER_SEC; 107 | 108 | fev_timespec_assert_valid(abs_time); 109 | } 110 | 111 | FEV_NONNULL(1) static inline uint64_t fev_timespec_to_ns(const struct timespec *ts) 112 | { 113 | uint64_t sec, nsec, x, ret; 114 | 115 | fev_timespec_assert_valid(ts); 116 | 117 | sec = (uint64_t)ts->tv_sec; 118 | nsec = (uint64_t)ts->tv_nsec; 119 | 120 | if (sec > UINT64_MAX / FEV_NSECS_PER_SEC) 121 | x = UINT64_MAX; 122 | else 123 | x = sec * FEV_NSECS_PER_SEC; 124 | 125 | ret = x + nsec; 126 | if (ret < x) 127 | ret = UINT64_MAX; 128 | 129 | return ret; 130 | } 131 | 132 | #endif /* !FEV_TIME_H */ 133 | -------------------------------------------------------------------------------- /src/fev_timers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_TIMERS_H 11 | #define FEV_TIMERS_H 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include "fev_compiler.h" 19 | #include "fev_spinlock_impl.h" 20 | #include "fev_thr_mutex.h" 21 | #include "fev_time.h" 22 | #include "fev_waiter_intf.h" 23 | 24 | #ifdef FEV_TIMERS_MIN_LOCK_MUTEX 25 | 26 | typedef struct fev_thr_mutex fev_timers_bucket_min_lock_t; 27 | #define fev_timers_bucket_min_lock_init fev_thr_mutex_init 28 | #define fev_timers_bucket_min_lock_fini fev_thr_mutex_fini 29 | #define fev_timers_bucket_min_lock fev_thr_mutex_lock 30 | #define fev_timers_bucket_min_unlock fev_thr_mutex_unlock 31 | 32 | #elif defined(FEV_TIMERS_MIN_LOCK_SPINLOCK) 33 | 34 | typedef struct fev_spinlock fev_timers_bucket_min_lock_t; 35 | #define fev_timers_bucket_min_lock_init fev_spinlock_init 36 | #define fev_timers_bucket_min_lock_fini fev_spinlock_fini 37 | #define fev_timers_bucket_min_lock fev_spinlock_lock 38 | #define fev_timers_bucket_min_unlock fev_spinlock_unlock 39 | 40 | #endif 41 | 42 | #if defined(FEV_TIMERS_BINHEAP) 43 | #include "fev_timers_binheap.h" 44 | #elif defined(FEV_TIMERS_RBTREE) 45 | #include "fev_timers_rbtree.h" 46 | #else 47 | #error Wrong timers strategy selected, define either FEV_TIMERS_BINHEAP or FEV_TIMERS_RBTREE. 48 | #endif 49 | 50 | /* TODO: Move it to config. */ 51 | #ifndef FEV_TIMERS_BUCKETS 52 | #define FEV_TIMERS_BUCKETS 64u 53 | #endif 54 | 55 | static_assert(FEV_TIMERS_BUCKETS > 0, "FEV_TIMERS_BUCKETS must be greater than 0"); 56 | static_assert((FEV_TIMERS_BUCKETS & (FEV_TIMERS_BUCKETS - 1)) == 0, 57 | "FEV_TIMERS_BUCKETS must be a power of 2"); 58 | 59 | #define FEV_TIMERS_BUCKET_MASK ((FEV_TIMERS_BUCKETS)-1) 60 | 61 | /* 62 | * fev_timed_wait() can fail with ENOMEM if fev_timers_bucket_add() can fail. 63 | * fev_timers_bucket_add() fails only with ENOMEM, the function should not return any other error 64 | * codes. 65 | */ 66 | #define FEV_TIMED_WAIT_CAN_RETURN_ENOMEM FEV_TIMERS_ADD_CAN_FAIL 67 | 68 | struct fev_timers { 69 | struct fev_timers_bucket buckets[FEV_TIMERS_BUCKETS]; 70 | }; 71 | 72 | FEV_NONNULL(1) FEV_PURE bool fev_timer_is_expired(const struct fev_timer *timer); 73 | 74 | FEV_NONNULL(1) void fev_timer_set_expired(struct fev_timer *timer); 75 | 76 | FEV_COLD FEV_NONNULL(1) int fev_timers_bucket_init(struct fev_timers_bucket *bucket); 77 | 78 | FEV_COLD FEV_NONNULL(1) void fev_timers_bucket_fini(struct fev_timers_bucket *bucket); 79 | 80 | FEV_NONNULL(1, 2) 81 | int fev_timers_bucket_add(struct fev_timers_bucket *bucket, struct fev_timer *timer); 82 | 83 | FEV_NONNULL(1, 2) 84 | int fev_timers_bucket_del(struct fev_timers_bucket *bucket, struct fev_timer *timer); 85 | 86 | FEV_NONNULL(1) void fev_timers_bucket_del_min(struct fev_timers_bucket *bucket); 87 | 88 | FEV_NONNULL(1) FEV_PURE bool fev_timers_bucket_empty(const struct fev_timers_bucket *bucket); 89 | 90 | FEV_NONNULL(1) 91 | FEV_PURE FEV_RETURNS_NONNULL struct fev_timer * 92 | fev_timers_bucket_min(const struct fev_timers_bucket *bucket); 93 | 94 | FEV_COLD FEV_NONNULL(1) FEV_WARN_UNUSED_RESULT int fev_timers_init(struct fev_timers *timers); 95 | 96 | FEV_COLD FEV_NONNULL(1) void fev_timers_fini(struct fev_timers *timers); 97 | 98 | FEV_NONNULL(1, 2) 99 | FEV_WARN_UNUSED_RESULT 100 | int fev_timed_wait(struct fev_waiter *waiter, const struct timespec *abs_time); 101 | 102 | #endif /* !FEV_TIMERS_H */ 103 | -------------------------------------------------------------------------------- /src/fev_timers_binheap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_TIMERS_BINHEAP_H 11 | #define FEV_TIMERS_BINHEAP_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fev_alloc.h" 18 | #include "fev_assert.h" 19 | #include "fev_compiler.h" 20 | #include "fev_ilock_intf.h" 21 | #include "fev_poller.h" 22 | #include "fev_time.h" 23 | #include "fev_waiter_intf.h" 24 | 25 | #ifdef FEV_ASSUME_MALLOC_NEVER_FAILS 26 | #define FEV_TIMERS_ADD_CAN_FAIL 0 27 | #else 28 | #define FEV_TIMERS_ADD_CAN_FAIL 1 29 | #endif 30 | 31 | struct fev_timer { 32 | struct timespec abs_time; 33 | size_t index; 34 | struct fev_waiter *waiter; 35 | }; 36 | 37 | struct fev_timers_bucket { 38 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_ilock lock; 39 | struct fev_timer **heap; 40 | size_t len; 41 | size_t capacity; 42 | struct fev_poller_timers_bucket_data poller_data; 43 | 44 | alignas(FEV_DCACHE_LINE_SIZE) fev_timers_bucket_min_lock_t min_lock; 45 | struct fev_timer *min; 46 | }; 47 | 48 | FEV_NONNULL(1) FEV_PURE static inline bool fev_timer_is_expired(const struct fev_timer *timer) 49 | { 50 | return timer->index == SIZE_MAX; 51 | } 52 | 53 | FEV_NONNULL(1) static inline void fev_timer_set_expired(struct fev_timer *timer) 54 | { 55 | timer->index = SIZE_MAX; 56 | } 57 | 58 | FEV_NONNULL(1) 59 | FEV_PURE static inline bool fev_timers_bucket_empty(const struct fev_timers_bucket *bucket) 60 | { 61 | return bucket->len == 0; 62 | } 63 | 64 | FEV_NONNULL(1) 65 | FEV_PURE FEV_RETURNS_NONNULL static inline struct fev_timer * 66 | fev_timers_bucket_min(const struct fev_timers_bucket *bucket) 67 | { 68 | struct fev_timer *min; 69 | 70 | FEV_ASSERT(!fev_timers_bucket_empty(bucket)); 71 | min = bucket->heap[0]; 72 | FEV_ASSERT(min != NULL); 73 | return min; 74 | } 75 | 76 | #endif /* !FEV_TIMERS_BINHEAP_H */ 77 | -------------------------------------------------------------------------------- /src/fev_timers_rbtree.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #include "fev_timers.h" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | #include "fev_assert.h" 19 | #include "fev_compiler.h" 20 | #include "fev_ilock_impl.h" 21 | #include "fev_time.h" 22 | 23 | /* 24 | * Entries in the rbtree implementation must be unique, comparing only timespec is not enough and 25 | * could crash the application. Thus, in addition, timer->waiter is compared, since it is always 26 | * unique at a given time. A waiter can be reused when the corresponding timer is removed. 27 | */ 28 | FEV_NONNULL(1, 2) 29 | FEV_PURE static int fev_timers_cmp(const struct fev_timer *lhs, const struct fev_timer *rhs) 30 | { 31 | long cmp; 32 | 33 | cmp = fev_timespec_cmp(&lhs->abs_time, &rhs->abs_time); 34 | if (cmp != 0) 35 | return cmp < 0 ? -1 : 1; 36 | 37 | /* The lhs & rhs pointers are unrelated. */ 38 | if ((uintptr_t)lhs->waiter == (uintptr_t)rhs->waiter) 39 | return 0; 40 | if ((uintptr_t)lhs->waiter < (uintptr_t)rhs->waiter) 41 | return -1; 42 | return 1; 43 | } 44 | 45 | RB_GENERATE_INSERT_COLOR(fev_timers_rbtree_head, fev_timer, entry, static); 46 | RB_GENERATE_INSERT(fev_timers_rbtree_head, fev_timer, entry, fev_timers_cmp, static); 47 | RB_GENERATE_REMOVE_COLOR(fev_timers_rbtree_head, fev_timer, entry, static); 48 | RB_GENERATE_REMOVE(fev_timers_rbtree_head, fev_timer, entry, static); 49 | RB_GENERATE_NEXT(fev_timers_rbtree_head, fev_timer, entry, FEV_PURE static); 50 | 51 | FEV_NONNULL(1, 2) 52 | int fev_timers_bucket_add(struct fev_timers_bucket *bucket, struct fev_timer *timer) 53 | { 54 | int min_changed = 0; 55 | 56 | /* The tree_min element should be minimal, thus it shouldn't have a left child. */ 57 | FEV_ASSERT(bucket->tree_min == NULL || RB_LEFT(bucket->tree_min, entry) == NULL); 58 | 59 | timer->expired = false; 60 | 61 | RB_INSERT(fev_timers_rbtree_head, &bucket->head, timer); 62 | 63 | /* If the left child changed, then the element is not minimal anymore. */ 64 | if (bucket->tree_min == NULL || RB_LEFT(bucket->tree_min, entry) != NULL) { 65 | FEV_ASSERT(bucket->tree_min == NULL || RB_LEFT(bucket->tree_min, entry) == timer); 66 | bucket->tree_min = timer; 67 | min_changed = 1; 68 | } 69 | 70 | return min_changed; 71 | } 72 | 73 | FEV_NONNULL(1, 2) 74 | int fev_timers_bucket_del(struct fev_timers_bucket *bucket, struct fev_timer *timer) 75 | { 76 | int min_changed = 0; 77 | 78 | FEV_ASSERT(!fev_timers_bucket_empty(bucket)); 79 | 80 | if (bucket->tree_min == timer) { 81 | bucket->tree_min = RB_NEXT(fev_timers_rbtree_head, &bucket->head, bucket->tree_min); 82 | min_changed = 1; 83 | } 84 | 85 | RB_REMOVE(fev_timers_rbtree_head, &bucket->head, timer); 86 | 87 | return min_changed; 88 | } 89 | 90 | FEV_NONNULL(1) void fev_timers_bucket_del_min(struct fev_timers_bucket *bucket) 91 | { 92 | struct fev_timer *tree_min; 93 | 94 | FEV_ASSERT(!fev_timers_bucket_empty(bucket)); 95 | 96 | tree_min = bucket->tree_min; 97 | bucket->tree_min = RB_NEXT(fev_timers_rbtree_head, &bucket->head, tree_min); 98 | RB_REMOVE(fev_timers_rbtree_head, &bucket->head, tree_min); 99 | } 100 | 101 | FEV_COLD FEV_NONNULL(1) int fev_timers_bucket_init(struct fev_timers_bucket *bucket) 102 | { 103 | int ret; 104 | 105 | ret = fev_ilock_init(&bucket->lock); 106 | if (FEV_UNLIKELY(ret != 0)) 107 | return ret; 108 | 109 | ret = fev_timers_bucket_min_lock_init(&bucket->min_lock); 110 | if (FEV_UNLIKELY(ret != 0)) { 111 | fev_ilock_fini(&bucket->lock); 112 | return ret; 113 | } 114 | 115 | RB_INIT(&bucket->head); 116 | bucket->tree_min = NULL; 117 | bucket->min = NULL; 118 | return 0; 119 | } 120 | 121 | FEV_COLD FEV_NONNULL(1) void fev_timers_bucket_fini(struct fev_timers_bucket *bucket) 122 | { 123 | fev_timers_bucket_min_lock_fini(&bucket->min_lock); 124 | fev_ilock_fini(&bucket->lock); 125 | } 126 | -------------------------------------------------------------------------------- /src/fev_timers_rbtree.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_TIMERS_RBTREE_H 11 | #define FEV_TIMERS_RBTREE_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | 19 | #include "fev_assert.h" 20 | #include "fev_compiler.h" 21 | #include "fev_ilock_intf.h" 22 | #include "fev_poller.h" 23 | #include "fev_time.h" 24 | #include "fev_waiter_intf.h" 25 | 26 | #define FEV_TIMERS_ADD_CAN_FAIL 0 27 | 28 | struct fev_timer { 29 | struct timespec abs_time; 30 | RB_ENTRY(fev_timer) entry; 31 | bool expired; 32 | struct fev_waiter *waiter; 33 | }; 34 | 35 | struct fev_timers_bucket { 36 | alignas(FEV_DCACHE_LINE_SIZE) struct fev_ilock lock; 37 | RB_HEAD(fev_timers_rbtree_head, fev_timer) head; 38 | struct fev_timer *tree_min; 39 | struct fev_poller_timers_bucket_data poller_data; 40 | 41 | alignas(FEV_DCACHE_LINE_SIZE) fev_timers_bucket_min_lock_t min_lock; 42 | struct fev_timer *min; 43 | }; 44 | 45 | FEV_NONNULL(1) FEV_PURE static inline bool fev_timer_is_expired(const struct fev_timer *timer) 46 | { 47 | return timer->expired; 48 | } 49 | 50 | FEV_NONNULL(1) 51 | static inline void fev_timer_set_expired(struct fev_timer *timer) { timer->expired = true; } 52 | 53 | FEV_NONNULL(1) 54 | FEV_PURE static inline bool fev_timers_bucket_empty(const struct fev_timers_bucket *bucket) 55 | { 56 | return bucket->tree_min == NULL; 57 | } 58 | 59 | FEV_NONNULL(1) 60 | FEV_PURE FEV_RETURNS_NONNULL static inline struct fev_timer * 61 | fev_timers_bucket_min(const struct fev_timers_bucket *bucket) 62 | { 63 | struct fev_timer *tree_min; 64 | 65 | FEV_ASSERT(!fev_timers_bucket_empty(bucket)); 66 | tree_min = bucket->tree_min; 67 | FEV_ASSERT(tree_min != NULL); 68 | return tree_min; 69 | } 70 | 71 | #endif /* !FEV_TIMERS_RBTREE_H */ 72 | -------------------------------------------------------------------------------- /src/fev_util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_UTIL_H 11 | #define FEV_UTIL_H 12 | 13 | #include 14 | #include 15 | 16 | #define FEV_CONTAINER_OF(ptr, type, member) ((type *)((uint8_t *)(ptr)-offsetof(type, member))) 17 | 18 | /* Lehmer RNG. */ 19 | #define FEV_RANDOM_MULTIPLIER UINT64_C(48271) 20 | #define FEV_RANDOM_MODULUS UINT64_C(2147483647) 21 | #define FEV_RANDOM_MAX (FEV_RANDOM_MODULUS - 1) 22 | #define FEV_RANDOM_NEXT(r) ((uint32_t)((uint64_t)(r)*FEV_RANDOM_MULTIPLIER % FEV_RANDOM_MODULUS)) 23 | 24 | #endif /* !FEV_UTIL_H */ 25 | -------------------------------------------------------------------------------- /src/fev_waiter_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_WAITER_INTF_H 11 | #define FEV_WAITER_INTF_H 12 | 13 | #include 14 | #include 15 | 16 | struct fev_fiber; 17 | 18 | /* Result of fev_waiter_wake() */ 19 | enum fev_waiter_wake_result { 20 | /* Setting reason failed, possibly someone else has set it before. */ 21 | FEV_WAITER_FAILED, 22 | 23 | /* 24 | * We managed to set wake reason, but someone else will wake up the fiber. This can happen when a 25 | * fiber wants to be woken up, but it is switching to scheduler right now and hasn't set `do_wake` 26 | * flag yet. In that case, it will be woken up just after the switch (see 27 | * fev_waiter_enable_wake_ups()). 28 | */ 29 | FEV_WAITER_SET_ONLY, 30 | 31 | /* We managed to set wake reason and we are responsible of waking up the fiber. */ 32 | FEV_WAITER_SET_AND_WAKE_UP, 33 | }; 34 | 35 | enum fev_waiter_wake_reason { 36 | /* The reason is not set yet. */ 37 | FEV_WAITER_NONE, 38 | 39 | /* The object is ready (e.g. an unlocked mutex, incoming data on a socket). */ 40 | FEV_WAITER_READY, 41 | 42 | /* 43 | * The timer has expired, the caller of fev_waiter_wait() is responsible for checking the timers 44 | * for other timeouts. This reason is issued by the underlying poller. 45 | */ 46 | FEV_WAITER_TIMED_OUT_CHECK, 47 | 48 | /* 49 | * The timer has expired, but the caller doesn't have to check for other timeouts. This is issued 50 | * by a fiber that is processing timers and thus we don't have to check them again. 51 | */ 52 | FEV_WAITER_TIMED_OUT_NO_CHECK, 53 | }; 54 | 55 | struct fev_waiter { 56 | /* 57 | * Reason (socket ready, timeout, etc.) that was set in attempt to wake up the fiber. The callers 58 | * of fev_waiter_wait() should set this to FEV_WAITER_NONE to indicate that we are ready to wait 59 | * for some events. Then, the event handlers will try to update it if the value is still 60 | * FEV_WAITER_NONE. If the handler succeeds, it will try to wake up the stored fiber. 61 | */ 62 | atomic_uint reason; 63 | 64 | /* Must the stored fiber be woken up after setting `reason`? */ 65 | atomic_uint do_wake; 66 | 67 | /* 68 | * Reason of wake up that is set by the worker that managed to change `do_wake` from 1 to 0. This 69 | * differs from `reason`, since `reason` can be updated by waiters (e.g. by fev_waiter_wait()) and 70 | * handlers (by fev_waiter_wake()). It is possible that `reason` is FEV_WAITER_NONE after a wake 71 | * up. In comparison, `wake_reason` should never be FEV_WAITER_NONE after a wake up. 72 | */ 73 | atomic_uint wake_reason; 74 | 75 | /* 76 | * Should a woken up fiber wait for some operations to finish? 77 | * A waiter can be allocated on the stack. In such case the woken up fiber cannot return from 78 | * fev_waiter_wait() while procedures that hold references to the waiter are still alive, 79 | * otherwise stack-use-after-return bugs are possible. To overcome this problem, we will spin in 80 | * fev_waiter_wait() until the procedures have finished, that is when `wait_for_post` and 81 | * `wait_for_wake` are both 0 (or equivalently `wait` is 0). 82 | */ 83 | union { 84 | struct { 85 | /* If 1, the woken fiber will wait for fev_waiter_enable_wake_ups() to finish. */ 86 | _Atomic uint8_t wait_for_post; 87 | 88 | /* If 1, the woken fiber will wait for fev_waiter_wake() to finish. */ 89 | _Atomic uint8_t wait_for_wake; 90 | }; 91 | _Atomic uint16_t wait; 92 | }; 93 | 94 | /* The fiber that must be woken up. */ 95 | struct fev_fiber *fiber; 96 | }; 97 | 98 | #endif /* !FEV_WAITER_INTF_H */ 99 | -------------------------------------------------------------------------------- /src/fev_waiters_queue_intf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Patryk Stefanski 3 | * 4 | * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be 7 | * copied, modified, or distributed except according to those terms. 8 | */ 9 | 10 | #ifndef FEV_WAITERS_QUEUE_INTF_H 11 | #define FEV_WAITERS_QUEUE_INTF_H 12 | 13 | #include 14 | 15 | #include 16 | 17 | #include "fev_ilock_intf.h" 18 | #include "fev_waiter_intf.h" 19 | 20 | struct fev_waiters_queue_node { 21 | struct fev_waiter waiter; 22 | TAILQ_ENTRY(fev_waiters_queue_node) tq_entry; 23 | bool deleted; 24 | }; 25 | 26 | struct fev_waiters_queue { 27 | struct fev_ilock lock; 28 | TAILQ_HEAD(, fev_waiters_queue_node) nodes; 29 | }; 30 | 31 | #endif /* !FEV_WAITERS_QUEUE_INTF_H */ 32 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(FEV_TESTS 2 | #sleep 3 | stress_cond 4 | stress_cond_with_timeout 5 | stress_ilock 6 | stress_mpmc_queue 7 | stress_mutex 8 | stress_mutex_with_timeout 9 | stress_qsbr_queue 10 | stress_sem 11 | stress_sem_with_timeout 12 | stress_thr_mutex 13 | timers_bucket 14 | ) 15 | foreach(target ${FEV_TESTS}) 16 | add_executable(${target} ${target}.c) 17 | target_include_directories(${target} PRIVATE ../third_party) 18 | target_link_libraries(${target} PRIVATE fev) 19 | set_property(TARGET ${target} PROPERTY C_STANDARD 11) 20 | fev_set_compile_options(${target}) 21 | endforeach() 22 | -------------------------------------------------------------------------------- /tests/sleep.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include "util.h" 6 | 7 | static void *test(void *arg) 8 | { 9 | (void)arg; 10 | 11 | for (int i = 0; i < 10; i++) { 12 | struct timespec now, rel_time, expected; 13 | 14 | clock_gettime(CLOCK_MONOTONIC, &now); 15 | 16 | rel_time.tv_sec = 0; 17 | rel_time.tv_nsec = i * 1000000; 18 | 19 | expected = now; 20 | expected.tv_sec += 21 | rel_time.tv_sec + (expected.tv_nsec + rel_time.tv_nsec) / (1000 * 1000 * 1000); 22 | expected.tv_nsec = (expected.tv_nsec + rel_time.tv_nsec) % (1000 * 1000 * 1000); 23 | 24 | fev_sleep_for(&rel_time); 25 | 26 | clock_gettime(CLOCK_MONOTONIC, &now); 27 | CHECK(now.tv_sec > expected.tv_sec || 28 | (now.tv_sec == expected.tv_sec && now.tv_nsec >= expected.tv_nsec), 29 | "Timer expired too soon"); 30 | } 31 | 32 | return NULL; 33 | } 34 | 35 | int main(void) 36 | { 37 | struct fev_sched *sched; 38 | int err; 39 | 40 | err = fev_sched_create(&sched, NULL); 41 | CHECK(err == 0, "Creating scheduler failed: err=%i", err); 42 | 43 | err = fev_fiber_spawn(sched, &test, NULL); 44 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 45 | 46 | err = fev_sched_run(sched); 47 | CHECK(err == 0, "Running scheduler failed: err=%i", err); 48 | 49 | fev_sched_destroy(sched); 50 | 51 | return 0; 52 | } 53 | -------------------------------------------------------------------------------- /tests/stress_cond.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | #include "util.h" 9 | 10 | static uint32_t num_fibers; 11 | static uint32_t num_iterations; 12 | 13 | struct fiber_data { 14 | struct fev_cond *cond; 15 | struct fev_mutex *mutex; 16 | uint64_t counter; 17 | int turn; 18 | }; 19 | 20 | static void *work(void *arg) 21 | { 22 | int turn = (uintptr_t)arg & 1; 23 | struct fiber_data *data = (void *)((uintptr_t)arg & ~(uintptr_t)1); 24 | 25 | for (uint32_t i = 0; i < num_iterations; i++) { 26 | fev_mutex_lock(data->mutex); 27 | 28 | while (data->turn != turn) 29 | fev_cond_wait(data->cond, data->mutex); 30 | 31 | data->counter++; 32 | data->turn = !turn; 33 | 34 | fev_mutex_unlock(data->mutex); 35 | 36 | fev_cond_notify_one(data->cond); 37 | } 38 | 39 | return NULL; 40 | } 41 | 42 | static void *test(void *arg) 43 | { 44 | struct fiber_data *data; 45 | struct fev_fiber **fibers; 46 | uint32_t n; 47 | 48 | (void)arg; 49 | 50 | /* Well, we need at least 2 fibers... */ 51 | n = (num_fibers + 1) / 2; 52 | 53 | data = malloc((size_t)n * sizeof(*data)); 54 | CHECK(data != NULL, "Allocating memory for shared data failed"); 55 | 56 | for (uint32_t i = 0; i < n; i++) { 57 | struct fiber_data *d = &data[i]; 58 | int err; 59 | 60 | err = fev_cond_create(&d->cond); 61 | CHECK(err == 0, "Creating condition variable failed: err=%i", err); 62 | 63 | err = fev_mutex_create(&d->mutex); 64 | CHECK(err == 0, "Creating mutex failed: err=%i", err); 65 | 66 | d->counter = 0; 67 | d->turn = 1; 68 | } 69 | 70 | fibers = malloc((size_t)num_fibers * sizeof(*fibers)); 71 | CHECK(fibers != NULL, "Allocating memory for fibers failed"); 72 | 73 | for (uint32_t i = 0; i < n; i++) { 74 | void *arg = &data[i]; 75 | int err = fev_fiber_create(&fibers[2 * i], NULL, &work, arg, NULL); 76 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 77 | 78 | arg = (void *)((uintptr_t)arg | 1); 79 | err = fev_fiber_create(&fibers[2 * i + 1], NULL, &work, arg, NULL); 80 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 81 | } 82 | 83 | for (uint32_t i = 0; i < num_fibers; i++) 84 | fev_fiber_join(fibers[i], NULL); 85 | 86 | free(fibers); 87 | 88 | for (uint32_t i = 0; i < n; i++) { 89 | struct fiber_data *d = &data[i]; 90 | uint64_t expected; 91 | 92 | expected = (uint64_t)num_iterations * 2; 93 | CHECK(d->counter == expected, 94 | "The counter value is incorrect: counter=%" PRIu64 " expected=%" PRIu64, d->counter, 95 | expected); 96 | 97 | CHECK(d->turn == 1, "The turn is incorrect"); 98 | 99 | fev_mutex_destroy(d->mutex); 100 | fev_cond_destroy(d->cond); 101 | } 102 | 103 | free(data); 104 | 105 | return NULL; 106 | } 107 | 108 | int main(int argc, char **argv) 109 | { 110 | struct fev_sched_attr *sched_attr; 111 | struct fev_sched *sched; 112 | uint32_t num_workers; 113 | int err; 114 | 115 | CHECK(argc == 4, "Usage: %s ", argv[0]); 116 | 117 | num_workers = parse_uint32_t(argv[1], "num_workers", &(uint32_t){1}); 118 | num_fibers = parse_uint32_t(argv[2], "num_fibers", &(uint32_t){1}); 119 | num_iterations = parse_uint32_t(argv[3], "num_iterations", &(uint32_t){1}); 120 | 121 | err = fev_sched_attr_create(&sched_attr); 122 | CHECK(err == 0, "Creating scheduler attributes failed: err=%i", err); 123 | 124 | fev_sched_attr_set_num_workers(sched_attr, num_workers); 125 | 126 | err = fev_sched_create(&sched, sched_attr); 127 | CHECK(err == 0, "Creating scheduler failed: err=%i", err); 128 | 129 | fev_sched_attr_destroy(sched_attr); 130 | 131 | err = fev_fiber_spawn(sched, &test, NULL); 132 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 133 | 134 | err = fev_sched_run(sched); 135 | CHECK(err == 0, "Running scheduler failed: err=%i", err); 136 | 137 | fev_sched_destroy(sched); 138 | 139 | return 0; 140 | } 141 | -------------------------------------------------------------------------------- /tests/stress_ilock.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "../src/fev_ilock_impl.h" 8 | #include 9 | 10 | #include "util.h" 11 | 12 | static uint32_t num_fibers; 13 | static uint32_t num_iterations; 14 | 15 | static struct fev_ilock ilock; 16 | static uint64_t counter; 17 | 18 | static void *work(void *arg) 19 | { 20 | uint32_t i; 21 | 22 | (void)arg; 23 | 24 | for (i = 0; i < num_iterations; i++) { 25 | fev_ilock_lock(&ilock); 26 | counter++; 27 | fev_ilock_unlock_and_wake(&ilock); 28 | } 29 | 30 | return NULL; 31 | } 32 | 33 | static void *test(void *arg) 34 | { 35 | struct fev_fiber **fibers; 36 | int ret; 37 | 38 | (void)arg; 39 | 40 | ret = fev_ilock_init(&ilock); 41 | CHECK(ret == 0, "Initializing ilock failed: err=%i", ret); 42 | 43 | fibers = malloc(num_fibers * sizeof(*fibers)); 44 | CHECK(fibers != NULL, "Allocating memory for fibers failed"); 45 | 46 | for (uint32_t i = 0; i < num_fibers; i++) { 47 | ret = fev_fiber_create(&fibers[i], NULL, &work, NULL, NULL); 48 | CHECK(ret == 0, "Creating fiber failed: err=%i", ret); 49 | } 50 | 51 | for (uint32_t i = 0; i < num_fibers; i++) 52 | fev_fiber_join(fibers[i], NULL); 53 | 54 | free(fibers); 55 | 56 | return NULL; 57 | } 58 | 59 | int main(int argc, char **argv) 60 | { 61 | struct fev_sched_attr *sched_attr; 62 | struct fev_sched *sched; 63 | uint64_t expected; 64 | uint32_t num_workers; 65 | int err; 66 | 67 | CHECK(argc == 4, "Usage: %s ", argv[0]); 68 | 69 | num_workers = parse_uint32_t(argv[1], "num_workers", &(uint32_t){1}); 70 | num_fibers = parse_uint32_t(argv[2], "num_fibers", &(uint32_t){1}); 71 | num_iterations = parse_uint32_t(argv[3], "num_iterations", &(uint32_t){1}); 72 | 73 | err = fev_sched_attr_create(&sched_attr); 74 | CHECK(err == 0, "Creating scheduler attributes failed: err=%i", err); 75 | 76 | fev_sched_attr_set_num_workers(sched_attr, num_workers); 77 | 78 | err = fev_sched_create(&sched, sched_attr); 79 | CHECK(err == 0, "Creating scheduler failed: err=%i", err); 80 | 81 | fev_sched_attr_destroy(sched_attr); 82 | 83 | err = fev_fiber_spawn(sched, &test, NULL); 84 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 85 | 86 | err = fev_sched_run(sched); 87 | CHECK(err == 0, "Running scheduler failed: err=%i", err); 88 | 89 | fev_sched_destroy(sched); 90 | 91 | expected = (uint64_t)num_fibers * (uint64_t)num_iterations; 92 | printf("counter: %" PRIu64 ", expected: %" PRIu64 "\n", counter, expected); 93 | 94 | return 0; 95 | } 96 | -------------------------------------------------------------------------------- /tests/stress_mpmc_queue.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "../src/fev_alloc.h" 9 | #include "../src/fev_simple_mpmc_pool.h" 10 | #include "../src/fev_simple_mpmc_queue.h" 11 | #include "../src/fev_thr.h" 12 | #include "../src/fev_util.h" 13 | 14 | #include "util.h" 15 | 16 | static uint32_t num_tries; 17 | static struct fev_simple_mpmc_queue queue; 18 | static atomic_uint barrier; 19 | static _Atomic uint64_t total_sum; 20 | 21 | static struct fev_simple_mpmc_pool_global pool_global; 22 | 23 | static void *worker_proc(void *arg) 24 | { 25 | uint64_t sum = 0; 26 | uint32_t num_enqueue = 0, num_dequeue = 0; 27 | uint32_t r = (uint32_t)(uintptr_t)arg; 28 | 29 | struct fev_simple_mpmc_pool_local pool_local; 30 | fev_simple_mpmc_pool_init_local(&pool_local, &pool_global, 1024); 31 | 32 | /* Wait until all threads are created. */ 33 | atomic_fetch_sub(&barrier, 1); 34 | while (atomic_load_explicit(&barrier, memory_order_acquire) > 0) 35 | ; 36 | 37 | while (num_enqueue < num_tries || num_dequeue < num_tries) { 38 | uint32_t iters; 39 | 40 | r = FEV_RANDOM_NEXT(r); 41 | iters = r % 1024; 42 | while (iters-- > 0 && num_enqueue < num_tries) { 43 | struct fev_simple_mpmc_queue_node *node = fev_simple_mpmc_pool_alloc_local(&pool_local); 44 | CHECK(node != NULL, "Allocating node failed"); 45 | fev_simple_mpmc_queue_push(&queue, node, (void *)(uintptr_t)(num_enqueue + 1)); 46 | num_enqueue++; 47 | } 48 | 49 | r = FEV_RANDOM_NEXT(r); 50 | iters = r % 1024; 51 | while (iters-- > 0 && num_dequeue < num_tries) { 52 | struct fev_simple_mpmc_queue_node *node; 53 | void *value; 54 | bool popped = fev_simple_mpmc_queue_pop(&queue, &value, &node); 55 | if (!popped) 56 | continue; 57 | fev_simple_mpmc_pool_free_local(&pool_local, node); 58 | sum += (uint32_t)(uintptr_t)value; 59 | num_dequeue++; 60 | } 61 | } 62 | 63 | fev_simple_mpmc_pool_fini_local(&pool_local); 64 | 65 | atomic_fetch_add(&total_sum, sum); 66 | 67 | return NULL; 68 | } 69 | 70 | int main(int argc, char **argv) 71 | { 72 | struct fev_thr *threads; 73 | struct fev_simple_mpmc_queue_node *node; 74 | uint64_t expected_sum; 75 | uint32_t seed, num_workers, i; 76 | uint32_t r; 77 | 78 | CHECK(argc == 4, "Usage: %s ", argv[0]); 79 | 80 | seed = parse_uint32_t(argv[1], "seed", &(uint32_t){1}); 81 | num_workers = parse_uint32_t(argv[2], "num_workers", NULL); 82 | num_tries = parse_uint32_t(argv[3], "num_tries", NULL); 83 | 84 | threads = fev_malloc((size_t)num_workers * sizeof(*threads)); 85 | if (threads == NULL) { 86 | fputs("Allocating memory for threads failed\n", stderr); 87 | return 1; 88 | } 89 | 90 | fev_simple_mpmc_pool_init_global(&pool_global); 91 | 92 | node = fev_simple_mpmc_pool_alloc_global(&pool_global); 93 | CHECK(node != NULL, "Allocating node failed"); 94 | fev_simple_mpmc_queue_init(&queue, node); 95 | 96 | atomic_store(&barrier, num_workers); 97 | 98 | r = seed; 99 | for (i = 0; i < num_workers; i++) { 100 | r = FEV_RANDOM_NEXT(r); 101 | int err = fev_thr_create(&threads[i], worker_proc, (void *)(uintptr_t)r); 102 | if (err != 0) { 103 | fprintf(stderr, "Creating thread failed, err=%d\n", err); 104 | return 1; 105 | } 106 | } 107 | 108 | for (i = 0; i < num_workers; i++) 109 | fev_thr_join(&threads[i], NULL); 110 | 111 | fev_free(threads); 112 | 113 | fev_simple_mpmc_queue_fini(&queue, &node); 114 | fev_simple_mpmc_pool_free_global(&pool_global, node); 115 | 116 | fev_simple_mpmc_pool_fini_global(&pool_global); 117 | 118 | expected_sum = ((uint64_t)num_tries * ((uint64_t)num_tries + 1) / 2) * num_workers; 119 | printf("sum: %" PRIu64 ", expected: %" PRIu64 "\n", atomic_load(&total_sum), expected_sum); 120 | 121 | return total_sum != expected_sum; 122 | } 123 | -------------------------------------------------------------------------------- /tests/stress_mutex.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include "util.h" 10 | 11 | static uint32_t num_fibers; 12 | static uint32_t num_iterations; 13 | 14 | static struct fev_mutex *mutex; 15 | static uint64_t counter; 16 | 17 | static void *work(void *arg) 18 | { 19 | (void)arg; 20 | 21 | for (uint32_t i = 0; i < num_iterations; i++) { 22 | fev_mutex_lock(mutex); 23 | counter++; 24 | fev_mutex_unlock(mutex); 25 | } 26 | 27 | return NULL; 28 | } 29 | 30 | static void *test(void *arg) 31 | { 32 | struct fev_fiber **fibers; 33 | int err; 34 | 35 | (void)arg; 36 | 37 | err = fev_mutex_create(&mutex); 38 | CHECK(err == 0, "Creating mutex failed with: err=%i", err); 39 | 40 | fibers = malloc((size_t)num_fibers * sizeof(*fibers)); 41 | CHECK(fibers != NULL, "Allocating memory for fibers failed"); 42 | 43 | for (uint32_t i = 0; i < num_fibers; i++) { 44 | err = fev_fiber_create(&fibers[i], NULL, &work, NULL, NULL); 45 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 46 | } 47 | 48 | for (uint32_t i = 0; i < num_fibers; i++) 49 | fev_fiber_join(fibers[i], NULL); 50 | 51 | free(fibers); 52 | 53 | fev_mutex_destroy(mutex); 54 | 55 | return NULL; 56 | } 57 | 58 | int main(int argc, char **argv) 59 | { 60 | struct fev_sched_attr *sched_attr; 61 | struct fev_sched *sched; 62 | uint64_t expected; 63 | uint32_t num_workers; 64 | int err; 65 | 66 | CHECK(argc == 4, "Usage: %s ", argv[0]); 67 | 68 | num_workers = parse_uint32_t(argv[1], "num_workers", &(uint32_t){1}); 69 | num_fibers = parse_uint32_t(argv[2], "num_fibers", &(uint32_t){1}); 70 | num_iterations = parse_uint32_t(argv[3], "num_iterations", &(uint32_t){1}); 71 | 72 | err = fev_sched_attr_create(&sched_attr); 73 | CHECK(err == 0, "Creating scheduler attributes failed: err=%i", err); 74 | 75 | fev_sched_attr_set_num_workers(sched_attr, num_workers); 76 | 77 | err = fev_sched_create(&sched, sched_attr); 78 | CHECK(err == 0, "Creating scheduler failed: err=%i", err); 79 | 80 | fev_sched_attr_destroy(sched_attr); 81 | 82 | err = fev_fiber_spawn(sched, &test, NULL); 83 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 84 | 85 | err = fev_sched_run(sched); 86 | CHECK(err == 0, "Running scheduler failed: err=%i", err); 87 | 88 | fev_sched_destroy(sched); 89 | 90 | expected = (uint64_t)num_fibers * (uint64_t)num_iterations; 91 | printf("counter: %" PRIu64 ", expected: %" PRIu64 "\n", counter, expected); 92 | 93 | return counter != expected; 94 | } 95 | -------------------------------------------------------------------------------- /tests/stress_mutex_with_timeout.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "../src/fev_util.h" 13 | #include 14 | 15 | #include "util.h" 16 | 17 | static uint32_t num_fibers; 18 | static uint32_t num_iterations; 19 | static uint64_t timeout_ns; 20 | 21 | static struct fev_mutex *mutex; 22 | static uint64_t counter; 23 | static _Atomic uint64_t num_timeouts; 24 | 25 | static void *work(void *arg) 26 | { 27 | struct timespec rel_time; 28 | uint64_t timeouts = 0; 29 | uint32_t r; 30 | 31 | (void)arg; 32 | 33 | assert(timeout_ns <= LONG_MAX); 34 | rel_time.tv_sec = (time_t)timeout_ns / (1000 * 1000 * 1000); 35 | rel_time.tv_nsec = (long)timeout_ns % (1000 * 1000 * 1000); 36 | 37 | r = (uint32_t)rand(); 38 | 39 | for (uint32_t i = 0; i < num_iterations;) { 40 | r = FEV_RANDOM_NEXT(r); 41 | if (r % 2 == 0) { 42 | fev_mutex_lock(mutex); 43 | } else { 44 | int ret = fev_mutex_try_lock_for(mutex, &rel_time); 45 | if (ret == -ETIMEDOUT) { 46 | timeouts++; 47 | continue; 48 | } 49 | CHECK(ret == 0, "fev_mutex_try_lock_for() failed: err=%i", ret); 50 | } 51 | 52 | counter++; 53 | fev_mutex_unlock(mutex); 54 | 55 | i++; 56 | } 57 | 58 | atomic_fetch_add(&num_timeouts, timeouts); 59 | 60 | return NULL; 61 | } 62 | 63 | static void *test(void *arg) 64 | { 65 | struct fev_fiber **fibers; 66 | int err; 67 | 68 | (void)arg; 69 | 70 | err = fev_mutex_create(&mutex); 71 | CHECK(err == 0, "Creating mutex failed with: err=%i", err); 72 | 73 | fibers = malloc((size_t)num_fibers * sizeof(*fibers)); 74 | CHECK(fibers != NULL, "Allocating memory for fibers failed"); 75 | 76 | for (uint32_t i = 0; i < num_fibers; i++) { 77 | err = fev_fiber_create(&fibers[i], NULL, &work, NULL, NULL); 78 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 79 | } 80 | 81 | for (uint32_t i = 0; i < num_fibers; i++) 82 | fev_fiber_join(fibers[i], NULL); 83 | 84 | free(fibers); 85 | 86 | fev_mutex_destroy(mutex); 87 | 88 | return NULL; 89 | } 90 | 91 | int main(int argc, char **argv) 92 | { 93 | struct fev_sched_attr *sched_attr; 94 | struct fev_sched *sched; 95 | uint64_t expected; 96 | uint32_t num_workers; 97 | int err; 98 | 99 | CHECK(argc == 5, "Usage: %s ", argv[0]); 100 | 101 | num_workers = parse_uint32_t(argv[1], "num_workers", &(uint32_t){1}); 102 | num_fibers = parse_uint32_t(argv[2], "num_fibers", &(uint32_t){1}); 103 | num_iterations = parse_uint32_t(argv[3], "num_iterations", &(uint32_t){1}); 104 | timeout_ns = parse_uint64_t(argv[4], "timeout_ns", &(uint64_t){1}); 105 | 106 | err = fev_sched_attr_create(&sched_attr); 107 | CHECK(err == 0, "Creating scheduler attributes failed: err=%i", err); 108 | 109 | fev_sched_attr_set_num_workers(sched_attr, num_workers); 110 | 111 | err = fev_sched_create(&sched, sched_attr); 112 | CHECK(err == 0, "Creating scheduler failed: err=%i", err); 113 | 114 | fev_sched_attr_destroy(sched_attr); 115 | 116 | err = fev_fiber_spawn(sched, &test, NULL); 117 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 118 | 119 | err = fev_sched_run(sched); 120 | CHECK(err == 0, "Running scheduler failed: err=%i", err); 121 | 122 | fev_sched_destroy(sched); 123 | 124 | expected = (uint64_t)num_fibers * (uint64_t)num_iterations; 125 | printf("counter: %" PRIu64 ", expected: %" PRIu64 ", num_timeouts: %" PRIu64 "\n", counter, 126 | expected, atomic_load(&num_timeouts)); 127 | 128 | return counter != expected; 129 | } 130 | -------------------------------------------------------------------------------- /tests/stress_qsbr_queue.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "../src/fev_alloc.h" 9 | #include "../src/fev_qsbr.h" 10 | #include "../src/fev_qsbr_queue.h" 11 | #include "../src/fev_thr.h" 12 | #include "../src/fev_util.h" 13 | 14 | #include "util.h" 15 | 16 | static uint32_t num_workers; 17 | static uint32_t num_tries; 18 | static struct fev_qsbr_queue queue; 19 | static atomic_uint barrier; 20 | static _Atomic uint64_t total_sum; 21 | static struct fev_qsbr_global qsbr_global; 22 | 23 | static struct fev_qsbr_queue_node *alloc_node(void) 24 | { 25 | struct fev_qsbr_queue_node *node; 26 | 27 | node = fev_malloc(sizeof(*node)); 28 | CHECK(node != NULL, "Allocating node failed"); 29 | 30 | CHECK((uintptr_t)node % 8 == 0, "Bad alignment"); 31 | 32 | return node; 33 | } 34 | 35 | static void free_node(struct fev_qsbr_queue_node *node) { fev_free(node); } 36 | 37 | static void free_qsbr_nodes(struct fev_qsbr_entry *head) 38 | { 39 | struct fev_qsbr_entry *next; 40 | 41 | while (head != NULL) { 42 | next = atomic_load_explicit(&head->next, memory_order_relaxed); 43 | free_node(FEV_CONTAINER_OF(head, struct fev_qsbr_queue_node, qsbr_entry)); 44 | head = next; 45 | } 46 | } 47 | 48 | static void *worker_proc(void *arg) 49 | { 50 | struct fev_qsbr_local qsbr_local; 51 | struct fev_qsbr_entry *to_free; 52 | uint64_t sum = 0; 53 | uint32_t num_enqueue = 0, num_dequeue = 0; 54 | uint32_t r = (uint32_t)(uintptr_t)arg; 55 | 56 | fev_qsbr_init_local(&qsbr_local); 57 | 58 | /* Wait until all threads are created. */ 59 | atomic_fetch_sub(&barrier, 1); 60 | while (atomic_load_explicit(&barrier, memory_order_acquire) > 0) 61 | ; 62 | 63 | while (num_enqueue < num_tries || num_dequeue < num_tries) { 64 | uint32_t iters; 65 | 66 | r = FEV_RANDOM_NEXT(r); 67 | iters = r % 1024; 68 | while (iters-- > 0 && num_enqueue < num_tries) { 69 | struct fev_qsbr_queue_node *node = alloc_node(); 70 | fev_qsbr_queue_push(&queue, node, (void *)(uintptr_t)(num_enqueue + 1)); 71 | num_enqueue++; 72 | } 73 | 74 | r = FEV_RANDOM_NEXT(r); 75 | iters = r % 1024; 76 | while (iters-- > 0 && num_dequeue < num_tries) { 77 | struct fev_qsbr_queue_node *node; 78 | void *value; 79 | bool popped = fev_qsbr_queue_pop(&queue, &node, &value); 80 | if (!popped) 81 | continue; 82 | if (num_workers == 1) 83 | free_node(node); 84 | else 85 | fev_qsbr_free(&qsbr_global, &qsbr_local, &node->qsbr_entry); 86 | sum += (uint32_t)(uintptr_t)value; 87 | num_dequeue++; 88 | } 89 | 90 | to_free = fev_qsbr_quiescent(&qsbr_global, &qsbr_local); 91 | free_qsbr_nodes(to_free); 92 | } 93 | 94 | atomic_fetch_add(&total_sum, sum); 95 | 96 | return NULL; 97 | } 98 | 99 | int main(int argc, char **argv) 100 | { 101 | struct fev_thr *threads; 102 | struct fev_qsbr_queue_node *node; 103 | struct fev_qsbr_entry *to_free1, *to_free2; 104 | uint64_t expected_sum; 105 | uint32_t seed, i; 106 | uint32_t r; 107 | 108 | CHECK(argc == 4, "Usage: %s ", argv[0]); 109 | 110 | seed = parse_uint32_t(argv[1], "seed", &(uint32_t){1}); 111 | num_workers = parse_uint32_t(argv[2], "num_workers", &(uint32_t){1}); 112 | num_tries = parse_uint32_t(argv[3], "num_tries", NULL); 113 | 114 | threads = fev_malloc((size_t)num_workers * sizeof(*threads)); 115 | if (threads == NULL) { 116 | fputs("Allocating memory for threads failed\n", stderr); 117 | return 1; 118 | } 119 | 120 | fev_qsbr_init_global(&qsbr_global, num_workers); 121 | 122 | node = alloc_node(); 123 | fev_qsbr_queue_init(&queue, node); 124 | 125 | atomic_store(&barrier, num_workers); 126 | 127 | r = seed; 128 | for (i = 0; i < num_workers; i++) { 129 | r = FEV_RANDOM_NEXT(r); 130 | int err = fev_thr_create(&threads[i], worker_proc, (void *)(uintptr_t)r); 131 | if (err != 0) { 132 | fprintf(stderr, "Creating thread failed, err=%d\n", err); 133 | return 1; 134 | } 135 | } 136 | 137 | for (i = 0; i < num_workers; i++) 138 | fev_thr_join(&threads[i], NULL); 139 | 140 | fev_free(threads); 141 | 142 | fev_qsbr_queue_fini(&queue, &node); 143 | free_node(node); 144 | 145 | fev_qsbr_fini_global(&qsbr_global, &to_free1, &to_free2); 146 | free_qsbr_nodes(to_free1); 147 | free_qsbr_nodes(to_free2); 148 | 149 | expected_sum = ((uint64_t)num_tries * ((uint64_t)num_tries + 1) / 2) * num_workers; 150 | printf("sum: %" PRIu64 ", expected: %" PRIu64 "\n", atomic_load(&total_sum), expected_sum); 151 | 152 | return total_sum != expected_sum; 153 | } 154 | -------------------------------------------------------------------------------- /tests/stress_sem.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | #include "util.h" 9 | 10 | static uint32_t num_fibers; 11 | static uint32_t num_iterations; 12 | 13 | static struct fev_sem *sem; 14 | static uint64_t counter; 15 | 16 | static void *work(void *arg) 17 | { 18 | (void)arg; 19 | 20 | for (uint32_t i = 0; i < num_iterations; i++) { 21 | fev_sem_wait(sem); 22 | counter++; 23 | fev_sem_post(sem); 24 | } 25 | 26 | return NULL; 27 | } 28 | 29 | static void *test(void *arg) 30 | { 31 | struct fev_fiber **fibers; 32 | int err; 33 | 34 | (void)arg; 35 | 36 | err = fev_sem_create(&sem, 1); 37 | CHECK(err == 0, "Creating sem failed with: err=%i", err); 38 | 39 | fibers = malloc((size_t)num_fibers * sizeof(*fibers)); 40 | CHECK(fibers != NULL, "Allocating memory for fibers failed"); 41 | 42 | for (uint32_t i = 0; i < num_fibers; i++) { 43 | err = fev_fiber_create(&fibers[i], NULL, &work, NULL, NULL); 44 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 45 | } 46 | 47 | for (uint32_t i = 0; i < num_fibers; i++) 48 | fev_fiber_join(fibers[i], NULL); 49 | 50 | free(fibers); 51 | 52 | fev_sem_destroy(sem); 53 | 54 | return NULL; 55 | } 56 | 57 | int main(int argc, char **argv) 58 | { 59 | struct fev_sched_attr *sched_attr; 60 | struct fev_sched *sched; 61 | uint64_t expected; 62 | uint32_t num_workers; 63 | int err; 64 | 65 | CHECK(argc == 4, "Usage: %s ", argv[0]); 66 | 67 | num_workers = parse_uint32_t(argv[1], "num_workers", &(uint32_t){1}); 68 | num_fibers = parse_uint32_t(argv[2], "num_fibers", &(uint32_t){1}); 69 | num_iterations = parse_uint32_t(argv[3], "num_iterations", &(uint32_t){1}); 70 | 71 | err = fev_sched_attr_create(&sched_attr); 72 | CHECK(err == 0, "Creating scheduler attributes failed: err=%i", err); 73 | 74 | fev_sched_attr_set_num_workers(sched_attr, num_workers); 75 | 76 | err = fev_sched_create(&sched, sched_attr); 77 | CHECK(err == 0, "Creating scheduler failed: err=%i", err); 78 | 79 | fev_sched_attr_destroy(sched_attr); 80 | 81 | err = fev_fiber_spawn(sched, &test, NULL); 82 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 83 | 84 | err = fev_sched_run(sched); 85 | CHECK(err == 0, "Running scheduler failed: err=%i", err); 86 | 87 | fev_sched_destroy(sched); 88 | 89 | expected = (uint64_t)num_fibers * (uint64_t)num_iterations; 90 | printf("counter: %" PRIu64 ", expected: %" PRIu64 "\n", counter, expected); 91 | 92 | return 0; 93 | } 94 | -------------------------------------------------------------------------------- /tests/stress_sem_with_timeout.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "../src/fev_util.h" 13 | #include 14 | 15 | #include "util.h" 16 | 17 | static uint32_t num_fibers; 18 | static uint32_t num_iterations; 19 | static uint64_t timeout_ns; 20 | 21 | static struct fev_sem *sem; 22 | static uint64_t counter; 23 | static _Atomic uint64_t num_timeouts; 24 | 25 | static void *work(void *arg) 26 | { 27 | struct timespec rel_time; 28 | uint64_t timeouts = 0; 29 | uint32_t r; 30 | 31 | (void)arg; 32 | 33 | assert(timeout_ns <= LONG_MAX); 34 | rel_time.tv_sec = (time_t)timeout_ns / (1000 * 1000 * 1000); 35 | rel_time.tv_nsec = (long)timeout_ns % (1000 * 1000 * 1000); 36 | 37 | r = (uint32_t)rand(); 38 | 39 | for (uint32_t i = 0; i < num_iterations;) { 40 | r = FEV_RANDOM_NEXT(r); 41 | if (r % 2 == 0) { 42 | fev_sem_wait(sem); 43 | } else { 44 | int ret = fev_sem_wait_for(sem, &rel_time); 45 | if (ret == -ETIMEDOUT) { 46 | timeouts++; 47 | continue; 48 | } 49 | CHECK(ret == 0, "fev_sem_wait_for() failed: err=%i", ret); 50 | } 51 | 52 | counter++; 53 | fev_sem_post(sem); 54 | 55 | i++; 56 | } 57 | 58 | atomic_fetch_add(&num_timeouts, timeouts); 59 | 60 | return NULL; 61 | } 62 | 63 | static void *test(void *arg) 64 | { 65 | struct fev_fiber **fibers; 66 | int err; 67 | 68 | (void)arg; 69 | 70 | err = fev_sem_create(&sem, 1); 71 | CHECK(err == 0, "Creating sem failed with: err=%i", err); 72 | 73 | fibers = malloc((size_t)num_fibers * sizeof(*fibers)); 74 | CHECK(fibers != NULL, "Allocating memory for fibers failed"); 75 | 76 | for (uint32_t i = 0; i < num_fibers; i++) { 77 | err = fev_fiber_create(&fibers[i], NULL, &work, NULL, NULL); 78 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 79 | } 80 | 81 | for (uint32_t i = 0; i < num_fibers; i++) 82 | fev_fiber_join(fibers[i], NULL); 83 | 84 | free(fibers); 85 | 86 | fev_sem_destroy(sem); 87 | 88 | return NULL; 89 | } 90 | 91 | int main(int argc, char **argv) 92 | { 93 | struct fev_sched_attr *sched_attr; 94 | struct fev_sched *sched; 95 | uint64_t expected; 96 | uint32_t num_workers; 97 | int err; 98 | 99 | CHECK(argc == 5, "Usage: %s ", argv[0]); 100 | 101 | num_workers = parse_uint32_t(argv[1], "num_workers", &(uint32_t){1}); 102 | num_fibers = parse_uint32_t(argv[2], "num_fibers", &(uint32_t){1}); 103 | num_iterations = parse_uint32_t(argv[3], "num_iterations", &(uint32_t){1}); 104 | timeout_ns = parse_uint64_t(argv[4], "timeout_ns", &(uint64_t){1}); 105 | 106 | err = fev_sched_attr_create(&sched_attr); 107 | CHECK(err == 0, "Creating scheduler attributes failed: err=%i", err); 108 | 109 | fev_sched_attr_set_num_workers(sched_attr, num_workers); 110 | 111 | err = fev_sched_create(&sched, sched_attr); 112 | CHECK(err == 0, "Creating scheduler failed: err=%i", err); 113 | 114 | fev_sched_attr_destroy(sched_attr); 115 | 116 | err = fev_fiber_spawn(sched, &test, NULL); 117 | CHECK(err == 0, "Creating fiber failed: err=%i", err); 118 | 119 | err = fev_sched_run(sched); 120 | CHECK(err == 0, "Running scheduler failed: err=%i", err); 121 | 122 | fev_sched_destroy(sched); 123 | 124 | expected = (uint64_t)num_fibers * (uint64_t)num_iterations; 125 | printf("counter: %" PRIu64 ", expected: %" PRIu64 ", num_timeouts: %" PRIu64 "\n", counter, 126 | expected, atomic_load(&num_timeouts)); 127 | 128 | return counter != expected; 129 | } 130 | -------------------------------------------------------------------------------- /tests/stress_thr_mutex.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "../src/fev_thr.h" 7 | #include "../src/fev_thr_mutex.h" 8 | #include 9 | 10 | #include "util.h" 11 | 12 | static struct fev_thr_mutex mutex; 13 | static _Atomic uint32_t barrier; 14 | static uint32_t num_iterations; 15 | static uint64_t counter; 16 | 17 | static void *work(void *arg) 18 | { 19 | (void)arg; 20 | 21 | /* Wait until all threads are created. */ 22 | atomic_fetch_sub(&barrier, 1); 23 | while (atomic_load_explicit(&barrier, memory_order_acquire) > 0) 24 | ; 25 | 26 | for (uint32_t i = 0; i < num_iterations; i++) { 27 | fev_thr_mutex_lock(&mutex); 28 | counter++; 29 | fev_thr_mutex_unlock(&mutex); 30 | } 31 | 32 | return NULL; 33 | } 34 | 35 | int main(int argc, char **argv) 36 | { 37 | struct fev_thr *threads; 38 | uint64_t expected; 39 | uint32_t num_threads, i; 40 | int err; 41 | 42 | CHECK(argc == 3, "Usage: %s ", argv[0]); 43 | 44 | num_threads = parse_uint32_t(argv[1], "num_threads", &(uint32_t){1}); 45 | num_iterations = parse_uint32_t(argv[2], "num_iterations", NULL); 46 | 47 | atomic_store(&barrier, num_threads); 48 | 49 | threads = malloc((size_t)num_threads * sizeof(*threads)); 50 | CHECK(threads != NULL, "Allocating memory for threads failed"); 51 | 52 | for (i = 0; i < num_threads; i++) { 53 | err = fev_thr_create(&threads[i], &work, NULL); 54 | CHECK(err == 0, "Creating thread failed, err=%d", err); 55 | } 56 | 57 | for (i = 0; i < num_threads; i++) 58 | fev_thr_join(&threads[i], NULL); 59 | 60 | free(threads); 61 | 62 | expected = (uint64_t)num_threads * num_iterations; 63 | printf("counter: %" PRIu64 ", expected: %" PRIu64 "\n", counter, expected); 64 | 65 | return counter != expected; 66 | } 67 | -------------------------------------------------------------------------------- /tests/util.h: -------------------------------------------------------------------------------- 1 | #ifndef FEV_TESTS_UTIL_H 2 | #define FEV_TESTS_UTIL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define FATAL(fmt, ...) \ 11 | do { \ 12 | fprintf(stderr, fmt "\n", ##__VA_ARGS__); \ 13 | abort(); \ 14 | } while (0) 15 | 16 | #define CHECK(expr, fmt, ...) \ 17 | do { \ 18 | if (!(expr)) \ 19 | FATAL("Asseration \"%s\" failed in %s (%s:%u): " fmt, #expr, __func__, __FILE__, __LINE__, \ 20 | ##__VA_ARGS__); \ 21 | } while (0) 22 | 23 | #define DEF_PARSE(type, scan_fmt, print_fmt) \ 24 | static inline type parse_##type(const char *str, const char *name, type *min_val) \ 25 | { \ 26 | type val; \ 27 | int num_scanned = sscanf(str, "%" scan_fmt, &val); \ 28 | CHECK(num_scanned == 1, "Failed to parse '%s' as %s", str, name); \ 29 | CHECK(min_val == NULL || val >= *min_val, "%s must be at least %" print_fmt, name, *min_val); \ 30 | return val; \ 31 | } 32 | 33 | DEF_PARSE(uint32_t, SCNu32, PRIu32) 34 | DEF_PARSE(uint64_t, SCNu64, PRIu64) 35 | 36 | #undef DEF_PARSE 37 | 38 | #endif /* !FEV_TESTS_UTIL_H */ 39 | --------------------------------------------------------------------------------