├── LICENSE ├── README.md ├── mpsc_queue.h ├── shm_mpsc_queue.h └── test ├── .gitignore ├── AppendFile.h ├── LogStream.cc ├── LogStream.h ├── Logging.cc ├── Logging.h ├── README.md ├── async_logging ├── async_logging.cc ├── build.sh ├── cpupin.h ├── mpsc_test.cc ├── shm_logging_client.cc ├── shm_logging_server.cc ├── shm_mpsc_test.cc ├── shmmap.h └── timestamp.h /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Meng Rao 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MPSC_Queue 2 | MPSC_Queue is based on single linked list. Producers have the queue allocate a msg object, set msg content and push it back to the queue; Consumer pops all msgs from the queue at a time, processes them and gives the objects back to the queue for later allocation from producers. So neither producer nor consumer needs to allocate memory themselves and no memory copy is needed in any of the operations. 3 | 4 | There're two versions of implementation: 5 | ## MPSCQueue(mpsc_queue.h) 6 | MPSCQueue is for single process usage, it allows for configurating a pre-allocated msg size and max allocated msg size at run time. 7 | 8 | ## SHMMPSCQueue(shm_mpsc_queue.h) 9 | SHMMPSCQueue can reside in shared memory, thus suitable for IPC, and the msg size is fixed at compile time. 10 | 11 | # Examples 12 | [test](https://github.com/MengRao/MPSC_Queue/tree/master/test) provides a simple test program and a full fledged async logging implementation(based on [muduo](https://github.com/chenshuo/muduo)), for both versions. 13 | -------------------------------------------------------------------------------- /mpsc_queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright (c) 2018 Meng Rao 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | #pragma once 26 | #include 27 | 28 | template 29 | class MPSCQueue 30 | { 31 | public: 32 | struct Entry 33 | { 34 | Entry* next; 35 | EntryData data; 36 | }; 37 | 38 | MPSCQueue(uint32_t init_size, uint32_t max_size) 39 | : pending_tail((Entry*)&pending_head) 40 | , max_entry(max_size) { 41 | while(init_size--) { 42 | Entry* cur = _Alloc(); 43 | cur->next = empty_top; 44 | empty_top = cur; 45 | } 46 | } 47 | 48 | Entry* Alloc() { 49 | Entry* top = FetchAndLock(empty_top, (Entry*)EmptyLock); 50 | if(top == nullptr) { 51 | empty_top = nullptr; 52 | return _Alloc(); 53 | } 54 | empty_top = top->next; 55 | return top; 56 | } 57 | 58 | // A correct and the most efficient implementation that I've tested 59 | // In normal case pending_tail saves the address of tail entry(if empty it save the address of head) 60 | // contention winner will temporarily set pending_tail to PendingLock 61 | void Push(Entry* entry) { 62 | Entry* tail = FetchAndLock(pending_tail, (Entry*)PendingLock); 63 | tail->next = entry; 64 | asm volatile("" : : "m"(tail->next), "m"(pending_tail) :); // memory fence 65 | pending_tail = entry; 66 | } 67 | 68 | // Dont use. An incorrect implemnetation similar to Push, supposed to be faster but indeed slower, who can tell me 69 | // why? it's incorrect in that when pending_tail is changed to the new entry, its next is not guaranteed to point to 70 | // it so the consumer could get a corrupt list 71 | void Push2(Entry* entry) { 72 | Entry* tail = pending_tail; 73 | while(true) { 74 | Entry* cur_tail = __sync_val_compare_and_swap(&pending_tail, tail, entry); 75 | if(__builtin_expect(cur_tail == tail, 1)) break; 76 | tail = cur_tail; 77 | } 78 | tail->next = entry; 79 | } 80 | 81 | // Dont use. Another correct implementation of Push, but turned out to be slightly slower 82 | // In normal case pending_tail's next is PendingLock, 83 | // and any other entry's next must not be PendingLock regardless of whether it's on the pending list 84 | // contention winner will set pending_tail's next to the new entry and pending_tail to the new entry 85 | void Push3(Entry* entry) { 86 | entry->next = (Entry*)PendingLock; 87 | while(__builtin_expect(!__sync_bool_compare_and_swap(&pending_tail->next, PendingLock, entry), 0)) 88 | ; 89 | pending_tail = entry; 90 | } 91 | 92 | // Similar to Push(), but it directly set pending_tail to the address of pending_head 93 | Entry* PopAll() { 94 | if(pending_tail == (Entry*)&pending_head) return nullptr; 95 | Entry* tail = FetchAndLock(pending_tail, (Entry*)PendingLock); 96 | Entry* ret = pending_head; 97 | asm volatile("" : : "m"(pending_tail) :); // memory fence 98 | pending_tail = (Entry*)&pending_head; 99 | tail->next = nullptr; 100 | return ret; 101 | } 102 | 103 | void Recycle(Entry* first, Entry* last) { 104 | Entry* top = FetchAndLock(empty_top, (Entry*)EmptyLock); 105 | last->next = top; 106 | asm volatile("" : : "m"(last->next), "m"(empty_top) :); // memory fence 107 | empty_top = first; 108 | } 109 | 110 | private: 111 | template 112 | static T FetchAndLock(volatile T& var, T lock_val) { 113 | T val = var; 114 | while(true) { 115 | while(__builtin_expect(val == lock_val, 0)) val = var; 116 | T new_val = __sync_val_compare_and_swap(&var, val, lock_val); 117 | if(__builtin_expect(new_val == val, 1)) break; 118 | val = new_val; 119 | } 120 | return val; 121 | } 122 | 123 | Entry* _Alloc() { 124 | if(entry_cnt >= max_entry) return nullptr; 125 | int cnt = FetchAndLock(entry_cnt, -1); 126 | if(__builtin_expect(cnt >= max_entry, 0)) { 127 | entry_cnt = cnt; 128 | return nullptr; 129 | } 130 | deq.emplace_back(); 131 | Entry* ret = &deq.back(); 132 | asm volatile("" : : "m"(entry_cnt) :); // memory fence 133 | entry_cnt = deq.size(); 134 | return ret; 135 | } 136 | 137 | private: 138 | static constexpr long PendingLock = 0x8; 139 | static constexpr long EmptyLock = 0x10; 140 | 141 | alignas(64) Entry* volatile pending_tail; 142 | Entry* pending_head; 143 | 144 | alignas(64) Entry* volatile empty_top = nullptr; 145 | 146 | alignas(64) volatile int entry_cnt = 0; 147 | std::deque deq; 148 | const int max_entry; 149 | }; 150 | -------------------------------------------------------------------------------- /shm_mpsc_queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright (c) 2018 Meng Rao 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | #pragma once 26 | 27 | // SHMMPSCQueue is not "crash safe", which means: 28 | // Participant process crash could cause memory leak or even deadlock of other participants 29 | // If deadlock happens(under theoretical possibility), kill all participants and remove the shm file under /dev/shm/ 30 | template 31 | class SHMMPSCQueue 32 | { 33 | public: 34 | struct Entry 35 | { 36 | uint32_t next; 37 | EntryData data; 38 | }; 39 | 40 | // SHMMPSCQueue's memory must be zero initialized, e.g. as a global variable or by shm_open/mmap 41 | // so init_state will be 0 at first 42 | SHMMPSCQueue() { 43 | if(!__sync_bool_compare_and_swap(&init_state, 0, 1)) { 44 | while(init_state != 2) 45 | ; 46 | return; 47 | } 48 | pending_tail = 0; 49 | empty_top = 1; 50 | for(uint32_t i = 1; i < SIZE; i++) { 51 | entries[i].next = i + 1; 52 | } 53 | asm volatile("" : : : "memory"); // memory fence 54 | init_state = 2; 55 | } 56 | 57 | Entry* Alloc() { 58 | uint32_t top = FetchAndLock(empty_top, EmptyLock); 59 | if(top == SIZE) { 60 | empty_top = SIZE; 61 | return nullptr; 62 | } 63 | empty_top = entries[top].next; 64 | return &entries[top]; 65 | } 66 | 67 | void Push(Entry* entry) { 68 | uint32_t entry_idx = entry - entries; 69 | uint32_t tail = FetchAndLock(pending_tail, PendingLock); 70 | entries[tail].next = entry_idx; 71 | asm volatile("" : : "m"(entries[tail].next), "m"(pending_tail) :); // memory fence 72 | pending_tail = entry_idx; 73 | } 74 | 75 | Entry* PopAll() { 76 | if(pending_tail == 0) return nullptr; 77 | uint32_t tail = FetchAndLock(pending_tail, PendingLock); 78 | Entry* ret = &entries[entries[0].next]; 79 | asm volatile("" : : "m"(pending_tail) :); // memory fence 80 | pending_tail = 0; 81 | entries[tail].next = SIZE; 82 | return ret; 83 | } 84 | 85 | Entry* NextEntry(Entry* entry) { 86 | if(entry->next == SIZE) return nullptr; 87 | return &entries[entry->next]; 88 | } 89 | 90 | void Recycle(Entry* first, Entry* last) { 91 | uint32_t top = FetchAndLock(empty_top, EmptyLock); 92 | last->next = top; 93 | asm volatile("" : : "m"(last->next), "m"(empty_top) :); // memory fence 94 | empty_top = first - entries; 95 | } 96 | 97 | private: 98 | template 99 | static T FetchAndLock(volatile T& var, T lock_val) { 100 | T val = var; 101 | while(true) { 102 | while(__builtin_expect(val == lock_val, 0)) val = var; 103 | T new_val = __sync_val_compare_and_swap(&var, val, lock_val); 104 | if(__builtin_expect(new_val == val, 1)) break; 105 | val = new_val; 106 | } 107 | return val; 108 | } 109 | 110 | private: 111 | static constexpr uint32_t PendingLock = SIZE + 1; 112 | static constexpr uint32_t EmptyLock = SIZE + 2; 113 | 114 | int volatile init_state; 115 | 116 | alignas(64) uint32_t volatile pending_tail; 117 | 118 | alignas(64) uint32_t volatile empty_top; 119 | 120 | alignas(64) Entry entries[SIZE]; 121 | }; 122 | 123 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | clear.sh 2 | mpsc_test 3 | shm_mpsc_test 4 | async_logging 5 | async_logging.log 6 | shm_logging_client 7 | shm_logging_server 8 | shm_logging_server.log 9 | -------------------------------------------------------------------------------- /test/AppendFile.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | // Muduo - A reactor-based C++ network library for Linux 3 | // Copyright (c) 2010, Shuo Chen. All rights reserved. 4 | // http://code.google.com/p/muduo/ 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions 8 | // are met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above copyright 13 | // notice, this list of conditions and the following disclaimer in the 14 | // documentation and/or other materials provided with the distribution. 15 | // * Neither the name of Shuo Chen nor the names of other contributors 16 | // may be used to endorse or promote products derived from this software 17 | // without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | #include 31 | 32 | class AppendFile 33 | { 34 | public: 35 | explicit AppendFile(const char* filename) 36 | : fp_(::fopen(filename, "ae")) 37 | , writtenBytes_(0) { 38 | ::setbuffer(fp_, buffer_, sizeof buffer_); 39 | } 40 | 41 | ~AppendFile() { 42 | ::fclose(fp_); 43 | } 44 | 45 | void append(const char* logline, const size_t len) { 46 | size_t n = write(logline, len); 47 | size_t remain = len - n; 48 | while(remain > 0) { 49 | size_t x = write(logline + n, remain); 50 | if(x == 0) { 51 | int err = ferror(fp_); 52 | if(err) { 53 | fprintf(stderr, "AppendFile::append() failed %d\n", err); 54 | } 55 | break; 56 | } 57 | n += x; 58 | remain = len - n; // remain -= x 59 | } 60 | 61 | writtenBytes_ += len; 62 | } 63 | 64 | void flush() { 65 | ::fflush(fp_); 66 | } 67 | 68 | off_t writtenBytes() const { 69 | return writtenBytes_; 70 | } 71 | 72 | private: 73 | size_t write(const char* logline, size_t len) { 74 | return ::fwrite_unlocked(logline, 1, len, fp_); 75 | } 76 | 77 | FILE* fp_; 78 | char buffer_[64 * 1024]; 79 | off_t writtenBytes_; 80 | }; 81 | -------------------------------------------------------------------------------- /test/LogStream.cc: -------------------------------------------------------------------------------- 1 | // Muduo - A reactor-based C++ network library for Linux 2 | // Copyright (c) 2010, Shuo Chen. All rights reserved. 3 | // http://code.google.com/p/muduo/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions 7 | // are met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above copyright 12 | // notice, this list of conditions and the following disclaimer in the 13 | // documentation and/or other materials provided with the distribution. 14 | // * Neither the name of Shuo Chen nor the names of other contributors 15 | // may be used to endorse or promote products derived from this software 16 | // without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | #include "LogStream.h" 30 | 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | using namespace muduo; 39 | using namespace muduo::detail; 40 | 41 | #if defined(__clang__) 42 | #pragma clang diagnostic ignored "-Wtautological-compare" 43 | #else 44 | #pragma GCC diagnostic ignored "-Wtype-limits" 45 | #endif 46 | 47 | namespace muduo 48 | { 49 | namespace detail 50 | { 51 | 52 | const char digits[] = "9876543210123456789"; 53 | const char* zero = digits + 9; 54 | // BOOST_STATIC_ASSERT(sizeof(digits) == 20); 55 | 56 | const char digitsHex[] = "0123456789ABCDEF"; 57 | // BOOST_STATIC_ASSERT(sizeof digitsHex == 17); 58 | 59 | // Efficient Integer to String Conversions, by Matthew Wilson. 60 | template 61 | size_t convert(char buf[], T value) 62 | { 63 | T i = value; 64 | char* p = buf; 65 | 66 | do 67 | { 68 | int lsd = static_cast(i % 10); 69 | i /= 10; 70 | *p++ = zero[lsd]; 71 | } while (i != 0); 72 | 73 | if (value < 0) 74 | { 75 | *p++ = '-'; 76 | } 77 | *p = '\0'; 78 | std::reverse(buf, p); 79 | 80 | return p - buf; 81 | } 82 | 83 | size_t convertHex(char buf[], uintptr_t value) 84 | { 85 | uintptr_t i = value; 86 | char* p = buf; 87 | 88 | do 89 | { 90 | int lsd = static_cast(i % 16); 91 | i /= 16; 92 | *p++ = digitsHex[lsd]; 93 | } while (i != 0); 94 | 95 | *p = '\0'; 96 | std::reverse(buf, p); 97 | 98 | return p - buf; 99 | } 100 | 101 | template class FixedBuffer; 102 | template class FixedBuffer; 103 | 104 | } 105 | } 106 | 107 | template 108 | const char* FixedBuffer::debugString() 109 | { 110 | *cur_ = '\0'; 111 | return data_; 112 | } 113 | 114 | template 115 | void FixedBuffer::cookieStart() 116 | { 117 | } 118 | 119 | template 120 | void FixedBuffer::cookieEnd() 121 | { 122 | } 123 | 124 | void LogStream::staticCheck() 125 | { 126 | // BOOST_STATIC_ASSERT(kMaxNumericSize - 10 > std::numeric_limits::digits10); 127 | // BOOST_STATIC_ASSERT(kMaxNumericSize - 10 > std::numeric_limits::digits10); 128 | // BOOST_STATIC_ASSERT(kMaxNumericSize - 10 > std::numeric_limits::digits10); 129 | // BOOST_STATIC_ASSERT(kMaxNumericSize - 10 > std::numeric_limits::digits10); 130 | } 131 | 132 | template 133 | void LogStream::formatInteger(T v) 134 | { 135 | if (buffer_.avail() >= kMaxNumericSize) 136 | { 137 | size_t len = convert(buffer_.current(), v); 138 | buffer_.add(len); 139 | } 140 | } 141 | 142 | LogStream& LogStream::operator<<(short v) 143 | { 144 | *this << static_cast(v); 145 | return *this; 146 | } 147 | 148 | LogStream& LogStream::operator<<(unsigned short v) 149 | { 150 | *this << static_cast(v); 151 | return *this; 152 | } 153 | 154 | LogStream& LogStream::operator<<(int v) 155 | { 156 | formatInteger(v); 157 | return *this; 158 | } 159 | 160 | LogStream& LogStream::operator<<(unsigned int v) 161 | { 162 | formatInteger(v); 163 | return *this; 164 | } 165 | 166 | LogStream& LogStream::operator<<(long v) 167 | { 168 | formatInteger(v); 169 | return *this; 170 | } 171 | 172 | LogStream& LogStream::operator<<(unsigned long v) 173 | { 174 | formatInteger(v); 175 | return *this; 176 | } 177 | 178 | LogStream& LogStream::operator<<(long long v) 179 | { 180 | formatInteger(v); 181 | return *this; 182 | } 183 | 184 | LogStream& LogStream::operator<<(unsigned long long v) 185 | { 186 | formatInteger(v); 187 | return *this; 188 | } 189 | 190 | LogStream& LogStream::operator<<(const void* p) 191 | { 192 | uintptr_t v = reinterpret_cast(p); 193 | if (buffer_.avail() >= kMaxNumericSize) 194 | { 195 | char* buf = buffer_.current(); 196 | buf[0] = '0'; 197 | buf[1] = 'x'; 198 | size_t len = convertHex(buf+2, v); 199 | buffer_.add(len+2); 200 | } 201 | return *this; 202 | } 203 | 204 | // FIXME: replace this with Grisu3 by Florian Loitsch. 205 | LogStream& LogStream::operator<<(double v) 206 | { 207 | if (buffer_.avail() >= kMaxNumericSize) 208 | { 209 | int len = snprintf(buffer_.current(), kMaxNumericSize, "%.12g", v); 210 | buffer_.add(len); 211 | } 212 | return *this; 213 | } 214 | 215 | template 216 | Fmt::Fmt(const char* fmt, T val) 217 | { 218 | // BOOST_STATIC_ASSERT(boost::is_arithmetic::value == true); 219 | 220 | length_ = snprintf(buf_, sizeof buf_, fmt, val); 221 | assert(static_cast(length_) < sizeof buf_); 222 | } 223 | 224 | // Explicit instantiations 225 | 226 | template Fmt::Fmt(const char* fmt, char); 227 | 228 | template Fmt::Fmt(const char* fmt, short); 229 | template Fmt::Fmt(const char* fmt, unsigned short); 230 | template Fmt::Fmt(const char* fmt, int); 231 | template Fmt::Fmt(const char* fmt, unsigned int); 232 | template Fmt::Fmt(const char* fmt, long); 233 | template Fmt::Fmt(const char* fmt, unsigned long); 234 | template Fmt::Fmt(const char* fmt, long long); 235 | template Fmt::Fmt(const char* fmt, unsigned long long); 236 | 237 | template Fmt::Fmt(const char* fmt, float); 238 | template Fmt::Fmt(const char* fmt, double); 239 | -------------------------------------------------------------------------------- /test/LogStream.h: -------------------------------------------------------------------------------- 1 | #ifndef MUDUO_BASE_LOGSTREAM_H 2 | #define MUDUO_BASE_LOGSTREAM_H 3 | // Muduo - A reactor-based C++ network library for Linux 4 | // Copyright (c) 2010, Shuo Chen. All rights reserved. 5 | // http://code.google.com/p/muduo/ 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions 9 | // are met: 10 | // 11 | // * Redistributions of source code must retain the above copyright 12 | // notice, this list of conditions and the following disclaimer. 13 | // * Redistributions in binary form must reproduce the above copyright 14 | // notice, this list of conditions and the following disclaimer in the 15 | // documentation and/or other materials provided with the distribution. 16 | // * Neither the name of Shuo Chen nor the names of other contributors 17 | // may be used to endorse or promote products derived from this software 18 | // without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | #include 33 | #include // memcpy 34 | #ifndef MUDUO_STD_STRING 35 | #include 36 | #endif 37 | 38 | namespace muduo 39 | { 40 | 41 | namespace detail 42 | { 43 | 44 | const int kSmallBuffer = 4000; 45 | const int kLargeBuffer = 4000*1000; 46 | 47 | template 48 | class FixedBuffer 49 | { 50 | public: 51 | FixedBuffer() 52 | : cur_(data_) 53 | { 54 | setCookie(cookieStart); 55 | } 56 | 57 | void setData(char* data) { 58 | cur_ = data_ = data; 59 | } 60 | 61 | ~FixedBuffer() 62 | { 63 | setCookie(cookieEnd); 64 | } 65 | 66 | void append(const char* /*restrict*/ buf, size_t len) 67 | { 68 | // FIXME: append partially 69 | if((size_t)(avail()) > len) { 70 | memcpy(cur_, buf, len); 71 | cur_ += len; 72 | } 73 | } 74 | 75 | const char* data() const { return data_; } 76 | int length() const { return static_cast(cur_ - data_); } 77 | 78 | // write to data_ directly 79 | char* current() { return cur_; } 80 | int avail() const { return static_cast(end() - cur_); } 81 | void add(size_t len) { cur_ += len; } 82 | 83 | void reset() { cur_ = data_; } 84 | void bzero() { 85 | ::bzero(data_, /*sizeof data_*/ SIZE); 86 | } 87 | 88 | // for used by GDB 89 | const char* debugString(); 90 | void setCookie(void (*cookie)()) { cookie_ = cookie; } 91 | // for used by unit test 92 | // string toString() const { return string(data_, length()); } 93 | // StringPiece toStringPiece() const { return StringPiece(data_, length()); } 94 | 95 | private: 96 | const char* end() const { 97 | return data_ + /*sizeof data_*/ SIZE; 98 | } 99 | // Must be outline function for cookies. 100 | static void cookieStart(); 101 | static void cookieEnd(); 102 | 103 | void (*cookie_)(); 104 | // char data_[SIZE]; 105 | char* data_; 106 | char* cur_; 107 | }; 108 | 109 | } 110 | 111 | class LogStream // : boost::noncopyable 112 | { 113 | typedef LogStream self; 114 | public: 115 | typedef detail::FixedBuffer Buffer; 116 | 117 | void setBuf(char* buf) { 118 | buffer_.setData(buf); 119 | } 120 | 121 | void addBuf(size_t len) { 122 | buffer_.add(len); 123 | } 124 | 125 | self& operator<<(bool v) 126 | { 127 | buffer_.append(v ? "1" : "0", 1); 128 | return *this; 129 | } 130 | 131 | self& operator<<(short); 132 | self& operator<<(unsigned short); 133 | self& operator<<(int); 134 | self& operator<<(unsigned int); 135 | self& operator<<(long); 136 | self& operator<<(unsigned long); 137 | self& operator<<(long long); 138 | self& operator<<(unsigned long long); 139 | 140 | self& operator<<(const void*); 141 | 142 | self& operator<<(float v) 143 | { 144 | *this << static_cast(v); 145 | return *this; 146 | } 147 | self& operator<<(double); 148 | // self& operator<<(long double); 149 | 150 | self& operator<<(char v) 151 | { 152 | buffer_.append(&v, 1); 153 | return *this; 154 | } 155 | 156 | // self& operator<<(signed char); 157 | // self& operator<<(unsigned char); 158 | 159 | self& operator<<(const char* str) 160 | { 161 | if (str) 162 | { 163 | buffer_.append(str, strlen(str)); 164 | } 165 | else 166 | { 167 | buffer_.append("(null)", 6); 168 | } 169 | return *this; 170 | } 171 | 172 | self& operator<<(const unsigned char* str) 173 | { 174 | return operator<<(reinterpret_cast(str)); 175 | } 176 | 177 | /* 178 | self& operator<<(const string& v) 179 | { 180 | buffer_.append(v.c_str(), v.size()); 181 | return *this; 182 | } 183 | */ 184 | 185 | #ifndef MUDUO_STD_STRING 186 | self& operator<<(const std::string& v) 187 | { 188 | buffer_.append(v.c_str(), v.size()); 189 | return *this; 190 | } 191 | #endif 192 | 193 | /* 194 | self& operator<<(const StringPiece& v) 195 | { 196 | buffer_.append(v.data(), v.size()); 197 | return *this; 198 | } 199 | */ 200 | 201 | /* 202 | self& operator<<(const Buffer& v) 203 | { 204 | *this << v.toStringPiece(); 205 | return *this; 206 | } 207 | */ 208 | 209 | void append(const char* data, int len) { buffer_.append(data, len); } 210 | const Buffer& buffer() const { return buffer_; } 211 | void resetBuffer() { buffer_.reset(); } 212 | 213 | private: 214 | void staticCheck(); 215 | 216 | template 217 | void formatInteger(T); 218 | 219 | Buffer buffer_; 220 | 221 | static const int kMaxNumericSize = 32; 222 | }; 223 | 224 | class Fmt // : boost::noncopyable 225 | { 226 | public: 227 | template 228 | Fmt(const char* fmt, T val); 229 | 230 | const char* data() const { return buf_; } 231 | int length() const { return length_; } 232 | 233 | private: 234 | char buf_[32]; 235 | int length_; 236 | }; 237 | 238 | inline LogStream& operator<<(LogStream& s, const Fmt& fmt) 239 | { 240 | s.append(fmt.data(), fmt.length()); 241 | return s; 242 | } 243 | 244 | } 245 | #endif // MUDUO_BASE_LOGSTREAM_H 246 | 247 | -------------------------------------------------------------------------------- /test/Logging.cc: -------------------------------------------------------------------------------- 1 | // Muduo - A reactor-based C++ network library for Linux 2 | // Copyright (c) 2010, Shuo Chen. All rights reserved. 3 | // http://code.google.com/p/muduo/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions 7 | // are met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above copyright 12 | // notice, this list of conditions and the following disclaimer in the 13 | // documentation and/or other materials provided with the distribution. 14 | // * Neither the name of Shuo Chen nor the names of other contributors 15 | // may be used to endorse or promote products derived from this software 16 | // without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | #include "Logging.h" 30 | 31 | #include 32 | #include 33 | #include 34 | 35 | #include 36 | #include 37 | #include 38 | 39 | __thread char t_time[64]; 40 | __thread time_t t_lastSecond; 41 | 42 | namespace muduo 43 | { 44 | 45 | /* 46 | class LoggerImpl 47 | { 48 | public: 49 | typedef Logger::LogLevel LogLevel; 50 | LoggerImpl(LogLevel level, int old_errno, const char* file, int line); 51 | void finish(); 52 | 53 | Timestamp time_; 54 | LogStream stream_; 55 | LogLevel level_; 56 | int line_; 57 | const char* fullname_; 58 | const char* basename_; 59 | }; 60 | */ 61 | 62 | __thread char t_errnobuf[512]; 63 | __thread int t_cachedTid; 64 | __thread char t_tidString[32]; 65 | __thread int t_tidStringLength; 66 | 67 | const char* strerror_tl(int savedErrno) 68 | { 69 | return strerror_r(savedErrno, t_errnobuf, sizeof t_errnobuf); 70 | } 71 | 72 | Logger::LogLevel initLogLevel() 73 | { 74 | if (::getenv("MUDUO_LOG_TRACE")) 75 | return Logger::TRACE; 76 | else if (::getenv("MUDUO_LOG_DEBUG")) 77 | return Logger::DEBUG; 78 | else 79 | return Logger::INFO; 80 | } 81 | 82 | Logger::LogLevel g_logLevel = initLogLevel(); 83 | 84 | const char* LogLevelName[Logger::NUM_LOG_LEVELS] = 85 | { 86 | "TRACE ", 87 | "DEBUG ", 88 | "INFO ", 89 | "WARN ", 90 | "ERROR ", 91 | "FATAL ", 92 | }; 93 | 94 | // helper class for known string length at compile time 95 | class T 96 | { 97 | public: 98 | T(const char* str, unsigned len) 99 | :str_(str), 100 | len_(len) 101 | { 102 | assert(strlen(str) == len_); 103 | } 104 | 105 | const char* str_; 106 | const unsigned len_; 107 | }; 108 | 109 | inline LogStream& operator<<(LogStream& s, T v) 110 | { 111 | s.append(v.str_, v.len_); 112 | return s; 113 | } 114 | 115 | inline LogStream& operator<<(LogStream& s, const Logger::SourceFile& v) 116 | { 117 | s.append(v.data_, v.size_); 118 | return s; 119 | } 120 | 121 | void defaultOutput(const char* msg, int len) 122 | { 123 | size_t n = fwrite(msg, 1, len, stdout); 124 | //FIXME check n 125 | (void)n; 126 | } 127 | 128 | void defaultFlush() 129 | { 130 | fflush(stdout); 131 | } 132 | 133 | Logger::OutputFunc g_output = defaultOutput; 134 | Logger::FlushFunc g_flush = defaultFlush; 135 | 136 | } 137 | 138 | using namespace muduo; 139 | 140 | Logger::Impl::Impl(LogLevel level, int savedErrno, const SourceFile& file, int line) 141 | : stream_() 142 | , level_(level) 143 | , line_(line) 144 | , basename_(file) { 145 | // formatTime(); 146 | while((entry_ = g_logq->Alloc()) == nullptr) 147 | ; 148 | stream_.setBuf(entry_->data.buf); 149 | entry_->data.stampTime(); 150 | if(!g_delay_format_ts) { 151 | entry_->data.formatTime(); 152 | } 153 | stream_.addBuf(LogEntry::kFormatTimeLen); 154 | if(__builtin_expect(t_cachedTid == 0, 0)) { 155 | t_cachedTid = static_cast(::syscall(SYS_gettid)); 156 | t_tidStringLength = snprintf(t_tidString, sizeof t_tidString, "%5d ", t_cachedTid); 157 | } 158 | stream_ << T(t_tidString, t_tidStringLength); 159 | stream_ << T(LogLevelName[level], 6); 160 | if(savedErrno != 0) { 161 | stream_ << strerror_tl(savedErrno) << " (errno=" << savedErrno << ") "; 162 | } 163 | } 164 | 165 | /* 166 | void Logger::Impl::formatTime() 167 | { 168 | int64_t microSecondsSinceEpoch = time_.microSecondsSinceEpoch(); 169 | time_t seconds = static_cast(microSecondsSinceEpoch / Timestamp::kMicroSecondsPerSecond); 170 | int microseconds = static_cast(microSecondsSinceEpoch % Timestamp::kMicroSecondsPerSecond); 171 | if (seconds != t_lastSecond) 172 | { 173 | t_lastSecond = seconds; 174 | struct tm tm_time; 175 | { 176 | ::gmtime_r(&seconds, &tm_time); // FIXME TimeZone::fromUtcTime 177 | } 178 | 179 | int len = snprintf(t_time, sizeof(t_time), "%4d%02d%02d %02d:%02d:%02d", 180 | tm_time.tm_year + 1900, tm_time.tm_mon + 1, tm_time.tm_mday, 181 | tm_time.tm_hour, tm_time.tm_min, tm_time.tm_sec); 182 | assert(len == 17); (void)len; 183 | } 184 | 185 | { 186 | Fmt us(".%06dZ ", microseconds); 187 | assert(us.length() == 9); 188 | stream_ << T(t_time, 17) << T(us.data(), 9); 189 | } 190 | } 191 | */ 192 | 193 | void Logger::Impl::finish() 194 | { 195 | stream_ << " - " << basename_ << ':' << line_ << '\n'; 196 | entry_->data.buflen = stream_.buffer().length(); 197 | g_logq->Push(entry_); 198 | } 199 | 200 | Logger::Logger(SourceFile file, int line) 201 | : impl_(INFO, 0, file, line) 202 | { 203 | } 204 | 205 | Logger::Logger(SourceFile file, int line, LogLevel level, const char* func) 206 | : impl_(level, 0, file, line) 207 | { 208 | impl_.stream_ << func << ' '; 209 | } 210 | 211 | Logger::Logger(SourceFile file, int line, LogLevel level) 212 | : impl_(level, 0, file, line) 213 | { 214 | } 215 | 216 | Logger::Logger(SourceFile file, int line, bool toAbort) 217 | : impl_(toAbort?FATAL:ERROR, errno, file, line) 218 | { 219 | } 220 | 221 | Logger::~Logger() 222 | { 223 | impl_.finish(); 224 | // const LogStream::Buffer& buf(stream().buffer()); 225 | // g_output(buf.data(), buf.length()); 226 | if (impl_.level_ == FATAL) 227 | { 228 | g_flush(); 229 | abort(); 230 | } 231 | } 232 | 233 | void Logger::setLogLevel(Logger::LogLevel level) 234 | { 235 | g_logLevel = level; 236 | } 237 | 238 | void Logger::setOutput(OutputFunc out) 239 | { 240 | g_output = out; 241 | } 242 | 243 | void Logger::setFlush(FlushFunc flush) 244 | { 245 | g_flush = flush; 246 | } 247 | 248 | -------------------------------------------------------------------------------- /test/Logging.h: -------------------------------------------------------------------------------- 1 | #ifndef MUDUO_BASE_LOGGING_H 2 | #define MUDUO_BASE_LOGGING_H 3 | // Muduo - A reactor-based C++ network library for Linux 4 | // Copyright (c) 2010, Shuo Chen. All rights reserved. 5 | // http://code.google.com/p/muduo/ 6 | // 7 | // Redistribution and use in source and binary forms, with or without 8 | // modification, are permitted provided that the following conditions 9 | // are met: 10 | // 11 | // * Redistributions of source code must retain the above copyright 12 | // notice, this list of conditions and the following disclaimer. 13 | // * Redistributions in binary form must reproduce the above copyright 14 | // notice, this list of conditions and the following disclaimer in the 15 | // documentation and/or other materials provided with the distribution. 16 | // * Neither the name of Shuo Chen nor the names of other contributors 17 | // may be used to endorse or promote products derived from this software 18 | // without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | #include "LogStream.h" 33 | #include "timestamp.h" 34 | 35 | extern __thread char t_time[64]; 36 | extern __thread time_t t_lastSecond; 37 | 38 | struct LogEntry 39 | { 40 | static const int kFormatTimeLen = 17 + 9; 41 | LogEntry() { 42 | // memset(buf, 0, sizeof(buf)); 43 | } 44 | void stampTime() { 45 | timestamp = now(); 46 | } 47 | 48 | void formatTime() { 49 | int64_t microSecondsSinceEpoch = timestamp; 50 | time_t seconds = static_cast(microSecondsSinceEpoch / kMicroSecondsPerSecond); 51 | int microseconds = static_cast(microSecondsSinceEpoch % kMicroSecondsPerSecond); 52 | if(seconds != t_lastSecond) { 53 | t_lastSecond = seconds; 54 | struct tm tm_time; 55 | { 56 | ::gmtime_r(&seconds, &tm_time); // FIXME TimeZone::fromUtcTime 57 | } 58 | 59 | int len = snprintf(t_time, 60 | sizeof(t_time), 61 | "%4d%02d%02d %02d:%02d:%02d", 62 | tm_time.tm_year + 1900, 63 | tm_time.tm_mon + 1, 64 | tm_time.tm_mday, 65 | tm_time.tm_hour, 66 | tm_time.tm_min, 67 | tm_time.tm_sec); 68 | assert(len == 17); 69 | (void)len; 70 | } 71 | 72 | { 73 | muduo::Fmt us(".%06dZ ", microseconds); 74 | assert(us.length() == 9); 75 | memcpy(buf, t_time, 17); 76 | memcpy(buf + 17, us.data(), 9); 77 | } 78 | } 79 | 80 | int64_t timestamp; 81 | uint32_t buflen; 82 | char buf[muduo::detail::kSmallBuffer]; 83 | }; 84 | 85 | #ifdef USE_SHM 86 | #include "../shm_mpsc_queue.h" 87 | typedef SHMMPSCQueue LogQueue; 88 | #else 89 | #include "../mpsc_queue.h" 90 | typedef MPSCQueue LogQueue; 91 | #endif 92 | 93 | extern LogQueue* g_logq; 94 | extern bool g_delay_format_ts; 95 | 96 | namespace muduo { 97 | 98 | class Logger 99 | { 100 | public: 101 | enum LogLevel 102 | { 103 | TRACE, 104 | DEBUG, 105 | INFO, 106 | WARN, 107 | ERROR, 108 | FATAL, 109 | NUM_LOG_LEVELS, 110 | }; 111 | 112 | // compile time calculation of basename of source file 113 | class SourceFile 114 | { 115 | public: 116 | template 117 | inline SourceFile(const char (&arr)[N]) 118 | : data_(arr), 119 | size_(N-1) 120 | { 121 | const char* slash = strrchr(data_, '/'); // builtin function 122 | if (slash) 123 | { 124 | data_ = slash + 1; 125 | size_ -= static_cast(data_ - arr); 126 | } 127 | } 128 | 129 | explicit SourceFile(const char* filename) 130 | : data_(filename) 131 | { 132 | const char* slash = strrchr(filename, '/'); 133 | if (slash) 134 | { 135 | data_ = slash + 1; 136 | } 137 | size_ = static_cast(strlen(data_)); 138 | } 139 | 140 | const char* data_; 141 | int size_; 142 | }; 143 | 144 | Logger(SourceFile file, int line); 145 | Logger(SourceFile file, int line, LogLevel level); 146 | Logger(SourceFile file, int line, LogLevel level, const char* func); 147 | Logger(SourceFile file, int line, bool toAbort); 148 | ~Logger(); 149 | 150 | LogStream& stream() { return impl_.stream_; } 151 | 152 | static LogLevel logLevel(); 153 | static void setLogLevel(LogLevel level); 154 | 155 | typedef void (*OutputFunc)(const char* msg, int len); 156 | typedef void (*FlushFunc)(); 157 | static void setOutput(OutputFunc); 158 | static void setFlush(FlushFunc); 159 | 160 | private: 161 | 162 | class Impl 163 | { 164 | public: 165 | typedef Logger::LogLevel LogLevel; 166 | Impl(LogLevel level, int old_errno, const SourceFile& file, int line); 167 | // void formatTime(); 168 | void finish(); 169 | 170 | LogStream stream_; 171 | LogLevel level_; 172 | int line_; 173 | SourceFile basename_; 174 | LogQueue::Entry* entry_; 175 | }; 176 | 177 | Impl impl_; 178 | 179 | }; 180 | 181 | extern Logger::LogLevel g_logLevel; 182 | 183 | inline Logger::LogLevel Logger::logLevel() 184 | { 185 | return g_logLevel; 186 | } 187 | 188 | // 189 | // CAUTION: do not write: 190 | // 191 | // if (good) 192 | // LOG_INFO << "Good news"; 193 | // else 194 | // LOG_WARN << "Bad news"; 195 | // 196 | // this expends to 197 | // 198 | // if (good) 199 | // if (logging_INFO) 200 | // logInfoStream << "Good news"; 201 | // else 202 | // logWarnStream << "Bad news"; 203 | // 204 | #define LOG_TRACE if (muduo::Logger::logLevel() <= muduo::Logger::TRACE) \ 205 | muduo::Logger(__FILE__, __LINE__, muduo::Logger::TRACE, __func__).stream() 206 | #define LOG_DEBUG if (muduo::Logger::logLevel() <= muduo::Logger::DEBUG) \ 207 | muduo::Logger(__FILE__, __LINE__, muduo::Logger::DEBUG, __func__).stream() 208 | #define LOG_INFO if (muduo::Logger::logLevel() <= muduo::Logger::INFO) \ 209 | muduo::Logger(__FILE__, __LINE__).stream() 210 | #define LOG_WARN muduo::Logger(__FILE__, __LINE__, muduo::Logger::WARN).stream() 211 | #define LOG_ERROR muduo::Logger(__FILE__, __LINE__, muduo::Logger::ERROR).stream() 212 | #define LOG_FATAL muduo::Logger(__FILE__, __LINE__, muduo::Logger::FATAL).stream() 213 | #define LOG_SYSERR muduo::Logger(__FILE__, __LINE__, false).stream() 214 | #define LOG_SYSFATAL muduo::Logger(__FILE__, __LINE__, true).stream() 215 | 216 | const char* strerror_tl(int savedErrno); 217 | 218 | // Taken from glog/logging.h 219 | // 220 | // Check that the input is non NULL. This very useful in constructor 221 | // initializer lists. 222 | 223 | #define CHECK_NOTNULL(val) \ 224 | ::muduo::CheckNotNull(__FILE__, __LINE__, "'" #val "' Must be non NULL", (val)) 225 | 226 | // A small helper for CHECK_NOTNULL(). 227 | template 228 | T* CheckNotNull(Logger::SourceFile file, int line, const char *names, T* ptr) 229 | { 230 | if (ptr == NULL) 231 | { 232 | Logger(file, line, Logger::FATAL).stream() << names; 233 | } 234 | return ptr; 235 | } 236 | 237 | } 238 | 239 | #endif // MUDUO_BASE_LOGGING_H 240 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | ## mpsc_test/shm_mpsc_test 2 | Simple multi-threaded test program for both versions of queue. 3 | 4 | ## async_logging 5 | Multi-threaded logging test program based on MPSCQueue. 6 | 7 | ## shm_logging_client/shm_logging_server 8 | Multi-process logging test program based on SHMMPSCQueue, where client is producer and server is consumer. 9 | 10 | -------------------------------------------------------------------------------- /test/async_logging: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MengRao/MPSC_Queue/9fd34e45241f04e8331b8246f26e60b11df90150/test/async_logging -------------------------------------------------------------------------------- /test/async_logging.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "Logging.h" 3 | #include "AppendFile.h" 4 | #include "cpupin.h" 5 | #include 6 | #include 7 | using namespace std; 8 | 9 | LogQueue __logq(2000, 4000); 10 | LogQueue* g_logq = &__logq; 11 | bool g_delay_format_ts = true; 12 | 13 | volatile bool running = true; 14 | 15 | void bench(bool longLog) { 16 | int cnt = 0; 17 | const int kBatch = 1000; 18 | string empty = " "; 19 | string longStr(3000, 'X'); 20 | longStr += " "; 21 | 22 | for(int t = 0; t < 30; ++t) { 23 | int64_t start = now(); 24 | for(int i = 0; i < kBatch; ++i) { 25 | LOG_INFO << "Hello 0123456789" 26 | << " abcdefghijklmnopqrstuvwxyz " << (longLog ? longStr : empty) << cnt; 27 | ++cnt; 28 | } 29 | int64_t end = now(); 30 | printf("%f\n", static_cast(end - start) / kBatch); 31 | struct timespec ts = {0, 500 * 1000 * 1000}; 32 | nanosleep(&ts, NULL); 33 | } 34 | } 35 | 36 | void logServer(const string& logfile) { 37 | AppendFile file(logfile.c_str()); 38 | while(running) { 39 | LogQueue::Entry* list = g_logq->PopAll(); 40 | if(!list) { 41 | this_thread::yield(); 42 | continue; 43 | } 44 | auto cur = list; 45 | while(true) { 46 | if(g_delay_format_ts) { 47 | cur->data.formatTime(); 48 | } 49 | file.append(cur->data.buf, cur->data.buflen); 50 | if(!cur->next) break; 51 | cur = cur->next; 52 | } 53 | g_logq->Recycle(list, cur); 54 | file.flush(); 55 | } 56 | } 57 | 58 | int main(int argc, char* argv[]) { 59 | cpupin(1); 60 | { 61 | // set max virtual memory to 2GB. 62 | size_t kOneGB = 1000 * 1024 * 1024; 63 | rlimit rl = {2 * kOneGB, 2 * kOneGB}; 64 | setrlimit(RLIMIT_AS, &rl); 65 | } 66 | printf("pid = %d\n", getpid()); 67 | char name[256] = {0}; 68 | strncpy(name, argv[0], sizeof name - 1); 69 | thread srv_thr(logServer, string(::basename(name)) + ".log"); 70 | 71 | bool longLog = argc > 1; 72 | bench(longLog); 73 | running = false; 74 | srv_thr.join(); 75 | } 76 | 77 | 78 | -------------------------------------------------------------------------------- /test/build.sh: -------------------------------------------------------------------------------- 1 | set -x 2 | g++ -std=c++11 -O3 -o mpsc_test mpsc_test.cc -pthread 3 | g++ -std=c++11 -O3 -o shm_mpsc_test shm_mpsc_test.cc -pthread 4 | g++ -std=c++11 -O3 -o async_logging async_logging.cc Logging.cc LogStream.cc -pthread 5 | g++ -std=c++11 -O3 -DUSE_SHM=1 -o shm_logging_client shm_logging_client.cc Logging.cc LogStream.cc -lrt 6 | g++ -std=c++11 -O3 -DUSE_SHM=1 -o shm_logging_server shm_logging_server.cc Logging.cc LogStream.cc -lrt 7 | 8 | -------------------------------------------------------------------------------- /test/cpupin.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | bool cpupin(int cpuid, int prio = 99) { 4 | if(prio > 0) { 5 | sched_param param; 6 | param.sched_priority = prio; 7 | if(sched_setscheduler(0, SCHED_FIFO, ¶m)) { 8 | std::cout << "sched_setscheduler error: " << strerror(errno) << std::endl; 9 | return false; 10 | } 11 | } 12 | 13 | cpu_set_t my_set; 14 | CPU_ZERO(&my_set); 15 | CPU_SET(cpuid, &my_set); 16 | if(sched_setaffinity(0, sizeof(cpu_set_t), &my_set)) { 17 | std::cout << "sched_setaffinity error: " << strerror(errno) << std::endl; 18 | return false; 19 | } 20 | 21 | return true; 22 | } 23 | -------------------------------------------------------------------------------- /test/mpsc_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "../mpsc_queue.h" 3 | #include "timestamp.h" 4 | #include "cpupin.h" 5 | using namespace std; 6 | 7 | typedef MPSCQueue Q; 8 | Q q(1000, 2000); 9 | int64_t produce_total = 0; 10 | int64_t produce_alloc_miss = 0; 11 | volatile int ready = 0; 12 | volatile int done = 0; 13 | const int nthread = 4; 14 | const int num_per_thr = 1000000; 15 | 16 | void produce(int nth) { 17 | cpupin(nth); 18 | 19 | random_device rd; 20 | mt19937 generator(rd()); 21 | uniform_int_distribution dis(0, 100000); 22 | 23 | __sync_fetch_and_add(&ready, 1); 24 | while(ready != nthread) 25 | ; 26 | int64_t sum = 0; 27 | int cnt = num_per_thr; 28 | int alloc_miss_cnt = 0; 29 | while(cnt--) { 30 | Q::Entry* entry; 31 | while((entry = q.Alloc()) == nullptr) alloc_miss_cnt++; 32 | int newval = dis(generator); 33 | entry->data = newval; 34 | q.Push(entry); 35 | sum += newval; 36 | } 37 | 38 | __sync_fetch_and_add(&produce_total, sum); 39 | __sync_fetch_and_add(&produce_alloc_miss, alloc_miss_cnt); 40 | __sync_fetch_and_add(&done, 1); 41 | } 42 | 43 | 44 | int64_t doConsume() { 45 | Q::Entry* list = q.PopAll(); 46 | if(!list) return 0; 47 | int64_t sum = 0; 48 | auto cur = list; 49 | while(true) { 50 | sum += cur->data; 51 | if(!cur->next) break; 52 | cur = cur->next; 53 | } 54 | q.Recycle(list, cur); 55 | return sum; 56 | } 57 | 58 | void consume() { 59 | cpupin(nthread); 60 | while(ready != nthread) 61 | ; 62 | auto before = now(); 63 | cout << "started" << endl; 64 | 65 | int64_t sum = 0; 66 | while(done != nthread) { 67 | sum += doConsume(); 68 | // std::this_thread::yield(); 69 | } 70 | sum += doConsume(); 71 | auto after = now(); 72 | cout << "latency: " << (after - before) << " produce total: " << produce_total << " consume total: " << sum 73 | << " alloc miss: " << produce_alloc_miss << endl; 74 | } 75 | 76 | int main() { 77 | vector thr; 78 | for(int i = 0; i < nthread; i++) { 79 | thr.emplace_back(produce, i); 80 | } 81 | consume(); 82 | for(auto& cur : thr) { 83 | cur.join(); 84 | } 85 | return 0; 86 | } 87 | 88 | -------------------------------------------------------------------------------- /test/shm_logging_client.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "Logging.h" 3 | #include "AppendFile.h" 4 | #include "cpupin.h" 5 | #include "shmmap.h" 6 | #include 7 | #include 8 | using namespace std; 9 | 10 | LogQueue* g_logq = nullptr; 11 | bool g_delay_format_ts = true; 12 | 13 | void bench(bool longLog) { 14 | int cnt = 0; 15 | const int kBatch = 1000; 16 | string empty = " "; 17 | string longStr(3000, 'X'); 18 | longStr += " "; 19 | 20 | for(int t = 0; t < 30; ++t) { 21 | int64_t start = now(); 22 | for(int i = 0; i < kBatch; ++i) { 23 | LOG_INFO << "Hello 0123456789" 24 | << " abcdefghijklmnopqrstuvwxyz " << (longLog ? longStr : empty) << cnt; 25 | ++cnt; 26 | } 27 | int64_t end = now(); 28 | printf("%f\n", static_cast(end - start) / kBatch); 29 | struct timespec ts = {0, 500 * 1000 * 1000}; 30 | nanosleep(&ts, NULL); 31 | } 32 | } 33 | 34 | int main(int argc, char* argv[]) { 35 | // cpupin(1); 36 | g_logq = shmmap("/LogQueue.shm"); 37 | if(!g_logq) return 1; 38 | 39 | { 40 | // set max virtual memory to 2GB. 41 | size_t kOneGB = 1000 * 1024 * 1024; 42 | rlimit rl = {2 * kOneGB, 2 * kOneGB}; 43 | setrlimit(RLIMIT_AS, &rl); 44 | } 45 | printf("pid = %d\n", getpid()); 46 | 47 | bool longLog = argc > 1; 48 | bench(longLog); 49 | } 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /test/shm_logging_server.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "Logging.h" 3 | #include "AppendFile.h" 4 | #include "cpupin.h" 5 | #include "shmmap.h" 6 | #include 7 | #include 8 | using namespace std; 9 | 10 | LogQueue* g_logq = nullptr; 11 | bool g_delay_format_ts = true; 12 | 13 | volatile bool running = true; 14 | 15 | void logServer(const string& logfile) { 16 | AppendFile file(logfile.c_str()); 17 | while(running) { 18 | LogQueue::Entry* list = g_logq->PopAll(); 19 | if(!list) { 20 | this_thread::yield(); 21 | continue; 22 | } 23 | auto cur = list; 24 | while(true) { 25 | if(g_delay_format_ts) { 26 | cur->data.formatTime(); 27 | } 28 | file.append(cur->data.buf, cur->data.buflen); 29 | LogQueue::Entry* next = g_logq->NextEntry(cur); 30 | if(!next) break; 31 | cur = next; 32 | } 33 | g_logq->Recycle(list, cur); 34 | file.flush(); 35 | } 36 | } 37 | 38 | int main(int argc, char* argv[]) { 39 | // cpupin(2); 40 | g_logq = shmmap("/LogQueue.shm"); 41 | if(!g_logq) return 1; 42 | { 43 | // set max virtual memory to 2GB. 44 | size_t kOneGB = 1000 * 1024 * 1024; 45 | rlimit rl = {2 * kOneGB, 2 * kOneGB}; 46 | setrlimit(RLIMIT_AS, &rl); 47 | } 48 | printf("pid = %d\n", getpid()); 49 | char name[256] = {0}; 50 | strncpy(name, argv[0], sizeof name - 1); 51 | logServer(string(::basename(name)) + ".log"); 52 | } 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /test/shm_mpsc_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "../shm_mpsc_queue.h" 3 | #include "timestamp.h" 4 | #include "cpupin.h" 5 | using namespace std; 6 | 7 | typedef SHMMPSCQueue Q; 8 | Q q; 9 | int64_t produce_total = 0; 10 | int64_t produce_alloc_miss = 0; 11 | volatile int ready = 0; 12 | volatile int done = 0; 13 | const int nthread = 4; 14 | const int num_per_thr = 1000000; 15 | 16 | void produce(int nth) { 17 | cpupin(nth); 18 | 19 | random_device rd; 20 | mt19937 generator(rd()); 21 | uniform_int_distribution dis(0, 100000); 22 | 23 | __sync_fetch_and_add(&ready, 1); 24 | while(ready != nthread) 25 | ; 26 | int64_t sum = 0; 27 | int cnt = num_per_thr; 28 | int alloc_miss_cnt = 0; 29 | while(cnt--) { 30 | Q::Entry* entry; 31 | while((entry = q.Alloc()) == nullptr) alloc_miss_cnt++; 32 | int newval = dis(generator); 33 | entry->data = newval; 34 | q.Push(entry); 35 | sum += newval; 36 | } 37 | 38 | __sync_fetch_and_add(&produce_total, sum); 39 | __sync_fetch_and_add(&produce_alloc_miss, alloc_miss_cnt); 40 | __sync_fetch_and_add(&done, 1); 41 | } 42 | 43 | 44 | int64_t doConsume() { 45 | Q::Entry* list = q.PopAll(); 46 | if(!list) return 0; 47 | int64_t sum = 0; 48 | auto cur = list; 49 | while(true) { 50 | sum += cur->data; 51 | Q::Entry* next = q.NextEntry(cur); 52 | if(!next) break; 53 | cur = next; 54 | } 55 | q.Recycle(list, cur); 56 | return sum; 57 | } 58 | 59 | void consume() { 60 | cpupin(nthread); 61 | while(ready != nthread) 62 | ; 63 | auto before = now(); 64 | cout << "started" << endl; 65 | 66 | int64_t sum = 0; 67 | while(done != nthread) { 68 | sum += doConsume(); 69 | // std::this_thread::yield(); 70 | } 71 | sum += doConsume(); 72 | auto after = now(); 73 | cout << "latency: " << (after - before) << " produce total: " << produce_total << " consume total: " << sum 74 | << " alloc miss: " << produce_alloc_miss << endl; 75 | } 76 | 77 | int main() { 78 | vector thr; 79 | for(int i = 0; i < nthread; i++) { 80 | thr.emplace_back(produce, i); 81 | } 82 | consume(); 83 | for(auto& cur : thr) { 84 | cur.join(); 85 | } 86 | return 0; 87 | } 88 | 89 | 90 | -------------------------------------------------------------------------------- /test/shmmap.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | template 10 | T* shmmap(const char * filename) { 11 | int fd = shm_open(filename, O_CREAT | O_RDWR, 0666); 12 | if(fd == -1) { 13 | std::cerr << "shm_open failed: " << strerror(errno) << std::endl; 14 | return nullptr; 15 | } 16 | /* 17 | struct stat st; 18 | bool need_construct = false; 19 | if(fstat(fd, &st) == -1) { 20 | std::cerr << "fstat failed: " << strerror(errno) << std::endl; 21 | close(fd); 22 | return nullptr; 23 | } 24 | if(st.st_size != sizeof(T)) { 25 | */ 26 | if(ftruncate(fd, sizeof(T))) { 27 | std::cerr << "ftruncate failed: " << strerror(errno) << std::endl; 28 | close(fd); 29 | return nullptr; 30 | } 31 | /* 32 | need_construct = true; 33 | } 34 | */ 35 | T* ret = (T*)mmap(0, sizeof(T), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 36 | close(fd); 37 | if(ret == MAP_FAILED) { 38 | std::cerr << "mmap failed: " << strerror(errno) << std::endl; 39 | return nullptr; 40 | } 41 | // T's constuctor must be thread safe 42 | new(ret) T; 43 | return ret; 44 | } 45 | -------------------------------------------------------------------------------- /test/timestamp.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | static const int kMicroSecondsPerSecond = 1000 * 1000; 4 | 5 | inline int64_t now() { 6 | struct timeval tv; 7 | gettimeofday(&tv, NULL); 8 | int64_t seconds = tv.tv_sec; 9 | return seconds * kMicroSecondsPerSecond + tv.tv_usec; 10 | } 11 | 12 | inline unsigned long long rdtsc() { 13 | return __builtin_ia32_rdtsc(); 14 | } 15 | 16 | inline unsigned long long rdtscp() { 17 | unsigned int dummy; 18 | return __builtin_ia32_rdtscp(&dummy); 19 | } 20 | 21 | --------------------------------------------------------------------------------