├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── README.md ├── build.sh ├── clear.sh ├── fmtlog-inl.h ├── fmtlog.cc ├── fmtlog.h └── test ├── CMakeLists.txt ├── enc_dec_test.cc ├── lib.cc ├── lib.h ├── link_test.cc ├── log_test.cc └── multithread_test.cc /.gitignore: -------------------------------------------------------------------------------- 1 | .build 2 | out 3 | CMakeSettings.json 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "fmt"] 2 | path = fmt 3 | url = https://github.com/fmtlib/fmt 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | project(fmtlog CXX) 4 | 5 | if(MSVC) 6 | add_compile_options(/std:c++latest) 7 | else() 8 | add_compile_options(-Wall -O3 -std=c++17) 9 | #add_compile_options(-Wall -Ofast -std=c++2a -DNDEBUG -march=skylake -flto -fno-exceptions -fno-rtti -fno-unwind-tables -fno-asynchronous-unwind-tables -DFMTLOG_NO_CHECK_LEVEL=1) 10 | #SET(CMAKE_AR "gcc-ar") 11 | #SET(CMAKE_RANLIB "gcc-ranlib") 12 | link_libraries(pthread) 13 | endif() 14 | 15 | link_directories(.) 16 | include_directories(fmt/include) 17 | 18 | add_library(fmtlog-shared SHARED fmtlog.cc) 19 | if(MSVC) 20 | target_link_libraries(fmtlog-shared fmt) 21 | endif() 22 | install(TARGETS fmtlog-shared) 23 | 24 | add_library(fmtlog-static fmtlog.cc) 25 | if(MSVC) 26 | target_link_libraries(fmtlog-static fmt) 27 | endif() 28 | install(TARGETS fmtlog-static) 29 | 30 | add_subdirectory(fmt) 31 | add_subdirectory(test) 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Meng Rao 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fmtlog 2 | fmtlog is a performant asynchronous logging library using [fmt](https://github.com/fmtlib/fmt) library format. 3 | 4 | ## Features 5 | * Faster - lower runtime latency than [NanoLog](https://github.com/PlatformLab/NanoLog) and higher throughput than [spdlog](https://github.com/gabime/spdlog) (see [Performance](https://github.com/MengRao/fmtlog#Performance) below). 6 | * Headers only or compiled 7 | * Feature rich formatting on top of excellent fmt library. 8 | * Asynchronous multi-threaded logging **in time order** and can also be used synchronously in single thread. 9 | * Custom formatting 10 | * Custom handling - user can set a callback function to handle log msgs in addition to writing into file. 11 | * Log filtering - log levels can be modified in runtime as well as in compile time. 12 | * Log frequency limitation - specific logs can be set a minimum logging interval. 13 | 14 | ## Platforms 15 | * Linux (GCC 10.2 tested) 16 | * Windows (MSVC 2019 tested) 17 | 18 | ## Install 19 | C++17 is required, and fmtlog is dependent on [fmtlib](https://github.com/fmtlib/fmt), you need to install fmtlib first if you haven't. 20 | #### Header only version 21 | Just copy `fmtlog.h` and `fmtlog-inl.h` to your project, and: 22 | * Either define macro `FMTLOG_HEADER_ONLY` before including fmtlog.h. 23 | * Or include `fmtlog-inl.h` in one of your source files. 24 | 25 | #### Static/Shared lib version built by CMake 26 | ```console 27 | $ git clone https://github.com/MengRao/fmtlog.git 28 | $ cd fmtlog 29 | $ git submodule init 30 | $ git submodule update 31 | $ ./build.sh 32 | ``` 33 | Then copy `fmtlog.h` and `libfmtlog-static.a`/`libfmtlog-shared.so` generated in `.build` dir. 34 | 35 | ## Usage 36 | ```c++ 37 | #include "fmtlog/fmtlog.h" 38 | int main() 39 | { 40 | FMTLOG(fmtlog::INF, "The answer is {}.", 42); 41 | } 42 | ``` 43 | There're also shortcut macros `logd`, `logi`, `logw` and `loge` defined for logging `DBG`, `INF`, `WRN` and `ERR` msgs respectively: 44 | ```c++ 45 | logi("A info msg"); 46 | logd("This msg will not be logged as the default log level is INF"); 47 | fmtlog::setLogLevel(fmtlog::DBG); 48 | logd("Now debug msg is shown"); 49 | ``` 50 | Note that fmtlog is asynchronous in nature, msgs are not written into file/console immediately after the log statements: they are simply pushed into a queue. You need to call `fmtlog::poll()` to collect data from log queues, format and write it out: 51 | ```c++ 52 | fmtlog::setThreadName("aaa"); 53 | logi("Thread name is bbb in this msg"); 54 | fmtlog::setThreadName("bbb"); 55 | fmtlog::poll(); 56 | fmtlog::setThreadName("ccc"); 57 | logi("Thread name is ccc in this msg"); 58 | fmtlog::poll(); 59 | fmtlog::setThreadName("ddd"); 60 | 61 | ``` 62 | fmtlog supports multi-threaded logging, but can only have one thread calling `fmtlog::poll()`. By default, fmtlog doesn't create a polling thread internally, it requires the user to poll it periodically. The idea is that this allows users to manage the threads in their own way, and have full control of polling/flushing behavior. However, you can ask fmtlog to create a background polling thread for you by `fmtlog::startPollingThread(interval)` with a polling interval, but you can't call `fmtlog::poll()` yourself when the thread is running. 63 | 64 | ## Format 65 | fmtlog is based on fmtlib, almost all fmtlib features are supported(except for color): 66 | ```c++ 67 | #include "fmt/ranges.h" 68 | using namespace fmt::literals; 69 | 70 | logi("I'd rather be {1} than {0}.", "right", "happy"); 71 | logi("Hello, {name}! The answer is {number}. Goodbye, {name}.", "name"_a = "World", "number"_a = 42); 72 | 73 | std::vector v = {1, 2, 3}; 74 | logi("ranges: {}", v); 75 | 76 | logi("std::move can be used for objects with non-trivial destructors: {}", std::move(v)); 77 | assert(v.size() == 0); 78 | 79 | std::tuple t = {1, 'a'}; 80 | logi("tuples: {}", fmt::join(t, ", ")); 81 | 82 | enum class color {red, green, blue}; 83 | template <> struct fmt::formatter: formatter { 84 | // parse is inherited from formatter. 85 | template 86 | auto format(color c, FormatContext& ctx) { 87 | string_view name = "unknown"; 88 | switch (c) { 89 | case color::red: name = "red"; break; 90 | case color::green: name = "green"; break; 91 | case color::blue: name = "blue"; break; 92 | } 93 | return formatter::format(name, ctx); 94 | } 95 | }; 96 | logi("user defined type: {:>10}", color::blue); 97 | logi("{:*^30}", "centered"); 98 | logi("int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}", 42); 99 | logi("dynamic precision: {:.{}f}", 3.14, 1); 100 | 101 | // This gives a compile-time error because d is an invalid format specifier for a string. 102 | // FMT_STRING() is not needed from C++20 onward 103 | logi(FMT_STRING("{:d}"), "I am not a number"); 104 | ``` 105 | As an asynchronous logging library, fmtlog provides additional support for passing arguments by pointers(which is seldom needed for fmtlib and it only supports void and char pointers). User can pass a pointer of any type as argument to avoid copy overhead if the lifetime of referred object is assured(otherwise the polling thread will refer to a dangling pointer!). For string arg as an example, fmtlog copies string content for type `std::string` by default, but only a pointer for type `std::string*`: 106 | ```c++ 107 | std::string str = "aaa"; 108 | logi("str: {}, pstr: {}", str, &str); 109 | str = "bbb"; 110 | fmtlog::poll(); 111 | // output: str: aaa, pstr: bbb 112 | ``` 113 | In addition to raw pointers, fmtlog supports `std::shared_ptr` and `std::unique_ptr` as well, which makes object lifetime management much easier: 114 | ```c++ 115 | int a = 4; 116 | auto sptr = std::make_shared(5); 117 | auto uptr = std::make_unique(6); 118 | logi("void ptr: {}, ptr: {}, sptr: {}, uptr: {}", (void*)&a, &a, sptr, std::move(uptr)); 119 | a = 7; 120 | *sptr = 8; 121 | fmtlog::poll(); 122 | // output: void ptr: 0x7ffd08ac53ac, ptr: 7, sptr: 8, uptr: 6 123 | ``` 124 | 125 | Log header pattern can also be customized with `fmtlog::setHeaderPattern()` and the argument is a fmtlib format string with named arguments. The default header pattern is "{HMSf} {s:<16} {l}[{t:<6}] " (example: "15:46:19.149844 log_test.cc:43 INF[448050] "). All supported named arguments in header are as below: 126 | | Name | Meaning| Example | 127 | | :------ | :-------: | :-----: | 128 | |`l`|Log level|INF| 129 | |`s`|File base name and line num|log_test.cc:48| 130 | |`g`|File path and line num|/home/raomeng/fmtlog/log_test.cc:48| 131 | |`t`|Thread id by default, can be reset by `fmt::setThreadName()`|main| 132 | |`a`|Weekday|Mon| 133 | |`b`|Month name|May| 134 | |`Y`|Year|2021| 135 | |`C`|Short year|21| 136 | |`m`|Month|05| 137 | |`d`|Day|03| 138 | |`H`|Hour|16| 139 | |`M`|Minute|08| 140 | |`S`|Second|09| 141 | |`e`|Millisecond|796| 142 | |`f`|Microsecond|796341| 143 | |`F`|Nanosecond|796341126| 144 | |`Ymd`|Year-Month-Day|2021-05-03| 145 | |`HMS`|Hour:Minute:Second|16:08:09| 146 | |`HMSe`|Hour:Minute:Second.Millisecond|16:08:09.796| 147 | |`HMSf`|Hour:Minute:Second.Microsecond|16:08:09.796341| 148 | |`HMSF`|Hour:Minute:Second.Nanosecond|16:08:09.796341126| 149 | |`YmdHMS`|Year-Month-Day Hour:Minute:Second|2021-05-03 16:08:09| 150 | |`YmdHMSe`|Year-Month-Day Hour:Minute:Second.Millisecond|2021-05-03 16:08:09.796| 151 | |`YmdHMSf`|Year-Month-Day Hour:Minute:Second.Microsecond|2021-05-03 16:08:09.796341| 152 | |`YmdHMSF`|Year-Month-Day Hour:Minute:Second.Nanosecond|2021-05-03 16:08:09.796341126| 153 | 154 | Note that using concatenated named args is more efficient than seperated ones, e.g. `{YmdHMS}` is faster than `{Y}-{m}-{d} {H}:{M}:{S}`. 155 | 156 | ## Output 157 | By default, fmtlog output to stdout. Normally users want to write to a log file instead, this is accomplished by `fmtlog::setLogFile(filename,truncate)`. For performance, fmtlog internally buffer data, and under certain conditions will the buffer be flushed into the underlying file. The flushing conditions are: 158 | * The underlying FILE* is not managed by fmtlog, then fmtlog will not buffer at all. For example, the default stdout FILE* will not be buffered. User can also pass an existing FILE* and indicate whether fmtlog should manage it by `fmtlog::setLogFile(fp, manageFp)`, e.g. `fmtlog::setLogFile(stderr, false)`, then fmtlog will log into stderr without buffering. 159 | * The buffer size is larger than 8 KB, this number can be reset by `fmtlog::setFlushBufSize(bytes)`. 160 | * The oldest data in the buffer has passed a specified duration. The duration is by default 3 seconds, and can be set by `fmtlog::setFlushDelay(ns)`. 161 | * The new log has at least a specified flush log level. The default flush log level can't be reached by any log, but it can be set by `fmtlog::flushOn(logLevel)`. 162 | * User can actively ask fmtlog to flush by `fmtlog::poll(true)`. 163 | 164 | Optionally, user can ask fmtlog to close the log file by `fmtlog::closeLogFile()`, and subsequent log msgs will not be output. 165 | 166 | In addition to writing to a FILE*, user can register a callback function to handle log msgs by `fmtlog::setLogCB(cb, minCBLogLevel)`. This can be useful in circumstances where warning/error msgs need to be published out in real time for alerting purposes. Log callback will not be buffered as log file, and can be triggered even when the file is closed. 167 | The signature of callback function is: 168 | ```c++ 169 | // callback signature user can register 170 | // ns: nanosecond timestamp 171 | // level: logLevel 172 | // location: full file path with line num, e.g: /home/raomeng/fmtlog/fmtlog.h:45 173 | // basePos: file base index in the location 174 | // threadName: thread id or the name user set with setThreadName 175 | // msg: full log msg with header 176 | // bodyPos: log body index in the msg 177 | // logFilePos: log file position of this msg 178 | typedef void (*LogCBFn)(int64_t ns, LogLevel level, fmt::string_view location, size_t basePos, 179 | fmt::string_view threadName, fmt::string_view msg, size_t bodyPos, size_t logFilePos); 180 | ``` 181 | 182 | ## Performance 183 | Benchmark is done in terms of both front-end latency and throughput, with comparisons to Nanolog and spdlog basic_logger_st. Test log messages use [NanoLog benchmark Log-Messages-Map](https://github.com/PlatformLab/NanoLog#Log-Messages-Map), and header pattern uses spdlog default pattern(e.g. "[2021-05-04 10:36:38.098] [spdlog] [info] [bench.cc:111] "), check [bench.cc](https://github.com/MengRao/fmtlog/blob/main/bench/bench.cc) for details. 184 | 185 | The results on a linux server with "Intel(R) Xeon(R) Gold 6144 CPU @ 3.50GHz" is: 186 | | Message | fmtlog | Nanolog | spdlog | 187 | |---------|:--------:|:--------:|:--------:| 188 | |staticString |6.4 ns, 7.08 M/s|6.5 ns, 33.10 M/s|156.4 ns, 6.37 M/s| 189 | |stringConcat |6.4 ns, 6.05 M/s|7.5 ns, 14.20 M/s|209.4 ns, 4.77 M/s| 190 | |singleInteger |6.3 ns, 6.22 M/s|6.5 ns, 50.29 M/s|202.3 ns, 4.94 M/s| 191 | |twoIntegers |6.4 ns, 4.87 M/s|6.6 ns, 39.25 M/s|257.2 ns, 3.89 M/s| 192 | |singleDouble |6.2 ns, 5.37 M/s|6.5 ns, 39.62 M/s|225.0 ns, 4.44 M/s| 193 | |complexFormat |6.4 ns, 2.95 M/s|6.7 ns, 24.30 M/s|390.9 ns, 2.56 M/s| 194 | 195 | Note that the throughput of Nanolog is not comparable here because it outputs to binary log file instead of human-readable text format, e.g. it saves an int64 timestamp instead of a long formatted date time string. 196 | 197 | How can fmtlog achieve such low and stable latency? Two key optimization techniques are employed inspired by Nanolog: 198 | 199 | One is allocating a single producer single consumer queue for each logging thread, and have the background thread polling for all these queues. This avoids threads contention and performance will not deteriorate when thread number increases. The queue is automatically created on the first log msg of a thread, so queue is not created for threads that don't use fmtlog. The thread queue has a default size of 1 MB(can be changed by macro `FMTLOG_QUEUE_SIZE`), and it takes a little time to allocate the queue. It's recommended that user actively calls `fmt::preallocate()` once the thread is created, so even the first log can have low latency. 200 | 201 | What happens when the queue is full? By default, fmtlog simply dump addtional log msgs and return. Alternatively, front-end logging can be blocked while the queue is full by defining macro `FMTLOG_BLOCK=1`, then no log will be missing. User can register a callback function when log queue is full by `fmtlog::setLogQFullCB(cb, userData)`, by which user can be aware that the consumer(polling thread) is not keeping up. Normally, the queue being full is seldom a problem, but incautious user could leave log statements that are invoked in an unexpected high frequency, e.g. a tcp client spamming with "Connection refused" errors without a connection retry delay. To handle this problem in an elegant way, fmtlog provides a log macro which limits frequency of this log: `FMTLOG_LIMIT` and 4 shortcuts `logdl`, `logil`, `logwl` and `logel` respectively, user needs to pass the mininum interval in nanosecond as the first argument, e.g. 202 | ```c++ 203 | logil(1e9, "this log will be displayed at most once per second"). 204 | ``` 205 | 206 | The other optimization is that static information of a log(such as format string, log level and location) is saved in a table at its first call, and fmtlog simply pushes the index of the static info table entry with dynamic arguments to the queue, minimizing the msg size. In addition, fmtlog defines a decoding function for each log statment, which is invoked in `fmtlog::poll()` when the log msg is popped from the queue. 207 | 208 | However, these decoding functions bloat program size with each function consuming around 50 bytes. In addition, the static infomation entry also consumes 50-ish bytes runtime memory for each log statement. Such memory overhead may not be worthwhile for those infrequent and latency insensitive logs(e.g. program initialization info), thus fmtlog provides user with another log macro which disables this optimization: `FMTLOG_ONCE` and of couse shortcuts: `logdo`, `logio`, `logwo`and `logeo`. `FMTLOG_ONCE` will not create a static info table entry, nor add a decoding function: it pushes static info along with formatted msg body onto the queue. Note that passing argument by pointer is not supported by `FMTLOG_ONCE`. 209 | 210 | For those who prefer to further optimize memory usage by filtering log at compile time, macro `FMTLOG_ACTIVE_LEVEL` is applied with a default value `FMTLOG_LEVEL_INF`, meaning debug logs will simply be discarded at compile time. Note that `FMTLOG_ACTIVE_LEVEL` only applies to log shortcut macros, e.g. `logi`, but not `FMTLOG`. Similarly, runtime log level filtering can be disabled by defining macro `FMTLOG_NO_CHECK_LEVEL`, which will increase performance and reduce generated code size a bit. 211 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | BUILD_DIR=${BUILD_DIR:-.build} 2 | 3 | mkdir -p "$BUILD_DIR" \ 4 | && cd "$BUILD_DIR" \ 5 | && cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..\ 6 | && cmake --build . -j \ 7 | && cmake --install . --prefix . 8 | -------------------------------------------------------------------------------- /clear.sh: -------------------------------------------------------------------------------- 1 | rm fmtlog.txt spdlog.txt nanalog.bin compressedLog 2 | -------------------------------------------------------------------------------- /fmtlog-inl.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright (c) 2021 Meng Rao 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #include "fmtlog.h" 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #ifdef _WIN32 31 | #ifndef NOMINMAX 32 | #define NOMINMAX 33 | #endif 34 | #include 35 | #include 36 | #else 37 | #include 38 | #include 39 | #endif 40 | 41 | namespace { 42 | void fmtlogEmptyFun(void*) { 43 | } 44 | } // namespace 45 | 46 | template 47 | class fmtlogDetailT 48 | { 49 | public: 50 | // https://github.com/MengRao/str 51 | template 52 | class Str 53 | { 54 | public: 55 | static const int Size = SIZE; 56 | char s[SIZE]; 57 | 58 | Str() {} 59 | Str(const char* p) { *this = *(const Str*)p; } 60 | 61 | char& operator[](int i) { return s[i]; } 62 | char operator[](int i) const { return s[i]; } 63 | 64 | template 65 | void fromi(T num) { 66 | if constexpr (Size & 1) { 67 | s[Size - 1] = '0' + (num % 10); 68 | num /= 10; 69 | } 70 | switch (Size & -2) { 71 | case 18: *(uint16_t*)(s + 16) = *(uint16_t*)(digit_pairs + ((num % 100) << 1)); num /= 100; 72 | case 16: *(uint16_t*)(s + 14) = *(uint16_t*)(digit_pairs + ((num % 100) << 1)); num /= 100; 73 | case 14: *(uint16_t*)(s + 12) = *(uint16_t*)(digit_pairs + ((num % 100) << 1)); num /= 100; 74 | case 12: *(uint16_t*)(s + 10) = *(uint16_t*)(digit_pairs + ((num % 100) << 1)); num /= 100; 75 | case 10: *(uint16_t*)(s + 8) = *(uint16_t*)(digit_pairs + ((num % 100) << 1)); num /= 100; 76 | case 8: *(uint16_t*)(s + 6) = *(uint16_t*)(digit_pairs + ((num % 100) << 1)); num /= 100; 77 | case 6: *(uint16_t*)(s + 4) = *(uint16_t*)(digit_pairs + ((num % 100) << 1)); num /= 100; 78 | case 4: *(uint16_t*)(s + 2) = *(uint16_t*)(digit_pairs + ((num % 100) << 1)); num /= 100; 79 | case 2: *(uint16_t*)(s + 0) = *(uint16_t*)(digit_pairs + ((num % 100) << 1)); num /= 100; 80 | } 81 | } 82 | 83 | static constexpr const char* digit_pairs = "00010203040506070809" 84 | "10111213141516171819" 85 | "20212223242526272829" 86 | "30313233343536373839" 87 | "40414243444546474849" 88 | "50515253545556575859" 89 | "60616263646566676869" 90 | "70717273747576777879" 91 | "80818283848586878889" 92 | "90919293949596979899"; 93 | }; 94 | 95 | fmtlogDetailT() 96 | : flushDelay(3000000000) { 97 | args.reserve(4096); 98 | args.resize(parttenArgSize); 99 | 100 | fmtlogWrapper<>::impl.init(); 101 | resetDate(); 102 | fmtlog::setLogFile(stdout); 103 | setHeaderPattern("{HMSf} {s:<16} {l}[{t:<6}] "); 104 | logInfos.reserve(32); 105 | bgLogInfos.reserve(128); 106 | bgLogInfos.emplace_back(nullptr, nullptr, fmtlog::DBG, fmt::string_view()); 107 | bgLogInfos.emplace_back(nullptr, nullptr, fmtlog::INF, fmt::string_view()); 108 | bgLogInfos.emplace_back(nullptr, nullptr, fmtlog::WRN, fmt::string_view()); 109 | bgLogInfos.emplace_back(nullptr, nullptr, fmtlog::ERR, fmt::string_view()); 110 | threadBuffers.reserve(8); 111 | bgThreadBuffers.reserve(8); 112 | memset(membuf.data(), 0, membuf.capacity()); 113 | } 114 | 115 | ~fmtlogDetailT() { 116 | stopPollingThread(); 117 | poll(true); 118 | closeLogFile(); 119 | } 120 | 121 | void setHeaderPattern(const char* pattern) { 122 | if (shouldDeallocateHeader) delete[] headerPattern.data(); 123 | using namespace fmt::literals; 124 | for (int i = 0; i < parttenArgSize; i++) { 125 | reorderIdx[i] = parttenArgSize - 1; 126 | } 127 | headerPattern = fmtlog::unNameFormat( 128 | pattern, reorderIdx, "a"_a = "", "b"_a = "", "C"_a = "", "Y"_a = "", "m"_a = "", "d"_a = "", 129 | "t"_a = "thread name", "F"_a = "", "f"_a = "", "e"_a = "", "S"_a = "", "M"_a = "", "H"_a = "", 130 | "l"_a = fmtlog::LogLevel(), "s"_a = "fmtlog.cc:123", "g"_a = "/home/raomeng/fmtlog/fmtlog.cc:123", "Ymd"_a = "", 131 | "HMS"_a = "", "HMSe"_a = "", "HMSf"_a = "", "HMSF"_a = "", "YmdHMS"_a = "", "YmdHMSe"_a = "", "YmdHMSf"_a = "", 132 | "YmdHMSF"_a = ""); 133 | shouldDeallocateHeader = headerPattern.data() != pattern; 134 | 135 | setArg<0>(fmt::string_view(weekdayName.s, 3)); 136 | setArg<1>(fmt::string_view(monthName.s, 3)); 137 | setArg<2>(fmt::string_view(&year[2], 2)); 138 | setArg<3>(fmt::string_view(year.s, 4)); 139 | setArg<4>(fmt::string_view(month.s, 2)); 140 | setArg<5>(fmt::string_view(day.s, 2)); 141 | setArg<6>(fmt::string_view()); 142 | setArg<7>(fmt::string_view(nanosecond.s, 9)); 143 | setArg<8>(fmt::string_view(nanosecond.s, 6)); 144 | setArg<9>(fmt::string_view(nanosecond.s, 3)); 145 | setArg<10>(fmt::string_view(second.s, 2)); 146 | setArg<11>(fmt::string_view(minute.s, 2)); 147 | setArg<12>(fmt::string_view(hour.s, 2)); 148 | setArg<13>(fmt::string_view(logLevel.s, 3)); 149 | setArg<14>(fmt::string_view()); 150 | setArg<15>(fmt::string_view()); 151 | setArg<16>(fmt::string_view(year.s, 10)); // Ymd 152 | setArg<17>(fmt::string_view(hour.s, 8)); // HMS 153 | setArg<18>(fmt::string_view(hour.s, 12)); // HMSe 154 | setArg<19>(fmt::string_view(hour.s, 15)); // HMSf 155 | setArg<20>(fmt::string_view(hour.s, 18)); // HMSF 156 | setArg<21>(fmt::string_view(year.s, 19)); // YmdHMS 157 | setArg<22>(fmt::string_view(year.s, 23)); // YmdHMSe 158 | setArg<23>(fmt::string_view(year.s, 26)); // YmdHMSf 159 | setArg<24>(fmt::string_view(year.s, 29)); // YmdHMSF 160 | } 161 | 162 | class ThreadBufferDestroyer 163 | { 164 | public: 165 | explicit ThreadBufferDestroyer() {} 166 | 167 | void threadBufferCreated() {} 168 | 169 | ~ThreadBufferDestroyer() { 170 | if (fmtlog::threadBuffer != nullptr) { 171 | fmtlog::threadBuffer->shouldDeallocate = true; 172 | fmtlog::threadBuffer = nullptr; 173 | } 174 | } 175 | }; 176 | 177 | struct StaticLogInfo 178 | { 179 | // Constructor 180 | constexpr StaticLogInfo(fmtlog::FormatToFn fn, const char* loc, fmtlog::LogLevel level, fmt::string_view fmtString) 181 | : formatToFn(fn) 182 | , formatString(fmtString) 183 | , location(loc) 184 | , logLevel(level) 185 | , argIdx(-1) {} 186 | 187 | void processLocation() { 188 | size_t size = strlen(location); 189 | const char* p = location + size; 190 | if (size > 255) { 191 | location = p - 255; 192 | } 193 | endPos = p - location; 194 | const char* base = location; 195 | while (p > location) { 196 | char c = *--p; 197 | if (c == '/' || c == '\\') { 198 | base = p + 1; 199 | break; 200 | } 201 | } 202 | basePos = base - location; 203 | } 204 | 205 | inline fmt::string_view getBase() { return fmt::string_view(location + basePos, endPos - basePos); } 206 | 207 | inline fmt::string_view getLocation() { return fmt::string_view(location, endPos); } 208 | 209 | fmtlog::FormatToFn formatToFn; 210 | fmt::string_view formatString; 211 | const char* location; 212 | uint8_t basePos; 213 | uint8_t endPos; 214 | fmtlog::LogLevel logLevel; 215 | int argIdx; 216 | }; 217 | 218 | static thread_local ThreadBufferDestroyer sbc; 219 | int64_t midnightNs; 220 | fmt::string_view headerPattern; 221 | bool shouldDeallocateHeader = false; 222 | FILE* outputFp = nullptr; 223 | bool manageFp = false; 224 | size_t fpos = 0; // file position of membuf, used only when manageFp == true 225 | int64_t flushDelay; 226 | int64_t nextFlushTime = (std::numeric_limits::max)(); 227 | uint32_t flushBufSize = 8 * 1024; 228 | fmtlog::LogLevel flushLogLevel = fmtlog::OFF; 229 | std::mutex bufferMutex; 230 | std::vector threadBuffers; 231 | struct HeapNode 232 | { 233 | HeapNode(fmtlog::ThreadBuffer* buffer) 234 | : tb(buffer) {} 235 | 236 | fmtlog::ThreadBuffer* tb; 237 | const fmtlog::SPSCVarQueueOPT::MsgHeader* header = nullptr; 238 | }; 239 | std::vector bgThreadBuffers; 240 | std::mutex logInfoMutex; 241 | std::vector logInfos; 242 | std::vector bgLogInfos; 243 | 244 | fmtlog::LogCBFn logCB = nullptr; 245 | fmtlog::LogLevel minCBLogLevel; 246 | fmtlog::LogQFullCBFn logQFullCB = fmtlogEmptyFun; 247 | void* logQFullCBArg = nullptr; 248 | 249 | fmtlog::MemoryBuffer membuf; 250 | 251 | const static int parttenArgSize = 25; 252 | uint32_t reorderIdx[parttenArgSize]; 253 | Str<3> weekdayName; 254 | Str<3> monthName; 255 | Str<4> year; 256 | char dash1 = '-'; 257 | Str<2> month; 258 | char dash2 = '-'; 259 | Str<2> day; 260 | char space = ' '; 261 | Str<2> hour; 262 | char colon1 = ':'; 263 | Str<2> minute; 264 | char colon2 = ':'; 265 | Str<2> second; 266 | char dot1 = '.'; 267 | Str<9> nanosecond; 268 | Str<3> logLevel; 269 | std::vector> args; 270 | 271 | volatile bool threadRunning = false; 272 | std::thread thr; 273 | 274 | void resetDate() { 275 | time_t rawtime = fmtlogWrapper<>::impl.tscns.rdns() / 1000000000; 276 | struct tm* timeinfo = localtime(&rawtime); 277 | timeinfo->tm_sec = timeinfo->tm_min = timeinfo->tm_hour = 0; 278 | midnightNs = mktime(timeinfo) * 1000000000; 279 | year.fromi(1900 + timeinfo->tm_year); 280 | month.fromi(1 + timeinfo->tm_mon); 281 | day.fromi(timeinfo->tm_mday); 282 | const char* weekdays[7] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; 283 | weekdayName = weekdays[timeinfo->tm_wday]; 284 | const char* monthNames[12] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; 285 | monthName = monthNames[timeinfo->tm_mon]; 286 | } 287 | 288 | void preallocate() { 289 | if (fmtlog::threadBuffer) return; 290 | fmtlog::threadBuffer = new fmtlog::ThreadBuffer(); 291 | #ifdef _WIN32 292 | uint32_t tid = static_cast(::GetCurrentThreadId()); 293 | #else 294 | uint32_t tid = static_cast(::syscall(SYS_gettid)); 295 | #endif 296 | fmtlog::threadBuffer->nameSize = 297 | fmt::format_to_n(fmtlog::threadBuffer->name, sizeof(fmtlog::threadBuffer->name), "{}", tid).size; 298 | sbc.threadBufferCreated(); 299 | 300 | std::unique_lock guard(bufferMutex); 301 | threadBuffers.push_back(fmtlog::threadBuffer); 302 | } 303 | 304 | template 305 | inline void setArg(const T& arg) { 306 | args[reorderIdx[I]] = arg; 307 | } 308 | 309 | template 310 | inline void setArgVal(const T& arg) { 311 | fmt::detail::value& value_ = *(fmt::detail::value*)&args[reorderIdx[I]]; 312 | value_ = arg; 313 | } 314 | 315 | void flushLogFile() { 316 | if (outputFp) { 317 | fwrite(membuf.data(), 1, membuf.size(), outputFp); 318 | if (!manageFp) fflush(outputFp); 319 | else 320 | fpos += membuf.size(); 321 | } 322 | membuf.clear(); 323 | nextFlushTime = (std::numeric_limits::max)(); 324 | } 325 | 326 | void closeLogFile() { 327 | if (membuf.size()) flushLogFile(); 328 | if (manageFp) fclose(outputFp); 329 | outputFp = nullptr; 330 | manageFp = false; 331 | } 332 | 333 | void startPollingThread(int64_t pollInterval) { 334 | stopPollingThread(); 335 | threadRunning = true; 336 | thr = std::thread([pollInterval, this]() { 337 | while (threadRunning) { 338 | int64_t before = fmtlogWrapper<>::impl.tscns.rdns(); 339 | poll(false); 340 | int64_t delay = fmtlogWrapper<>::impl.tscns.rdns() - before; 341 | if (delay < pollInterval) { 342 | std::this_thread::sleep_for(std::chrono::nanoseconds(pollInterval - delay)); 343 | } 344 | } 345 | poll(true); 346 | }); 347 | } 348 | 349 | void stopPollingThread() { 350 | if (!threadRunning) return; 351 | threadRunning = false; 352 | if (thr.joinable()) thr.join(); 353 | } 354 | 355 | void handleLog(fmt::string_view threadName, const fmtlog::SPSCVarQueueOPT::MsgHeader* header) { 356 | setArgVal<6>(threadName); 357 | StaticLogInfo& info = bgLogInfos[header->logId]; 358 | const char* data = (const char*)(header + 1); 359 | const char* end = (const char*)header + header->size; 360 | int64_t tsc = *(int64_t*)data; 361 | data += 8; 362 | if (!info.formatToFn) { // log once 363 | info.location = *(const char**)data; 364 | data += 8; 365 | info.processLocation(); 366 | } 367 | int64_t ts = fmtlogWrapper<>::impl.tscns.tsc2ns(tsc); 368 | // the date could go back when polling different threads 369 | uint64_t t = (ts > midnightNs) ? (ts - midnightNs) : 0; 370 | nanosecond.fromi(t % 1000000000); 371 | t /= 1000000000; 372 | second.fromi(t % 60); 373 | t /= 60; 374 | minute.fromi(t % 60); 375 | t /= 60; 376 | uint32_t h = t; // hour 377 | if (h > 23) { 378 | h %= 24; 379 | resetDate(); 380 | } 381 | hour.fromi(h); 382 | setArgVal<14>(info.getBase()); 383 | setArgVal<15>(info.getLocation()); 384 | logLevel = (const char*)"DBG INF WRN ERR OFF" + (info.logLevel << 2); 385 | 386 | size_t headerPos = membuf.size(); 387 | fmtlog::vformat_to(membuf, headerPattern, fmt::basic_format_args(args.data(), parttenArgSize)); 388 | size_t bodyPos = membuf.size(); 389 | 390 | if (info.formatToFn) { 391 | info.formatToFn(info.formatString, data, membuf, info.argIdx, args); 392 | } 393 | else { // log once 394 | membuf.append(fmt::string_view(data, end - data)); 395 | } 396 | 397 | if (logCB && info.logLevel >= minCBLogLevel) { 398 | logCB(ts, info.logLevel, info.getLocation(), info.basePos, threadName, 399 | fmt::string_view(membuf.data() + headerPos, membuf.size() - headerPos), bodyPos - headerPos, 400 | fpos + headerPos); 401 | } 402 | membuf.push_back('\n'); 403 | if (membuf.size() >= flushBufSize || info.logLevel >= flushLogLevel) { 404 | flushLogFile(); 405 | } 406 | } 407 | 408 | void adjustHeap(size_t i) { 409 | while (true) { 410 | size_t min_i = i; 411 | for (size_t ch = i * 2 + 1, end = std::min(ch + 2, bgThreadBuffers.size()); ch < end; ch++) { 412 | auto h_ch = bgThreadBuffers[ch].header; 413 | auto h_min = bgThreadBuffers[min_i].header; 414 | if (h_ch && (!h_min || *(int64_t*)(h_ch + 1) < *(int64_t*)(h_min + 1))) min_i = ch; 415 | } 416 | if (min_i == i) break; 417 | std::swap(bgThreadBuffers[i], bgThreadBuffers[min_i]); 418 | i = min_i; 419 | } 420 | } 421 | 422 | void poll(bool forceFlush) { 423 | fmtlogWrapper<>::impl.tscns.calibrate(); 424 | int64_t tsc = fmtlogWrapper<>::impl.tscns.rdtsc(); 425 | if (logInfos.size()) { 426 | std::unique_lock lock(logInfoMutex); 427 | for (auto& info : logInfos) { 428 | info.processLocation(); 429 | } 430 | bgLogInfos.insert(bgLogInfos.end(), logInfos.begin(), logInfos.end()); 431 | logInfos.clear(); 432 | } 433 | if (threadBuffers.size()) { 434 | std::unique_lock lock(bufferMutex); 435 | for (auto tb : threadBuffers) { 436 | bgThreadBuffers.emplace_back(tb); 437 | } 438 | threadBuffers.clear(); 439 | } 440 | 441 | for (size_t i = 0; i < bgThreadBuffers.size(); i++) { 442 | auto& node = bgThreadBuffers[i]; 443 | if (node.header) continue; 444 | node.header = node.tb->varq.front(); 445 | if (!node.header && node.tb->shouldDeallocate) { 446 | delete node.tb; 447 | node = bgThreadBuffers.back(); 448 | bgThreadBuffers.pop_back(); 449 | i--; 450 | } 451 | } 452 | 453 | if (bgThreadBuffers.empty()) return; 454 | 455 | // build heap 456 | for (int i = bgThreadBuffers.size() / 2; i >= 0; i--) { 457 | adjustHeap(i); 458 | } 459 | 460 | while (true) { 461 | auto h = bgThreadBuffers[0].header; 462 | if (!h || h->logId >= bgLogInfos.size() || *(int64_t*)(h + 1) >= tsc) break; 463 | auto tb = bgThreadBuffers[0].tb; 464 | handleLog(fmt::string_view(tb->name, tb->nameSize), h); 465 | tb->varq.pop(); 466 | bgThreadBuffers[0].header = tb->varq.front(); 467 | adjustHeap(0); 468 | } 469 | 470 | if (membuf.size() == 0) return; 471 | if (!manageFp || forceFlush) { 472 | flushLogFile(); 473 | return; 474 | } 475 | int64_t now = fmtlogWrapper<>::impl.tscns.tsc2ns(tsc); 476 | if (now > nextFlushTime) { 477 | flushLogFile(); 478 | } 479 | else if (nextFlushTime == (std::numeric_limits::max)()) { 480 | nextFlushTime = now + flushDelay; 481 | } 482 | } 483 | }; 484 | 485 | template 486 | thread_local typename fmtlogDetailT<_>::ThreadBufferDestroyer fmtlogDetailT<_>::sbc; 487 | 488 | template 489 | struct fmtlogDetailWrapper 490 | { static fmtlogDetailT<> impl; }; 491 | 492 | template 493 | fmtlogDetailT<> fmtlogDetailWrapper<_>::impl; 494 | 495 | template 496 | void fmtlogT<_>::registerLogInfo(uint32_t& logId, FormatToFn fn, const char* location, 497 | LogLevel level, fmt::string_view fmtString) noexcept { 498 | auto& d = fmtlogDetailWrapper<>::impl; 499 | std::lock_guard lock(d.logInfoMutex); 500 | if (logId) return; 501 | logId = d.logInfos.size() + d.bgLogInfos.size(); 502 | d.logInfos.emplace_back(fn, location, level, fmtString); 503 | } 504 | 505 | template 506 | void fmtlogT<_>::vformat_to(fmtlog::MemoryBuffer& out, fmt::string_view fmt, 507 | fmt::format_args args) { 508 | fmt::detail::vformat_to(out, fmt, args); 509 | } 510 | 511 | template 512 | size_t fmtlogT<_>::formatted_size(fmt::string_view fmt, fmt::format_args args) { 513 | auto buf = fmt::detail::counting_buffer<>(); 514 | fmt::detail::vformat_to(buf, fmt, args); 515 | return buf.count(); 516 | } 517 | 518 | template 519 | void fmtlogT<_>::vformat_to(char* out, fmt::string_view fmt, fmt::format_args args) { 520 | fmt::vformat_to(out, fmt, args); 521 | } 522 | 523 | template 524 | typename fmtlogT<_>::SPSCVarQueueOPT::MsgHeader* fmtlogT<_>::allocMsg(uint32_t size, 525 | bool q_full_cb) noexcept { 526 | auto& d = fmtlogDetailWrapper<>::impl; 527 | if (threadBuffer == nullptr) preallocate(); 528 | auto ret = threadBuffer->varq.alloc(size); 529 | if ((ret == nullptr) & q_full_cb) d.logQFullCB(d.logQFullCBArg); 530 | return ret; 531 | } 532 | 533 | template 534 | typename fmtlogT<_>::SPSCVarQueueOPT::MsgHeader* 535 | fmtlogT<_>::SPSCVarQueueOPT::allocMsg(uint32_t size) noexcept { 536 | return alloc(size); 537 | } 538 | 539 | template 540 | void fmtlogT<_>::preallocate() noexcept { 541 | fmtlogDetailWrapper<>::impl.preallocate(); 542 | } 543 | 544 | template 545 | void fmtlogT<_>::setLogFile(const char* filename, bool truncate) { 546 | auto& d = fmtlogDetailWrapper<>::impl; 547 | FILE* newFp = fopen(filename, truncate ? "w" : "a"); 548 | if (!newFp) { 549 | std::string err = fmt::format("Unable to open file: {}: {}", filename, strerror(errno)); 550 | fmt::report_error(err.c_str()); 551 | } 552 | setbuf(newFp, nullptr); 553 | d.fpos = ftell(newFp); 554 | 555 | closeLogFile(); 556 | d.outputFp = newFp; 557 | d.manageFp = true; 558 | } 559 | 560 | template 561 | void fmtlogT<_>::setLogFile(FILE* fp, bool manageFp) { 562 | auto& d = fmtlogDetailWrapper<>::impl; 563 | closeLogFile(); 564 | if (manageFp) { 565 | setbuf(fp, nullptr); 566 | d.fpos = ftell(fp); 567 | } 568 | else 569 | d.fpos = 0; 570 | d.outputFp = fp; 571 | d.manageFp = manageFp; 572 | } 573 | 574 | template 575 | void fmtlogT<_>::setFlushDelay(int64_t ns) noexcept { 576 | fmtlogDetailWrapper<>::impl.flushDelay = ns; 577 | } 578 | 579 | template 580 | void fmtlogT<_>::flushOn(LogLevel flushLogLevel) noexcept { 581 | fmtlogDetailWrapper<>::impl.flushLogLevel = flushLogLevel; 582 | } 583 | 584 | template 585 | void fmtlogT<_>::setFlushBufSize(uint32_t bytes) noexcept { 586 | fmtlogDetailWrapper<>::impl.flushBufSize = bytes; 587 | } 588 | 589 | template 590 | void fmtlogT<_>::closeLogFile() noexcept { 591 | fmtlogDetailWrapper<>::impl.closeLogFile(); 592 | } 593 | 594 | template 595 | void fmtlogT<_>::poll(bool forceFlush) { 596 | fmtlogDetailWrapper<>::impl.poll(forceFlush); 597 | } 598 | 599 | template 600 | void fmtlogT<_>::setThreadName(const char* name) noexcept { 601 | preallocate(); 602 | threadBuffer->nameSize = fmt::format_to_n(threadBuffer->name, sizeof(fmtlog::threadBuffer->name), "{}", name).size; 603 | } 604 | 605 | template 606 | void fmtlogT<_>::setLogCB(LogCBFn cb, LogLevel minCBLogLevel_) noexcept { 607 | auto& d = fmtlogDetailWrapper<>::impl; 608 | d.logCB = cb; 609 | d.minCBLogLevel = minCBLogLevel_; 610 | } 611 | 612 | template 613 | void fmtlogT<_>::setLogQFullCB(LogQFullCBFn cb, void* userData) noexcept { 614 | auto& d = fmtlogDetailWrapper<>::impl; 615 | d.logQFullCB = cb; 616 | d.logQFullCBArg = userData; 617 | } 618 | 619 | template 620 | void fmtlogT<_>::setHeaderPattern(const char* pattern) { 621 | fmtlogDetailWrapper<>::impl.setHeaderPattern(pattern); 622 | } 623 | 624 | template 625 | void fmtlogT<_>::startPollingThread(int64_t pollInterval) noexcept { 626 | fmtlogDetailWrapper<>::impl.startPollingThread(pollInterval); 627 | } 628 | 629 | template 630 | void fmtlogT<_>::stopPollingThread() noexcept { 631 | fmtlogDetailWrapper<>::impl.stopPollingThread(); 632 | } 633 | 634 | template class fmtlogT<0>; 635 | 636 | -------------------------------------------------------------------------------- /fmtlog.cc: -------------------------------------------------------------------------------- 1 | #include "fmtlog-inl.h" 2 | -------------------------------------------------------------------------------- /fmtlog.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright (c) 2021 Meng Rao 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | #pragma once 25 | //#define FMT_HEADER_ONLY 26 | #include "fmt/format.h" 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #ifdef _MSC_VER 35 | #include 36 | #endif 37 | 38 | #ifdef _WIN32 39 | #define FAST_THREAD_LOCAL thread_local 40 | #else 41 | #define FAST_THREAD_LOCAL __thread 42 | #endif 43 | 44 | // define FMTLOG_BLOCK=1 if log statment should be blocked when queue is full, instead of discarding the msg 45 | #ifndef FMTLOG_BLOCK 46 | #define FMTLOG_BLOCK 0 47 | #endif 48 | 49 | #define FMTLOG_LEVEL_DBG 0 50 | #define FMTLOG_LEVEL_INF 1 51 | #define FMTLOG_LEVEL_WRN 2 52 | #define FMTLOG_LEVEL_ERR 3 53 | #define FMTLOG_LEVEL_OFF 4 54 | 55 | // define FMTLOG_ACTIVE_LEVEL to turn off low log level in compile time 56 | #ifndef FMTLOG_ACTIVE_LEVEL 57 | #define FMTLOG_ACTIVE_LEVEL FMTLOG_LEVEL_DBG 58 | #endif 59 | 60 | #ifndef FMTLOG_QUEUE_SIZE 61 | #define FMTLOG_QUEUE_SIZE (1 << 20) 62 | #endif 63 | 64 | namespace fmtlogdetail { 65 | template 66 | struct UnrefPtr : std::false_type 67 | { using type = Arg; }; 68 | 69 | template<> 70 | struct UnrefPtr : std::false_type 71 | { using type = char*; }; 72 | 73 | template<> 74 | struct UnrefPtr : std::false_type 75 | { using type = void*; }; 76 | 77 | template 78 | struct UnrefPtr> : std::true_type 79 | { using type = Arg; }; 80 | 81 | template 82 | struct UnrefPtr> : std::true_type 83 | { using type = Arg; }; 84 | 85 | template 86 | struct UnrefPtr : std::true_type 87 | { using type = Arg; }; 88 | 89 | }; // namespace fmtlogdetail 90 | 91 | template 92 | class fmtlogT 93 | { 94 | public: 95 | enum LogLevel : uint8_t 96 | { 97 | DBG = 0, 98 | INF, 99 | WRN, 100 | ERR, 101 | OFF 102 | }; 103 | 104 | // Preallocate thread queue for current thread 105 | static void preallocate() noexcept; 106 | 107 | // Set the file for logging 108 | static void setLogFile(const char* filename, bool truncate = false); 109 | 110 | // Set an existing FILE* for logging, if manageFp is false fmtlog will not buffer log internally 111 | // and will not close the FILE* 112 | static void setLogFile(FILE* fp, bool manageFp = false); 113 | 114 | // Collect log msgs from all threads and write to log file 115 | // If forceFlush = true, internal file buffer is flushed 116 | // User need to call poll() repeatedly if startPollingThread is not used 117 | static void poll(bool forceFlush = false); 118 | 119 | // Set flush delay in nanosecond 120 | // If there's msg older than ns in the buffer, flush will be triggered 121 | static void setFlushDelay(int64_t ns) noexcept; 122 | 123 | // If current msg has level >= flushLogLevel, flush will be triggered 124 | static void flushOn(LogLevel flushLogLevel) noexcept; 125 | 126 | // If file buffer has more than specified bytes, flush will be triggered 127 | static void setFlushBufSize(uint32_t bytes) noexcept; 128 | 129 | // callback signature user can register 130 | // ns: nanosecond timestamp 131 | // level: logLevel 132 | // location: full file path with line num, e.g: /home/raomeng/fmtlog/fmtlog.h:45 133 | // basePos: file base index in the location 134 | // threadName: thread id or the name user set with setThreadName 135 | // msg: full log msg with header 136 | // bodyPos: log body index in the msg 137 | // logFilePos: log file position of this msg 138 | typedef void (*LogCBFn)(int64_t ns, LogLevel level, fmt::string_view location, size_t basePos, 139 | fmt::string_view threadName, fmt::string_view msg, size_t bodyPos, 140 | size_t logFilePos); 141 | 142 | // Set a callback function for all log msgs with a mininum log level 143 | static void setLogCB(LogCBFn cb, LogLevel minCBLogLevel) noexcept; 144 | 145 | typedef void (*LogQFullCBFn)(void* userData); 146 | static void setLogQFullCB(LogQFullCBFn cb, void* userData) noexcept; 147 | 148 | // Close the log file and subsequent msgs will not be written into the file, 149 | // but callback function can still be used 150 | static void closeLogFile() noexcept; 151 | 152 | // Set log header pattern with fmt named arguments 153 | static void setHeaderPattern(const char* pattern); 154 | 155 | // Set a name for current thread, it'll be shown in {t} part in header pattern 156 | static void setThreadName(const char* name) noexcept; 157 | 158 | // Set current log level, lower level log msgs will be discarded 159 | static inline void setLogLevel(LogLevel logLevel) noexcept; 160 | 161 | // Get current log level 162 | static inline LogLevel getLogLevel() noexcept; 163 | 164 | // return true if passed log level is not lower than current log level 165 | static inline bool checkLogLevel(LogLevel logLevel) noexcept; 166 | 167 | // Run a polling thread in the background with a polling interval in ns 168 | // Note that user must not call poll() himself when the thread is running 169 | static void startPollingThread(int64_t pollInterval = 1000000000) noexcept; 170 | 171 | // Stop the polling thread 172 | static void stopPollingThread() noexcept; 173 | 174 | // https://github.com/MengRao/SPSC_Queue 175 | class SPSCVarQueueOPT 176 | { 177 | public: 178 | struct MsgHeader 179 | { 180 | inline void push(uint32_t sz) { *(volatile uint32_t*)&size = sz + sizeof(MsgHeader); } 181 | 182 | uint32_t size; 183 | uint32_t logId; 184 | }; 185 | static constexpr uint32_t BLK_CNT = FMTLOG_QUEUE_SIZE / sizeof(MsgHeader); 186 | 187 | MsgHeader* allocMsg(uint32_t size) noexcept; 188 | 189 | MsgHeader* alloc(uint32_t size) { 190 | size += sizeof(MsgHeader); 191 | uint32_t blk_sz = (size + sizeof(MsgHeader) - 1) / sizeof(MsgHeader); 192 | if (blk_sz >= free_write_cnt) { 193 | uint32_t read_idx_cache = *(volatile uint32_t*)&read_idx; 194 | if (read_idx_cache <= write_idx) { 195 | free_write_cnt = BLK_CNT - write_idx; 196 | if (blk_sz >= free_write_cnt && read_idx_cache != 0) { // wrap around 197 | blk[0].size = 0; 198 | blk[write_idx].size = 1; 199 | write_idx = 0; 200 | free_write_cnt = read_idx_cache; 201 | } 202 | } 203 | else { 204 | free_write_cnt = read_idx_cache - write_idx; 205 | } 206 | if (free_write_cnt <= blk_sz) { 207 | return nullptr; 208 | } 209 | } 210 | MsgHeader* ret = &blk[write_idx]; 211 | write_idx += blk_sz; 212 | free_write_cnt -= blk_sz; 213 | blk[write_idx].size = 0; 214 | return ret; 215 | } 216 | 217 | inline const MsgHeader* front() { 218 | uint32_t size = blk[read_idx].size; 219 | if (size == 1) { // wrap around 220 | read_idx = 0; 221 | size = blk[0].size; 222 | } 223 | if (size == 0) return nullptr; 224 | return &blk[read_idx]; 225 | } 226 | 227 | inline void pop() { 228 | uint32_t blk_sz = (blk[read_idx].size + sizeof(MsgHeader) - 1) / sizeof(MsgHeader); 229 | *(volatile uint32_t*)&read_idx = read_idx + blk_sz; 230 | } 231 | 232 | private: 233 | alignas(64) MsgHeader blk[BLK_CNT] = {}; 234 | uint32_t write_idx = 0; 235 | uint32_t free_write_cnt = BLK_CNT; 236 | 237 | alignas(128) uint32_t read_idx = 0; 238 | }; 239 | 240 | struct ThreadBuffer 241 | { 242 | SPSCVarQueueOPT varq; 243 | bool shouldDeallocate = false; 244 | char name[32]; 245 | size_t nameSize; 246 | }; 247 | 248 | // https://github.com/MengRao/tscns 249 | class TSCNS 250 | { 251 | public: 252 | static const int64_t NsPerSec = 1000000000; 253 | 254 | void init(int64_t init_calibrate_ns = 20000000, int64_t calibrate_interval_ns = 3 * NsPerSec) { 255 | calibate_interval_ns_ = calibrate_interval_ns; 256 | int64_t base_tsc, base_ns; 257 | syncTime(base_tsc, base_ns); 258 | int64_t expire_ns = base_ns + init_calibrate_ns; 259 | while (rdsysns() < expire_ns) std::this_thread::yield(); 260 | int64_t delayed_tsc, delayed_ns; 261 | syncTime(delayed_tsc, delayed_ns); 262 | double init_ns_per_tsc = (double)(delayed_ns - base_ns) / (delayed_tsc - base_tsc); 263 | saveParam(base_tsc, base_ns, 0, init_ns_per_tsc); 264 | } 265 | 266 | void calibrate() { 267 | if (rdtsc() < next_calibrate_tsc_) return; 268 | int64_t tsc, ns; 269 | syncTime(tsc, ns); 270 | int64_t ns_err = tsc2ns(tsc) - ns; 271 | if (ns_err > 1000000) ns_err = 1000000; 272 | if (ns_err < -1000000) ns_err = -1000000; 273 | double new_ns_per_tsc = 274 | ns_per_tsc_ * (1.0 - (ns_err + ns_err - base_ns_err_) / ((tsc - base_tsc_) * ns_per_tsc_)); 275 | saveParam(tsc, ns, ns_err, new_ns_per_tsc); 276 | } 277 | 278 | static inline int64_t rdtsc() { 279 | #ifdef _MSC_VER 280 | return __rdtsc(); 281 | #elif defined(__i386__) || defined(__x86_64__) || defined(__amd64__) 282 | return __builtin_ia32_rdtsc(); 283 | #else 284 | return rdsysns(); 285 | #endif 286 | } 287 | 288 | inline int64_t tsc2ns(int64_t tsc) const { 289 | while (true) { 290 | uint32_t before_seq = param_seq_.load(std::memory_order_acquire) & ~1; 291 | std::atomic_signal_fence(std::memory_order_acq_rel); 292 | int64_t ns = base_ns_ + (int64_t)((tsc - base_tsc_) * ns_per_tsc_); 293 | std::atomic_signal_fence(std::memory_order_acq_rel); 294 | uint32_t after_seq = param_seq_.load(std::memory_order_acquire); 295 | if (before_seq == after_seq) return ns; 296 | } 297 | } 298 | 299 | inline int64_t rdns() const { return tsc2ns(rdtsc()); } 300 | 301 | static inline int64_t rdsysns() { 302 | using namespace std::chrono; 303 | return duration_cast(system_clock::now().time_since_epoch()).count(); 304 | } 305 | 306 | double getTscGhz() const { return 1.0 / ns_per_tsc_; } 307 | 308 | // Linux kernel sync time by finding the first trial with tsc diff < 50000 309 | // We try several times and return the one with the mininum tsc diff. 310 | // Note that MSVC has a 100ns resolution clock, so we need to combine those ns with the same 311 | // value, and drop the first and the last value as they may not scan a full 100ns range 312 | static void syncTime(int64_t& tsc_out, int64_t& ns_out) { 313 | #ifdef _MSC_VER 314 | const int N = 15; 315 | #else 316 | const int N = 3; 317 | #endif 318 | int64_t tsc[N + 1]; 319 | int64_t ns[N + 1]; 320 | 321 | tsc[0] = rdtsc(); 322 | for (int i = 1; i <= N; i++) { 323 | ns[i] = rdsysns(); 324 | tsc[i] = rdtsc(); 325 | } 326 | 327 | #ifdef _MSC_VER 328 | int j = 1; 329 | for (int i = 2; i <= N; i++) { 330 | if (ns[i] == ns[i - 1]) continue; 331 | tsc[j - 1] = tsc[i - 1]; 332 | ns[j++] = ns[i]; 333 | } 334 | j--; 335 | #else 336 | int j = N + 1; 337 | #endif 338 | 339 | int best = 1; 340 | for (int i = 2; i < j; i++) { 341 | if (tsc[i] - tsc[i - 1] < tsc[best] - tsc[best - 1]) best = i; 342 | } 343 | tsc_out = (tsc[best] + tsc[best - 1]) >> 1; 344 | ns_out = ns[best]; 345 | } 346 | 347 | void saveParam(int64_t base_tsc, int64_t sys_ns, int64_t base_ns_err, double new_ns_per_tsc) { 348 | base_ns_err_ = base_ns_err; 349 | next_calibrate_tsc_ = base_tsc + (int64_t)((calibate_interval_ns_ - 1000) / new_ns_per_tsc); 350 | uint32_t seq = param_seq_.load(std::memory_order_relaxed); 351 | param_seq_.store(++seq, std::memory_order_release); 352 | std::atomic_signal_fence(std::memory_order_acq_rel); 353 | base_tsc_ = base_tsc; 354 | base_ns_ = sys_ns + base_ns_err; 355 | ns_per_tsc_ = new_ns_per_tsc; 356 | std::atomic_signal_fence(std::memory_order_acq_rel); 357 | param_seq_.store(++seq, std::memory_order_release); 358 | } 359 | 360 | alignas(64) std::atomic param_seq_ = 0; 361 | double ns_per_tsc_; 362 | int64_t base_tsc_; 363 | int64_t base_ns_; 364 | int64_t calibate_interval_ns_; 365 | int64_t base_ns_err_; 366 | int64_t next_calibrate_tsc_; 367 | }; 368 | 369 | void init() { 370 | tscns.init(); 371 | currentLogLevel = INF; 372 | } 373 | 374 | using Context = fmt::format_context; 375 | using MemoryBuffer = fmt::basic_memory_buffer; 376 | typedef const char* (*FormatToFn)(fmt::string_view format, const char* data, MemoryBuffer& out, 377 | int& argIdx, std::vector>& args); 378 | 379 | static void registerLogInfo(uint32_t& logId, FormatToFn fn, const char* location, LogLevel level, 380 | fmt::string_view fmtString) noexcept; 381 | 382 | static void vformat_to(MemoryBuffer& out, fmt::string_view fmt, fmt::format_args args); 383 | 384 | static size_t formatted_size(fmt::string_view fmt, fmt::format_args args); 385 | 386 | static void vformat_to(char* out, fmt::string_view fmt, fmt::format_args args); 387 | 388 | static typename SPSCVarQueueOPT::MsgHeader* allocMsg(uint32_t size, bool logQFullCB) noexcept; 389 | 390 | TSCNS tscns; 391 | 392 | volatile LogLevel currentLogLevel; 393 | static FAST_THREAD_LOCAL ThreadBuffer* threadBuffer; 394 | 395 | template 396 | static inline constexpr bool isNamedArg() { 397 | return fmt::detail::is_named_arg>::value; 398 | } 399 | 400 | template 401 | struct unNamedType 402 | { using type = Arg; }; 403 | 404 | template 405 | struct unNamedType> 406 | { using type = Arg; }; 407 | 408 | #if FMT_USE_NONTYPE_TEMPLATE_ARGS 409 | template Str> 410 | struct unNamedType> 411 | { using type = Arg; }; 412 | #endif 413 | 414 | template 415 | static inline constexpr bool isCstring() { 416 | return fmt::detail::mapped_type_constant::value == fmt::detail::type::cstring_type; 417 | } 418 | 419 | template 420 | static inline constexpr bool isString() { 421 | return fmt::detail::mapped_type_constant::value == fmt::detail::type::string_type; 422 | } 423 | 424 | template 425 | static inline constexpr bool needCallDtor() { 426 | using ArgType = fmt::remove_cvref_t; 427 | if constexpr (isNamedArg()) { 428 | return needCallDtor::type>(); 429 | } 430 | if constexpr (isString()) return false; 431 | return !std::is_trivially_destructible::value; 432 | } 433 | 434 | template 435 | static inline constexpr size_t getArgSizes(size_t* cstringSize) { 436 | return 0; 437 | } 438 | 439 | template 440 | static inline constexpr size_t getArgSizes(size_t* cstringSize, const Arg& arg, 441 | const Args&... args) { 442 | if constexpr (isNamedArg()) { 443 | return getArgSizes(cstringSize, arg.value, args...); 444 | } 445 | else if constexpr (isCstring()) { 446 | size_t len = strlen(arg) + 1; 447 | cstringSize[CstringIdx] = len; 448 | return len + getArgSizes(cstringSize, args...); 449 | } 450 | else if constexpr (isString()) { 451 | size_t len = arg.size() + 1; 452 | return len + getArgSizes(cstringSize, args...); 453 | } 454 | else { 455 | return sizeof(Arg) + getArgSizes(cstringSize, args...); 456 | } 457 | } 458 | 459 | template 460 | static inline constexpr char* encodeArgs(size_t* cstringSize, char* out) { 461 | return out; 462 | } 463 | 464 | template 465 | static inline constexpr char* encodeArgs(size_t* cstringSize, char* out, Arg&& arg, 466 | Args&&... args) { 467 | if constexpr (isNamedArg()) { 468 | return encodeArgs(cstringSize, out, arg.value, std::forward(args)...); 469 | } 470 | else if constexpr (isCstring()) { 471 | memcpy(out, arg, cstringSize[CstringIdx]); 472 | return encodeArgs(cstringSize, out + cstringSize[CstringIdx], 473 | std::forward(args)...); 474 | } 475 | else if constexpr (isString()) { 476 | size_t len = arg.size(); 477 | memcpy(out, arg.data(), len); 478 | out[len] = 0; 479 | return encodeArgs(cstringSize, out + len + 1, std::forward(args)...); 480 | } 481 | else { 482 | // If Arg has alignment >= 16, gcc could emit aligned move instructions(e.g. movdqa) for 483 | // placement new even if the *out* is misaligned, which would cause segfault. So we use memcpy 484 | // when possible 485 | if constexpr (std::is_trivially_copyable_v>) { 486 | memcpy(out, &arg, sizeof(Arg)); 487 | } 488 | else { 489 | new (out) fmt::remove_cvref_t(std::forward(arg)); 490 | } 491 | return encodeArgs(cstringSize, out + sizeof(Arg), std::forward(args)...); 492 | } 493 | } 494 | 495 | template 496 | static inline constexpr void storeNamedArgs(fmt::detail::named_arg_info* named_args_store) { 497 | } 498 | 499 | template 500 | static inline constexpr void storeNamedArgs(fmt::detail::named_arg_info* named_args_store, 501 | const Arg& arg, const Args&... args) { 502 | if constexpr (isNamedArg()) { 503 | named_args_store[NamedIdx] = {arg.name, Idx}; 504 | storeNamedArgs(named_args_store, args...); 505 | } 506 | else { 507 | storeNamedArgs(named_args_store, args...); 508 | } 509 | } 510 | 511 | template 512 | static inline const char* decodeArgs(const char* in, fmt::basic_format_arg* args, 513 | const char** destruct_args) { 514 | return in; 515 | } 516 | 517 | template 518 | static inline const char* decodeArgs(const char* in, fmt::basic_format_arg* args, 519 | const char** destruct_args) { 520 | using namespace fmtlogdetail; 521 | using ArgType = fmt::remove_cvref_t; 522 | if constexpr (isNamedArg()) { 523 | return decodeArgs::type, Args...>( 524 | in, args, destruct_args); 525 | } 526 | else if constexpr (isCstring() || isString()) { 527 | size_t size = strlen(in); 528 | fmt::string_view v(in, size); 529 | if constexpr (ValueOnly) { 530 | fmt::detail::value& value_ = *(fmt::detail::value*)(args + Idx); 531 | value_ = v; 532 | } 533 | else { 534 | args[Idx] = v; 535 | } 536 | return decodeArgs(in + size + 1, args, 537 | destruct_args); 538 | } 539 | else { 540 | if constexpr (ValueOnly) { 541 | fmt::detail::value& value_ = *(fmt::detail::value*)(args + Idx); 542 | if constexpr (UnrefPtr::value) { 543 | value_ = **(ArgType*)in; 544 | } 545 | else { 546 | value_ = *(ArgType*)in; 547 | } 548 | } 549 | else { 550 | if constexpr (UnrefPtr::value) { 551 | args[Idx] = **(ArgType*)in; 552 | } 553 | else { 554 | args[Idx] = *(ArgType*)in; 555 | } 556 | } 557 | 558 | if constexpr (needCallDtor()) { 559 | destruct_args[DestructIdx] = in; 560 | return decodeArgs(in + sizeof(ArgType), args, 561 | destruct_args); 562 | } 563 | else { 564 | return decodeArgs(in + sizeof(ArgType), args, 565 | destruct_args); 566 | } 567 | } 568 | } 569 | 570 | template 571 | static inline void destructArgs(const char** destruct_args) {} 572 | 573 | template 574 | static inline void destructArgs(const char** destruct_args) { 575 | using ArgType = fmt::remove_cvref_t; 576 | if constexpr (isNamedArg()) { 577 | destructArgs::type, Args...>(destruct_args); 578 | } 579 | else if constexpr (needCallDtor()) { 580 | ((ArgType*)destruct_args[DestructIdx])->~ArgType(); 581 | destructArgs(destruct_args); 582 | } 583 | else { 584 | destructArgs(destruct_args); 585 | } 586 | } 587 | 588 | template 589 | static const char* formatTo(fmt::string_view format, const char* data, MemoryBuffer& out, 590 | int& argIdx, std::vector>& args) { 591 | constexpr size_t num_args = sizeof...(Args); 592 | constexpr size_t num_dtors = fmt::detail::count()...>(); 593 | const char* dtor_args[std::max(num_dtors, (size_t)1)]; 594 | const char* ret; 595 | if (argIdx < 0) { 596 | argIdx = (int)args.size(); 597 | args.resize(argIdx + num_args); 598 | ret = decodeArgs(data, args.data() + argIdx, dtor_args); 599 | } 600 | else { 601 | ret = decodeArgs(data, args.data() + argIdx, dtor_args); 602 | } 603 | vformat_to(out, format, fmt::basic_format_args(args.data() + argIdx, num_args)); 604 | destructArgs<0, Args...>(dtor_args); 605 | 606 | return ret; 607 | } 608 | 609 | template 610 | static fmt::string_view unNameFormat(fmt::string_view in, uint32_t* reorderIdx, 611 | const Args&... args) { 612 | constexpr size_t num_named_args = fmt::detail::count()...>(); 613 | if constexpr (num_named_args == 0) { 614 | return in; 615 | } 616 | const char* begin = in.data(); 617 | const char* p = begin; 618 | std::unique_ptr unnamed_str(new char[in.size() + 1 + num_named_args * 5]); 619 | fmt::detail::named_arg_info named_args[std::max(num_named_args, (size_t)1)]; 620 | storeNamedArgs<0, 0>(named_args, args...); 621 | 622 | char* out = (char*)unnamed_str.get(); 623 | uint8_t arg_idx = 0; 624 | while (true) { 625 | auto c = *p++; 626 | if (!c) { 627 | size_t copy_size = p - begin - 1; 628 | memcpy(out, begin, copy_size); 629 | out += copy_size; 630 | break; 631 | } 632 | if (c != '{') continue; 633 | size_t copy_size = p - begin; 634 | memcpy(out, begin, copy_size); 635 | out += copy_size; 636 | begin = p; 637 | c = *p++; 638 | if (!c) fmt::report_error("invalid format string"); 639 | if (fmt::detail::is_name_start(c)) { 640 | while ((fmt::detail::is_name_start(c = *p) || ('0' <= c && c <= '9'))) { 641 | ++p; 642 | } 643 | fmt::string_view name(begin, p - begin); 644 | int id = -1; 645 | for (size_t i = 0; i < num_named_args; ++i) { 646 | if (named_args[i].name == name) { 647 | id = named_args[i].id; 648 | break; 649 | } 650 | } 651 | if (id < 0) fmt::report_error("invalid format string"); 652 | if constexpr (Reorder) { 653 | reorderIdx[id] = arg_idx++; 654 | } 655 | else { 656 | out = fmt::format_to(out, "{}", id); 657 | } 658 | } 659 | else { 660 | *out++ = c; 661 | } 662 | begin = p; 663 | } 664 | const char* ptr = unnamed_str.release(); 665 | return fmt::string_view(ptr, out - ptr); 666 | } 667 | 668 | public: 669 | template 670 | inline void log( 671 | uint32_t& logId, int64_t tsc, const char* location, LogLevel level, 672 | fmt::format_string>::type...> format, 673 | Args&&... args) noexcept { 674 | if (!logId) { 675 | auto unnamed_format = unNameFormat(fmt::string_view(format), nullptr, args...); 676 | registerLogInfo(logId, formatTo, location, level, unnamed_format); 677 | } 678 | constexpr size_t num_cstring = fmt::detail::count()...>(); 679 | size_t cstringSizes[std::max(num_cstring, (size_t)1)]; 680 | uint32_t alloc_size = 8 + (uint32_t)getArgSizes<0>(cstringSizes, args...); 681 | bool q_full_cb = true; 682 | do { 683 | if (auto header = allocMsg(alloc_size, q_full_cb)) { 684 | header->logId = logId; 685 | char* out = (char*)(header + 1); 686 | *(int64_t*)out = tsc; 687 | out += 8; 688 | encodeArgs<0>(cstringSizes, out, std::forward(args)...); 689 | header->push(alloc_size); 690 | break; 691 | } 692 | q_full_cb = false; 693 | } while (FMTLOG_BLOCK); 694 | } 695 | 696 | template 697 | inline void logOnce(const char* location, LogLevel level, fmt::format_string format, 698 | Args&&... args) { 699 | fmt::string_view sv(format); 700 | auto&& fmt_args = fmt::make_format_args(args...); 701 | uint32_t fmt_size = formatted_size(sv, fmt_args); 702 | uint32_t alloc_size = 8 + 8 + fmt_size; 703 | bool q_full_cb = true; 704 | do { 705 | if (auto header = allocMsg(alloc_size, q_full_cb)) { 706 | header->logId = (uint32_t)level; 707 | char* out = (char*)(header + 1); 708 | *(int64_t*)out = tscns.rdtsc(); 709 | out += 8; 710 | *(const char**)out = location; 711 | out += 8; 712 | vformat_to(out, sv, fmt_args); 713 | header->push(alloc_size); 714 | break; 715 | } 716 | q_full_cb = false; 717 | } while (FMTLOG_BLOCK); 718 | } 719 | }; 720 | 721 | using fmtlog = fmtlogT<>; 722 | 723 | template 724 | FAST_THREAD_LOCAL typename fmtlogT<_>::ThreadBuffer* fmtlogT<_>::threadBuffer; 725 | 726 | template 727 | struct fmtlogWrapper 728 | { static fmtlog impl; }; 729 | 730 | template 731 | fmtlog fmtlogWrapper<_>::impl; 732 | 733 | template 734 | inline void fmtlogT<_>::setLogLevel(LogLevel logLevel) noexcept { 735 | fmtlogWrapper<>::impl.currentLogLevel = logLevel; 736 | } 737 | 738 | template 739 | inline typename fmtlogT<_>::LogLevel fmtlogT<_>::getLogLevel() noexcept { 740 | return fmtlogWrapper<>::impl.currentLogLevel; 741 | } 742 | 743 | template 744 | inline bool fmtlogT<_>::checkLogLevel(LogLevel logLevel) noexcept { 745 | #ifdef FMTLOG_NO_CHECK_LEVEL 746 | return true; 747 | #else 748 | return logLevel >= fmtlogWrapper<>::impl.currentLogLevel; 749 | #endif 750 | } 751 | 752 | #define __FMTLOG_S1(x) #x 753 | #define __FMTLOG_S2(x) __FMTLOG_S1(x) 754 | #define __FMTLOG_LOCATION __FILE__ ":" __FMTLOG_S2(__LINE__) 755 | 756 | #define FMTLOG(level, format, ...) \ 757 | do { \ 758 | static uint32_t logId = 0; \ 759 | if (!fmtlog::checkLogLevel(level)) break; \ 760 | fmtlogWrapper<>::impl.log(logId, fmtlogWrapper<>::impl.tscns.rdtsc(), __FMTLOG_LOCATION, \ 761 | level, format, ##__VA_ARGS__); \ 762 | } while (0) 763 | 764 | #define FMTLOG_LIMIT(min_interval, level, format, ...) \ 765 | do { \ 766 | static uint32_t logId = 0; \ 767 | static int64_t limitNs = 0; \ 768 | if (!fmtlog::checkLogLevel(level)) break; \ 769 | int64_t tsc = fmtlogWrapper<>::impl.tscns.rdtsc(); \ 770 | int64_t ns = fmtlogWrapper<>::impl.tscns.tsc2ns(tsc); \ 771 | if (ns < limitNs) break; \ 772 | limitNs = ns + min_interval; \ 773 | fmtlogWrapper<>::impl.log(logId, tsc, __FMTLOG_LOCATION, level, format, ##__VA_ARGS__); \ 774 | } while (0) 775 | 776 | #define FMTLOG_ONCE(level, format, ...) \ 777 | do { \ 778 | if (!fmtlog::checkLogLevel(level)) break; \ 779 | fmtlogWrapper<>::impl.logOnce(__FMTLOG_LOCATION, level, format, ##__VA_ARGS__); \ 780 | } while (0) 781 | 782 | #if FMTLOG_ACTIVE_LEVEL <= FMTLOG_LEVEL_DBG 783 | #define logd(format, ...) FMTLOG(fmtlog::DBG, format, ##__VA_ARGS__) 784 | #define logdo(format, ...) FMTLOG_ONCE(fmtlog::DBG, format, ##__VA_ARGS__) 785 | #define logdl(min_interval, format, ...) FMTLOG_LIMIT(min_interval, fmtlog::DBG, format, ##__VA_ARGS__) 786 | #else 787 | #define logd(format, ...) (void)0 788 | #define logdo(format, ...) (void)0 789 | #define logdl(min_interval, format, ...) (void)0 790 | #endif 791 | 792 | #if FMTLOG_ACTIVE_LEVEL <= FMTLOG_LEVEL_INF 793 | #define logi(format, ...) FMTLOG(fmtlog::INF, format, ##__VA_ARGS__) 794 | #define logio(format, ...) FMTLOG_ONCE(fmtlog::INF, format, ##__VA_ARGS__) 795 | #define logil(min_interval, format, ...) FMTLOG_LIMIT(min_interval, fmtlog::INF, format, ##__VA_ARGS__) 796 | #else 797 | #define logi(format, ...) (void)0 798 | #define logio(format, ...) (void)0 799 | #define logil(min_interval, format, ...) (void)0 800 | #endif 801 | 802 | #if FMTLOG_ACTIVE_LEVEL <= FMTLOG_LEVEL_WRN 803 | #define logw(format, ...) FMTLOG(fmtlog::WRN, format, ##__VA_ARGS__) 804 | #define logwo(format, ...) FMTLOG_ONCE(fmtlog::WRN, format, ##__VA_ARGS__) 805 | #define logwl(min_interval, format, ...) FMTLOG_LIMIT(min_interval, fmtlog::WRN, format, ##__VA_ARGS__) 806 | #else 807 | #define logw(format, ...) (void)0 808 | #define logwo(format, ...) (void)0 809 | #define logwl(min_interval, format, ...) (void)0 810 | #endif 811 | 812 | #if FMTLOG_ACTIVE_LEVEL <= FMTLOG_LEVEL_ERR 813 | #define loge(format, ...) FMTLOG(fmtlog::ERR, format, ##__VA_ARGS__) 814 | #define logeo(format, ...) FMTLOG_ONCE(fmtlog::ERR, format, ##__VA_ARGS__) 815 | #define logel(min_interval, format, ...) FMTLOG_LIMIT(min_interval, fmtlog::ERR, format, ##__VA_ARGS__) 816 | #else 817 | #define loge(format, ...) (void)0 818 | #define logeo(format, ...) (void)0 819 | #define logel(min_interval, format, ...) (void)0 820 | #endif 821 | 822 | #ifdef FMTLOG_HEADER_ONLY 823 | #include "fmtlog-inl.h" 824 | #endif 825 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if(NOT MSVC) 2 | add_library(static_lib lib.cc) 3 | 4 | add_library(static_header_lib lib.cc) 5 | target_compile_definitions(static_header_lib PUBLIC FMTLOG_HEADER_ONLY) 6 | 7 | add_library(shared_lib SHARED lib.cc) 8 | install(TARGETS shared_lib) 9 | 10 | add_library(shared_header_lib SHARED lib.cc) 11 | target_compile_definitions(shared_header_lib PUBLIC FMTLOG_HEADER_ONLY) 12 | install(TARGETS shared_header_lib) 13 | 14 | add_executable(link_static_static link_test.cc) 15 | target_link_libraries(link_static_static fmtlog-static static_lib fmt) 16 | install(TARGETS link_static_static) 17 | 18 | add_executable(link_static_shared link_test.cc) 19 | target_link_libraries(link_static_shared fmtlog-static shared_lib fmt) 20 | install(TARGETS link_static_shared) 21 | 22 | add_executable(link_shared_static link_test.cc) 23 | target_link_libraries(link_shared_static fmtlog-shared static_lib fmt) 24 | install(TARGETS link_shared_static) 25 | 26 | add_executable(link_shared_shared link_test.cc) 27 | target_link_libraries(link_shared_shared fmtlog-shared shared_lib fmt) 28 | install(TARGETS link_shared_shared) 29 | 30 | add_executable(link_header_static link_test.cc) 31 | target_link_libraries(link_header_static static_header_lib fmt) 32 | install(TARGETS link_header_static) 33 | 34 | add_executable(link_header_shared link_test.cc) 35 | target_link_libraries(link_header_shared shared_header_lib fmt) 36 | install(TARGETS link_header_shared) 37 | 38 | endif() 39 | 40 | add_executable(log_test log_test.cc) 41 | target_link_libraries(log_test fmtlog-static fmt) 42 | #target_compile_definitions(log_test PUBLIC FMTLOG_HEADER_ONLY) 43 | install(TARGETS log_test) 44 | 45 | add_executable(enc_dec_test enc_dec_test.cc) 46 | target_link_libraries(enc_dec_test fmtlog-static fmt) 47 | install(TARGETS enc_dec_test) 48 | 49 | add_executable(multithread_test multithread_test.cc) 50 | target_compile_definitions(multithread_test PUBLIC FMTLOG_HEADER_ONLY) 51 | #target_link_libraries(multithread_test fmtlog-static fmt) 52 | target_link_libraries(multithread_test fmt) 53 | install(TARGETS multithread_test) 54 | 55 | -------------------------------------------------------------------------------- /test/enc_dec_test.cc: -------------------------------------------------------------------------------- 1 | #include "../fmtlog.h" 2 | #include "fmt/ranges.h" 3 | #include 4 | #include 5 | using namespace std; 6 | using namespace fmt::literals; 7 | 8 | struct MyType 9 | { 10 | MyType(int val) 11 | : v(val) {} 12 | ~MyType() { 13 | dtor_cnt++; 14 | // fmt::print("dtor_cnt: {}\n", dtor_cnt); 15 | } 16 | int v; 17 | static int dtor_cnt; 18 | }; 19 | 20 | int MyType::dtor_cnt = 0; 21 | 22 | template<> 23 | struct fmt::formatter : formatter 24 | { 25 | // parse is inherited from formatter. 26 | template 27 | auto format(const MyType& val, FormatContext& ctx) const { 28 | return formatter::format(val.v, ctx); 29 | } 30 | }; 31 | 32 | struct MovableType 33 | { 34 | public: 35 | MovableType(int v = 0) 36 | : val{MyType(v)} {} 37 | 38 | std::vector val; 39 | }; 40 | 41 | template<> 42 | struct fmt::formatter : formatter 43 | { 44 | // parse is inherited from formatter. 45 | // template 46 | auto format(const MovableType& val, format_context& ctx) const { 47 | return formatter::format(val.val[0].v, ctx); 48 | } 49 | }; 50 | 51 | template 52 | void test(const S& format, Args&&... args) { 53 | auto sv = fmt::string_view(format); 54 | size_t formatted_size = fmt::formatted_size(fmt::runtime(sv), std::forward(args)...); 55 | string ans = fmt::format(fmt::runtime(sv), std::forward(args)...); 56 | assert(ans.size() == formatted_size); 57 | 58 | auto unnamed_format = fmtlog::unNameFormat(sv, nullptr, args...); 59 | fmt::print("unnamed_format: {}\n", unnamed_format); 60 | size_t cstringSizes[1000]; 61 | char buf[1024]; 62 | int allocSize = fmtlog::getArgSizes<0>(cstringSizes, args...); 63 | const char* ret = fmtlog::encodeArgs<0>(cstringSizes, buf, std::forward(args)...); 64 | // fmt::print("=========\n"); 65 | assert(ret - buf == allocSize); 66 | fmtlog::MemoryBuffer buffer; 67 | int argIdx = -1; 68 | std::vector> format_args; 69 | 70 | ret = fmtlog::formatTo(unnamed_format, buf, buffer, argIdx, format_args); 71 | assert(ret - buf == allocSize); 72 | 73 | string_view res(buffer.data(), buffer.size()); 74 | fmt::print("res: {}\n", res); 75 | fmt::print("ans: {}\n", ans); 76 | assert(res == ans); 77 | } 78 | 79 | int main() { 80 | char cstring[100] = "cstring cstring"; 81 | const char* p = "haha"; 82 | const char* pcstring = cstring; 83 | string str("str"); 84 | char ch = 'f'; 85 | char& ch2 = ch; 86 | int i = 5; 87 | int& ri = i; 88 | double d = 3.45; 89 | float f = 55.2; 90 | uint16_t short_int = 2222; 91 | 92 | test("test basic types: {}, {}, {}, {}, {}, {}, {}, {}, {:.1f}, {}, {}, {}, {}, {}, {}, {}", cstring, p, pcstring, 93 | "wow", 'a', 5, str, string_view(str), 1.34, ch, ch2, i, ri, d, f, short_int); 94 | 95 | test("test positional, {one}, {two:>5}, {three}, {four}, {0:.1f}", 5.012, "three"_a = 3, "two"_a = "two", 96 | "one"_a = string("one"), "four"_a = string("4")); 97 | test("test dynamic spec: {:.{}f}, {:*^30}", 3.14, 1, "centered"); 98 | test("test positional spec: int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}", 42); 99 | 100 | test("test custom types: {}, {}, {}", MyType(1), MyType(2), MovableType(3)); 101 | 102 | test("test ranges: {}, {}", vector{1, 2, 3}, vector{4, 5, 6}); 103 | 104 | int dtor_cnt; 105 | { 106 | MovableType val(123); 107 | dtor_cnt = MyType::dtor_cnt; 108 | test("test copy: {}", val); 109 | assert(MyType::dtor_cnt == dtor_cnt + 1); 110 | assert(val.val.size() == 1); 111 | dtor_cnt = MyType::dtor_cnt; 112 | } 113 | assert(MyType::dtor_cnt == dtor_cnt + 1); 114 | { 115 | MovableType val(456); 116 | dtor_cnt = MyType::dtor_cnt; 117 | test("test move: {}", std::move(val)); 118 | assert(MyType::dtor_cnt == dtor_cnt + 1); 119 | assert(val.val.size() == 0); 120 | dtor_cnt = MyType::dtor_cnt; 121 | } 122 | assert(MyType::dtor_cnt == dtor_cnt); 123 | 124 | fmt::print("tests passed\n"); 125 | 126 | return 0; 127 | } 128 | 129 | -------------------------------------------------------------------------------- /test/lib.cc: -------------------------------------------------------------------------------- 1 | #include "lib.h" 2 | #include "../fmtlog.h" 3 | 4 | void libFun(int i) { 5 | logi("libFun: {}", i); 6 | } 7 | 8 | -------------------------------------------------------------------------------- /test/lib.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | void libFun(int i); 3 | -------------------------------------------------------------------------------- /test/link_test.cc: -------------------------------------------------------------------------------- 1 | #include "lib.h" 2 | #include "../fmtlog.h" 3 | 4 | int main() { 5 | logi("link test: {}", 123); 6 | libFun(321); 7 | return 0; 8 | } 9 | -------------------------------------------------------------------------------- /test/log_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "../fmtlog.h" 5 | 6 | void runBenchmark(); 7 | 8 | void logcb(int64_t ns, fmtlog::LogLevel level, fmt::string_view location, size_t basePos, fmt::string_view threadName, 9 | fmt::string_view msg, size_t bodyPos, size_t logFilePos) { 10 | fmt::print("callback full msg: {}, logFilePos: {}\n", msg, logFilePos); 11 | msg.remove_prefix(bodyPos); 12 | fmt::print("callback msg body: {}\n", msg); 13 | } 14 | 15 | void logQFullCB(void* userData) { 16 | fmt::print("log q full\n"); 17 | } 18 | 19 | int main() { 20 | char randomString[] = "Hello World"; 21 | logi("A string, pointer, number, and float: '{}', {}, {}, {}", randomString, (void*)&randomString, 22 | 512, 3.14159); 23 | 24 | int a = 4; 25 | auto sptr = std::make_shared(5); 26 | auto uptr = std::make_unique(6); 27 | logi("void ptr: {}, ptr: {}, sptr: {}, uptr: {}", (void*)&a, &a, sptr, std::move(uptr)); 28 | a = 7; 29 | *sptr = 8; 30 | 31 | char strarr[10] = "111"; 32 | char* cstr = strarr; 33 | std::string str = "aaa"; 34 | logi("str: {}, pstr: {}, strarr: {}, pstrarr: {}, cstr: {}, pcstr: {}", str, &str, strarr, &strarr, cstr, &cstr); 35 | str = "bbb"; 36 | strcpy(cstr, "222"); 37 | 38 | // logi(FMT_STRING("This msg will trigger compile error: {:d}"), "I am not a number"); 39 | // FMT_STRING() above is not needed for c++20 40 | 41 | logd("This message wont be logged since it is lower " 42 | "than the current log level."); 43 | fmtlog::setLogLevel(fmtlog::DBG); 44 | logd("Now debug msg is shown"); 45 | 46 | fmtlog::poll(); 47 | 48 | for (int i = 0; i < 3; i++) { 49 | logio("log once: {}", i); 50 | } 51 | 52 | fmtlog::setThreadName("main"); 53 | logi("Thread name changed"); 54 | 55 | fmtlog::poll(); 56 | 57 | fmtlog::setHeaderPattern("{YmdHMSF} {s} {l}[{t}] "); 58 | logi("Header pattern is changed, full date time info is shown"); 59 | 60 | fmtlog::poll(); 61 | 62 | for (int i = 0; i < 10; i++) { 63 | logil(10, "This msg will be logged at an interval of at least 10 ns: {}.", i); 64 | } 65 | 66 | fmtlog::poll(); 67 | 68 | fmtlog::setLogCB(logcb, fmtlog::WRN); 69 | logw("This msg will be called back"); 70 | 71 | fmtlog::setLogFile("/tmp/wow", false); 72 | for (int i = 0; i < 10; i++) { 73 | logw("test logfilepos: {}.", i); 74 | } 75 | 76 | fmtlog::setLogQFullCB(logQFullCB, nullptr); 77 | for (int i = 0; i < 1024; i++) { 78 | std::string str(1000, ' '); 79 | logi("log q full cb test: {}", str); 80 | } 81 | 82 | fmtlog::poll(); 83 | runBenchmark(); 84 | 85 | return 0; 86 | } 87 | 88 | void runBenchmark() { 89 | const int RECORDS = 10000; 90 | // fmtlog::setLogFile("/dev/null", false); 91 | fmtlog::closeLogFile(); 92 | fmtlog::setLogCB(nullptr, fmtlog::WRN); 93 | 94 | std::chrono::high_resolution_clock::time_point t0, t1; 95 | 96 | t0 = std::chrono::high_resolution_clock::now(); 97 | for (int i = 0; i < RECORDS; ++i) { 98 | logi("Simple log message with one parameters, {}", i); 99 | } 100 | t1 = std::chrono::high_resolution_clock::now(); 101 | 102 | double span = std::chrono::duration_cast>(t1 - t0).count(); 103 | fmt::print("benchmark, front latency: {:.1f} ns/msg average\n", (span / RECORDS) * 1e9); 104 | } 105 | -------------------------------------------------------------------------------- /test/multithread_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "../fmtlog.h" 6 | using namespace std; 7 | 8 | const int thr_cnt = 4; 9 | atomic start_cnt; 10 | 11 | size_t msg_cnt = 0; 12 | size_t cb_size = 0; 13 | 14 | void logcb(int64_t ns, fmtlog::LogLevel level, fmt::string_view location, size_t basePos, fmt::string_view threadName, 15 | fmt::string_view msg, size_t bodyPos, size_t logFilePos) { 16 | msg_cnt++; 17 | cb_size += msg.size(); 18 | } 19 | 20 | void threadRun(int id) { 21 | fmtlog::setThreadName(fmt::format("thread {}", id).c_str()); 22 | start_cnt++; 23 | while (start_cnt < thr_cnt) 24 | ; 25 | for (int i = 0; i < 100000; i++) { 26 | logi("msg : {}, i: {}", id, i); 27 | if (i % 1000 == 0) std::this_thread::sleep_for(std::chrono::milliseconds(1)); 28 | } 29 | } 30 | 31 | int main() { 32 | fmtlog::setLogCB(logcb, fmtlog::INF); 33 | // fmtlog::closeLogFile(); 34 | fmtlog::setLogFile("multithread.txt"); 35 | // fmtlog::setHeaderPattern(""); 36 | vector thrs; 37 | fmtlog::startPollingThread(1); 38 | for (int i = 0; i < thr_cnt; i++) { 39 | thrs.emplace_back([i]() { threadRun(i); }); 40 | } 41 | 42 | while (start_cnt < thr_cnt) 43 | ; 44 | std::chrono::high_resolution_clock::time_point t0, t1; 45 | t0 = std::chrono::high_resolution_clock::now(); 46 | 47 | for (auto& t : thrs) { 48 | t.join(); 49 | } 50 | 51 | fmtlog::stopPollingThread(); 52 | 53 | t1 = std::chrono::high_resolution_clock::now(); 54 | double span = std::chrono::duration_cast>(t1 - t0).count(); 55 | 56 | fmt::print("total msg_cnt: {}, cb_size: {}, throughput: {:.1f} MB/second, {} msg/sec\n", msg_cnt, cb_size, 57 | (cb_size / 1000.0 / 1000 / span), (msg_cnt) / span); 58 | 59 | return 0; 60 | } 61 | --------------------------------------------------------------------------------