├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── build └── .empty ├── include ├── flb_data_file.h ├── flb_network.h ├── flb_proc.h ├── flb_report.h └── mk_list.h ├── src ├── CMakeLists.txt ├── flb-tail-writer.c ├── flb-tcp-writer.c ├── flb_data_file.c ├── flb_network.c ├── flb_proc.c ├── flb_report.c └── flb_utils.c └── test-scenario ├── fluent-bit-tail-tcp.yaml ├── fluentd.conf ├── otel-collector.yaml ├── performance_test_data.txt ├── run.sh └── vector.toml /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | *~ 3 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | project(fluent-bit-perf) 3 | 4 | # Change output path for binaries 5 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin") 6 | 7 | # Headers path 8 | include_directories(include/) 9 | 10 | # Tools 11 | add_subdirectory(src) 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Fluent Bit Performance Test Tools 2 | 3 | [Fluent Bit](https://fluentbit.io) is a fast and lightweight log processor. As part of our continuous development and testing model, we provide specific tools to test performance under different data load scenarios. 4 | 5 | The objective of our performance tooling is to gather the following insights: 6 | 7 | - Upon N number of records/data ingestion, measure: 8 | - CPU usage, user time and system time 9 | - Memory usage 10 | - Total time required to process the data 11 | 12 | Tests aim to run for a fixed number of time, data load can be increased per round. Every tool is able to monitor and collect resource usage metrics from the tested Fluent Bit or another logging tool as a target. 13 | 14 | > we focused in a specific set of metrics only which are enough for the purpose of the testing. 15 | 16 | ## How it Works 17 | 18 | Every tool available is written on top of a generic framework that provides interfaces to load data files, gather metrics nad generate a report from a running process. 19 | 20 | In the following diagram, using ```flb-tail-writer``` tool as an example, it writes N amount of records (lines) to a custom log file, in a separate session, Fluent Bit through [Tail input plugin](https://docs.fluentbit.io/manual/input/tail) reads information from the file generated. Internally the Linux Kernel exposes Fluent Bit process metrics through ProcFS, where flb-tail-writer _before_ and _after_ every write/round operation gather metrics and provides insights of resources consumption. 21 | 22 | ``` 23 | +-----------------+ +----------------+ 24 | | Proc FS (/proc) +<----------------+ Linux Kernel | 25 | +-----+-----+-----+ +-----+----+-----+ 26 | | ^ | ^ 27 | v | v | 28 | +-----+-----+-----+ +-----+----+-----+ 29 | | | | | 30 | | FLB Tail Writer +-----+ +-----+ Fluent Bit | 31 | | | | | | | 32 | +--------+--------+ | | +----------------+ 33 | | | | 34 | v v v 35 | +------+------+ +-+-----+----------+ 36 | | Test Report | | /var/log/out.log | 37 | +-------------+ +------------------+ 38 | ``` 39 | 40 | As an example, consider the following test using ```flb-tail-writer``` where: 41 | 42 | - Reads samples of data from _data.log_ file 43 | - Output data will be written to _out.log_ file 44 | - Write 1000000 records (log lines) every second, 10 times (called as 10 seconds). 45 | - Monitor resources usage of Fluent Bit process ID (PID). 46 | - Stop Monitoring Fluent Bit process once the process becomes almost idle for 3 seconds. 47 | 48 | ```bash 49 | records write (b) write secs | % cpu user (ms) sys (ms) Mem (bytes) Mem 50 | -------- ---------- -------- ----- + ------ --------- -------- ----------- ------- 51 | 1000000 131881447 125.77M 1.05 | 40.05 370 50 152735744 145.66M 52 | 1000000 131881447 125.77M 1.05 | 50.34 480 50 5758976 5.49M 53 | 1000000 131881447 125.77M 1.05 | 45.54 410 70 3915776 3.73M 54 | 1000000 131881447 125.77M 1.06 | 46.40 420 70 3915776 3.73M 55 | 1000000 131881447 125.77M 1.05 | 44.61 410 60 3915776 3.73M 56 | 1000000 131881447 125.77M 1.06 | 45.39 430 50 3915776 3.73M 57 | 1000000 131881447 125.77M 1.05 | 45.59 430 50 3915776 3.73M 58 | 1000000 131881447 125.77M 1.06 | 44.53 440 30 3915776 3.73M 59 | 1000000 131881447 125.77M 1.09 | 47.75 460 60 3915776 3.73M 60 | 1000000 131881447 125.77M 1.06 | 45.45 440 40 3915776 3.73M 61 | 0 0 0 b 1.00 | 0.00 0 0 3915776 3.73M 62 | 0 0 0 b 1.00 | 0.00 0 0 3915776 3.73M 63 | 0 0 0 b 1.00 | 0.00 0 0 3915776 3.73M 64 | 65 | - Summary 66 | - Process : fluent-bit 67 | - PID : 1660 68 | - Elapsed Time: 10.58 seconds 69 | - Avg Memory : 14.79M 70 | - Avg CPU : 45.56% 71 | - Avg Rate : 92.63M/sec 72 | ``` 73 | 74 | ## Report Details 75 | 76 | The report have two panes, left side belongs to the information provided by the perf tools in terms 77 | of data ingestion and the right side the metrics collected from the monitored process. 78 | 79 | #### Left Pane 80 | 81 | The information on the left side of the report belongs to a summary of data samples sent to the target service. 82 | 83 | | Column | Description | 84 | | --------- | ------------------------------------------------- | 85 | | records | Number of records ingested in the specific round. | 86 | | write (b) | Total number of bytes written. | 87 | | write | Human readable version of written bytes. | 88 | | secs | Elapsed time on writing the data. | 89 | 90 | #### Right Pane 91 | 92 | Overall metrics from the monitored process when the ```-p PID``` parameter is used. 93 | 94 | | Column | Description | 95 | | --------- | ------------------------------------------------------------ | 96 | | % cpu | Represents the CPU time used by the process in user and system space during the time (_secs_) that the performance tool was writing data. | 97 | | user (ms) | CPU time spent in milliseconds in user time (user space) | 98 | | sys (ms) | CPU time spent in milliseconds in system time (kernel space). | 99 | | Memory | Number of bytes in memory (RSS) currently used by the process after writing the data and waiting for one second. | 100 | | Mem | Human readable version of Memory used. | 101 | 102 | ## Tools Available 103 | 104 | | Tool | Fluent Bit Target | Description | 105 | | ----------- | :-------------------------------------------------------: | -------------------------------------------- | 106 | | Tail Writer | [Tail input](https://docs.fluentbit.io/manual/input/tail) | Writes large amount of data into a log file. | 107 | | TCP Writer | [TCP input](https://docs.fluentbit.io/manual/input/tail), [Syslog input](https://docs.fluentbit.io/manual/input/syslog) (tcp mode) | Writes large amount of data over a TCP socket. | 108 | 109 | ## Build Instructions 110 | 111 | ### Requirements 112 | 113 | - C compiler 114 | - CMake3 115 | - Linux environment 116 | 117 | ### Build 118 | 119 | Run the following command to compile the tools: 120 | 121 | ```bash 122 | $ cd build/ 123 | $ make 124 | ``` 125 | 126 | ## Comments about Performance and Benchmarking 127 | 128 | Performance is always critical and when managing data at high scale there are many corners where is possible to improve and make it better. 129 | 130 | When measuring performance is important to understand the variables that can affect a running _monitored_ service. If you are comparing same tool like Fluent Bit v/s Fluent Bit is not a hard task, but if you aim to compare Fluent Bit against other solution in the same space, you have to do an extra work and make sure that the setup and conditions are the same, e.g: make sure buffer sizes are the same on both tools. 131 | 132 | > Running a performance test using default options in different services will lead to unreliable results. 133 | 134 | Story: some years ago I was working in one of my [HTTP servers](http://monkey-project.com) projects. We got into a benchmark virtual battle against a proprietary web server. They claimed aims to be faster that all open source options available (e.g: Nginx, Lighttpd, Apache, etc)... and benchmark results shows that their project was _outstanding_ leaving every other project behind. 135 | 136 | After digging a bit more and starting measuring what was doing that web server from an operating system level, we ended up discovering that it was _caching_ every HTTP request and response without extra checks, so if it get one million request for the same end-point in a Keep-Alive session, it sent the same response over and over, without the expected processing. Basically it was _prepared_ before hand to cheat if it was benchmarked. 137 | 138 | > Upon sending a HTTP request with an URI that changed the query string variable (e.g: /?a=1..) every time, it was slow as hell :) 139 | 140 | On that moment I learn how important was to measure every aspect of a running service. That's why the simple metrics of CPU time in user/kernel space and memory usage are really important. 141 | 142 | final tip: if you are the user, try to do your own benchmarks for your own conditions and scenario. Trust in our performance tooling but don't trust in benchmarks reports made by us (maintainers) or vendors XD . 143 | 144 | ## License 145 | 146 | This program is under the terms of the [Apache License v2.0](http://www.apache.org/licenses/LICENSE-2.0). 147 | 148 | ## Authors 149 | 150 | - [Eduardo Silva ](https://twitter.com/edsiper) 151 | -------------------------------------------------------------------------------- /build/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluent/fluent-bit-perf/28cd53f6ced578ead8340541eea912053314288e/build/.empty -------------------------------------------------------------------------------- /include/flb_data_file.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2015-2018 Treasure Data Inc. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | #ifndef FLB_DATA_FILE_H 22 | #define FLB_DATA_FILE_H 23 | 24 | int flb_data_file_load(char *path, char **out_buf, size_t *out_size); 25 | void flb_data_file_unload(void *map, size_t size); 26 | int flb_data_file_offset_records(int n_records, char *buf, 27 | size_t size, off_t *offset); 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /include/flb_network.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2015-2018 Treasure Data Inc. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | #ifndef FLB_NETWORK_H 22 | #define FLB_NETWORK_H 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | int flb_net_socket_create(int family); 33 | int flb_net_tcp_connect(char *host, char *port); 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /include/flb_proc.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2014-2019 Eduardo Silva 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | /* 22 | * FYI: I wrote this code back in 2014, it's being modified for this project 23 | * purposes, original code is located here: 24 | * 25 | * https://github.com/edsiper/wr 26 | */ 27 | 28 | #ifndef FLB_PROC_H 29 | #define FLB_PROC_H 30 | 31 | #define PROC_PID_SIZE 1024 32 | #define PROC_STAT_BUF_SIZE 1024 33 | 34 | /* Our tast struct to read the /proc/PID/stat values */ 35 | struct flb_proc_task { 36 | /* Stat creation timestamp */ 37 | struct timespec ts; 38 | 39 | /* Process data */ 40 | char name[256]; 41 | unsigned long utime; /* %lu */ 42 | unsigned long stime; /* %lu */ 43 | long rss; /* %ld */ 44 | 45 | /* Internal resource conversion */ 46 | long r_rss; /* bytes = (rss * PAGESIZE) */ 47 | unsigned long r_utime_s; /* seconds = (utime / _SC_CLK_TCK) */ 48 | unsigned long r_utime_ms; /* milliseconds = ((utime * 1000) / CPU_HZ) */ 49 | unsigned long r_stime_s; /* seconds = (utime / _SC_CLK_TCK) */ 50 | unsigned long r_stime_ms; /* milliseconds = ((utime * 1000) / CPU_HZ) */ 51 | }; 52 | 53 | struct flb_proc_task *flb_proc_stat_create(pid_t pid); 54 | void flb_proc_stat_destroy(struct flb_proc_task *t); 55 | void flb_proc_stat_print(struct flb_proc_task *t); 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /include/flb_report.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2015-2018 Treasure Data Inc. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | #ifndef FLB_REPORT_H 22 | #define FLB_REPORT_H 23 | 24 | #define FLB_REPORT_TXT 0 25 | #define FLB_REPORT_MARKDOWN 1 26 | #define FLB_REPORT_CSV 2 27 | 28 | #include "flb_proc.h" 29 | 30 | struct flb_report { 31 | int format; /* report output format */ 32 | int fd; /* report file descriptor */ 33 | int cpu_count; /* number of CPUs */ 34 | int cpu_ticks; /* cpu clock ticks */ 35 | int snapshots; /* number of stats snapshots */ 36 | int wait_time; /* monitoring wait time */ 37 | 38 | /* Process info */ 39 | pid_t pid; /* monitored process ID */ 40 | char *name; /* process name */ 41 | 42 | /* General stats */ 43 | size_t sum_bytes; /* Total number of bytes */ 44 | size_t sum_mem; /* total number of bytes reported per snapshot */ 45 | size_t sum_records; /* total number of log records */ 46 | int sum_cpu_count; /* CPU snapshots summarized */ 47 | double sum_cpu; /* total %CPU usage */ 48 | double sum_duration; /* total elapsed time of tests */ 49 | 50 | }; 51 | 52 | struct flb_report *flb_report_create(char *out, int format, int pid, int wait); 53 | int flb_report_stats(struct flb_report *r, int records, 54 | size_t bytes, 55 | struct flb_proc_task *t1, struct flb_proc_task *t2); 56 | 57 | char *flb_report_human_readable_size(long size); 58 | double flb_report_cpu_usage(struct flb_report *r, 59 | struct flb_proc_task *t1, struct flb_proc_task *t2); 60 | 61 | int flb_report_summary(struct flb_report *r); 62 | int flb_report_destroy(struct flb_report *r); 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /include/mk_list.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Monkey HTTP Server 4 | * ================== 5 | * Copyright 2001-2017 Eduardo Silva 6 | * Copyright (C) 2010, Jonathan Gonzalez V. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | #ifndef MK_LIST_H_ 22 | #define MK_LIST_H_ 23 | 24 | #include 25 | 26 | #ifdef _WIN32 27 | /* Windows */ 28 | #define container_of(address, type, field) ((type *)( \ 29 | (PCHAR)(address) - \ 30 | (ULONG_PTR)(&((type *)0)->field))) 31 | #else 32 | /* Rest of the world */ 33 | #ifndef offsetof 34 | #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) 35 | #endif 36 | 37 | #define container_of(ptr, type, member) ({ \ 38 | const typeof( ((type *)0)->member ) *__mptr = (ptr); \ 39 | (type *)( (char *)__mptr - offsetof(type,member) );}) 40 | #endif 41 | 42 | struct mk_list 43 | { 44 | struct mk_list *prev, *next; 45 | }; 46 | 47 | static inline void mk_list_init(struct mk_list *list) 48 | { 49 | list->next = list; 50 | list->prev = list; 51 | } 52 | 53 | static inline void __mk_list_add(struct mk_list *_new, struct mk_list *prev, 54 | struct mk_list *next) 55 | { 56 | next->prev = _new; 57 | _new->next = next; 58 | _new->prev = prev; 59 | prev->next = _new; 60 | } 61 | 62 | static inline void mk_list_add(struct mk_list *_new, struct mk_list *head) 63 | { 64 | __mk_list_add(_new, head->prev, head); 65 | } 66 | 67 | static inline void mk_list_add_after(struct mk_list *_new, 68 | struct mk_list *prev, 69 | struct mk_list *head) 70 | { 71 | struct mk_list *next; 72 | 73 | if (head->prev == head->next || head->prev == prev) { 74 | mk_list_add(_new, head); 75 | return; 76 | } 77 | 78 | next = prev->next; 79 | next->prev = prev; 80 | _new->next = next; 81 | _new->prev = prev; 82 | prev->next = _new; 83 | } 84 | 85 | static inline void __mk_list_del(struct mk_list *prev, struct mk_list *next) 86 | { 87 | prev->next = next; 88 | next->prev = prev; 89 | } 90 | 91 | static inline void mk_list_del(struct mk_list *entry) 92 | { 93 | __mk_list_del(entry->prev, entry->next); 94 | entry->prev = NULL; 95 | entry->next = NULL; 96 | } 97 | 98 | static inline int mk_list_is_empty(struct mk_list *head) 99 | { 100 | if (head->next == head) return 0; 101 | else return -1; 102 | } 103 | 104 | static inline int mk_list_is_set(struct mk_list *head) 105 | { 106 | if (head->next && head->prev) { 107 | return 0; 108 | } 109 | 110 | return -1; 111 | } 112 | 113 | static inline int mk_list_size(struct mk_list *head) 114 | { 115 | int ret = 0; 116 | struct mk_list *it; 117 | for (it = head->next; it != head; it = it->next, ret++); 118 | return ret; 119 | } 120 | 121 | static inline int mk_list_entry_orphan(struct mk_list *head) 122 | { 123 | if (head->next && head->prev) { 124 | return 0; 125 | } 126 | 127 | return -1; 128 | } 129 | 130 | static inline void mk_list_cat(struct mk_list *list, struct mk_list *head) 131 | { 132 | struct mk_list *last; 133 | 134 | last = head->prev; 135 | last->next = list->next; 136 | list->next->prev = last; 137 | list->prev->next = head; 138 | head->prev = list->prev; 139 | } 140 | 141 | #define mk_list_foreach(curr, head) for( curr = (head)->next; curr != (head); curr = curr->next ) 142 | #define mk_list_foreach_safe(curr, n, head) \ 143 | for (curr = (head)->next, n = curr->next; curr != (head); curr = n, n = curr->next) 144 | 145 | 146 | #define mk_list_foreach_r(curr, head) for( curr = (head)->prev; curr != (head); curr = curr->prev ) 147 | #define mk_list_foreach_safe_r(curr, n, head) \ 148 | for (curr = (head)->prev, n = curr->prev; curr != (head); curr = n, n = curr->prev) 149 | 150 | #define mk_list_entry( ptr, type, member ) container_of( ptr, type, member ) 151 | 152 | /* 153 | * First node of the list 154 | * ---------------------- 155 | * Be careful with this Macro, its intended to be used when some node is already linked 156 | * to the list (ptr). If the list is empty it will return the list address as it points 157 | * to it self: list == list->prev == list->next. 158 | * 159 | * If exists some possiblity that your code handle an empty list, use mk_list_is_empty() 160 | * previously to check if its empty or not. 161 | */ 162 | #define mk_list_entry_first(ptr, type, member) container_of((ptr)->next, type, member) 163 | 164 | /* First node of the list 165 | * --------------------- 166 | * Be careful with this Macro, its intended to be used when some node is already linked 167 | * to the list (ptr). If the list is empty it will return the list address as it points 168 | * to it self: list == list->prev == list->next. 169 | * 170 | * If exists some possiblity that your code handle an empty list, use mk_list_is_empty() 171 | * previously to check if its empty or not. 172 | */ 173 | #define mk_list_entry_last(ptr, type, member) container_of((ptr)->prev, type, member) 174 | 175 | /* Next node */ 176 | #define mk_list_entry_next(ptr, type, member, head) \ 177 | (ptr)->next == (head) ? container_of((head)->next, type, member) : \ 178 | container_of((ptr)->next, type, member); 179 | 180 | #endif /* !MK_LIST_H_ */ 181 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Helper interfaces 2 | set(src_helpers 3 | flb_data_file.c 4 | flb_report.c 5 | flb_proc.c 6 | flb_network.c 7 | ) 8 | 9 | # flb-tail-writer 10 | set(src_tail_writer 11 | ${src_helpers} 12 | flb-tail-writer.c) 13 | 14 | # flb-tcp-writer 15 | set(src_tcp_writer 16 | ${src_helpers} 17 | flb-tcp-writer.c) 18 | 19 | add_executable(flb-tail-writer ${src_tail_writer}) 20 | add_executable(flb-tcp-writer ${src_tcp_writer}) 21 | -------------------------------------------------------------------------------- /src/flb-tail-writer.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2015-2018 Treasure Data Inc. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | /* local headers */ 34 | #include "mk_list.h" 35 | #include "flb_data_file.h" 36 | #include "flb_proc.h" 37 | #include "flb_report.h" 38 | 39 | /* Default values */ 40 | #define DEFAULT_RECORDS 1000 /* 1000 records per second */ 41 | #define DEFAULT_INC_BY 0 /* no increase */ 42 | #define DEFAULT_SECONDS 10 /* test time: 10 seconds */ 43 | 44 | static int flb_help(int rc) 45 | { 46 | printf("Usage: flb-tail-writer [OPTIONS]\n\n"); 47 | printf("Available options\n"); 48 | printf(" -d, --datafile=PATH\t\tspecify source data file\n"); 49 | printf(" -p --pid=FLB_PID\t\tFluent Bit PID used gather metrics\n"); 50 | printf(" -o, --output=PATH\t\tset output file name\n"); 51 | printf(" -i, --increase_by=N\t\tincrease N number of records per second (default: %i)\n", 52 | DEFAULT_INC_BY); 53 | printf(" -r, --records=RECORDS\t\trecords per second (default: %i)\n", 54 | DEFAULT_RECORDS); 55 | printf(" -s, --seconds=SECONDS\t\ttotal test time meassured in seconds (default: %i)\n", 56 | DEFAULT_SECONDS); 57 | printf(" -R, --report\t\t\tset report output file (default: stdout)\n"); 58 | printf(" -F, --format\t\t\treport format, text (default) or markdown)\n"); 59 | printf(" -D, --delta-stop\t\tstop the test when the delta between two snapshots is near this value\n"); 60 | printf(" -h, --help\t\t\tprint this help"); 61 | printf("\n\n"); 62 | exit(rc); 63 | } 64 | 65 | static void close_report(int fd) 66 | { 67 | if (fd == STDOUT_FILENO) { 68 | return; 69 | } 70 | 71 | if (fd != -1) { 72 | close(fd); 73 | } 74 | } 75 | 76 | static int run_fs_writer(pid_t pid, 77 | char *report, 78 | int fmt_report, 79 | char *in_data_file, char *out_data_file, 80 | int records, int increase_by, 81 | int seconds, int delta_stop) 82 | { 83 | int i; 84 | int x; 85 | int in_fd; 86 | int out_fd; 87 | int ret; 88 | int round_records; 89 | int report_fd = -1; 90 | int wait_time = 3; 91 | size_t round_bytes; 92 | off_t off = 0; 93 | off_t off_rec; 94 | off_t off_inc; 95 | char *data_buf; 96 | size_t data_size; 97 | ssize_t bytes; 98 | ssize_t total_cpu = 0; 99 | ssize_t total_mem = 0; 100 | size_t total_records = 0; 101 | ssize_t total_bytes = 0; 102 | char *proc_name = NULL; 103 | struct flb_proc_task *t1; 104 | struct flb_proc_task *t2; 105 | struct flb_report *r = NULL; 106 | time_t start_time; 107 | time_t end_time; 108 | 109 | /* Report file for process monitoring */ 110 | if (pid >= 0) { 111 | r = flb_report_create(report, fmt_report, pid, wait_time); 112 | if (!r) { 113 | fprintf(stderr, "error: cannot initialize report"); 114 | return -1; 115 | } 116 | } 117 | 118 | /* Open output target file */ 119 | out_fd = open(out_data_file, O_WRONLY | O_CREAT | O_TRUNC, 0666); 120 | if (out_fd == -1) { 121 | perror("open"); 122 | fprintf(stderr, "error: cannot open/create output data file '%s'\n", 123 | out_data_file); 124 | if (r) { 125 | flb_report_destroy(r); 126 | } 127 | return -1; 128 | } 129 | 130 | /* Load input data file in-memory */ 131 | in_fd = flb_data_file_load(in_data_file, &data_buf, &data_size); 132 | if (in_fd == -1) { 133 | close(out_fd); 134 | fprintf(stderr, "error: cannot load input data file '%s'\n", 135 | in_data_file); 136 | if (r) { 137 | flb_report_destroy(r); 138 | } 139 | return -1; 140 | } 141 | 142 | /* Retrieve the final offset of 'records' */ 143 | ret = flb_data_file_offset_records(records, data_buf, data_size, &off_rec); 144 | if (ret == -1) { 145 | fprintf(stderr, "error: cannot find %i number of records\n", records); 146 | flb_data_file_unload(data_buf, data_size); 147 | close(in_fd); 148 | close(out_fd); 149 | if (r) { 150 | flb_report_destroy(r); 151 | } 152 | return -1; 153 | } 154 | 155 | /* Retrieve the final offset of 'records' */ 156 | ret = flb_data_file_offset_records(increase_by, data_buf, data_size, 157 | &off_inc); 158 | if (ret == -1) { 159 | fprintf(stderr, "error: cannot find %i number of records\n", records); 160 | flb_data_file_unload(data_buf, data_size); 161 | close(in_fd); 162 | close(out_fd); 163 | if (r) { 164 | flb_report_destroy(r); 165 | } 166 | return -1; 167 | } 168 | 169 | /* Get Process name */ 170 | if (pid >= 0) { 171 | t1 = flb_proc_stat_create(pid); 172 | proc_name = strndup(t1->name + 1, strlen(t1->name) - 2); 173 | } 174 | 175 | /* Set the start time */ 176 | start_time = time(NULL); 177 | 178 | /* 179 | * Just write data chunks every second. Since a write operation will 180 | * put the data into a kernel buffer (or zero copy) and likely return 181 | * immediately, we don't care about the time invested in that operation, 182 | * just the wait time between writes. 183 | */ 184 | for (i = 0; i < seconds; i++) { 185 | round_bytes = 0; 186 | round_records = 0; 187 | 188 | if (pid >= 0) { 189 | t1 = flb_proc_stat_create(pid); 190 | if (!t1) { 191 | fprintf(stderr, "error gathering stats for PID %i\n", 192 | (int) pid); 193 | } 194 | } 195 | 196 | /* 197 | * Use zero-copy strategy with sendfile(2). In benchmarking we want 198 | * to avoid extra Kernel work, this is a Linux specific feature. 199 | */ 200 | off = 0; 201 | /* Dispatch the records chunk */ 202 | bytes = sendfile(out_fd, in_fd, &off, off_rec); 203 | if (bytes == -1) { 204 | perror("sendfile"); 205 | fprintf(stderr, "error: exception on writing records chunk\n"); 206 | } 207 | else { 208 | total_bytes += bytes; 209 | total_records += records; 210 | round_records = records; 211 | round_bytes = bytes; 212 | } 213 | 214 | if (increase_by > 0 && i > 0) { 215 | /* 216 | * Send incremental records, yeah, this involve a second system 217 | * call but it's better than writev() and data-copy. 218 | */ 219 | for (x = 0; x < i; x++) { 220 | off = 0; 221 | bytes = sendfile(out_fd, in_fd, &off, off_inc); 222 | if (bytes == -1) { 223 | perror("sendfile"); 224 | fprintf(stderr, "error: cannot write inc records chunk\n"); 225 | } 226 | else { 227 | total_bytes += bytes; 228 | total_records += increase_by; 229 | round_records += increase_by; 230 | round_bytes += bytes; 231 | } 232 | } 233 | } 234 | /* Dummy sleep */ 235 | sleep(1); 236 | 237 | /* Get stats */ 238 | if (pid >= 0) { 239 | t2 = flb_proc_stat_create(pid); 240 | if (!t2) { 241 | fprintf(stderr, "error gathering stats for PID %i\n", 242 | (int) pid); 243 | } 244 | 245 | if (r) { 246 | flb_report_stats(r, round_records, round_bytes, t1, t2); 247 | } 248 | 249 | /* Count total number of bytes in memory */ 250 | total_cpu += flb_report_cpu_usage(r, t1, t2); 251 | total_mem += t2->r_rss; 252 | 253 | flb_proc_stat_destroy(t1); 254 | flb_proc_stat_destroy(t2); 255 | } 256 | } 257 | 258 | /* 259 | * Create continuos snapshots until resources consumption (CPU) stabilize, 260 | * we assume that after two seconds without deltas in user time the process 261 | * finished processing our records. 262 | */ 263 | if (pid >= 0) { 264 | int count = 0; 265 | int test_time; 266 | char *tmp; 267 | 268 | while (1) { 269 | t1 = flb_proc_stat_create(pid); 270 | sleep(1); 271 | t2 = flb_proc_stat_create(pid); 272 | flb_report_stats(r, 0, 0, t1, t2); 273 | 274 | if ((t2->r_utime_ms - t1->r_utime_ms) <= delta_stop) { 275 | count++; 276 | } 277 | else { 278 | if (count > 0) { 279 | count = 0; 280 | } 281 | } 282 | 283 | 284 | if (count >= wait_time) { 285 | flb_proc_stat_destroy(t1); 286 | flb_proc_stat_destroy(t2); 287 | break; 288 | } 289 | 290 | flb_proc_stat_destroy(t1); 291 | flb_proc_stat_destroy(t2); 292 | 293 | } 294 | r->sum_records = total_records; 295 | flb_report_summary(r); 296 | } 297 | 298 | if (r) { 299 | flb_report_destroy(r); 300 | } 301 | 302 | return 0; 303 | } 304 | 305 | int main(int argc, char **argv) 306 | { 307 | int ret; 308 | int opt; 309 | int records = DEFAULT_RECORDS; 310 | int seconds = DEFAULT_SECONDS; 311 | int increase_by = DEFAULT_INC_BY; 312 | int delta_stop = 0; 313 | int out_fd; 314 | int pid = -1; 315 | int fd_report; 316 | int fmt_report = FLB_REPORT_TXT; 317 | char *format = NULL; 318 | char *report = NULL; 319 | char *out_file = NULL; 320 | char *data_file = NULL; 321 | 322 | /* Setup long-options */ 323 | static const struct option long_opts[] = { 324 | { "datafile" , required_argument, NULL, 'd' }, 325 | { "pid" , required_argument, NULL, 'p' }, 326 | { "output" , required_argument, NULL, 'o' }, 327 | { "records" , required_argument, NULL, 'r' }, 328 | { "increase_by", required_argument, NULL, 'i' }, 329 | { "seconds" , required_argument, NULL, 's' }, 330 | { "report" , required_argument, NULL, 'R' }, 331 | { "format" , required_argument, NULL, 'F' }, 332 | { "delta_stop" , required_argument, NULL, 'D' }, 333 | { "help" , no_argument , NULL, 'h' }, 334 | }; 335 | 336 | while ((opt = getopt_long(argc, argv, 337 | "d:p:o:r:i:s:R:F:D:h", long_opts, NULL)) != -1) { 338 | switch (opt) { 339 | case 'd': 340 | data_file = strdup(optarg); 341 | break; 342 | case 'p': 343 | pid = atoi(optarg); 344 | break; 345 | case 'o': 346 | out_file = strdup(optarg); 347 | break; 348 | case 'r': 349 | records = atoi(optarg); 350 | break; 351 | case 'i': 352 | increase_by = atoi(optarg); 353 | break; 354 | case 's': 355 | seconds = atoi(optarg); 356 | break; 357 | case 'R': 358 | report = strdup(optarg); 359 | break; 360 | case 'F': 361 | format = strdup(optarg); 362 | break; 363 | case 'D': 364 | delta_stop = atoi(optarg); 365 | break; 366 | case 'h': 367 | flb_help(EXIT_SUCCESS); 368 | break; 369 | }; 370 | }; 371 | 372 | if (!data_file) { 373 | fprintf(stderr, "error: no data file specified\n"); 374 | exit(EXIT_FAILURE); 375 | } 376 | 377 | if (records < 1) { 378 | fprintf(stderr, "error: invalid number of records '%i'\n", records); 379 | exit(EXIT_FAILURE); 380 | } 381 | 382 | if (seconds < 1) { 383 | fprintf(stderr, "error: invalid number of seconds '%i'\n", seconds); 384 | exit(EXIT_FAILURE); 385 | } 386 | 387 | if (!out_file) { 388 | fprintf(stderr, "warn: no output file has been specified, data will be send to " 389 | "STDOUT\n"); 390 | out_file = strdup("/dev/stdout"); 391 | } 392 | 393 | if (format) { 394 | if (strcasecmp(format, "markdown") == 0) { 395 | fmt_report = FLB_REPORT_MARKDOWN; 396 | } 397 | else if (strcasecmp(format, "text") == 0) { 398 | fmt_report = FLB_REPORT_TXT; 399 | } 400 | else if (strcasecmp(format, "csv") == 0) { 401 | fmt_report = FLB_REPORT_CSV; 402 | } 403 | else { 404 | fprintf(stderr, "error: invalid format type"); 405 | exit(EXIT_FAILURE); 406 | } 407 | } 408 | 409 | ret = run_fs_writer(pid, report, fmt_report, data_file, out_file, 410 | records, increase_by, seconds, delta_stop); 411 | if (ret == -1) { 412 | exit(EXIT_FAILURE); 413 | } 414 | 415 | return 0; 416 | } 417 | -------------------------------------------------------------------------------- /src/flb-tcp-writer.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2015-2018 Treasure Data Inc. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | /* local headers */ 34 | #include "mk_list.h" 35 | #include "flb_data_file.h" 36 | #include "flb_proc.h" 37 | #include "flb_report.h" 38 | #include "flb_network.h" 39 | 40 | /* Default values */ 41 | #define DEFAULT_RECORDS 1000 /* 1000 records per second */ 42 | #define DEFAULT_INC_BY 0 /* no increase */ 43 | #define DEFAULT_SECONDS 10 /* test time: 10 seconds */ 44 | #define DEFAULT_CONCURRENCY 1 /* one active connection */ 45 | 46 | /* Default network host and port */ 47 | #define DEFAULT_PORT "5170" 48 | #define DEFAULT_HOST "127.0.0.1" 49 | 50 | struct tcp_conn { 51 | int fd; 52 | struct mk_list _head; 53 | }; 54 | 55 | static void tcp_connect_destroy(struct mk_list *list) 56 | { 57 | struct mk_list *tmp; 58 | struct mk_list *head; 59 | struct tcp_conn *conn; 60 | 61 | mk_list_foreach_safe(head, tmp, list) { 62 | conn = mk_list_entry(head, struct tcp_conn, _head); 63 | mk_list_del(&conn->_head); 64 | if (conn->fd > 0) { 65 | close(conn->fd); 66 | } 67 | free(conn); 68 | } 69 | 70 | free(list); 71 | } 72 | 73 | static struct mk_list *tcp_connect_create(int connections, char *host, char *port) 74 | { 75 | int i; 76 | int fd; 77 | struct mk_list *list; 78 | struct tcp_conn *conn; 79 | 80 | list = malloc(sizeof(struct mk_list)); 81 | if (!list) { 82 | perror("malloc"); 83 | return NULL; 84 | } 85 | mk_list_init(list); 86 | 87 | for (i = 0; i < connections; i++) { 88 | conn = calloc(1, sizeof(struct tcp_conn)); 89 | if (!conn) { 90 | perror("calloc"); 91 | fprintf(stderr, "error creating connection #%i to %s:%s\n", 92 | i, host, port); 93 | tcp_connect_destroy(list); 94 | return NULL; 95 | } 96 | 97 | /* Perform TCP connection */ 98 | fd = flb_net_tcp_connect(host, port); 99 | if (fd == -1) { 100 | fprintf(stderr, "error creating connection #%i to %s:%s\n", 101 | i, host, port); 102 | tcp_connect_destroy(list); 103 | return NULL; 104 | } 105 | 106 | conn->fd = fd; 107 | mk_list_add(&conn->_head, list); 108 | } 109 | 110 | return list; 111 | } 112 | 113 | static int flb_help(int rc) 114 | { 115 | printf("Usage: flb-tcp-writer [OPTIONS]\n\n"); 116 | printf("Available options\n"); 117 | printf(" -c, --concurrency=N\t\tconcurrency level (default: %i)\n", 118 | DEFAULT_CONCURRENCY); 119 | printf(" -d, --datafile=PATH\t\tspecify source data file\n"); 120 | printf(" -p --pid=FLB_PID\t\tFluent Bit PID used gather metrics\n"); 121 | printf(" -o, --output=HOST:PORT\tset remote TCP Host and Port\n"); 122 | printf(" -i, --increase_by=N\t\tincrease N number of records per second (default: %i)\n", 123 | DEFAULT_INC_BY); 124 | printf(" -r, --records=RECORDS\t\trecords per second (default: %i)\n", 125 | DEFAULT_RECORDS); 126 | printf(" -s, --seconds=SECONDS\t\ttotal test time meassured in seconds (default: %i)\n", 127 | DEFAULT_SECONDS); 128 | printf(" -R, --report\t\t\tset report output file (default: stdout)\n"); 129 | printf(" -F, --format\t\t\treport format: text (default) or markdown\n"); 130 | printf(" -h, --help\t\t\tprint this help"); 131 | printf("\n\n"); 132 | exit(rc); 133 | } 134 | 135 | static void close_report(int fd) 136 | { 137 | if (fd == STDOUT_FILENO) { 138 | return; 139 | } 140 | 141 | if (fd != -1) { 142 | close(fd); 143 | } 144 | } 145 | 146 | static int run_tcp_writer(pid_t pid, 147 | char *report, 148 | int fmt_report, 149 | char *in_data_file, 150 | char *host, char *port, 151 | int n_cons, 152 | int records, int increase_by, 153 | int seconds) 154 | { 155 | int i; 156 | int x; 157 | int in_fd; 158 | int out_fd; 159 | int ret; 160 | int conn_records; 161 | int round_records; 162 | int report_fd = -1; 163 | int wait_time = 3; 164 | size_t round_bytes; 165 | off_t off = 0; 166 | off_t off_rec; 167 | off_t off_inc; 168 | char *data_buf; 169 | size_t data_size; 170 | ssize_t bytes; 171 | double total_cpu = 0; 172 | ssize_t total_mem = 0; 173 | size_t total_records = 0; 174 | ssize_t total_bytes = 0; 175 | char *proc_name = NULL; 176 | struct flb_proc_task *t1; 177 | struct flb_proc_task *t2; 178 | struct flb_report *r = NULL; 179 | struct mk_list *head; 180 | struct mk_list *connections; 181 | struct tcp_conn *conn; 182 | time_t start_time; 183 | time_t end_time; 184 | 185 | /* Report file for process monitoring */ 186 | if (pid >= 0) { 187 | r = flb_report_create(report, fmt_report, pid, wait_time); 188 | if (!r) { 189 | fprintf(stderr, "error: cannot initialize report"); 190 | return -1; 191 | } 192 | } 193 | 194 | /* Create TCP connections */ 195 | connections = tcp_connect_create(n_cons, host, port); 196 | if (!connections) { 197 | return -1; 198 | } 199 | 200 | /* Load input data file in-memory */ 201 | in_fd = flb_data_file_load(in_data_file, &data_buf, &data_size); 202 | if (in_fd == -1) { 203 | close(out_fd); 204 | fprintf(stderr, "error: cannot load input data file '%s'\n", 205 | in_data_file); 206 | if (r) { 207 | flb_report_destroy(r); 208 | } 209 | return -1; 210 | } 211 | 212 | /* Get the number of records that will be send per connection */ 213 | conn_records = (records / n_cons); 214 | 215 | /* Retrieve the final offset of 'records' */ 216 | ret = flb_data_file_offset_records(conn_records, data_buf, data_size, &off_rec); 217 | if (ret == -1) { 218 | fprintf(stderr, "error: cannot find %i number of records\n", conn_records); 219 | flb_data_file_unload(data_buf, data_size); 220 | close(in_fd); 221 | close(out_fd); 222 | if (r) { 223 | flb_report_destroy(r); 224 | } 225 | return -1; 226 | } 227 | 228 | /* Retrieve the final offset of 'records' */ 229 | ret = flb_data_file_offset_records(increase_by, data_buf, data_size, 230 | &off_inc); 231 | if (ret == -1) { 232 | fprintf(stderr, "error: cannot find %i number of records\n", conn_records); 233 | flb_data_file_unload(data_buf, data_size); 234 | close(in_fd); 235 | close(out_fd); 236 | if (r) { 237 | flb_report_destroy(r); 238 | } 239 | return -1; 240 | } 241 | 242 | /* Get Process name */ 243 | if (pid >= 0) { 244 | t1 = flb_proc_stat_create(pid); 245 | proc_name = strndup(t1->name + 1, strlen(t1->name) - 2); 246 | } 247 | 248 | /* Set the start time */ 249 | start_time = time(NULL); 250 | 251 | /* 252 | * Just write data chunks every second. Since a write operation will 253 | * put the data into a kernel buffer (or zero copy) and likely return 254 | * immediately, we don't care about the time invested in that operation, 255 | * just the wait time between writes. 256 | */ 257 | for (i = 0; i < seconds; i++) { 258 | round_bytes = 0; 259 | round_records = 0; 260 | 261 | if (pid >= 0 && i == 0) { 262 | t1 = flb_proc_stat_create(pid); 263 | if (!t1) { 264 | fprintf(stderr, "error gathering stats for PID %i\n", 265 | (int) pid); 266 | } 267 | } 268 | 269 | mk_list_foreach(head, connections) { 270 | conn = mk_list_entry(head, struct tcp_conn, _head); 271 | out_fd = conn->fd; 272 | 273 | /* 274 | * Use zero-copy strategy with sendfile(2). In benchmarking we want 275 | * to avoid extra Kernel work, this is a Linux specific feature. 276 | */ 277 | off = 0; 278 | /* Dispatch the records chunk */ 279 | bytes = sendfile(out_fd, in_fd, &off, off_rec); 280 | if (bytes == -1) { 281 | perror("sendfile"); 282 | fprintf(stderr, "error: exception on writing records chunk\n"); 283 | } 284 | else { 285 | total_bytes += bytes; 286 | total_records += records; 287 | round_records += records; 288 | round_bytes += bytes; 289 | } 290 | 291 | if (increase_by > 0 && i > 0) { 292 | /* 293 | * Send incremental records, yeah, this involve a second system 294 | * call but it's better than writev() and data-copy. 295 | */ 296 | for (x = 0; x < i; x++) { 297 | off = 0; 298 | bytes = sendfile(out_fd, in_fd, &off, off_inc); 299 | if (bytes == -1) { 300 | perror("sendfile"); 301 | fprintf(stderr, "error: cannot write inc records chunk\n"); 302 | } 303 | else { 304 | total_bytes += bytes; 305 | total_records += increase_by; 306 | round_records += increase_by; 307 | round_bytes += bytes; 308 | } 309 | } 310 | } 311 | } 312 | 313 | /* Dummy sleep */ 314 | sleep(1); 315 | 316 | /* Get stats */ 317 | if (pid >= 0) { 318 | t2 = flb_proc_stat_create(pid); 319 | if (!t2) { 320 | fprintf(stderr, "error gathering stats for PID %i\n", 321 | (int) pid); 322 | } 323 | 324 | if (r) { 325 | flb_report_stats(r, round_records, round_bytes, t1, t2); 326 | } 327 | 328 | /* Count total number of bytes in memory */ 329 | total_cpu += flb_report_cpu_usage(r, t1, t2); 330 | total_mem += t2->r_rss; 331 | 332 | flb_proc_stat_destroy(t1); 333 | t1 = t2; 334 | 335 | } 336 | } 337 | 338 | if (t1) { 339 | flb_proc_stat_destroy(t1); 340 | } 341 | 342 | /* 343 | * Create continuos snapshots until resources consumption (CPU) stabilize, 344 | * we assume that after two seconds without deltas in user time the process 345 | * finished processing our records. 346 | */ 347 | if (pid >= 0) { 348 | int count = 0; 349 | int loops = 0; 350 | 351 | while (1) { 352 | t1 = flb_proc_stat_create(pid); 353 | sleep(1); 354 | t2 = flb_proc_stat_create(pid); 355 | flb_report_stats(r, 0, 0, t1, t2); 356 | loops++; 357 | 358 | if ((t2->r_utime_ms - t1->r_utime_ms) == 0) { 359 | count++; 360 | } 361 | else { 362 | if (count > 0) { 363 | count = 0; 364 | } 365 | } 366 | 367 | if (count >= wait_time) { 368 | flb_proc_stat_destroy(t1); 369 | flb_proc_stat_destroy(t2); 370 | break; 371 | } 372 | 373 | /* Count total number of bytes in memory */ 374 | total_cpu += flb_report_cpu_usage(r, t1, t2); 375 | total_mem += t2->r_rss; 376 | 377 | flb_proc_stat_destroy(t1); 378 | flb_proc_stat_destroy(t2); 379 | 380 | } 381 | r->sum_records = total_records; 382 | flb_report_summary(r); 383 | } 384 | 385 | if (r) { 386 | flb_report_destroy(r); 387 | } 388 | 389 | if (proc_name) { 390 | free(proc_name); 391 | } 392 | 393 | close(out_fd); 394 | } 395 | 396 | int main(int argc, char **argv) 397 | { 398 | int ret; 399 | int opt; 400 | int concurrency = DEFAULT_CONCURRENCY; 401 | int records = DEFAULT_RECORDS; 402 | int seconds = DEFAULT_SECONDS; 403 | int increase_by = DEFAULT_INC_BY; 404 | int out_fd; 405 | int pid = -1; 406 | int fd_report; 407 | int fmt_report = FLB_REPORT_TXT; 408 | char *format = NULL; 409 | char *report = NULL; 410 | char *out_host = NULL; 411 | char *data_file = NULL; 412 | char *host = NULL; 413 | char *port = NULL; 414 | 415 | /* Setup long-options */ 416 | static const struct option long_opts[] = { 417 | { "concurrency", required_argument, NULL, 'c' }, 418 | { "datafile" , required_argument, NULL, 'd' }, 419 | { "pid" , required_argument, NULL, 'p' }, 420 | { "output" , required_argument, NULL, 'o' }, 421 | { "records" , required_argument, NULL, 'r' }, 422 | { "increase_by", required_argument, NULL, 'i' }, 423 | { "seconds" , required_argument, NULL, 's' }, 424 | { "report" , required_argument, NULL, 'R' }, 425 | { "format" , required_argument, NULL, 'F' }, 426 | { "help" , no_argument , NULL, 'h' }, 427 | }; 428 | 429 | while ((opt = getopt_long(argc, argv, 430 | "c:d:p:o:r:i:s:R:F:h", long_opts, NULL)) != -1) { 431 | switch (opt) { 432 | case 'c': 433 | concurrency = atoi(optarg); 434 | break; 435 | case 'd': 436 | data_file = strdup(optarg); 437 | break; 438 | case 'p': 439 | pid = atoi(optarg); 440 | break; 441 | case 'o': 442 | out_host = strdup(optarg); 443 | break; 444 | case 'r': 445 | records = atoi(optarg); 446 | break; 447 | case 'i': 448 | increase_by = atoi(optarg); 449 | break; 450 | case 's': 451 | seconds = atoi(optarg); 452 | break; 453 | case 'R': 454 | report = strdup(optarg); 455 | break; 456 | case 'F': 457 | format = strdup(optarg); 458 | break; 459 | case 'h': 460 | flb_help(EXIT_SUCCESS); 461 | break; 462 | }; 463 | }; 464 | 465 | if (!data_file) { 466 | fprintf(stderr, "error: no data file specified\n"); 467 | exit(EXIT_FAILURE); 468 | } 469 | 470 | if (records < 1) { 471 | fprintf(stderr, "error: invalid number of records '%i'\n", records); 472 | exit(EXIT_FAILURE); 473 | } 474 | 475 | if (seconds < 1) { 476 | fprintf(stderr, "error: invalid number of seconds '%i'\n", seconds); 477 | exit(EXIT_FAILURE); 478 | } 479 | 480 | if (!out_host) { 481 | host = strdup(DEFAULT_HOST); 482 | port = strdup(DEFAULT_PORT); 483 | } 484 | else { 485 | /* Parse host and port */ 486 | char *p; 487 | char *sep; 488 | 489 | p = strchr(out_host, ':'); 490 | if (!p) { 491 | host = strdup(out_host); 492 | port = strdup(DEFAULT_PORT); 493 | } 494 | else { 495 | host = strndup(out_host, p - out_host); 496 | *p++; 497 | if (!p) { 498 | port = strdup(DEFAULT_PORT); 499 | } 500 | else { 501 | port = strdup(p); 502 | } 503 | } 504 | } 505 | 506 | if (format) { 507 | if (strcasecmp(format, "markdown") == 0) { 508 | fmt_report = FLB_REPORT_MARKDOWN; 509 | } 510 | else if (strcasecmp(format, "text") == 0) { 511 | fmt_report = FLB_REPORT_TXT; 512 | } 513 | else { 514 | fprintf(stderr, "error: invalid format type"); 515 | exit(EXIT_FAILURE); 516 | } 517 | } 518 | 519 | ret = run_tcp_writer(pid, report, fmt_report, data_file, 520 | host, port, 521 | concurrency, records, increase_by, seconds); 522 | 523 | free(report); 524 | free(format); 525 | free(data_file); 526 | free(host); 527 | free(port); 528 | 529 | if (ret == -1) { 530 | exit(EXIT_FAILURE); 531 | } 532 | 533 | return 0; 534 | } 535 | -------------------------------------------------------------------------------- /src/flb_data_file.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2015-2018 Treasure Data Inc. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | static void *mmap_file(int fd, size_t *file_size) 31 | { 32 | int ret; 33 | int oflags = 0; 34 | size_t size = 0; 35 | void *map; 36 | struct stat fst; 37 | 38 | /* Validate file descriptor */ 39 | ret = fstat(fd, &fst); 40 | if (ret == -1) { 41 | fprintf(stderr, "error: loading mmap file, file descriptor = '%i'\n", 42 | fd); 43 | return NULL; 44 | } 45 | size = fst.st_size; 46 | 47 | if (size == 0) { 48 | fprintf(stderr, "error: data file size is zero\n"); 49 | return NULL; 50 | } 51 | 52 | /* Mmap flags */ 53 | oflags = PROT_READ; 54 | 55 | map = mmap(0, size, oflags, MAP_PRIVATE, fd, 0); 56 | if (map == MAP_FAILED) { 57 | fprintf(stderr, "error: mapping file, file descriptor = '%i'\n", fd); 58 | return NULL; 59 | } 60 | 61 | *file_size = size; 62 | 63 | return map; 64 | } 65 | 66 | int flb_data_file_load(char *path, char **out_buf, size_t *out_size) 67 | { 68 | int fd; 69 | char *buf; 70 | size_t size; 71 | 72 | fd = open(path, O_RDONLY); 73 | if (fd == -1) { 74 | perror("open"); 75 | fprintf(stderr, "error: cannot open data file '%s'\n", path); 76 | return -1; 77 | } 78 | 79 | buf = mmap_file(fd, &size); 80 | if (!buf) { 81 | close(fd); 82 | return -1; 83 | } 84 | 85 | *out_buf = buf; 86 | *out_size = size; 87 | 88 | return fd; 89 | } 90 | 91 | void flb_data_file_unload(void *map, size_t size) 92 | { 93 | munmap(map, size); 94 | } 95 | 96 | /* 97 | * Given a buffer and it size, retrieve the offset position that have N number 98 | * of records (lines). 99 | */ 100 | int flb_data_file_offset_records(int n_records, char *buf, 101 | size_t size, off_t *offset) 102 | { 103 | int total = 0; 104 | char *p; 105 | char *start; 106 | char *end; 107 | 108 | p = buf; 109 | start = buf; 110 | end = buf + size; 111 | while (start < end) { 112 | p = strchr(start, '\n'); 113 | if (p) { 114 | total++; 115 | } 116 | else { 117 | break; 118 | } 119 | 120 | if (total == n_records) { 121 | break; 122 | } 123 | 124 | start = p + 1; 125 | } 126 | 127 | if (total < n_records) { 128 | return -1; 129 | } 130 | 131 | *offset = (p - buf) + 1; 132 | return 0; 133 | } 134 | -------------------------------------------------------------------------------- /src/flb_network.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2015-2018 Treasure Data Inc. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | int flb_net_socket_create(int family) 30 | { 31 | int fd; 32 | 33 | /* create the socket and set the nonblocking flag status */ 34 | fd = socket(family, SOCK_STREAM, 0); 35 | if (fd == -1) { 36 | perror("socket"); 37 | return -1; 38 | } 39 | 40 | return fd; 41 | } 42 | 43 | int flb_net_tcp_connect(char *host, char *port) 44 | { 45 | int fd = -1; 46 | int ret; 47 | struct addrinfo hints; 48 | struct addrinfo *res, *rp; 49 | 50 | memset(&hints, 0, sizeof hints); 51 | hints.ai_family = AF_UNSPEC; 52 | hints.ai_socktype = SOCK_STREAM; 53 | 54 | ret = getaddrinfo(host, port, &hints, &res); 55 | if (ret != 0) { 56 | fprintf(stderr, "net_tcp_connect: getaddrinfo(host='%s'): %s", 57 | host, gai_strerror(ret)); 58 | return -1; 59 | } 60 | 61 | for (rp = res; rp != NULL; rp = rp->ai_next) { 62 | fd = flb_net_socket_create(rp->ai_family); 63 | if (fd == -1) { 64 | fprintf(stderr, "net_tcp_connect: error creating client socket"); 65 | continue; 66 | } 67 | 68 | if (connect(fd, rp->ai_addr, rp->ai_addrlen) == -1) { 69 | fprintf(stderr, "net_tcp_connect: cannot connect to %s:%s", 70 | host, port); 71 | close(fd); 72 | continue; 73 | } 74 | break; 75 | } 76 | 77 | freeaddrinfo(res); 78 | 79 | if (rp == NULL) { 80 | return -1; 81 | } 82 | 83 | return fd; 84 | } 85 | -------------------------------------------------------------------------------- /src/flb_proc.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2014-2019 Eduardo Silva 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | /* 22 | * FYI: I wrote this code back in 2014, it's being modified for this project 23 | * purposes, original code is located here: 24 | * 25 | * https://github.com/edsiper/wr 26 | */ 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #include "flb_proc.h" 36 | 37 | /* 38 | * Read file content to a memory buffer, 39 | * Use this function just for really SMALL files 40 | */ 41 | static char *file_to_buffer(const char *path) 42 | { 43 | FILE *fp; 44 | char *buffer; 45 | long bytes; 46 | 47 | if (access(path, R_OK) != 0) { 48 | perror("access"); 49 | fprintf(stderr, "error: cannot open '%s'\n", path); 50 | return NULL; 51 | } 52 | 53 | if (!(fp = fopen(path, "r"))) { 54 | perror("fopen"); 55 | fprintf(stderr, "error: cannot fopen '%s'\n", path); 56 | return NULL; 57 | } 58 | 59 | buffer = malloc(PROC_STAT_BUF_SIZE); 60 | if (!buffer) { 61 | perror("malloc"); 62 | fprintf(stderr, "error: could not allocate memory to open '%s'\n", 63 | path); 64 | fclose(fp); 65 | exit(EXIT_FAILURE); 66 | } 67 | 68 | bytes = fread(buffer, PROC_STAT_BUF_SIZE, 1, fp); 69 | if (bytes < 0) { 70 | free(buffer); 71 | fclose(fp); 72 | printf("Error: could not read '%s'\n", path); 73 | exit(EXIT_FAILURE); 74 | } 75 | 76 | fclose(fp); 77 | return buffer; 78 | } 79 | 80 | struct flb_proc_task *flb_proc_stat_create(pid_t pid) 81 | { 82 | int ret; 83 | int fields; 84 | int cpu_hz = sysconf(_SC_CLK_TCK); 85 | char *p, *q; 86 | char *buf; 87 | char pid_path[PROC_PID_SIZE]; 88 | char tmp[256]; 89 | struct flb_proc_task *t; 90 | 91 | t = calloc(1, sizeof(struct flb_proc_task)); 92 | if (!t) { 93 | perror("calloc"); 94 | return NULL; 95 | } 96 | 97 | /* Compose path for /proc/PID/stat */ 98 | ret = snprintf(pid_path, PROC_PID_SIZE, "/proc/%i/stat", pid); 99 | if (ret < 0) { 100 | fprintf(stderr, "error: could not compose PID path: %s\n", pid_path); 101 | free(t); 102 | return NULL; 103 | } 104 | 105 | buf = file_to_buffer(pid_path); 106 | if (!buf) { 107 | fprintf(stderr, "error: could not read stat file data: %s\n", pid_path); 108 | free(t); 109 | return NULL; 110 | } 111 | 112 | p = buf; 113 | while (*p != ')') *p++; 114 | *p++; 115 | 116 | /* Read pending values */ 117 | char *x = "%*d %s %*c %*d %*d %*d %*d %*d %*lu %*lu %*lu %*lu %*lu %lu %lu %*ld %*ld %*ld %*ld %*ld %*ld %*lu %*lu %ld "; 118 | 119 | fields = sscanf(buf, x, 120 | &tmp, 121 | &t->utime, 122 | &t->stime, 123 | &t->rss); 124 | 125 | p = strndup(tmp + 1, strlen(tmp + 1) - 1); 126 | memcpy(t->name, p, strlen(p)); 127 | 128 | /* Internal conversion */ 129 | t->r_rss = (t->rss * getpagesize()); 130 | t->r_utime_s = (t->utime / cpu_hz); 131 | t->r_utime_ms = ((t->utime * 1000) / cpu_hz); 132 | t->r_stime_s = (t->stime / cpu_hz); 133 | t->r_stime_ms = ((t->stime * 1000) / cpu_hz); 134 | 135 | /* Set timestamp */ 136 | clock_gettime(CLOCK_REALTIME, &t->ts); 137 | 138 | free(buf); 139 | return t; 140 | } 141 | 142 | void flb_proc_stat_destroy(struct flb_proc_task *t) 143 | { 144 | free(t); 145 | } 146 | 147 | void flb_proc_stat_print(struct flb_proc_task *t) 148 | { 149 | printf("utime = %lu\n", t->utime); 150 | printf("stime = %lu\n", t->stime); 151 | printf("rss = %ld\n", t->rss); 152 | fflush(stdout); 153 | } 154 | -------------------------------------------------------------------------------- /src/flb_report.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2015-2018 Treasure Data Inc. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include "flb_proc.h" 30 | #include "flb_report.h" 31 | 32 | #define BILLION 1000000000.0 33 | 34 | char *flb_report_human_readable_size(long size) 35 | { 36 | long u = 1024, i, len = 128; 37 | char *buf = malloc(len); 38 | static const char *__units[] = { "b", "K", "M", "G", 39 | "T", "P", "E", "Z", "Y", NULL 40 | }; 41 | 42 | for (i = 0; __units[i] != NULL; i++) { 43 | if ((size / u) == 0) { 44 | break; 45 | } 46 | u *= 1024; 47 | } 48 | if (!i) { 49 | snprintf(buf, len, "%ld %s", size, __units[0]); 50 | } 51 | else { 52 | float fsize = (float) ((double) size / (u / 1024)); 53 | snprintf(buf, len, "%.2f%s", fsize, __units[i]); 54 | } 55 | 56 | return buf; 57 | } 58 | 59 | static void report_txt_header(struct flb_report *r) 60 | { 61 | dprintf(r->fd, 62 | " records write (b) write secs | %% cpu user (ms) " 63 | "sys (ms) Mem (bytes) Mem\n"); 64 | dprintf(r->fd, 65 | "-------- ---------- -------- ----- + ------ --------- " 66 | "-------- ----------- -------\n"); 67 | } 68 | 69 | static void report_markdown_header(struct flb_report *r) 70 | { 71 | dprintf(r->fd, 72 | "| records | write (b) | write | secs | %%cpu | user (ms) " 73 | "| sys (ms) | Mem (bytes) | Mem |\n"); 74 | dprintf(r->fd, 75 | "| ---: | ---: | ---: | ---: | ---: | ---: " 76 | "| ---: | ---: |---: |\n"); 77 | } 78 | 79 | static void report_csv_header(struct flb_report *r) 80 | { 81 | dprintf(r->fd, 82 | "records,write_bytes,write_human,secs,cpu,user_ms,sys_ms," 83 | "mem_bytes,mem_human\n"); 84 | } 85 | 86 | struct flb_report *flb_report_create(char *out, int format, int pid, int wait) 87 | { 88 | int fd; 89 | char *target; 90 | struct flb_report *r; 91 | struct flb_proc_task *t; 92 | 93 | if (out) { 94 | fd = open(out, O_CREAT | O_TRUNC | O_WRONLY, 0666); 95 | if (fd == -1) { 96 | perror("open"); 97 | fprintf(stderr, "error: cannot open/create report file '%s'\n", 98 | out); 99 | return NULL; 100 | } 101 | } 102 | else { 103 | fd = STDOUT_FILENO; 104 | } 105 | 106 | r = calloc(1, sizeof(struct flb_report)); 107 | if (!r) { 108 | perror("malloc"); 109 | close(fd); 110 | return NULL; 111 | } 112 | r->fd = fd; 113 | r->format = format; 114 | r->cpu_count = sysconf(_SC_NPROCESSORS_ONLN); 115 | r->cpu_ticks = sysconf(_SC_CLK_TCK); 116 | r->snapshots = 0; 117 | r->wait_time = wait; 118 | r->pid = pid; 119 | r->sum_bytes = 0; 120 | r->sum_mem = 0; 121 | r->sum_cpu = 0.0; 122 | r->sum_records = 0; 123 | 124 | if (r->pid >= 0) { 125 | t = flb_proc_stat_create(r->pid); 126 | if (!t) { 127 | fprintf(stderr, "error: cannot get process stats"); 128 | close(fd); 129 | return NULL; 130 | } 131 | r->name = strdup(t->name); 132 | flb_proc_stat_destroy(t); 133 | } 134 | 135 | if (r->format == FLB_REPORT_TXT) { 136 | report_txt_header(r); 137 | } 138 | else if (r->format == FLB_REPORT_MARKDOWN) { 139 | report_markdown_header(r); 140 | } 141 | else if (r->format == FLB_REPORT_CSV) { 142 | report_csv_header(r); 143 | } 144 | 145 | return r; 146 | } 147 | 148 | double flb_report_cpu_usage(struct flb_report *r, 149 | struct flb_proc_task *t1, struct flb_proc_task *t2) 150 | { 151 | double diff; 152 | double cpu; 153 | double delta_sec; 154 | double delta_nsec; 155 | 156 | /* Calculate overall CPU usage */ 157 | diff = ((t2->utime + t2->stime) - (t1->utime + t1->stime)); 158 | cpu = ((diff * 100.0) / (double) r->cpu_ticks); 159 | 160 | /* Check and adjust based on time invested in the test */ 161 | delta_sec = (t2->ts.tv_sec - t1->ts.tv_sec); 162 | delta_nsec = (t2->ts.tv_nsec - t1->ts.tv_nsec); 163 | cpu /= (delta_sec + 1e-9 * delta_nsec); 164 | 165 | return cpu; 166 | } 167 | 168 | int flb_report_stats(struct flb_report *r, int records, 169 | size_t bytes, 170 | struct flb_proc_task *t1, struct flb_proc_task *t2) 171 | { 172 | char *rss_hr; 173 | char *bytes_hr; 174 | double cpu; 175 | double duration; 176 | 177 | r->snapshots++; 178 | 179 | /* Calculate CPU usage */ 180 | cpu = flb_report_cpu_usage(r, t1, t2); 181 | 182 | /* Duration: elapsed time between time snapshots */ 183 | duration = (t2->ts.tv_sec - t1->ts.tv_sec) + 184 | (t2->ts.tv_nsec - t1->ts.tv_nsec) / BILLION; 185 | 186 | /* Convert bytes to human readable size */ 187 | rss_hr = flb_report_human_readable_size(t2->r_rss); 188 | bytes_hr = flb_report_human_readable_size(bytes); 189 | 190 | /* Add values */ 191 | r->sum_mem += t2->r_rss; 192 | if (cpu > 0.0) { 193 | r->sum_cpu += cpu; 194 | r->sum_cpu_count++; 195 | } 196 | r->sum_duration += duration; 197 | r->sum_bytes += bytes; 198 | 199 | if (r->format == FLB_REPORT_TXT) { 200 | dprintf(r->fd, "%8d %10zu %8s %5.2lf | %6.2lf %9ld %8ld %12ld %8s\n", 201 | records, 202 | bytes, 203 | bytes_hr, 204 | duration, 205 | cpu, 206 | (t2->r_utime_ms - t1->r_utime_ms), 207 | (t2->r_stime_ms - t1->r_stime_ms), 208 | t2->r_rss, 209 | rss_hr); 210 | } 211 | else if (r->format == FLB_REPORT_MARKDOWN) { 212 | dprintf(r->fd, 213 | "| %d | %zu | %s | %.2lf | %.2lf | %ld | %ld | " 214 | "%ld | %s |\n", 215 | records, 216 | bytes, 217 | bytes_hr, 218 | duration, 219 | cpu, 220 | (t2->r_utime_ms - t1->r_utime_ms), 221 | (t2->r_stime_ms - t1->r_stime_ms), 222 | t2->r_rss, 223 | rss_hr); 224 | } 225 | else if (r->format == FLB_REPORT_CSV) { 226 | dprintf(r->fd, "%d,%zu,%s,%.2lf,%.2lf,%ld,%ld,%ld,%s\n", 227 | records, 228 | bytes, 229 | bytes_hr, 230 | duration, 231 | cpu, 232 | (t2->r_utime_ms - t1->r_utime_ms), 233 | (t2->r_stime_ms - t1->r_stime_ms), 234 | t2->r_rss, 235 | rss_hr); 236 | } 237 | 238 | free(rss_hr); 239 | free(bytes_hr); 240 | } 241 | 242 | int flb_report_summary(struct flb_report *r) 243 | { 244 | char *tmp; 245 | char unit[32]; 246 | double duration = r->sum_duration - r->wait_time; 247 | 248 | dprintf(r->fd, "\n"); 249 | dprintf(r->fd, "- Summary\n"); 250 | dprintf(r->fd, " - Process : %s\n", r->name); 251 | dprintf(r->fd, " - PID : %i\n", r->pid); 252 | dprintf(r->fd, " - Elapsed Time: %.2lf seconds\n", duration); 253 | 254 | tmp = flb_report_human_readable_size(r->sum_mem / r->snapshots); 255 | dprintf(r->fd, " - Avg Memory : %s\n", tmp); 256 | free(tmp); 257 | 258 | if (r->sum_cpu_count == 0) { 259 | r->sum_cpu_count = r->snapshots; 260 | } 261 | 262 | dprintf(r->fd, " - Avg CPU : %.2lf%%\n", r->sum_cpu / r->sum_cpu_count); 263 | 264 | tmp = flb_report_human_readable_size(r->sum_bytes / r->sum_duration); 265 | dprintf(r->fd, " - Avg Rate : %s/sec\n", tmp); 266 | tmp = flb_report_human_readable_size(r->sum_records / r->sum_duration); 267 | dprintf(r->fd, " - Avg Records : %s/sec\n", tmp); 268 | 269 | free(tmp); 270 | } 271 | 272 | int flb_report_destroy(struct flb_report *r) 273 | { 274 | if (r->fd != STDOUT_FILENO) { 275 | close(r->fd); 276 | } 277 | 278 | if (r->name) { 279 | free(r->name); 280 | } 281 | free(r); 282 | return 0; 283 | } 284 | -------------------------------------------------------------------------------- /src/flb_utils.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ 2 | 3 | /* Fluent Bit 4 | * ========== 5 | * Copyright (C) 2019 The Fluent Bit Authors 6 | * Copyright (C) 2015-2018 Treasure Data Inc. 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); 9 | * you may not use this file except in compliance with the License. 10 | * You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | -------------------------------------------------------------------------------- /test-scenario/fluent-bit-tail-tcp.yaml: -------------------------------------------------------------------------------- 1 | service: 2 | flush: 1 3 | log_level: info 4 | 5 | parsers: 6 | - name: json 7 | format: json 8 | 9 | pipeline: 10 | inputs: 11 | - name: tail 12 | path: /tmp/test-scenario-fluentbit.log 13 | read_from_head: true 14 | refresh_interval: 1 15 | parser: json 16 | 17 | outputs: 18 | - name: null 19 | match: '*' 20 | -------------------------------------------------------------------------------- /test-scenario/fluentd.conf: -------------------------------------------------------------------------------- 1 | 2 | @type tail 3 | path /tmp/test-scenario-fluentd.log 4 | pos_file tmp-test-scenario.pos 5 | tag test.scenario 6 | read_from_head true 7 | refresh_interval 1 8 | 9 | @type json 10 | 11 | 12 | 13 | # 14 | # @type record_transformer 15 | # 16 | # custom_key_1 "value1" 17 | # custom_key_2 "value2" 18 | # custom_key_3 "value3" 19 | # custom_key_4 "value4" 20 | # 21 | # 22 | 23 | 24 | @type null 25 | 26 | -------------------------------------------------------------------------------- /test-scenario/otel-collector.yaml: -------------------------------------------------------------------------------- 1 | receivers: 2 | filelog: 3 | include: [/tmp/test-scenario-otel-contrib.log] 4 | operators: 5 | - type: json_parser 6 | 7 | # processors: 8 | # attributes: 9 | # actions: 10 | # - key: custom_key_1 11 | # value: "value1" 12 | # action: insert 13 | # - key: custom_key_2 14 | # value: "value2" 15 | # action: insert 16 | # - key: custom_key_3 17 | # value: "value3" 18 | # action: insert 19 | # - key: custom_key_4 20 | # value: "value4" 21 | # action: insert 22 | 23 | exporters: 24 | nop: {} 25 | 26 | service: 27 | pipelines: 28 | logs: 29 | receivers: [filelog] 30 | processors: [] 31 | exporters: [nop] 32 | telemetry: 33 | logs: 34 | level: "debug" 35 | 36 | -------------------------------------------------------------------------------- /test-scenario/run.sh: -------------------------------------------------------------------------------- 1 | 2 | rm -rf /tmp/test-sc*.log 3 | rm -rf reports/* 4 | 5 | FLB_PERF=/home/edsiper/c/fluent-bit-perf/build/bin/flb-tail-writer 6 | DATA_FILE=performance_test_data.txt 7 | 8 | RECORDS_PER_SECOND=5000 9 | INCREASE_RECORDS=5000 10 | SECONDS=60 11 | 12 | OPTIONS="-i $INCREASE_RECORDS -r $RECORDS_PER_SECOND -s $SECONDS -F csv" 13 | 14 | # Fluent Bit 15 | # ---------- 16 | 17 | # Check that Fluent Bit is running 18 | if ! pidof fluent-bit > /dev/null; then 19 | echo "Error: Fluent Bit is not running." 20 | exit 1 21 | fi 22 | 23 | echo "Testing Fluent Bit..." 24 | # run with: fluent-bit -c test-scenario/fluent-bit-tail-tcp.yaml 25 | $FLB_PERF -d $DATA_FILE -p `pidof fluent-bit` -o /tmp/test-scenario-fluentbit.log $OPTIONS > fluent-bit.csv 26 | 27 | 28 | # Vector 29 | # ------ 30 | echo "Testing Vector..." 31 | 32 | # Check that Vector is running 33 | if ! pidof vector > /dev/null; then 34 | echo "Error: Vector is not running." 35 | exit 1 36 | fi 37 | 38 | # run with: vector -c test-scenario/vector.toml 39 | $FLB_PERF -d $DATA_FILE -p `pidof vector` -o /tmp/test-scenario-vector.log $OPTIONS > vector.csv 40 | 41 | 42 | 43 | # Fluentd 44 | # ------- 45 | echo "Testing Fluentd..." 46 | # run with fluentd --no-supervisor -c test-scenario/fluentd.conf 47 | 48 | # Check that Fluentd is running 49 | FLUENTD_PID=`ps aux | grep fluentd | grep -v grep | awk '{print $2}'` 50 | 51 | if [ -z "$FLUENTD_PID" ]; then 52 | echo "Error: Fluentd is not running." 53 | exit 1 54 | fi 55 | 56 | # Fluentd must be run with --no-supervisor option 57 | $FLB_PERF -d $DATA_FILE -p $FLUENTD_PID -o /tmp/test-scenario-fluentd.log $OPTIONS > fluentd.csv 58 | 59 | # OpenTelemetry Collector Contrib 60 | # ------------------------------- 61 | echo "Testing OpenTelemetry Collector Contrib..." 62 | # run with: otelcol-contrib -config test-scenario/otel-collector.yaml 63 | 64 | # Check that OpenTelemetry Collector Contrib is running 65 | if ! pidof otelcol-contrib > /dev/null; then 66 | echo "Error: OpenTelemetry Collector Contrib is not running." 67 | exit 1 68 | fi 69 | 70 | $FLB_PERF -d $DATA_FILE -p `pidof otelcol-contrib` -o /tmp/test-scenario-otel-contrib.log $OPTIONS -D 200 > otelcol-contrib.csv 71 | 72 | echo "done." 73 | -------------------------------------------------------------------------------- /test-scenario/vector.toml: -------------------------------------------------------------------------------- 1 | data_dir = "/tmp/vector" 2 | 3 | # Data Sources 4 | [sources.file] 5 | type = "file" 6 | include = ["/tmp/test-scenario-vector.log"] 7 | read_from = "beginning" 8 | 9 | # Parse JSON 10 | [transforms.json_parser] 11 | type = "remap" 12 | inputs = ["file"] 13 | source = ''' 14 | . = parse_json!(.message) 15 | ''' 16 | 17 | # Add Non-Nested Keys 18 | #[transforms.add_keys] 19 | # type = "remap" 20 | # inputs = ["json_parser"] 21 | # source = ''' 22 | # .custom_key_1 = "value1" 23 | # .custom_key_2 = "value2" 24 | # .custom_key_3 = "value3" 25 | # .custom_key_4 = "value4" 26 | # ''' 27 | 28 | # Data Sinks 29 | [sinks.null] 30 | type = "blackhole" # Blackhole sink to discard the data 31 | inputs = ["json_parser"] 32 | healthcheck = false 33 | --------------------------------------------------------------------------------