├── .gitignore ├── CHANGELOG.md ├── CMakeLists.txt ├── LICENSE ├── README.md ├── performance.md ├── sample ├── CMakeLists.txt ├── log_config.c ├── log_config.h ├── log_post_logs_sample.c ├── log_producer_benchmark.c ├── log_producer_sample.c └── video_frame_producer_sample.c ├── save_send_buffer.md └── src ├── hmac-sha.c ├── hmac-sha.h ├── inner_log.c ├── inner_log.h ├── log_api.c ├── log_api.h ├── log_builder.c ├── log_builder.h ├── log_define.h ├── log_inner_include.h ├── log_multi_thread.h ├── log_producer_client.c ├── log_producer_client.h ├── log_producer_common.c ├── log_producer_common.h ├── log_producer_config.c ├── log_producer_config.h ├── log_producer_manager.c ├── log_producer_manager.h ├── log_producer_sender.c ├── log_producer_sender.h ├── log_queue.c ├── log_queue.h ├── log_util.c ├── log_util.h ├── lz4.c ├── lz4.h ├── md5.c ├── md5.h ├── sds.c ├── sds.h ├── sha1.c ├── sha1.h ├── sha256.c ├── sha256.h ├── zstd.c └── zstd.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac OS X 2 | .DS_Store 3 | 4 | # CMake 5 | CMakeCache.txt 6 | 7 | build/ 8 | .vscode/ 9 | CMakeFiles 10 | Makefile 11 | cmake_install.cmake 12 | install_manifest.txt -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log (Producer Lite) 2 | ## 0.1.0 (2018-02-27) 3 | Producer Lite版本发布 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | PROJECT(log_c_sdk) 2 | 3 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 4 | 5 | set(CMAKE_VERSION 2.0.0) 6 | 7 | # default Release C / CXX flags 8 | set(CMAKE_C_FLAGS " -O3 -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 9 | set(CMAKE_CXX_FLAGS " -O3 -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 10 | 11 | set(CMAKE_C_FLAGS_DEBUG " -g -ggdb -O0 -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 12 | set(CMAKE_CXX_FLAGS_DEBUG " -g -ggdb -O0 -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 13 | 14 | set(CMAKE_C_FLAGS_RELEASE " -O3 -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 15 | set(CMAKE_CXX_FLAGS_RELEASE " -O3 -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 16 | 17 | set(CMAKE_C_FLAGS_MINSIZEREF " -Os -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 18 | set(CMAKE_CXX_FLAGS_MINSIZEREF " -Os -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 19 | 20 | set(CMAKE_C_FLAGS_RELWITHDEBINFO " -O2 -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 21 | set(CMAKE_CXX_FLAGS_RELWITHDEBINFO " -O2 -Wall -Werror -fpic -fPIC -D_LARGEFILE64_SOURCE") 22 | 23 | set(CMAKE_C_FLAGS_COVERAGE " ${CMAKE_C_FLAGS_DEBUG} -fprofile-arcs -ftest-coverage") 24 | set(CMAKE_CXX_FLAGS_COVERAGE " ${CMAKE_C_FLAGS_DEBUG} -fprofile-arcs -ftest-coverag") 25 | 26 | set(LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/${CMAKE_BUILD_TYPE}/lib) 27 | set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/${CMAKE_BUILD_TYPE}/bin) 28 | set(CMAKE_C_OUTPUT_EXTENSION_REPLACE 1) 29 | set(CMAKE_CXX_OUTPUT_EXTENSION_REPLACE 1) 30 | 31 | option(ADD_LOG_KEY_VALUE_FUN "ON for generating add log key&value functions" OFF) 32 | option(FEATURE_ZSTD_COMPRESS "support zstd compress type, disable to get smaller lib size" OFF) 33 | 34 | IF(ADD_LOG_KEY_VALUE_FUN) 35 | add_definitions(-DLOG_KEY_VALUE_FLAG) 36 | ENDIF(ADD_LOG_KEY_VALUE_FUN) 37 | 38 | if(FEATURE_ZSTD_COMPRESS) 39 | add_definitions(-DLOG_FEATURE_ZSTD_COMPRESS) 40 | endif(FEATURE_ZSTD_COMPRESS) 41 | 42 | #curl-config 43 | FIND_PROGRAM(CURL_CONFIG_BIN NAMES curl-config) 44 | 45 | IF (CURL_CONFIG_BIN) 46 | EXECUTE_PROCESS( 47 | COMMAND ${CURL_CONFIG_BIN} --libs 48 | OUTPUT_VARIABLE CURL_LIBRARIES 49 | OUTPUT_STRIP_TRAILING_WHITESPACE 50 | ) 51 | ELSE() 52 | MESSAGE(FATAL_ERROR "Could not find curl-config") 53 | ENDIF() 54 | 55 | # Compile and link lib_log_c_sdk 56 | include_directories(${CURL_INCLUDE_DIR}) 57 | 58 | aux_source_directory(src SRC_LIST) 59 | 60 | 61 | add_library(${CMAKE_PROJECT_NAME}_static STATIC ${SRC_LIST}) 62 | INSTALL(TARGETS ${CMAKE_PROJECT_NAME}_static ARCHIVE DESTINATION lib) 63 | 64 | IF (APPLE) 65 | MESSAGE(STATUS "For mac os, skip build dynamic library") 66 | ELSE () 67 | add_library(${CMAKE_PROJECT_NAME} SHARED ${SRC_LIST}) 68 | target_compile_definitions(${CMAKE_PROJECT_NAME} PUBLIC BUILD_SHARED_LIB) 69 | set_target_properties(${CMAKE_PROJECT_NAME} PROPERTIES VERSION ${CMAKE_VERSION} SOVERSION ${CMAKE_VERSION}) 70 | INSTALL(TARGETS ${CMAKE_PROJECT_NAME} LIBRARY DESTINATION lib) 71 | ENDIF () 72 | 73 | 74 | 75 | 76 | 77 | file(GLOB HEADER_FILES src/*.h) 78 | 79 | INSTALL(FILES 80 | ${HEADER_FILES} 81 | DESTINATION include/log_c_sdk) 82 | 83 | add_subdirectory(sample) 84 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Aliyun inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 日志服务C Producer是用纯C编写的日志采集客户端，提供更加精简的环境依赖以及更低的资源占用，试用于各类嵌入式/智能设备的日志采集。 2 | 3 | # 功能特点 4 | 5 | * 异步 6 | * 异步写入，客户端线程无阻塞 7 | * 聚合&压缩 上传 8 | * 支持按超时时间、日志数、日志size聚合数据发送 9 | * 支持lz4压缩 10 | * 支持上下文查询 11 | * 同一个客户端产生的日志在同一上下文中，支持查看某条日志前后相关日志 12 | * 并发发送 13 | * 支持可配置的线程池发送 14 | * 缓存 15 | * 支持缓存上线可设置 16 | * 超过上限后日志写入失败 17 | * 自定义标识 18 | * 日志上传时默认会带上ip 19 | * 支持设置自定义tag、topic 20 | 21 | # 功能优势 22 | 23 | * 客户端高并发写入：可配置的发送线程池，支持每秒数十万条日志写入，详情参见[性能测试](performance.md)。 24 | * 低资源消耗：每秒20W日志写入只消耗30% CPU；同时在低性能硬件（例如树莓派）上，每秒产生100条日志对资源基本无影响。详情参见[性能测试](performance.md)。 25 | * 客户端日志不落盘：数据产生后直接通过网络发往服务端。 26 | * 客户端计算与 I/O 逻辑分离：日志异步输出，不阻塞工作线程。 27 | * 可靠退出：程序退出时，会调用接口将日志持久化，待下次应用启动时将数据发送，保证数据可靠性。详情参见[程序可靠退出方案](save_send_buffer.md)。 28 | 29 | 在以上场景中，C Producer 会简化您程序开发的步骤，您无需关心日志采集细节实现、也不用担心日志采集会影响您的业务正常运行，大大降低数据采集门槛。 30 | 31 | # 与1.0版本（v1分支） C Producer对比 32 | ## 功能对比 33 | 相比1.0版本 C Producer，仅缺少以下功能： 34 | 1. 同一Producer只支持单个Client，不支持多Client设置 35 | 2. 不支持发送优先级设置 36 | 3. 不支持JSON方式的配置文件 37 | 4. 不提供日志相关宏定义包装 38 | 5. 不支持本地调试功能 39 | 40 | ## 性能对比 41 | 性能相对1.0版本 C Producer有30%以上提升。 42 | 43 | ## 资源占用对比 44 | 资源占用相比1.0版本 C Producer大大降低（详细请参考[性能测试](performance.md)）： 45 | * 除去libcurl占用，运行期间内存占用80KB左右。 46 | * 编译后的动态库仅有65KB（strip后） 47 | 48 | # 分支选择 49 | C Producer根据不同的设备类型和使用场景做了非常多的定制工作，因此分为几个分支来支持不同的场景，请根据实际需求选择。 50 | 51 | |分支|状态|功能优势|建议使用场景| 52 | |--|--|--|--| 53 | | master | 可用 | 原lite分支，相比v1（1.0f）版本依赖、资源占用、性能等有大幅度提升，是目前SLS性能最强的Producer，推荐使用 | Linux服务器、嵌入式Linux | 54 | | live | 可用 | 主要功能和master版本一致，增加最多平台的编译支持，包括Windows、Mac、Android、IOS等 | 非master支持的环境 | 55 | | bricks | 可用 | 极致精简版本，binary和内存占用极低，但是功能非常弱，建议在资源非常受限的场景中使用 | 资源占用在10KB以内的场景，例如RTOS | 56 | | persistent | 可用 | 相比master增加本地缓存功能，目前用于Android、IOS移动端版本的Native实现，本地缓存功能开启后只能单线程发送，不建议服务端使用 | 建议直接使用Android、IOS官方SDK | 57 | 58 | 59 | * 注意：除以上分支外，不建议使用其他分支 60 | 61 | # 安装方法 62 | ### 下载Producer代码 63 | 您可以使用如下命令获取代码： 64 | ```shell 65 | git clone https://github.com/aliyun/aliyun-log-c-sdk.git 66 | ``` 67 | 68 | ### 环境依赖 69 | 需要安装openssl 70 | C Producer使用curl进行网络操作，您需要确认这些库已经安装，并且将它们的头文件目录和库文件目录都加入到了项目中。 71 | 72 | #### libcurl下载以及安装 73 | 74 | libcurl建议 7.49.0 及以上版本 75 | 76 | 请从[这里](http://curl.haxx.se/download.html)下载，并参考[libcurl 安装指南](http://curl.haxx.se/docs/install.html)安装。典型的安装方式如下： 77 | ```shell 78 | ./configure 79 | make 80 | make install 81 | ``` 82 | 83 | * 注意： 84 | - 执行./configure时默认是配置安装目录为/usr/local/，如果需要指定安装目录，请使用 ./configure --prefix=/your/install/path/ 85 | 86 | #### LOG C SDK的安装 87 | 88 | 安装时请在cmake命令中指定第三方库头文件以及库文件的路径，典型的编译命令如下： 89 | ```shell 90 | cmake . 91 | make 92 | make install 93 | ``` 94 | 95 | 96 | * 注意： 97 | - 执行cmake . 时默认会到/usr/local/下面去寻找curl的头文件和库文件。 98 | - 默认编译是Release类型，可以指定以下几种编译类型： Debug, Release, RelWithDebInfo和MinSizeRel，如果要使用Debug类型编译，则执行cmake . -DCMAKE_BUILD_TYPE=Debug 99 | - 如果您在安装curl时指定了安装目录，则需要在执行cmake时指定这些库的路径，比如： 100 | ```shell 101 | cmake . -DCURL_INCLUDE_DIR=/usr/local/include/curl/ -DCURL_LIBRARY=/usr/local/lib/libcurl.a 102 | ``` 103 | - 如果要指定安装目录，则需要在cmake时增加： -DCMAKE_INSTALL_PREFIX=/your/install/path/usr/local/ 104 | 105 | ## 使用 106 | 一个应用可创建多个producer，每个producer可单独配置目的地址、缓存大小、发送线程数、自定义标识、topic等信息。 107 | 108 | ### 配置参数 109 | 所有参数通过`log_producer_config`结构设置，支持设置的参数如下： 110 | 111 | | 参数 | 说明 | 取值 | 112 | | ------------------------ | ------------------------------------------------------------------------------------ | --------------------------------------- | 113 | | project | 日志服务project | string | 114 | | logstore | 日志服务logstore | string | 115 | | endpoint | 日志服务endpoint | string，以`http:`开头 | 116 | | access_id | 云账号 access id | string | 117 | | access_key | 云账号 access key | string | 118 | | topic | topic名 | string，非必须设置。 | 119 | | add_tag | 自定义的tag | key value 模式，非必须设置。 | 120 | | packet_timeout | 指定被缓存日志的发送超时时间，如果缓存超时，则会被立即发送。 | 整数形式，单位为毫秒。默认为3000。 | 121 | | packet_log_count | 指定每个缓存的日志包中包含日志数量的最大值。 | 整数形式，取值为1~4096。默认2048。 | 122 | | packet_log_bytes | 指定每个缓存的日志包的大小上限。 | 整数形式，取值为1~5242880，单位为字节。默认为 3 * 1024 * 1024。 | 123 | | max_buffer_limit | 指定单个Producer Client实例可以使用的内存的上限。 | 整数形式，单位为字节。默认为64 * 1024 * 1024。 | 124 | | log_queue_size | 缓存队列的最大长度，默认通过 max_buffer_limit/packet_log_bytes + 10 自动计算，可以手动设置以获得更大的队列大小。 | 整数形式，支持32-128000。默认值max_buffer_limit/packet_log_bytes + 10。 | 125 | | send_thread_count | 指定发送线程数量，用于发送数据到日志服务。 | 整数形式，默认0，为0时发送与内部Flush公用一个线程，若您每秒日志发送不超过100条，无需设置此参数。 | 126 | | net_interface | 网络发送绑定的网卡名 | 字符串，为空时表示自动选择可用网卡 | 127 | | connect_timeout_sec | 网络连接超时时间 | 整数，单位秒，默认为10 | 128 | | send_timeout_sec | 日志发送超时时间 | 整数，单位秒，默认为15 | 129 | | destroy_flusher_wait_sec | flusher线程销毁最大等待时间 | 整数，单位秒，默认为1 | 130 | | destroy_sender_wait_sec | sender线程池销毁最大等待时间 | 整数，单位秒，默认为1 | 131 | | compress_type | 压缩类型 | 枚举值，默认为lz4。 | 132 | ### 相关限制 133 | * C Producer销毁时，会尽可能将缓存中的数据发送出去，若您不对未发送的数据进行处理，则有一定概率丢失数据。处理方法参见[程序可靠退出方案](save_send_buffer.md) 134 | * C Producer销毁的最长时间可能为 `send_timeout_sec` + `destroy_flusher_wait_sec` + `destroy_sender_wait_sec` 135 | ### 样例代码 136 | 137 | 请参考`sample`目录中的`log_producer_sample.c`。 138 | 139 | 140 | 141 | ## C-Producer系列介绍 142 | 143 | ![image.png](http://ata2-img.cn-hangzhou.img-pub.aliyun-inc.com/bb6a86424ebba0efeca634dc2cc2ffd3.png) 144 | 145 | > 关于C Producer Library的更多内容参见目录：[https://yq.aliyun.com/articles/304602](https://yq.aliyun.com/articles/304602) 146 | 目前针对不同的环境（例如网络服务器、ARM设备、以及RTOS等设备）从大到小我们提供了3种方案： 147 | 148 | 149 | ![image.png | left | 827x368](http://ata2-img.cn-hangzhou.img-pub.aliyun-inc.com/22e93dfe553b35a0d3c609840aa2db02.png "") 150 | 151 | 同时对于Producer我们进行了一系列的性能和资源优化，确保数据采集可以“塞”到任何IOT设备上，其中C Producer Bricks版本更是达到了极致的内存占用（库体积13KB，运行内存4KB以内）。 152 | 153 | 154 | ![image.png | left | 827x171](http://ata2-img.cn-hangzhou.img-pub.aliyun-inc.com/3371df540fcd57c18e943d3e44f30892.png "") 155 | 156 | 157 | 使用C Producer系列的客户有： 百万日活的天猫精灵、小朋友们最爱的故事机火火兔、 遍布全球的码牛、钉钉路由器、 兼容多平台的视频播放器、 实时传输帧图像的摄像头等。 158 | 159 | 这些智能SDK每天DAU超百万，遍布在全球各地的设备上，一天传输百TB数据。关于C Producer Library 的细节可以参考这篇文章: [智能设备日志利器：嵌入式日志客户端（C Producer）发布](https://yq.aliyun.com/articles/304602)。 160 | 161 | 162 | ![image.png | left | 827x264](http://ata2-img.cn-hangzhou.img-pub.aliyun-inc.com/470c6b49edd08278d42c4b8f836a6701.png "") 163 | 164 | 165 | ### 数据采集全球加速 166 | IOT设备作为典型的“端”设备，通常都会部署在全国、甚至全球各地，部署区域的网络条件难以保证，这会对数据采集产生一个巨大的问题：数据采集受网络质量影响，可靠性难以保证。 167 | 168 | 针对以上问题，日志服务联合阿里云CDN推出了一款[全球数据上传自动加速](https://help.aliyun.com/document_detail/86817.html)方案：“基于阿里云CDN硬件资源，全球数据就近接入边缘节点，通过内部高速通道路由至LogHub，大大降低网络延迟和抖动 ”。 169 | 该方案有如下特点： 170 | 171 | * 全网边缘节点覆盖：全球1000+节点，国内700+节点，分布60多个国家和地区，覆盖六大洲 172 | * 智能路由技术：实时探测网络质量，自动根据运营商、网络等状况选择最近接入 173 | * 传输协议优化：CDN节点之间走私有协议、高效安全 174 | * 使用便捷：只需1分钟即可开通加速服务，只需切换到专属加速域名即可获得加速效果 175 | 176 | ## ![image | left](http://ata2-img.cn-hangzhou.img-pub.aliyun-inc.com/e61177787e1c333103c9cd479c879070.png "image | left") 177 | 178 | 在我们的日志上传基准测试中，全球7个区域对比整体延时下降50%，在中东，欧洲、澳洲和新加坡等效果明显。除了平均延时下降外，整体稳定性也有较大提升（参见最下图，几乎没有任何抖动，而且超时请求基本为0）。确保无论在全球的何时何地，只要访问这个加速域名，就能够高效、便捷将数据采集到期望Region内。 179 | 180 | 关于全球采集加速的更多内容，可参考我们的文章：[数据采集新形态-全球加速](https://yq.aliyun.com/articles/620453) 。 181 | 182 | 183 | ![image.png | left | 827x396](http://ata2-img.cn-hangzhou.img-pub.aliyun-inc.com/a0b6820aa4a9b339edd9fa7615c1e13d.png "") 184 | 185 | 186 | 187 | ![image.png | left | 827x222](http://ata2-img.cn-hangzhou.img-pub.aliyun-inc.com/adec099e7475cd5e5d5ba7bca6e28b1e.png "") -------------------------------------------------------------------------------- /performance.md: -------------------------------------------------------------------------------- 1 | # 服务端测试 2 | ## 环境 3 | ### 软硬件环境 4 | * CPU : Intel(R) Xeon(R) CPU E5-2682 v4 @ 2.50GHz 5 | * MEM : 64GB 6 | * OS : Linux version 2.6.32-220.23.2.ali1113.el5.x86_64 7 | * GCC : 4.1.2 8 | * c-producer：动态库 65K、静态库59K (测试使用静态库，编译后的binary 69KB，所有都是strip后) 9 | 10 | ### c-producer配置 11 | * 缓存：64MB 12 | * 聚合时间：3秒 （聚合时间、聚合数据包大小、聚合日志数任一满足即打包发送） 13 | * 聚合数据包大小：4MB 14 | * 聚合日志数：4096 15 | * 发送线程：16 16 | * 自定义tag : 5 17 | 18 | ### 日志样例 19 | 日志样例为2种 20 | 1. 10个键值对，总数据量约为700字节 21 | 1. 4个键值对，数据量约为200字节 22 | 23 | ## 测试结果 24 | 25 | | 发送速率(logs/s) | 平均每次log耗时(us) | 原始数据速率(KB/s) | cpu(%) | mem(MB) | 26 | | ---------------- | ---------------- | ------------------- | ------------------ | ------ | 27 | | 200000 | 1.08 | 84495.5 | 28.8 | 10.58 | 28 | | 100000 | 1.10 | 42247.8 | 14.6 | 8.10 | 29 | | 20000 | 1.09 | 8449.6 | 3.0 | 7.35 | 30 | | 10000 | 1.08 | 4224.7 | 1.6 | 5.12 | 31 | | 2000 | 1.20 | 844.9 | 0.2 | 3.58 | 32 | | 1000 | 1.19 | 463.9 | 0.1 | 3.31 | 33 | | 200 | 1.45（不准确） | 84.5 | 0.0? | 2.81 | 34 | | 100 | 1.70（不准确） | 42.2 | 0.0? | 2.82 | 35 | 36 | # 树莓派测试 37 | ## 软硬件环境 38 | ### 硬件环境 39 | * 型号：树莓派3B 40 | * CPU：Broadcom BCM2837 1.2GHz A53 64位(使用主机USB供电，被降频到600MHz) 41 | * 内存：1GB DDR2 42 | * 网络：百兆网卡（台式机本地网卡提供共享网络，台式机网络为无线网，网络质量不是很好，经常还有timeout） 43 | 44 | ### OS&软件 45 | * OS: Linux 4.9.41-v7+ #1023 SMP armv71 GNU/Linux 46 | * GCC：6.3.0 (Raspbian 6.3.0-18+rpi1) 47 | * c-producer：动态库 72K、静态库65K (测试使用静态库，编译后的binary 87KB，所有都是strip后) 48 | 49 | ### c-producer配置 50 | * 缓存：10MB 51 | * 聚合时间：3秒 （聚合时间、聚合数据包大小、聚合日志数任一满足即打包发送） 52 | * 聚合数据包大小：1MB 53 | * 聚合日志数：1000 54 | * 发送线程：4 55 | * 自定义tag : 5 56 | 57 | ### 日志样例 58 | 日志样例为2种 59 | 1. 10个键值对，总数据量约为700字节 60 | 1. 4个键值对，数据量约为200字节 61 | 62 | ## 测试结果 63 | 64 | | 发送速率(logs/s) | 平均每次log耗时(us) | 原始数据速率(KB/s) | cpu(%) | mem(MB) | 65 | | ---------------- | ---------------- | ------------------- | ------------------ | ------ | 66 | | 20000 | 8.8 | 8449.6 | 31.5 | 5.46 | 67 | | 10000 | 9.3 | 4224.7 | 15.8 | 5.1 | 68 | | 2000 | 9.9 | 844.9 | 2.9 | 3.38 | 69 | | 1000 | 10.5 | 463.9 | 1.4 | 3.11 | 70 | | 200 | 10.1 | 84.5 | 0.3 | 2.41 | 71 | | 100 | 10.6 | 42.2 | 0.1 | 3.01 | 72 | | 20 | 10.3 | 8.4 | 0.0? | 2.96 | 73 | | 10 | 11.1 | 4.2 | 0.0? | 2.80 | 74 | 75 | # 总结 76 | * 服务器端的测试中，C Producer可以轻松到达90M/s的发送速度，每秒上传日志20W，占用CPU只有70%，内存12M 77 | * 在树莓派的测试中，由于CPU的频率只有600MHz，性能差不多是服务器的1/10左右，最高每秒可发送2W条日志 78 | * 服务器在200条/s、树莓派在20条/s的时候，发送数据对于cpu基本无影响（降低到0.01%以内） 79 | * 客户线程发送一条数据（输出一条log）的平均耗时为：服务器1.1us、树莓派10us左右 80 | 81 | 82 | # 附件 83 | 84 | ## 日志样例 85 | ``` 86 | Interconnection:Grafana and JDBC/SQL92 87 | LogHub:Real-time log collection and consumption 88 | Search/Analytics:Query and real-time analysis 89 | Visualized:dashboard and report functions 90 | __tag__:__pack_id__:F20B4E7940C80975-0 91 | __tag__:tag_1:val_1 92 | __tag__:tag_2:val_2 93 | __tag__:tag_3:val_3 94 | __tag__:tag_4:val_4 95 | __tag__:tag_5:val_5 96 | __topic__:test_topic 97 | ``` 98 | 99 | ``` 100 | __tag__:__pack_id__:F20B4E7940C80975-0 101 | __tag__:tag_1:val_1 102 | __tag__:tag_2:val_2 103 | __tag__:tag_3:val_3 104 | __tag__:tag_4:val_4 105 | __tag__:tag_5:val_5 106 | __topic__:test_topic 107 | content_key_1:1abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+ 108 | content_key_2:2abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+ 109 | content_key_3:3abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+ 110 | content_key_4:4abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+ 111 | content_key_5:5abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+ 112 | content_key_6:6abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+ 113 | content_key_7:7abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+ 114 | content_key_8:8abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+ 115 | content_key_9:9abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+ 116 | index:0 117 | ``` 118 | 119 | ## 测试代码 120 | ``` 121 | #include "inner_log.h" 122 | #include "log_api.h" 123 | #include "log_producer_config.h" 124 | #include "log_producer_client.h" 125 | #include 126 | #include 127 | #include 128 | #include "log_multi_thread.h" 129 | 130 | 131 | void on_log_send_done(const char * config_name, log_producer_result result, size_t log_bytes, size_t compressed_bytes, const char * req_id, const char * message) 132 | { 133 | if (result == LOG_PRODUCER_OK) 134 | { 135 | if (aos_log_level == AOS_LOG_DEBUG) 136 | { 137 | printf("send success, config : %s, result : %d, log bytes : %d, compressed bytes : %d, request id : %s \n", 138 | config_name, (result), 139 | (int)log_bytes, (int)compressed_bytes, req_id); 140 | } 141 | 142 | } 143 | else 144 | { 145 | printf("send fail, config : %s, result : %d, log bytes : %d, compressed bytes : %d, request id : %s, error message : %s\n", 146 | config_name, (result), 147 | (int)log_bytes, (int)compressed_bytes, req_id, message); 148 | } 149 | 150 | } 151 | 152 | log_producer * create_log_producer_wrapper(on_log_producer_send_done_function on_send_done) 153 | { 154 | log_producer_config * config = create_log_producer_config(); 155 | // endpoint list: https://help.aliyun.com/document_detail/29008.html 156 | log_producer_config_set_endpoint(config, "${your_endpoint}"); 157 | log_producer_config_set_project(config, "${your_project}"); 158 | log_producer_config_set_logstore(config, "${your_logstore}"); 159 | log_producer_config_set_access_id(config, "${your_access_key_id}"); 160 | log_producer_config_set_access_key(config, "${your_access_key_secret}"); 161 | 162 | // if you do not need topic or tag, comment it 163 | log_producer_config_set_topic(config, "test_topic"); 164 | log_producer_config_add_tag(config, "tag_1", "val_1"); 165 | log_producer_config_add_tag(config, "tag_2", "val_2"); 166 | log_producer_config_add_tag(config, "tag_3", "val_3"); 167 | log_producer_config_add_tag(config, "tag_4", "val_4"); 168 | log_producer_config_add_tag(config, "tag_5", "val_5"); 169 | 170 | // set resource params 171 | log_producer_config_set_packet_log_bytes(config, 4*1024*1024); 172 | log_producer_config_set_packet_log_count(config, 4096); 173 | log_producer_config_set_packet_timeout(config, 3000); 174 | log_producer_config_set_max_buffer_limit(config, 64*1024*1024); 175 | 176 | // set send thread 177 | log_producer_config_set_send_thread_count(config, 16); 178 | 179 | return create_log_producer(config, on_send_done); 180 | } 181 | 182 | void log_producer_post_logs(int logsPerSecond, int sendSec) 183 | { 184 | //aos_log_level = AOS_LOG_DEBUG; 185 | if (log_producer_env_init(LOG_GLOBAL_ALL) != LOG_PRODUCER_OK) { 186 | exit(1); 187 | } 188 | 189 | log_producer * producer = create_log_producer_wrapper(on_log_send_done); 190 | if (producer == NULL) 191 | { 192 | printf("create log producer by config file fail \n"); 193 | exit(1); 194 | } 195 | 196 | log_producer_client * client = get_log_producer_client(producer, NULL); 197 | if (client == NULL) 198 | { 199 | printf("create log producer client by config file fail \n"); 200 | exit(1); 201 | } 202 | 203 | 204 | int32_t i = 0; 205 | int32_t totalTime = 0; 206 | for (i = 0; i < sendSec; ++i) 207 | { 208 | int64_t startTime = GET_TIME_US(); 209 | int j = 0; 210 | for (; j < logsPerSecond; ++j) 211 | { 212 | char indexStr[32]; 213 | sprintf(indexStr, "%d", i * logsPerSecond + j); 214 | log_producer_client_add_log(client, 20, "content_key_1", "1abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 215 | "content_key_2", "2abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 216 | "content_key_3", "3abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 217 | "content_key_4", "4abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 218 | "content_key_5", "5abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 219 | "content_key_6", "6abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 220 | "content_key_7", "7abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 221 | "content_key_8", "8abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 222 | "content_key_9", "9abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 223 | "index", indexStr); 224 | 225 | log_producer_result rst = log_producer_client_add_log(client, 8, "LogHub", "Real-time log collection and consumption", 226 | "Search/Analytics", "Query and real-time analysis", 227 | "Visualized", "dashboard and report functions", 228 | "Interconnection", "Grafana and JDBC/SQL92"); 229 | if (rst != LOG_PRODUCER_OK) 230 | { 231 | printf("add log error %d \n", rst); 232 | } 233 | } 234 | int64_t endTime = GET_TIME_US(); 235 | aos_error_log("Done : %d %d time %f us \n", i, logsPerSecond, (float)(endTime - startTime) * 1.0 / logsPerSecond / 2); 236 | totalTime += endTime - startTime; 237 | if (endTime - startTime < 1000000) 238 | { 239 | usleep(1000000 - (endTime - startTime)); 240 | } 241 | } 242 | aos_error_log("Total done : %f us, avg %f us", (float)totalTime / 180, (float)totalTime / (180 * logsPerSecond * 2)); 243 | //sleep(10); 244 | 245 | destroy_log_producer(producer); 246 | 247 | log_producer_env_destroy(); 248 | } 249 | 250 | int main(int argc, char *argv[]) 251 | { 252 | int logsPerSec = 100; 253 | int sendSec = 180; 254 | if (argc == 3) 255 | { 256 | logsPerSec = atoi(argv[1]); 257 | sendSec = atoi(argv[2]); 258 | } 259 | log_producer_post_logs(logsPerSec, sendSec); 260 | return 0; 261 | } 262 | 263 | 264 | ``` -------------------------------------------------------------------------------- /sample/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project (log_c_sdk_sample) 2 | 3 | include_directories(${CMAKE_SOURCE_DIR}) 4 | 5 | set(POST_LOGS_SOURCE_FILES log_config.c 6 | log_post_logs_sample.c) 7 | 8 | 9 | set(PRODUCER_SAMPLE_SOURCE_FILES log_producer_sample.c) 10 | 11 | set(PRODUCER_BENCHMARK_SOURCE_FILES log_producer_benchmark.c) 12 | 13 | set(PRODUCER_FRAME_SOURCE_FILES video_frame_producer_sample.c) 14 | 15 | include_directories (${CURL_INCLUDE_DIR}) 16 | include_directories ("${CMAKE_SOURCE_DIR}/src") 17 | 18 | find_library(CURL_LIBRARY curl) 19 | find_library(PTHREAD_LIBRARY pthread) 20 | find_library(RT_LIBRARY rt) 21 | find_library(M_LIBRARY m) 22 | find_library(SSL_LIBRARY ssl) 23 | find_library(CRYPTO_LIBRARY crypto) 24 | find_library(ZLIB_LIBRARY z) 25 | find_library(DL_LIBRARY dl) 26 | 27 | function(_TARGET_SAMPLE_LIBRARIES SAMPLE_BIN_NAME SOURCE_FILES) 28 | add_executable(${SAMPLE_BIN_NAME} ${SOURCE_FILES}) 29 | target_link_libraries(${SAMPLE_BIN_NAME} log_c_sdk_static) 30 | target_link_libraries(${SAMPLE_BIN_NAME} ${CURL_LIBRARY}) 31 | target_link_libraries(${SAMPLE_BIN_NAME} ${PTHREAD_LIBRARY}) 32 | target_link_libraries(${SAMPLE_BIN_NAME} ${M_LIBRARY}) 33 | target_link_libraries(${SAMPLE_BIN_NAME} ${DL_LIBRARY}) 34 | target_link_libraries(${SAMPLE_BIN_NAME} ${SSL_LIBRARY}) 35 | target_link_libraries(${SAMPLE_BIN_NAME} ${CRYPTO_LIBRARY}) 36 | target_link_libraries(${SAMPLE_BIN_NAME} ${ZLIB_LIBRARY}) 37 | endfunction() 38 | 39 | _TARGET_SAMPLE_LIBRARIES(log_producer_sample "${PRODUCER_SAMPLE_SOURCE_FILES}") 40 | 41 | _TARGET_SAMPLE_LIBRARIES(log_post_logs_sample "${POST_LOGS_SOURCE_FILES}") 42 | 43 | _TARGET_SAMPLE_LIBRARIES(log_producer_benchmark "${PRODUCER_BENCHMARK_SOURCE_FILES}") 44 | 45 | _TARGET_SAMPLE_LIBRARIES(video_frame_producer_sample "${PRODUCER_FRAME_SOURCE_FILES}") -------------------------------------------------------------------------------- /sample/log_config.c: -------------------------------------------------------------------------------- 1 | #include "log_config.h" 2 | 3 | // your endpoint 4 | const char LOG_ENDPOINT[] = ""; 5 | // your access key id 6 | const char ACCESS_KEY_ID[] = ""; 7 | // your access key secret 8 | const char ACCESS_KEY_SECRET[] = ""; 9 | // your project name 10 | const char PROJECT_NAME[] = ""; 11 | // your logstore name 12 | const char LOGSTORE_NAME[] = ""; 13 | -------------------------------------------------------------------------------- /sample/log_config.h: -------------------------------------------------------------------------------- 1 | #ifndef LOG_TEST_CONFIG_H 2 | #define LOG_TEST_CONFIG_H 3 | 4 | #include "log_define.h" 5 | 6 | LOG_CPP_START 7 | 8 | extern const char LOG_ENDPOINT[]; 9 | extern const char ACCESS_KEY_ID[]; 10 | extern const char ACCESS_KEY_SECRET[]; 11 | extern const char PROJECT_NAME[]; 12 | extern const char LOGSTORE_NAME[]; 13 | 14 | LOG_CPP_END 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /sample/log_post_logs_sample.c: -------------------------------------------------------------------------------- 1 | #include "log_util.h" 2 | #include "log_api.h" 3 | #include "log_config.h" 4 | #include 5 | #include 6 | 7 | // @note this sample is just for SDK's developer, 99% of cases you should use log_producer_sample 8 | 9 | #define POST_LOG_COUNT 10 10 | #define POST_LOG_CONTENT_PAIRS 15 11 | 12 | void post_logs_with_http_cont_lz4_log_option() 13 | { 14 | 15 | sls_log_init(LOG_GLOBAL_ALL); 16 | 17 | int i = 0; 18 | for (i = 0; i < 10; ++i) 19 | { 20 | log_group_builder *bder = log_group_create(); 21 | add_source(bder, "10.230.201.117", sizeof("10.230.201.117")); 22 | add_topic(bder, "post_logs_with_http_cont_lz4_log_option", 23 | sizeof("post_logs_with_http_cont_lz4_log_option")); 24 | 25 | add_tag(bder, "taga_key", strlen("taga_key"), "taga_value", 26 | strlen("taga_value")); 27 | add_tag(bder, "tagb_key", strlen("tagb_key"), "tagb_value", 28 | strlen("tagb_value")); 29 | add_pack_id(bder, "123456789ABC", strlen("123456789ABC"), 0); 30 | 31 | int x; 32 | for (x = 0; x < POST_LOG_COUNT; x++) 33 | { 34 | char * keys[POST_LOG_CONTENT_PAIRS]; 35 | char * values[POST_LOG_CONTENT_PAIRS]; 36 | size_t key_lens[POST_LOG_CONTENT_PAIRS]; 37 | size_t val_lens[POST_LOG_CONTENT_PAIRS]; 38 | int j = 0; 39 | for (j = 0; j < POST_LOG_CONTENT_PAIRS; j++) 40 | { 41 | 42 | keys[j] = (char *)malloc(64); 43 | sprintf(keys[j], "key_%d", x * j); 44 | values[j] = (char *)malloc(64); 45 | sprintf(values[j], "value_%d", 2 * j); 46 | key_lens[j] = strlen(keys[j]); 47 | val_lens[j] = strlen(values[j]); 48 | } 49 | add_log_full(bder, time(NULL), POST_LOG_CONTENT_PAIRS, keys, key_lens, values, val_lens); 50 | for (j = 0; j < POST_LOG_CONTENT_PAIRS; j++) 51 | { 52 | free(keys[j]); 53 | free(values[j]); 54 | } 55 | } 56 | 57 | // if you want to use this functions, add `-DADD_LOG_KEY_VALUE_FUN=ON` for cmake. eg `cmake . -DADD_LOG_KEY_VALUE_FUN=ON` 58 | #ifdef LOG_KEY_VALUE_FLAG 59 | for (x = 0; x < POST_LOG_COUNT; x++) 60 | { 61 | int j = 0; 62 | add_log_begin(bder); 63 | for (j = 0; j < POST_LOG_CONTENT_PAIRS; j++) 64 | { 65 | char key[64]; 66 | sprintf(key, "add_key_%d", x * j); 67 | char value[64]; 68 | sprintf(value, "add_value_%d", 2 * j); 69 | add_log_key_value(bder, key, strlen(key), value, strlen(value)); 70 | if (j == POST_LOG_CONTENT_PAIRS / 2) 71 | { 72 | add_log_time(bder, time(NULL)); 73 | } 74 | } 75 | 76 | add_log_end(bder); 77 | } 78 | 79 | #endif 80 | 81 | log_post_option option; 82 | memset(&option, 0, sizeof(log_post_option)); 83 | option.interface = NULL; 84 | option.connect_timeout = 15; 85 | option.operation_timeout = 15; 86 | option.compress_type = i % 2 == 0; 87 | lz4_log_buf *pLZ4Buf = NULL; 88 | if (option.compress_type == 1) 89 | { 90 | printf("post log with lz4 \n"); 91 | pLZ4Buf = serialize_to_proto_buf_with_malloc_lz4(bder); 92 | } 93 | else 94 | { 95 | printf("post log with no compress \n"); 96 | pLZ4Buf = serialize_to_proto_buf_with_malloc_no_compress(bder); 97 | } 98 | 99 | log_group_destroy(bder); 100 | if (pLZ4Buf == NULL) 101 | { 102 | printf("serialize_to_proto_buf_with_malloc_lz4 failed\n"); 103 | exit(1); 104 | } 105 | post_log_result * rst = post_logs_from_lz4buf(LOG_ENDPOINT, ACCESS_KEY_ID, 106 | ACCESS_KEY_SECRET, NULL, 107 | PROJECT_NAME, LOGSTORE_NAME, 108 | pLZ4Buf, &option, 109 | AUTH_VERSION_1, NULL); 110 | printf("result %d %d \n", i, rst->statusCode); 111 | if (rst->errorMessage != NULL) 112 | { 113 | printf("error message %s \n", rst->errorMessage); 114 | } 115 | 116 | if (rst->requestID != NULL) 117 | { 118 | printf("requestID %s \n", rst->requestID); 119 | } 120 | post_log_result_destroy(rst); 121 | free_lz4_log_buf(pLZ4Buf); 122 | } 123 | 124 | sls_log_destroy(); 125 | } 126 | 127 | // @note this sample is just for SDK's developer, 99% of cases you should use log_producer_sample 128 | int main(int argc, char *argv[]) 129 | { 130 | post_logs_with_http_cont_lz4_log_option(); 131 | return 0; 132 | } 133 | -------------------------------------------------------------------------------- /sample/log_producer_benchmark.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 26/12/2017. 3 | // 4 | 5 | #include "inner_log.h" 6 | #include "log_api.h" 7 | #include "log_producer_config.h" 8 | #include "log_producer_client.h" 9 | #include 10 | #include 11 | #include 12 | #include "log_multi_thread.h" 13 | 14 | int producer_send_thread_count = 16; 15 | 16 | void builder_speed_test(int32_t logsPerGroup) 17 | { 18 | int32_t startTime = time(NULL); 19 | int32_t count = 0; 20 | for (; count < 1000000; ++count) 21 | { 22 | log_group_builder* bder = log_group_create(); 23 | add_source(bder,"mSource",sizeof("mSource")); 24 | add_topic(bder,"mTopic", sizeof("mTopic")); 25 | 26 | add_tag(bder, "taga_key", strlen("taga_key"), "taga_value", strlen("taga_value")); 27 | add_tag(bder, "tagb_key", strlen("tagb_key"), "tagb_value", strlen("tagb_value")); 28 | add_pack_id(bder, "123456789ABC", strlen("123456789ABC"), 0); 29 | 30 | int i = 0; 31 | for (; i < logsPerGroup; ++i) 32 | { 33 | char * keys[] = { 34 | "content_key_1", 35 | "content_key_1", 36 | "content_key_1", 37 | "content_key_1", 38 | "content_key_1", 39 | "content_key_1", 40 | "content_key_1", 41 | "content_key_1", 42 | "content_key_1", 43 | "content_key_1" 44 | }; 45 | char * vals[] = { 46 | "1abcdefghijklmnopqrstuvwxyz01234567891abcdefghijklmnopqrstuvwxyz01234567891abcdefghijklmnopqrstuvwxyz01234567891abcdefghijklmnopqrstuvwxyz0123456789", 47 | "2abcdefghijklmnopqrstuvwxyz0123456789", 48 | "2abcdefghijklmnopqrstuvwxyz0123456789", 49 | "2abcdefghijklmnopqrstuvwxyz0123456789", 50 | "2abcdefghijklmnopqrstuvwxyz0123456789", 51 | "2abcdefghijklmnopqrstuvwxyz0123456789", 52 | "2abcdefghijklmnopqrstuvwxyz0123456789", 53 | "2abcdefghijklmnopqrstuvwxyz0123456789", 54 | "2abcdefghijklmnopqrstuvwxyz0123456789", 55 | "xxxxxxxxxxxxxxxxxxxxx" 56 | }; 57 | size_t key_lens[] = { 58 | strlen("content_key_1"), 59 | strlen("content_key_1"), 60 | strlen("content_key_1"), 61 | strlen("content_key_1"), 62 | strlen("content_key_1"), 63 | strlen("content_key_1"), 64 | strlen("content_key_1"), 65 | strlen("content_key_1"), 66 | strlen("content_key_1"), 67 | strlen("index") 68 | }; 69 | size_t val_lens[] = { 70 | strlen("1abcdefghijklmnopqrstuvwxyz01234567891abcdefghijklmnopqrstuvwxyz01234567891abcdefghijklmnopqrstuvwxyz01234567891abcdefghijklmnopqrstuvwxyz0123456789"), 71 | strlen("2abcdefghijklmnopqrstuvwxyz0123456789"), 72 | strlen("2abcdefghijklmnopqrstuvwxyz0123456789"), 73 | strlen("2abcdefghijklmnopqrstuvwxyz0123456789"), 74 | strlen("2abcdefghijklmnopqrstuvwxyz0123456789"), 75 | strlen("2abcdefghijklmnopqrstuvwxyz0123456789"), 76 | strlen("2abcdefghijklmnopqrstuvwxyz0123456789"), 77 | strlen("2abcdefghijklmnopqrstuvwxyz0123456789"), 78 | strlen("2abcdefghijklmnopqrstuvwxyz0123456789"), 79 | strlen("xxxxxxxxxxxxxxxxxxxxx") 80 | }; 81 | add_log_full(bder, (uint32_t)time(NULL), 10, (char **)keys, (size_t *)key_lens, (char **)vals, (size_t *)val_lens); 82 | } 83 | log_buf buf = serialize_to_proto_buf_with_malloc(bder); 84 | 85 | if (count % 1000 == 0) 86 | { 87 | int32_t nowTime = time(NULL); 88 | aos_error_log("Done : %d %d %d %d\n", count, logsPerGroup, nowTime - startTime, (int32_t)buf.n_buffer); 89 | } 90 | log_group_destroy(bder); 91 | } 92 | aos_error_log("total time sec %d ", (time(NULL) - startTime)); 93 | } 94 | 95 | void on_log_send_done(const char * config_name, log_producer_result result, size_t log_bytes, size_t compressed_bytes, const char * req_id, const char * message, const unsigned char * raw_buffer) 96 | { 97 | if (result == LOG_PRODUCER_OK) 98 | { 99 | if (aos_log_level == AOS_LOG_DEBUG) 100 | { 101 | printf("send success, config : %s, result : %d, log bytes : %d, compressed bytes : %d, request id : %s \n", 102 | config_name, (result), 103 | (int)log_bytes, (int)compressed_bytes, req_id); 104 | } 105 | 106 | } 107 | else 108 | { 109 | printf("send fail, config : %s, result : %d, log bytes : %d, compressed bytes : %d, request id : %s, error message : %s\n", 110 | config_name, (result), 111 | (int)log_bytes, (int)compressed_bytes, req_id, message); 112 | } 113 | 114 | } 115 | 116 | typedef struct _multi_write_log_param 117 | { 118 | log_producer_client * client; 119 | size_t send_count; 120 | }multi_write_log_param; 121 | 122 | void * write_log_thread(void* param) 123 | { 124 | aos_error_log("Thread start"); 125 | multi_write_log_param * write_log_param = (multi_write_log_param *)param; 126 | size_t i = 0; 127 | char buffer[512]; 128 | for (; i < write_log_param->send_count; ++i) 129 | { 130 | sprintf(buffer, "The Logstore is a unit in Log Service for the collection, storage, and query of log data, %d", (int)i); 131 | log_producer_client_add_log(write_log_param->client, 8, "Log group", "A log group is a collection of logs and is the basic unit for writing and reading", 132 | "Log topic", "Logs in a Logstore can be classified by log topics", 133 | "Project", "The project is the resource management unit in Log Service and is used to isolate and control resources", 134 | "Logstore", "The Logstore is a unit in Log Service for the collection, storage, and query of log data"); 135 | } 136 | aos_error_log("Thread done"); 137 | return NULL; 138 | } 139 | 140 | log_producer * create_log_producer_wrapper(on_log_producer_send_done_function on_send_done) 141 | { 142 | log_producer_config * config = create_log_producer_config(); 143 | // endpoint list: https://help.aliyun.com/document_detail/29008.html 144 | log_producer_config_set_endpoint(config, "${your_endpoint}"); 145 | log_producer_config_set_project(config, "${your_project}"); 146 | log_producer_config_set_logstore(config, "${your_logstore}"); 147 | log_producer_config_set_access_id(config, "${your_access_key_id}"); 148 | log_producer_config_set_access_key(config, "${your_access_key_secret}"); 149 | //log_producer_config_set_remote_address(config, "192.168.12.12"); 150 | 151 | // if you do not need topic or tag, comment it 152 | log_producer_config_set_topic(config, "test_topic"); 153 | log_producer_config_add_tag(config, "tag_1", "val_1"); 154 | log_producer_config_add_tag(config, "tag_2", "val_2"); 155 | log_producer_config_add_tag(config, "tag_3", "val_3"); 156 | log_producer_config_add_tag(config, "tag_4", "val_4"); 157 | log_producer_config_add_tag(config, "tag_5", "val_5"); 158 | 159 | // set resource params 160 | log_producer_config_set_packet_log_bytes(config, 4*1024*1024); 161 | log_producer_config_set_packet_log_count(config, 4096); 162 | log_producer_config_set_packet_timeout(config, 3000); 163 | log_producer_config_set_max_buffer_limit(config, 64*1024*1024); 164 | 165 | // set send thread 166 | log_producer_config_set_send_thread_count(config, producer_send_thread_count); 167 | 168 | return create_log_producer(config, on_send_done); 169 | } 170 | 171 | #define MUTLI_THREAD_COUNT 16 172 | #define MUTLI_PRODUCER_COUNT 4 173 | 174 | #define GLOBAL_THREAD_FLAG 175 | 176 | void log_producer_multi_thread(size_t logsPerSecond) 177 | { 178 | logsPerSecond *= 100; 179 | if (log_producer_env_init(LOG_GLOBAL_ALL) != LOG_PRODUCER_OK) { 180 | exit(1); 181 | } 182 | 183 | #ifdef GLOBAL_THREAD_FLAG 184 | log_producer_global_send_thread_init(producer_send_thread_count, 100000); 185 | producer_send_thread_count = 0; 186 | #endif 187 | 188 | log_producer * producers[MUTLI_PRODUCER_COUNT]; 189 | log_producer_client * clients[MUTLI_PRODUCER_COUNT]; 190 | multi_write_log_param param[MUTLI_PRODUCER_COUNT]; 191 | 192 | int i = 0; 193 | for (; i < MUTLI_PRODUCER_COUNT; ++i) { 194 | producers[i] = create_log_producer_wrapper(on_log_send_done); 195 | if (producers[i] == NULL) 196 | { 197 | printf("create log producer by config file fail \n"); 198 | exit(1); 199 | } 200 | clients[i] = get_log_producer_client(producers[i], NULL); 201 | if (clients[i] == NULL) 202 | { 203 | printf("create log producer client by config file fail \n"); 204 | exit(1); 205 | } 206 | param[i].send_count = logsPerSecond; 207 | param[i].client = clients[i] ; 208 | } 209 | 210 | 211 | pthread_t allThread[MUTLI_THREAD_COUNT]; 212 | for (i = 0; i < MUTLI_THREAD_COUNT; ++i) 213 | { 214 | pthread_create(&allThread[i], NULL, write_log_thread, ¶m[i % MUTLI_PRODUCER_COUNT]); 215 | } 216 | 217 | 218 | for (i = 0; i < MUTLI_THREAD_COUNT; ++i) 219 | { 220 | pthread_join(allThread[i], NULL); 221 | } 222 | aos_error_log("All thread done"); 223 | 224 | for (i = 0; i < MUTLI_PRODUCER_COUNT; ++i) { 225 | destroy_log_producer(producers[i]); 226 | } 227 | 228 | log_producer_env_destroy(); 229 | 230 | } 231 | 232 | void log_producer_create_destroy() 233 | { 234 | int count = 0; 235 | while (1) 236 | { 237 | if (log_producer_env_init(LOG_GLOBAL_ALL) != LOG_PRODUCER_OK) { 238 | exit(1); 239 | } 240 | 241 | log_producer * producer = create_log_producer_wrapper(on_log_send_done); 242 | if (producer == NULL) 243 | { 244 | printf("create log producer by config file fail \n"); 245 | exit(1); 246 | } 247 | 248 | log_producer_client * client = get_log_producer_client(producer, NULL); 249 | if (client == NULL) 250 | { 251 | printf("create log producer client by config file fail \n"); 252 | exit(1); 253 | } 254 | 255 | destroy_log_producer(producer); 256 | 257 | log_producer_env_destroy(); 258 | printf("%d \n", count++); 259 | } 260 | } 261 | 262 | void log_producer_post_logs(int logsPerSecond, int sendSec) 263 | { 264 | //aos_log_level = AOS_LOG_DEBUG; 265 | if (log_producer_env_init(LOG_GLOBAL_ALL) != LOG_PRODUCER_OK) { 266 | exit(1); 267 | } 268 | 269 | log_producer * producer = create_log_producer_wrapper(on_log_send_done); 270 | if (producer == NULL) 271 | { 272 | printf("create log producer by config file fail \n"); 273 | exit(1); 274 | } 275 | 276 | log_producer_client * client = get_log_producer_client(producer, NULL); 277 | if (client == NULL) 278 | { 279 | printf("create log producer client by config file fail \n"); 280 | exit(1); 281 | } 282 | 283 | 284 | int32_t i = 0; 285 | int32_t totalTime = 0; 286 | for (i = 0; i < sendSec; ++i) 287 | { 288 | int64_t startTime = GET_TIME_US(); 289 | int j = 0; 290 | for (; j < logsPerSecond; ++j) 291 | { 292 | char indexStr[32]; 293 | sprintf(indexStr, "%d", i * logsPerSecond + j); 294 | log_producer_client_add_log(client, 20, "content_key_1", "1abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 295 | "content_key_2", "2abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 296 | "content_key_3", "3abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 297 | "content_key_4", "4abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 298 | "content_key_5", "5abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 299 | "content_key_6", "6abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 300 | "content_key_7", "7abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 301 | "content_key_8", "8abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 302 | "content_key_9", "9abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 303 | "index", indexStr); 304 | 305 | log_producer_result rst = log_producer_client_add_log(client, 8, "LogHub", "Real-time log collection and consumption", 306 | "Search/Analytics", "Query and real-time analysis", 307 | "Visualized", "dashboard and report functions", 308 | "Interconnection", "Grafana and JDBC/SQL92"); 309 | if (rst != LOG_PRODUCER_OK) 310 | { 311 | printf("add log error %d \n", rst); 312 | } 313 | } 314 | int64_t endTime = GET_TIME_US(); 315 | aos_error_log("Done : %d %d time %f us \n", i, logsPerSecond, (float)(endTime - startTime) * 1.0 / logsPerSecond / 2); 316 | totalTime += endTime - startTime; 317 | if (endTime - startTime < 1000000) 318 | { 319 | usleep(1000000 - (endTime - startTime)); 320 | } 321 | } 322 | aos_error_log("Total done : %f us, avg %f us", (float)totalTime / 180, (float)totalTime / (180 * logsPerSecond * 2)); 323 | //sleep(10); 324 | 325 | destroy_log_producer(producer); 326 | 327 | log_producer_env_destroy(); 328 | } 329 | 330 | int main(int argc, char *argv[]) 331 | { 332 | aos_log_level = AOS_LOG_DEBUG; 333 | log_producer_multi_thread(999llu); 334 | return 0; 335 | //aos_log_level = AOS_LOG_TRACE; 336 | int logsPerSec = 100; 337 | int sendSec = 180; 338 | if (argc == 3) 339 | { 340 | logsPerSec = atoi(argv[1]); 341 | sendSec = atoi(argv[2]); 342 | } 343 | //log_producer_create_destroy(); 344 | //log_producer_multi_thread(logsPerSec / 10); 345 | log_producer_post_logs(logsPerSec, sendSec); 346 | //builder_speed_test(logsPerSec); 347 | return 0; 348 | } 349 | 350 | -------------------------------------------------------------------------------- /sample/log_producer_sample.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 26/12/2017. 3 | // 4 | 5 | #include "log_api.h" 6 | #include "log_producer_config.h" 7 | #include "log_producer_client.h" 8 | 9 | void on_log_send_done(const char * config_name, log_producer_result result, size_t log_bytes, size_t compressed_bytes, const char * req_id, const char * message, const unsigned char * raw_buffer) 10 | { 11 | if (result == LOG_PRODUCER_OK) 12 | { 13 | // @note requst id may be NULL and printf %s NULL is undefined behavior 14 | if (req_id == NULL) 15 | { 16 | req_id = ""; 17 | } 18 | printf("send success, config : %s, result : %d, log bytes : %d, compressed bytes : %d, request id : %s \n", 19 | config_name, (result), 20 | (int)log_bytes, (int)compressed_bytes, req_id); 21 | } 22 | else 23 | { 24 | // @note message may be NULL and printf %s NULL is undefined behavior 25 | if (message == NULL) 26 | { 27 | message = ""; 28 | } 29 | // @note requst id may be NULL and printf %s NULL is undefined behavior 30 | if (req_id == NULL) 31 | { 32 | req_id = ""; 33 | } 34 | printf("send fail, config : %s, result : %d, log bytes : %d, compressed bytes : %d, request id : %s, error message : %s\n", 35 | config_name, (result), 36 | (int)log_bytes, (int)compressed_bytes, req_id, message); 37 | } 38 | 39 | } 40 | 41 | log_producer * create_log_producer_wrapper(on_log_producer_send_done_function on_send_done) 42 | { 43 | log_producer_config * config = create_log_producer_config(); 44 | // endpoint list: https://help.aliyun.com/document_detail/29008.html 45 | log_producer_config_set_endpoint(config, "${your_endpoint}"); 46 | log_producer_config_set_project(config, "${your_project}"); 47 | log_producer_config_set_logstore(config, "${your_logstore}"); 48 | log_producer_config_set_access_id(config, "${your_access_key_id}"); 49 | log_producer_config_set_access_key(config, "${your_access_key_secret}"); 50 | 51 | // @note, if using security token to send log, remember reset token before expired. this function is thread safe 52 | //log_producer_config_reset_security_token(config, "${your_access_key_id}", "${your_access_key_secret}", "${your_access_security_token}"); 53 | 54 | // if you do not need topic or tag, comment it 55 | log_producer_config_set_topic(config, "test_topic"); 56 | log_producer_config_add_tag(config, "tag_1", "val_1"); 57 | log_producer_config_add_tag(config, "tag_2", "val_2"); 58 | log_producer_config_add_tag(config, "tag_3", "val_3"); 59 | log_producer_config_add_tag(config, "tag_4", "val_4"); 60 | log_producer_config_add_tag(config, "tag_5", "val_5"); 61 | 62 | // set resource params 63 | log_producer_config_set_packet_log_bytes(config, 4*1024*1024); 64 | log_producer_config_set_packet_log_count(config, 4096); 65 | log_producer_config_set_packet_timeout(config, 3000); 66 | log_producer_config_set_max_buffer_limit(config, 64*1024*1024); 67 | 68 | // set send thread count 69 | log_producer_config_set_send_thread_count(config, 4); 70 | 71 | // set compress type : lz4 72 | log_producer_config_set_compress_type(config, 1); 73 | 74 | // set timeout 75 | log_producer_config_set_connect_timeout_sec(config, 10); 76 | log_producer_config_set_send_timeout_sec(config, 15); 77 | log_producer_config_set_destroy_flusher_wait_sec(config, 1); 78 | log_producer_config_set_destroy_sender_wait_sec(config, 1); 79 | 80 | // set interface 81 | log_producer_config_set_net_interface(config, NULL); 82 | 83 | return create_log_producer(config, on_send_done); 84 | } 85 | 86 | 87 | void log_producer_post_logs() 88 | { 89 | if (log_producer_env_init(LOG_GLOBAL_ALL) != LOG_PRODUCER_OK) { 90 | exit(1); 91 | } 92 | 93 | log_producer * producer = create_log_producer_wrapper(on_log_send_done); 94 | if (producer == NULL) 95 | { 96 | printf("create log producer by config fail \n"); 97 | exit(1); 98 | } 99 | 100 | log_producer_client * client = get_log_producer_client(producer, NULL); 101 | if (client == NULL) 102 | { 103 | printf("create log producer client by config fail \n"); 104 | exit(1); 105 | } 106 | 107 | int i = 0; 108 | for (; i < 10; ++i) 109 | { 110 | char indexStr[32]; 111 | sprintf(indexStr, "%d", i); 112 | log_producer_client_add_log(client, 20, "content_key_1", "1abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 113 | "content_key_2", "2abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 114 | "content_key_3", "3abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 115 | "content_key_4", "4abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 116 | "content_key_5", "5abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 117 | "content_key_6", "6abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 118 | "content_key_7", "7abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 119 | "content_key_8", "8abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 120 | "content_key_9", "9abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+", 121 | "index", indexStr); 122 | 123 | log_producer_result rst = log_producer_client_add_log(client, 8, "LogHub", "Real-time log collection and consumption", 124 | "Search/Analytics", "Query and real-time analysis", 125 | "Visualized", "dashboard and report functions", 126 | "Interconnection", "Grafana and JDBC/SQL92"); 127 | if (rst != LOG_PRODUCER_OK) 128 | { 129 | printf("add log error %d \n", rst); 130 | } 131 | } 132 | 133 | destroy_log_producer(producer); 134 | 135 | log_producer_env_destroy(); 136 | } 137 | 138 | int main(int argc, char *argv[]) 139 | { 140 | log_producer_post_logs(); 141 | return 0; 142 | } 143 | 144 | -------------------------------------------------------------------------------- /sample/video_frame_producer_sample.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 26/02/2018. 3 | // 4 | 5 | #include "log_api.h" 6 | #include "log_producer_config.h" 7 | #include "log_producer_client.h" 8 | #include "log_multi_thread.h" 9 | 10 | 11 | /** 12 | * 发送数据结果的回调函数 13 | * @param config_name 14 | * @param result 15 | * @param log_bytes 16 | * @param compressed_bytes 17 | * @param req_id 18 | * @param message 19 | */ 20 | void on_log_send_done(const char * config_name, log_producer_result result, size_t log_bytes, size_t compressed_bytes, const char * req_id, const char * message, const unsigned char * raw_buffer) 21 | { 22 | if (result == LOG_PRODUCER_OK) 23 | { 24 | printf("send success, config : %s, result : %d, log bytes : %d, compressed bytes : %d, request id : %s \n", 25 | config_name, (result), 26 | (int)log_bytes, (int)compressed_bytes, req_id); 27 | 28 | } 29 | else 30 | { 31 | printf("send fail, config : %s, result : %d, log bytes : %d, compressed bytes : %d, request id : %s, error message : %s\n", 32 | config_name, (result), 33 | (int)log_bytes, (int)compressed_bytes, req_id, message); 34 | } 35 | 36 | } 37 | 38 | /** 39 | * create producer 的一层封装 40 | * @param on_send_done 41 | * @return 42 | */ 43 | log_producer * create_log_producer_wrapper(on_log_producer_send_done_function on_send_done) 44 | { 45 | log_producer_config * config = create_log_producer_config(); 46 | // endpoint list: https://help.aliyun.com/document_detail/29008.html 47 | log_producer_config_set_endpoint(config, "${your_endpoint}"); 48 | log_producer_config_set_project(config, "${your_project}"); 49 | log_producer_config_set_logstore(config, "${your_logstore}"); 50 | log_producer_config_set_access_id(config, "${your_access_key_id}"); 51 | log_producer_config_set_access_key(config, "${your_access_key_secret}"); 52 | 53 | // if you do not need topic or tag, comment it 54 | // 设置主题 55 | log_producer_config_set_topic(config, "test_topic"); 56 | 57 | // 设置tag信息，此tag会附加在每个关键帧上，建议固定的meta信息使用tag设置 58 | log_producer_config_add_tag(config, "os", "linux"); 59 | log_producer_config_add_tag(config, "version", "0.1.0"); 60 | log_producer_config_add_tag(config, "type", "type_xxx"); 61 | 62 | // set resource param 63 | log_producer_config_set_packet_log_bytes(config, 4*1024*1024); 64 | log_producer_config_set_packet_log_count(config, 4096); 65 | // 设置后台数据聚合上传的时间，默认为3000ms，如有需求可自定义设置 66 | log_producer_config_set_packet_timeout(config, 3000); 67 | // 最大缓存的数据大小，超出缓存时add_log接口会立即返回失败 68 | log_producer_config_set_max_buffer_limit(config, 64*1024*1024); 69 | // 设置不使用压缩 70 | log_producer_config_set_compress_type(config, 0); 71 | 72 | // set send thread count 73 | // 发送线程数，为1即可 74 | log_producer_config_set_send_thread_count(config, 1); 75 | 76 | // on_send_done 为发送的回调函数（发送成功/失败都会返回） 77 | return create_log_producer(config, on_send_done); 78 | } 79 | 80 | #define FRAME_FLAG_KEY_FRAME 1 81 | #define FRAME_FLAG_NONE 0 82 | 83 | /** 84 | * 帧数据的一种封装形式，您可以根据需求自定义调整 85 | */ 86 | typedef struct _video_frame { 87 | uint64_t duration; 88 | uint8_t * frame_data; 89 | uint32_t frame_data_size; 90 | uint32_t frame_flag; 91 | uint64_t decoding_timestamp; 92 | uint64_t presentation_timestamp; 93 | uint8_t * extra_info; 94 | uint8_t extra_info_size; 95 | }video_frame; 96 | 97 | 98 | /** 99 | * 发送帧数据，这里是一种示例的封装形式，您可以根据需求进行更改 100 | * @param client producer Client 101 | * @param frame 帧数据 102 | */ 103 | void post_frame(log_producer_client * client, video_frame * frame) 104 | { 105 | if (client == NULL || frame == NULL) 106 | { 107 | return; 108 | } 109 | 110 | // 帧数据封装 start 111 | char * g_frame_keys[] = { 112 | (char *)"duration", 113 | (char *)"frame_data", 114 | (char *)"frame_flag", 115 | (char *)"decoding_timestamp", 116 | (char *)"presentation_timestamp", 117 | (char *)"extra_info" 118 | }; 119 | 120 | size_t g_frame_keys_len[] = { 121 | strlen("duration"), 122 | strlen("frame_data"), 123 | strlen("frame_flag"), 124 | strlen("decoding_timestamp"), 125 | strlen("presentation_timestamp"), 126 | strlen("extra_info") 127 | }; 128 | 129 | char * values[6]; 130 | 131 | char duration[32]; 132 | sprintf(duration, "%llu", (long long unsigned int)frame->duration); 133 | 134 | char frame_flag[32]; 135 | sprintf(frame_flag, "%u", frame->frame_flag); 136 | 137 | char decoding_timestamp[32]; 138 | sprintf(decoding_timestamp, "%llu", (long long unsigned int)frame->decoding_timestamp); 139 | 140 | char presentation_timestamp[32]; 141 | sprintf(presentation_timestamp, "%llu", (long long unsigned int)frame->presentation_timestamp); 142 | 143 | values[0] = duration; 144 | values[1] = (char *)frame->frame_data; 145 | values[2] = frame_flag; 146 | values[3] = decoding_timestamp; 147 | values[4] = presentation_timestamp; 148 | values[5] = (char *)frame->extra_info; 149 | 150 | size_t values_len[] = { 151 | strlen(duration), 152 | frame->frame_data_size, 153 | strlen(frame_flag), 154 | strlen(decoding_timestamp), 155 | strlen(presentation_timestamp), 156 | frame->extra_info_size 157 | }; 158 | 159 | // 帧数据封装 end 160 | 161 | 162 | // 调用producer接口发送数据 163 | log_producer_result rst = log_producer_client_add_log_with_len(client, 6, g_frame_keys, g_frame_keys_len, values, values_len); 164 | // 异常处理 165 | if (rst != LOG_PRODUCER_OK) 166 | { 167 | // 这边可以加入一些异常处理，如果是关键帧发送失败，可以每隔10ms重试一次，超过10次继续失败时再丢弃 168 | if ((frame->frame_flag & FRAME_FLAG_KEY_FRAME) != 0) 169 | { 170 | int try_time = 0; 171 | for (; try_time < 10; ++try_time) 172 | { 173 | usleep(10000); 174 | if (log_producer_client_add_log_with_len(client, 6, g_frame_keys, g_frame_keys_len, values, values_len) == LOG_PRODUCER_OK) 175 | { 176 | break; 177 | } 178 | } 179 | if (try_time == 10) 180 | { 181 | printf("add key frame error %d \n", rst); 182 | } 183 | } 184 | else 185 | { 186 | printf("add normal frame error %d \n", rst); 187 | } 188 | } 189 | 190 | } 191 | 192 | /** 193 | * 示例函数 194 | */ 195 | void producer_post_frames() 196 | { 197 | if (log_producer_env_init(LOG_GLOBAL_ALL) != LOG_PRODUCER_OK) { 198 | exit(1); 199 | } 200 | 201 | log_producer * producer = create_log_producer_wrapper(on_log_send_done); 202 | if (producer == NULL) 203 | { 204 | printf("create producer by config fail \n"); 205 | exit(1); 206 | } 207 | 208 | log_producer_client * client = get_log_producer_client(producer, NULL); 209 | if (client == NULL) 210 | { 211 | printf("create producer client by config fail \n"); 212 | exit(1); 213 | } 214 | 215 | srand(time(NULL)); 216 | 217 | int i = 0; 218 | for (; i < 100; ++i) 219 | { 220 | video_frame frame; 221 | // 每10帧有一个为关键帧 222 | frame.frame_flag = i % 10 == 0 ? FRAME_FLAG_KEY_FRAME : FRAME_FLAG_NONE; 223 | 224 | // mock 帧数据 225 | uint32_t data_size = 960*720; 226 | uint8_t * frame_data = (uint8_t *)malloc(data_size); 227 | memset(frame_data, i % 256, data_size); 228 | frame.frame_data_size = data_size; 229 | frame.frame_data = frame_data; 230 | 231 | frame.duration = rand() % 500; 232 | frame.decoding_timestamp = time(NULL); 233 | frame.presentation_timestamp = time(NULL); 234 | char extra_info[32]; 235 | sprintf(extra_info, "extra_%d", i); 236 | frame.extra_info = (uint8_t *)extra_info; 237 | frame.extra_info_size = strlen(extra_info); 238 | 239 | // 发送帧数据 240 | post_frame(client, &frame); 241 | 242 | free(frame_data); 243 | 244 | usleep(30000); 245 | } 246 | 247 | destroy_log_producer(producer); 248 | log_producer_env_destroy(); 249 | } 250 | 251 | int main(int argc, char *argv[]) 252 | { 253 | producer_post_frames(); 254 | return 0; 255 | } 256 | 257 | -------------------------------------------------------------------------------- /save_send_buffer.md: -------------------------------------------------------------------------------- 1 | # 程序可靠退出方案 2 | ## 简介 3 | 在使用日志服务C Producer时，若您的程序需要可靠退出，可以根据此文档介绍实现此功能。 4 | 5 | ## 问题背景 6 | 在程序退出时，C Producer会在给定时间范围内尽可能的将缓存中的日志发送出去，当等待超过`destroy_sender_wait_sec`若还没有发送成功，则会强制停止发送线程池。此时可能出现最后若干日志丢失的情况。 7 | 8 | ## 解决方法 9 | 针对以上问题，日志服务 C Producer提供了相应的数据导出以及导入接口，为应用程序在退出时提供日志数据可靠保存的方式。具体实现方式如下： 10 | 1. 在销毁Producer时，producer会将未发送完的数据通过发送回调函数`on_log_send_done`通知应用程序，错误码为`LOG_PRODUCER_SEND_EXIT_BUFFERED` 11 | 2. 应用程序在接收到此错误码时，将`log_bytes` `compressed_bytes` `buffer` 保存到本地 12 | 3. 在应用程序下一次启动时，读取本地文件，并调用`log_producer_client_add_raw_log_buffer`将上一次未发送完毕的数据发送出去 13 | 14 | -------------------------------------------------------------------------------- /src/hmac-sha.c: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include 4 | #include 5 | #include "sha1.h" 6 | #include "hmac-sha.h" 7 | 8 | #define IPAD 0x36 9 | #define OPAD 0x5C 10 | 11 | 12 | #ifndef HMAC_SHORTONLY 13 | 14 | void hmac_sha1_init(hmac_sha1_ctx_t *s, const void *key, uint16_t keylength_b){ 15 | uint8_t buffer[SHA1_BLOCK_BYTES]; 16 | uint8_t i; 17 | 18 | memset(buffer, 0, SHA1_BLOCK_BYTES); 19 | if (keylength_b > SHA1_BLOCK_BITS){ 20 | sha1((void*)buffer, key, keylength_b); 21 | } else { 22 | memcpy(buffer, key, (keylength_b+7)/8); 23 | } 24 | 25 | for (i=0; ia)); 29 | sha1_nextBlock(&(s->a), buffer); 30 | 31 | for (i=0; ib)); 35 | sha1_nextBlock(&(s->b), buffer); 36 | 37 | 38 | #if defined SECURE_WIPE_BUFFER 39 | memset(buffer, 0, SHA1_BLOCK_BYTES); 40 | #endif 41 | } 42 | 43 | void hmac_sha1_nextBlock(hmac_sha1_ctx_t *s, const void *block){ 44 | sha1_nextBlock(&(s->a), block); 45 | } 46 | void hmac_sha1_lastBlock(hmac_sha1_ctx_t *s, const void *block, uint16_t length_b){ 47 | while(length_b>=SHA1_BLOCK_BITS){ 48 | sha1_nextBlock(&s->a, block); 49 | block = (uint8_t*)block + SHA1_BLOCK_BYTES; 50 | length_b -= SHA1_BLOCK_BITS; 51 | } 52 | sha1_lastBlock(&s->a, block, length_b); 53 | } 54 | 55 | void hmac_sha1_final(void *dest, hmac_sha1_ctx_t *s){ 56 | sha1_ctx2hash(dest, &s->a); 57 | sha1_lastBlock(&s->b, dest, SHA1_HASH_BITS); 58 | sha1_ctx2hash(dest, &(s->b)); 59 | } 60 | 61 | #endif 62 | 63 | /* 64 | * keylength in bits! 65 | * message length in bits! 66 | */ 67 | void hmac_sha1(void *dest, const void *key, uint16_t keylength_b, const void *msg, uint32_t msglength_b){ /* a one-shot*/ 68 | sha1_ctx_t s; 69 | uint8_t i; 70 | uint8_t buffer[SHA1_BLOCK_BYTES]; 71 | 72 | memset(buffer, 0, SHA1_BLOCK_BYTES); 73 | 74 | /* if key is larger than a block we have to hash it*/ 75 | if (keylength_b > SHA1_BLOCK_BITS){ 76 | sha1((void*)buffer, key, keylength_b); 77 | } else { 78 | memcpy(buffer, key, (keylength_b+7)/8); 79 | } 80 | 81 | for (i=0; i= SHA1_BLOCK_BITS){ 87 | sha1_nextBlock(&s, msg); 88 | msg = (uint8_t*)msg + SHA1_BLOCK_BYTES; 89 | msglength_b -= SHA1_BLOCK_BITS; 90 | } 91 | sha1_lastBlock(&s, msg, msglength_b); 92 | /* since buffer still contains key xor ipad we can do ... */ 93 | for (i=0; i 3 | #include 4 | #include 5 | 6 | 7 | void aos_log_print_default(const char *message, int lens) 8 | { 9 | puts(message); 10 | } 11 | 12 | 13 | void aos_log_format_default(int level, 14 | const char *file, 15 | int line, 16 | const char *function, 17 | const char *fmt, ...); 18 | 19 | aos_log_level_e aos_log_level = AOS_LOG_WARN; 20 | aos_log_format_pt aos_log_format = aos_log_format_default; 21 | aos_log_print_pt aos_log_output = aos_log_print_default; 22 | 23 | 24 | static const char * _aos_log_level_str[] = { 25 | "NONE", 26 | "NONE", 27 | "FATAL", 28 | "ERROR", 29 | "WARN", 30 | "INFO", 31 | "DEBUG", 32 | "TRACE", 33 | "NONE" 34 | }; 35 | 36 | 37 | void aos_log_set_level(aos_log_level_e level) 38 | { 39 | aos_log_level = level; 40 | } 41 | 42 | 43 | void aos_log_format_default(int level, 44 | const char *file, 45 | int line, 46 | const char *function, 47 | const char *fmt, ...) 48 | { 49 | va_list args; 50 | char buffer[1024]; 51 | int maxLen = 1020; 52 | 53 | // @note return value maybe < 0 || > maxLen 54 | int len = snprintf(buffer, maxLen, "[%s] [%s:%d] ", 55 | _aos_log_level_str[level], 56 | file, line); 57 | // should never happen 58 | if (len < 0 || len > maxLen) { 59 | puts("[aos_log_format] error log fmt\n"); 60 | return; 61 | } 62 | 63 | va_start(args, fmt); 64 | // @note return value maybe < 0 || > maxLen 65 | int rst = vsnprintf(buffer + len, maxLen - len, fmt, args); 66 | va_end(args); 67 | if (rst < 0) { 68 | puts("[aos_log_format] error log fmt\n"); 69 | return; 70 | } 71 | 72 | if (rst > maxLen - len) { 73 | rst = maxLen - len; 74 | } 75 | len += rst; 76 | 77 | 78 | while (len > 0 && buffer[len -1] == '\n') 79 | { 80 | len--; 81 | } 82 | buffer[len++] = '\n'; 83 | buffer[len] = '\0'; 84 | 85 | aos_log_output(buffer, len); 86 | } 87 | 88 | void aos_log_set_format_callback(aos_log_format_pt func) 89 | { 90 | if (func == NULL) { 91 | return; 92 | } 93 | aos_log_format = func; 94 | } 95 | 96 | void aos_log_set_print_callback(aos_log_print_pt func) 97 | { 98 | if (func == NULL) { 99 | return; 100 | } 101 | aos_log_output = func; 102 | } 103 | -------------------------------------------------------------------------------- /src/inner_log.h: -------------------------------------------------------------------------------- 1 | #ifndef LIBAOS_LOG_H 2 | #define LIBAOS_LOG_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" 6 | { 7 | #endif 8 | 9 | typedef void (*aos_log_format_pt)(int level, 10 | const char *file, 11 | int line, 12 | const char *function, 13 | const char *fmt, ...); 14 | typedef void (*aos_log_print_pt)(const char *message, int len); 15 | 16 | typedef enum { 17 | AOS_LOG_OFF = 1, 18 | AOS_LOG_FATAL, 19 | AOS_LOG_ERROR, 20 | AOS_LOG_WARN, 21 | AOS_LOG_INFO, 22 | AOS_LOG_DEBUG, 23 | AOS_LOG_TRACE, 24 | AOS_LOG_ALL 25 | } aos_log_level_e; 26 | 27 | extern aos_log_level_e aos_log_level; 28 | extern aos_log_format_pt aos_log_format; 29 | 30 | /** 31 | * set inner log callback for format 32 | * @note if format callback is set, print callback has no effect 33 | * @param func callback function 34 | */ 35 | void aos_log_set_format_callback(aos_log_format_pt func); 36 | 37 | /** 38 | * set inner log callback for string log print 39 | * @param func callback function 40 | */ 41 | void aos_log_set_print_callback(aos_log_print_pt func); 42 | 43 | 44 | #define aos_fatal_log(format, args...) if(aos_log_level>=AOS_LOG_FATAL) \ 45 | aos_log_format(AOS_LOG_FATAL, __FILE__, __LINE__, __FUNCTION__, format, ## args) 46 | #define aos_error_log(format, args...) if(aos_log_level>=AOS_LOG_ERROR) \ 47 | aos_log_format(AOS_LOG_ERROR, __FILE__, __LINE__, __FUNCTION__, format, ## args) 48 | #define aos_warn_log(format, args...) if(aos_log_level>=AOS_LOG_WARN) \ 49 | aos_log_format(AOS_LOG_WARN, __FILE__, __LINE__, __FUNCTION__, format, ## args) 50 | #define aos_info_log(format, args...) if(aos_log_level>=AOS_LOG_INFO) \ 51 | aos_log_format(AOS_LOG_INFO, __FILE__, __LINE__, __FUNCTION__, format, ## args) 52 | #define aos_debug_log(format, args...) if(aos_log_level>=AOS_LOG_DEBUG) \ 53 | aos_log_format(AOS_LOG_DEBUG, __FILE__, __LINE__, __FUNCTION__, format, ## args) 54 | #define aos_trace_log(format, args...) if(aos_log_level>=AOS_LOG_TRACE) \ 55 | aos_log_format(AOS_LOG_TRACE, __FILE__, __LINE__, __FUNCTION__, format, ## args) 56 | 57 | #ifdef __cplusplus 58 | } 59 | #endif 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/log_api.h: -------------------------------------------------------------------------------- 1 | #ifndef LIBLOG_API_H 2 | #define LIBLOG_API_H 3 | 4 | #include "log_define.h" 5 | 6 | #include "log_builder.h" 7 | LOG_CPP_START 8 | 9 | struct _log_post_option 10 | { 11 | char * interface; // net interface to send log, NULL as default 12 | char * remote_address; // remote_address to send log, NULL as default 13 | int connect_timeout; // connection timeout seconds, 0 as default 14 | int operation_timeout; // operation timeout seconds, 0 as default 15 | int compress_type; // 0 no compress, 1 lz4 16 | int using_https; // 0 http, 1 https 17 | }; 18 | typedef struct _log_post_option log_post_option; 19 | 20 | log_status_t sls_log_init(int32_t log_global_flag); 21 | void sls_log_destroy(); 22 | 23 | post_log_result * post_logs_from_lz4buf(const char *endpoint, 24 | const char * accesskeyId, 25 | const char *accessKey, 26 | const char *stsToken, 27 | const char *project, 28 | const char *logstore, 29 | lz4_log_buf * buffer, 30 | log_post_option * option, 31 | auth_version version, 32 | const char *region); 33 | 34 | void post_log_result_destroy(post_log_result * result); 35 | 36 | LOG_CPP_END 37 | #endif 38 | -------------------------------------------------------------------------------- /src/log_builder.c: -------------------------------------------------------------------------------- 1 | #include "log_builder.h" 2 | #include "lz4.h" 3 | #include "sds.h" 4 | #include 5 | #include 6 | #include 7 | #include "inner_log.h" 8 | 9 | #ifdef LOG_FEATURE_ZSTD_COMPRESS 10 | #include "zstd.h" 11 | #define LOG_ZSTD_COMPRESS_LEVEL 1 12 | #endif 13 | 14 | // 1+3( 1 ---> header; 2 ---> 128 * 128 = 16KB) 15 | #define INIT_LOG_SIZE_BYTES 3 16 | 17 | /** 18 | * Return the number of bytes required to store a variable-length unsigned 19 | * 32-bit integer in base-128 varint encoding. 20 | * 21 | * \param v 22 | * Value to encode. 23 | * \return 24 | * Number of bytes required. 25 | */ 26 | static inline size_t uint32_size(uint32_t v) 27 | { 28 | if (v < (1UL << 7)) { 29 | return 1; 30 | } else if (v < (1UL << 14)) { 31 | return 2; 32 | } else if (v < (1UL << 21)) { 33 | return 3; 34 | } else if (v < (1UL << 28)) { 35 | return 4; 36 | } else { 37 | return 5; 38 | } 39 | } 40 | 41 | /** 42 | * Pack an unsigned 32-bit integer in base-128 varint encoding and return the 43 | * number of bytes written, which must be 5 or less. 44 | * 45 | * \param value 46 | * Value to encode. 47 | * \param[out] out 48 | * Packed value. 49 | * \return 50 | * Number of bytes written to `out`. 51 | */ 52 | static inline size_t uint32_pack(uint32_t value, uint8_t *out) 53 | { 54 | unsigned rv = 0; 55 | 56 | if (value >= 0x80) { 57 | out[rv++] = value | 0x80; 58 | value >>= 7; 59 | if (value >= 0x80) { 60 | out[rv++] = value | 0x80; 61 | value >>= 7; 62 | if (value >= 0x80) { 63 | out[rv++] = value | 0x80; 64 | value >>= 7; 65 | if (value >= 0x80) { 66 | out[rv++] = value | 0x80; 67 | value >>= 7; 68 | } 69 | } 70 | } 71 | } 72 | /* assert: value<128 */ 73 | out[rv++] = value; 74 | return rv; 75 | } 76 | 77 | static inline uint32_t parse_uint32(unsigned len, const uint8_t *data) 78 | { 79 | uint32_t rv = data[0] & 0x7f; 80 | if (len > 1) { 81 | rv |= ((uint32_t) (data[1] & 0x7f) << 7); 82 | if (len > 2) { 83 | rv |= ((uint32_t) (data[2] & 0x7f) << 14); 84 | if (len > 3) { 85 | rv |= ((uint32_t) (data[3] & 0x7f) << 21); 86 | if (len > 4) 87 | rv |= ((uint32_t) (data[4]) << 28); 88 | } 89 | } 90 | } 91 | return rv; 92 | } 93 | 94 | static unsigned scan_varint(unsigned len, const uint8_t *data) 95 | { 96 | unsigned i; 97 | if (len > 10) 98 | len = 10; 99 | for (i = 0; i < len; i++) 100 | if ((data[i] & 0x80) == 0) 101 | break; 102 | if (i == len) 103 | return 0; 104 | return i + 1; 105 | } 106 | 107 | log_group_builder* log_group_create() 108 | { 109 | log_group_builder* bder = (log_group_builder*)malloc(sizeof(log_group_builder)+sizeof(log_group)); 110 | memset(bder, 0, sizeof(log_group_builder)+sizeof(log_group)); 111 | bder->grp = (log_group*)((char *)(bder) + sizeof(log_group_builder)); 112 | bder->loggroup_size = sizeof(log_group) + sizeof(log_group_builder); 113 | bder->builder_time = time(NULL); 114 | return bder; 115 | } 116 | 117 | void log_group_destroy(log_group_builder* bder) 118 | { 119 | // free tag 120 | log_group* group = bder->grp; 121 | if (group->tags.buffer != NULL) 122 | { 123 | free(group->tags.buffer); 124 | } 125 | // free log 126 | if (group->logs.buffer != NULL) 127 | { 128 | free(group->logs.buffer); 129 | } 130 | if (group->topic != NULL) 131 | { 132 | sdsfree(group->topic); 133 | } 134 | if (group->source != NULL) 135 | { 136 | sdsfree(group->source); 137 | } 138 | // free self 139 | free(bder); 140 | } 141 | 142 | 143 | /** 144 | * adjust buffer, this function will ensure tag's buffer size >= tag->now_buffer_len + new_len 145 | * @param tag 146 | * @param new_len new buffer len 147 | */ 148 | void _adjust_buffer(log_tag * tag, uint32_t new_len) 149 | { 150 | if (tag->buffer == NULL) 151 | { 152 | tag->buffer = (char *)malloc(new_len << 2); 153 | tag->max_buffer_len = new_len << 2; 154 | tag->now_buffer = tag->buffer; 155 | tag->now_buffer_len = 0; 156 | return; 157 | } 158 | uint32_t new_buffer_len = tag->max_buffer_len << 1; 159 | 160 | if (new_buffer_len < tag->now_buffer_len + new_len) 161 | { 162 | new_buffer_len = tag->now_buffer_len + new_len; 163 | } 164 | 165 | tag->buffer = (char *)realloc(tag->buffer, new_buffer_len); 166 | tag->now_buffer = tag->buffer + tag->now_buffer_len; 167 | tag->max_buffer_len = new_buffer_len; 168 | } 169 | 170 | void add_log_full(log_group_builder* bder, uint32_t logTime, int32_t pair_count, char ** keys, size_t * key_lens, char ** values, size_t * val_lens) 171 | { 172 | ++bder->grp->n_logs; 173 | 174 | // limit logTime's min value, ensure varint size is 5 175 | if (logTime < 1263563523) 176 | { 177 | logTime = 1263563523; 178 | } 179 | 180 | int32_t i = 0; 181 | int32_t logSize = 6; 182 | for (; i < pair_count; ++i) 183 | { 184 | uint32_t contSize = uint32_size(key_lens[i]) + uint32_size(val_lens[i]) + key_lens[i] + val_lens[i] + 2; 185 | logSize += 1 + uint32_size(contSize) + contSize; 186 | } 187 | int32_t totalBufferSize = logSize + 1 + uint32_size(logSize); 188 | 189 | log_tag * log = &(bder->grp->logs); 190 | 191 | if (log->now_buffer == NULL || log->max_buffer_len < log->now_buffer_len + totalBufferSize) 192 | { 193 | _adjust_buffer(log, totalBufferSize); 194 | } 195 | 196 | 197 | bder->loggroup_size += totalBufferSize; 198 | uint8_t * buf = (uint8_t*)log->now_buffer; 199 | 200 | *buf++ = 0x0A; 201 | buf += uint32_pack(logSize, buf); 202 | 203 | // time 204 | *buf++=0x08; 205 | buf += uint32_pack(logTime, buf); 206 | 207 | // Content 208 | // header 209 | i = 0; 210 | for (; i < pair_count; ++i) 211 | { 212 | *buf++ = 0x12; 213 | buf += uint32_pack(uint32_size(key_lens[i]) + uint32_size(val_lens[i]) + 2 + key_lens[i] + val_lens[i], buf); 214 | *buf++ = 0x0A; 215 | buf += uint32_pack(key_lens[i], buf); 216 | memcpy(buf, keys[i], key_lens[i]); 217 | buf += key_lens[i]; 218 | *buf++ = 0x12; 219 | buf += uint32_pack(val_lens[i], buf); 220 | memcpy(buf, values[i], val_lens[i]); 221 | buf += val_lens[i]; 222 | } 223 | assert(buf - (uint8_t*)log->now_buffer == totalBufferSize); 224 | log->now_buffer_len += totalBufferSize; 225 | log->now_buffer = (char *)buf; 226 | } 227 | 228 | 229 | void add_source(log_group_builder* bder,const char* src,size_t len) 230 | { 231 | bder->loggroup_size += sizeof(char)*(len) + uint32_size((uint32_t)len) + 1; 232 | bder->grp->source = sdsnewlen(src, len); 233 | } 234 | 235 | void add_topic(log_group_builder* bder,const char* tpc,size_t len) 236 | { 237 | bder->loggroup_size += sizeof(char)*(len) + uint32_size((uint32_t)len) + 1; 238 | bder->grp->topic = sdsnewlen(tpc, len); 239 | } 240 | 241 | void add_pack_id(log_group_builder* bder, const char* pack, size_t pack_len, size_t packNum) 242 | { 243 | char packStr[128]; 244 | packStr[127] = '\0'; 245 | snprintf(packStr, 127, "%s-%X", pack, (unsigned int)packNum); 246 | add_tag(bder, "__pack_id__", strlen("__pack_id__"), packStr, strlen(packStr)); 247 | } 248 | 249 | 250 | void add_tag(log_group_builder* bder, const char* k, size_t k_len, const char* v, size_t v_len) 251 | { 252 | // use only 1 malloc 253 | uint32_t tag_size = sizeof(char) * (k_len + v_len) + uint32_size((uint32_t)k_len) + uint32_size((uint32_t)v_len) + 2; 254 | uint32_t n_buffer = 1 + uint32_size(tag_size) + tag_size; 255 | log_tag * tag = &(bder->grp->tags); 256 | if (tag->now_buffer == NULL || tag->now_buffer_len + n_buffer > tag->max_buffer_len) 257 | { 258 | _adjust_buffer(tag, n_buffer); 259 | } 260 | uint8_t * buf = (uint8_t *)tag->now_buffer; 261 | *buf++ = 0x32; 262 | buf += uint32_pack(tag_size, buf); 263 | *buf++ = 0x0A; 264 | buf += uint32_pack((uint32_t)k_len, buf); 265 | memcpy(buf, k, k_len); 266 | buf += k_len; 267 | *buf++ = 0x12; 268 | buf += uint32_pack((uint32_t)v_len, buf); 269 | memcpy(buf, v, v_len); 270 | buf += v_len; 271 | assert((uint8_t *)tag->now_buffer + n_buffer == buf); 272 | tag->now_buffer = (char *)buf; 273 | tag->now_buffer_len += n_buffer; 274 | bder->loggroup_size += n_buffer; 275 | } 276 | 277 | static uint32_t _log_pack(log_group * grp, uint8_t * buf) 278 | { 279 | uint8_t * start_buf = buf; 280 | 281 | if (grp->logs.buffer != NULL) 282 | { 283 | buf += grp->logs.now_buffer_len; 284 | } 285 | else 286 | { 287 | return 0; 288 | } 289 | 290 | if (grp->topic != NULL) 291 | { 292 | *buf++ = 0x1A; 293 | buf+= uint32_pack((uint32_t)sdslen(grp->topic), buf); 294 | memcpy(buf, grp->topic, sdslen(grp->topic)); 295 | buf += sdslen(grp->topic); 296 | } 297 | 298 | if (grp->source != NULL) 299 | { 300 | *buf++ = 0x22; 301 | buf+= uint32_pack((uint32_t)sdslen(grp->source), buf); 302 | memcpy(buf, grp->source, sdslen(grp->source)); 303 | buf += sdslen(grp->source); 304 | } 305 | 306 | if (grp->tags.buffer != NULL) 307 | { 308 | memcpy(buf, grp->tags.buffer, grp->tags.now_buffer_len); 309 | buf += grp->tags.now_buffer_len; 310 | } 311 | 312 | return buf - start_buf; 313 | } 314 | 315 | void fix_log_group_time(char * pb_buffer, size_t len, uint32_t new_time) 316 | { 317 | if (len == 0 || pb_buffer == NULL || new_time < 1263563523) 318 | { 319 | return; 320 | } 321 | if (pb_buffer[0] != 0x0A) 322 | { 323 | return; 324 | } 325 | uint8_t * buf = (uint8_t *)pb_buffer; 326 | uint8_t * startBuf = (uint8_t *)pb_buffer; 327 | while (buf - startBuf < len && *buf == 0x0A) 328 | { 329 | ++buf; 330 | unsigned logSizeLen = scan_varint(5, buf); 331 | uint32_t logSize = parse_uint32(logSizeLen, buf); 332 | buf += logSizeLen; 333 | // time 334 | if (*buf == 0x08) 335 | { 336 | uint32_pack(new_time, buf + 1); 337 | } 338 | buf += logSize; 339 | } 340 | 341 | } 342 | 343 | 344 | log_buf serialize_to_proto_buf_with_malloc(log_group_builder* bder) 345 | { 346 | log_buf buf; 347 | buf.buffer = NULL; 348 | buf.n_buffer = 0; 349 | log_tag * log = &(bder->grp->logs); 350 | if (log->buffer == NULL) 351 | { 352 | return buf; 353 | } 354 | if (log->max_buffer_len < bder->loggroup_size) 355 | { 356 | _adjust_buffer(log, bder->loggroup_size - log->now_buffer_len); 357 | } 358 | buf.n_buffer = _log_pack(bder->grp, (uint8_t *)log->buffer); 359 | buf.buffer = log->buffer; 360 | return buf; 361 | } 362 | 363 | lz4_log_buf* serialize_with_malloc_compressed(log_group_builder* bder, log_compress_type compress_type) 364 | { 365 | if (compress_type == LOG_COMPRESS_NONE) 366 | { 367 | log_buf buf = serialize_to_proto_buf_with_malloc(bder); 368 | lz4_log_buf* pLogbuf = (lz4_log_buf*)malloc(sizeof(lz4_log_buf) + buf.n_buffer); 369 | pLogbuf->length = buf.n_buffer; 370 | pLogbuf->raw_length = buf.n_buffer; 371 | pLogbuf->compress_type = compress_type; 372 | memcpy(pLogbuf->data, buf.buffer, buf.n_buffer); 373 | return pLogbuf; 374 | } 375 | 376 | log_tag * log = &(bder->grp->logs); 377 | if (log->buffer == NULL) 378 | { 379 | aos_error_log("fail to serialize to log buf: logs is empty, %d", compress_type); 380 | return NULL; 381 | } 382 | if (log->max_buffer_len < bder->loggroup_size) 383 | { 384 | _adjust_buffer(log, bder->loggroup_size - log->now_buffer_len); 385 | } 386 | size_t length = _log_pack(bder->grp, (uint8_t *)log->buffer); 387 | 388 | size_t compress_bound, compressed_size; 389 | char* compressed_data; 390 | if (compress_type == LOG_COMPRESS_LZ4) 391 | { 392 | compress_bound = LZ4_compressBound(length); 393 | compressed_data = (char *)malloc(compress_bound); 394 | compressed_size = LZ4_compress_default((char *)log->buffer, compressed_data, length, compress_bound); 395 | if(compressed_size <= 0) 396 | { 397 | free(compressed_data); 398 | aos_error_log("fail to serialize to log buf: fail to compress using lz4"); 399 | return NULL; 400 | } 401 | } 402 | else if (compress_type == LOG_COMPRESS_ZSTD) 403 | { 404 | #ifdef LOG_FEATURE_ZSTD_COMPRESS 405 | compress_bound = ZSTD_compressBound(length); 406 | compressed_data = (char *)malloc(compress_bound); 407 | compressed_size = ZSTD_compress(compressed_data, 408 | compress_bound, 409 | (char *)log->buffer, 410 | length, 411 | LOG_ZSTD_COMPRESS_LEVEL); 412 | if (ZSTD_isError(compressed_size)) 413 | { 414 | free(compressed_data); 415 | aos_error_log("fail to serialize to log buf: fail to compress using zstd"); 416 | return NULL; 417 | } 418 | #else 419 | aos_error_log("fail to serialize to log buf, unsupported compress type: zstd"); 420 | return NULL; 421 | #endif 422 | } 423 | else 424 | { 425 | aos_error_log("fail to serialize to log buf, unsupported compress type:%d", compress_type); 426 | return NULL; 427 | } 428 | lz4_log_buf* pLogbuf = (lz4_log_buf*)malloc(sizeof(lz4_log_buf) + compressed_size); 429 | pLogbuf->length = compressed_size; 430 | pLogbuf->raw_length = length; 431 | pLogbuf->compress_type = compress_type; 432 | memcpy(pLogbuf->data, compressed_data, compressed_size); 433 | free(compressed_data); 434 | return pLogbuf; 435 | } 436 | 437 | lz4_log_buf* serialize_to_proto_buf_with_malloc_no_compress(log_group_builder* bder) 438 | { 439 | return serialize_with_malloc_compressed(bder, LOG_COMPRESS_NONE); 440 | } 441 | 442 | lz4_log_buf* serialize_to_proto_buf_with_malloc_zstd(log_group_builder* bder) 443 | { 444 | return serialize_with_malloc_compressed(bder, LOG_COMPRESS_ZSTD); 445 | } 446 | 447 | lz4_log_buf* serialize_to_proto_buf_with_malloc_lz4(log_group_builder* bder) 448 | { 449 | return serialize_with_malloc_compressed(bder, LOG_COMPRESS_LZ4); 450 | } 451 | 452 | void free_lz4_log_buf(lz4_log_buf* pBuf) 453 | { 454 | free(pBuf); 455 | } 456 | 457 | #ifdef LOG_KEY_VALUE_FLAG 458 | 459 | void add_log_begin(log_group_builder * bder) 460 | { 461 | log_tag * logs = &bder->grp->logs; 462 | if (logs->buffer == NULL || logs->now_buffer_len + INIT_LOG_SIZE_BYTES > logs->max_buffer_len) 463 | { 464 | _adjust_buffer(logs, INIT_LOG_SIZE_BYTES); 465 | } 466 | bder->grp->log_now_buffer = logs->now_buffer + INIT_LOG_SIZE_BYTES; 467 | } 468 | 469 | void add_log_time(log_group_builder * bder, uint32_t logTime) 470 | { 471 | log_tag * logs = &bder->grp->logs; 472 | // 1 header and 5 time 473 | if (bder->grp->log_now_buffer - logs->buffer + 6 > logs->max_buffer_len) 474 | { 475 | // reset log_now_buffer 476 | size_t delta = bder->grp->log_now_buffer - logs->buffer; 477 | _adjust_buffer(logs, delta + 6); 478 | bder->grp->log_now_buffer = logs->buffer + delta; 479 | } 480 | 481 | uint8_t * buf = (uint8_t *)bder->grp->log_now_buffer; 482 | // time 483 | *buf++=0x08; 484 | buf += uint32_pack(logTime, buf); 485 | bder->grp->log_now_buffer = (char *)buf; 486 | } 487 | 488 | void add_log_key_value(log_group_builder *bder, char * key, size_t key_len, char * value, size_t value_len) 489 | { 490 | // sum total size 491 | uint32_t kv_size = sizeof(char) * (key_len + value_len) + uint32_size((uint32_t)key_len) + uint32_size((uint32_t)value_len) + 2; 492 | kv_size += 1 + uint32_size(kv_size); 493 | log_tag * logs = &bder->grp->logs; 494 | // ensure buffer size 495 | if (bder->grp->log_now_buffer - logs->buffer + kv_size > logs->max_buffer_len ) 496 | { 497 | // reset log_now_buffer 498 | size_t delta = bder->grp->log_now_buffer - logs->buffer; 499 | _adjust_buffer(logs, delta + kv_size); 500 | bder->grp->log_now_buffer = logs->buffer + delta; 501 | } 502 | uint8_t * buf = (uint8_t *)bder->grp->log_now_buffer; 503 | // key_value header 504 | *buf++ = 0x12; 505 | buf += uint32_pack(uint32_size(key_len) + uint32_size(value_len) + 2 + key_len + value_len, buf); 506 | // key len 507 | *buf++ = 0x0A; 508 | buf += uint32_pack(key_len, buf); 509 | // key 510 | memcpy(buf, key, key_len); 511 | buf += key_len; 512 | // value len 513 | *buf++ = 0x12; 514 | buf += uint32_pack(value_len, buf); 515 | // value 516 | memcpy(buf, value, value_len); 517 | buf += value_len; 518 | bder->grp->log_now_buffer = (char *)buf; 519 | } 520 | 521 | void add_log_end(log_group_builder * bder) 522 | { 523 | log_tag * logs = &bder->grp->logs; 524 | uint32_t log_size = bder->grp->log_now_buffer - logs->now_buffer - INIT_LOG_SIZE_BYTES; 525 | // check total size and uint32_size(total size) 526 | 527 | int32_t header_size = uint32_size(log_size) + 1; 528 | 529 | if (header_size != INIT_LOG_SIZE_BYTES) 530 | { 531 | int32_t delta_header_size = header_size - (int32_t)INIT_LOG_SIZE_BYTES; 532 | // need check buffer size 533 | if (delta_header_size > 0 && bder->grp->log_now_buffer - logs->buffer + delta_header_size > logs->max_buffer_len) 534 | { 535 | // reset log_now_buffer 536 | size_t delta = bder->grp->log_now_buffer - logs->buffer; 537 | _adjust_buffer(logs, delta + delta_header_size); 538 | bder->grp->log_now_buffer = logs->buffer + delta; 539 | } 540 | // move buffer 541 | memmove(logs->now_buffer + header_size, logs->now_buffer + INIT_LOG_SIZE_BYTES, log_size); 542 | } 543 | // set log header 544 | uint8_t * buf = (uint8_t *)logs->now_buffer; 545 | *buf++ = 0x0A; 546 | buf += uint32_pack(log_size, buf); 547 | // do not need to add header_size 548 | logs->now_buffer = (char *)buf + log_size; 549 | logs->now_buffer_len += header_size + log_size; 550 | // update loggroup size 551 | bder->loggroup_size += header_size + log_size; 552 | } 553 | 554 | #endif 555 | 556 | -------------------------------------------------------------------------------- /src/log_builder.h: -------------------------------------------------------------------------------- 1 | // 2 | // log_builder.h 3 | // log-c-sdk 4 | // 5 | // Created by 王佳玮 on 16/8/23. 6 | // Copyright © 2016年 wangjwchn. All rights reserved. 7 | // 8 | #ifndef LOG_BUILDER_H 9 | #define LOG_BUILDER_H 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "log_define.h" 16 | LOG_CPP_START 17 | 18 | typedef struct _log_tag{ 19 | char * buffer; 20 | char * now_buffer; 21 | uint32_t max_buffer_len; 22 | uint32_t now_buffer_len; 23 | }log_tag; 24 | 25 | 26 | typedef struct _log_group{ 27 | char * source; 28 | char * topic; 29 | log_tag tags; 30 | log_tag logs; 31 | size_t n_logs; 32 | #ifdef LOG_KEY_VALUE_FLAG 33 | char * log_now_buffer; 34 | #endif 35 | }log_group; 36 | 37 | // todo: rename to compressed_log_buf 38 | typedef struct _lz4_log_buf{ 39 | size_t length; 40 | size_t raw_length; 41 | log_compress_type compress_type; 42 | unsigned char data[0]; 43 | }lz4_log_buf; 44 | 45 | typedef struct _log_group_builder{ 46 | log_group* grp; 47 | size_t loggroup_size; 48 | void * private_value; 49 | uint32_t builder_time; 50 | }log_group_builder; 51 | 52 | typedef struct _log_buffer { 53 | char * buffer; 54 | uint32_t n_buffer; 55 | }log_buf; 56 | 57 | extern log_buf serialize_to_proto_buf_with_malloc(log_group_builder* bder); 58 | extern lz4_log_buf* serialize_with_malloc_compressed(log_group_builder* bder, log_compress_type compress_type); 59 | extern lz4_log_buf* serialize_to_proto_buf_with_malloc_lz4(log_group_builder* bder); 60 | extern lz4_log_buf* serialize_to_proto_buf_with_malloc_zstd(log_group_builder* bder); 61 | // no compress 62 | extern lz4_log_buf* serialize_to_proto_buf_with_malloc_no_compress(log_group_builder* bder); 63 | extern void free_lz4_log_buf(lz4_log_buf* pBuf); 64 | extern log_group_builder* log_group_create(); 65 | extern void log_group_destroy(log_group_builder* bder); 66 | extern void add_log_full(log_group_builder* bder, uint32_t logTime, int32_t pair_count, char ** keys, size_t * key_lens, char ** values, size_t * val_lens); 67 | extern void add_source(log_group_builder* bder,const char* src,size_t len); 68 | extern void add_topic(log_group_builder* bder,const char* tpc,size_t len); 69 | extern void add_tag(log_group_builder* bder,const char* k,size_t k_len,const char* v,size_t v_len); 70 | extern void add_pack_id(log_group_builder* bder, const char* pack, size_t pack_len, size_t packNum); 71 | extern void fix_log_group_time(char * pb_buffer, size_t len, uint32_t new_time); 72 | 73 | // if you want to use this functions, add `-DADD_LOG_KEY_VALUE_FUN=ON` for cmake. eg `cmake . -DADD_LOG_KEY_VALUE_FUN=ON` 74 | #ifdef LOG_KEY_VALUE_FLAG 75 | /** 76 | * call it when you want to add a new log 77 | * @note sequence must be : add_log_begin -> add_log_time/add_log_key_value..... -> add_log_end 78 | * @param bder 79 | */ 80 | extern void add_log_begin(log_group_builder * bder); 81 | 82 | /** 83 | * set log's time, must been called only once in one log 84 | * @param bder 85 | * @param logTime 86 | */ 87 | extern void add_log_time(log_group_builder * bder, uint32_t logTime); 88 | 89 | /** 90 | * add key&value pair to log tail 91 | * @param bder 92 | * @param key 93 | * @param key_len 94 | * @param value 95 | * @param value_len 96 | */ 97 | extern void add_log_key_value(log_group_builder *bder, char * key, size_t key_len, char * value, size_t value_len); 98 | 99 | /** 100 | * add log end, call it when you add time and key&value done 101 | * @param bder 102 | */ 103 | extern void add_log_end(log_group_builder * bder); 104 | 105 | #endif 106 | 107 | LOG_CPP_END 108 | #endif /* log_builder_h */ 109 | -------------------------------------------------------------------------------- /src/log_define.h: -------------------------------------------------------------------------------- 1 | #ifndef LIBLOG_DEFINE_H 2 | #define LIBLOG_DEFINE_H 3 | 4 | 5 | #if defined(WIN32) && defined(BUILD_SHARED_LIB) 6 | #define LOG_EXPORT _declspec(dllexport) 7 | #else 8 | #define LOG_EXPORT 9 | #endif 10 | 11 | #ifdef __cplusplus 12 | # define LOG_CPP_START extern "C" { 13 | # define LOG_CPP_END } 14 | #else 15 | # define LOG_CPP_START 16 | # define LOG_CPP_END 17 | #endif 18 | 19 | #define LOG_GLOBAL_SSL (1<<0) 20 | #define LOG_GLOBAL_WIN32 (1<<1) 21 | #define LOG_GLOBAL_ALL (LOG_GLOBAL_SSL|LOG_GLOBAL_WIN32) 22 | #define LOG_GLOBAL_NOTHING (0) 23 | 24 | typedef int log_status_t; 25 | 26 | struct _post_log_result 27 | { 28 | int statusCode; 29 | char * errorMessage; 30 | char * requestID; 31 | }; 32 | 33 | typedef struct _post_log_result post_log_result; 34 | 35 | enum _auth_version 36 | { 37 | AUTH_VERSION_1 = 1, 38 | AUTH_VERSION_4 39 | }; 40 | 41 | typedef enum _auth_version auth_version; 42 | 43 | typedef enum { 44 | LOG_COMPRESS_NONE = 0, 45 | LOG_COMPRESS_LZ4 = 1, 46 | LOG_COMPRESS_ZSTD = 2, // FEATURE_ZSTD_COMPRESS 47 | } log_compress_type; 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /src/log_inner_include.h: -------------------------------------------------------------------------------- 1 | #ifndef LOG_INNER_INCLUDE_H 2 | #define LOG_INNER_INCLUDE_H 3 | 4 | //操作系统头文件 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #ifdef WIN32 13 | #define inline __inline 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | typedef int socklen_t; 20 | 21 | 22 | 23 | #elif defined(_VXWORKS) 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include //for vx6 52 | #include 53 | #include 54 | 55 | #else 56 | 57 | #include 58 | #include 59 | #include 60 | #include 61 | #include 62 | #include 63 | #include 64 | 65 | #endif 66 | 67 | 68 | #if defined WIN32 || defined _VXWORKS 69 | 70 | typedef unsigned char u_char; 71 | typedef unsigned char u_int8; 72 | typedef unsigned short u_int16; 73 | typedef unsigned int u_int32; 74 | typedef unsigned __int64 u_int64; 75 | typedef signed char int8; 76 | typedef signed short int16; 77 | typedef signed int int32; 78 | typedef signed __int64 int64; 79 | 80 | #elif defined __linux__ 81 | 82 | typedef unsigned char u_char; 83 | typedef unsigned char u_int8; 84 | typedef unsigned short u_int16; 85 | typedef unsigned int u_int32; 86 | typedef unsigned long long u_int64; 87 | typedef signed char int8; 88 | typedef signed short int16; 89 | typedef signed int int32; 90 | typedef long long int64; 91 | 92 | typedef int SOCKET; 93 | #define SOCKET_ERROR -1 94 | #define INVALID_SOCKET -1 95 | #define __in // 参数输入 96 | #define __out // 参数输出 97 | #define closesocket close 98 | #define stricmp strcasecmp 99 | typedef int BOOL; 100 | #define TRUE 1 101 | #define FALSE 0 102 | 103 | #define Sleep(param) usleep(1000*(param)) 104 | #define strcpy_s(a, b, c) strcpy(a, c) 105 | #define sprintf_s(a, b, c) sprintf(a, c) 106 | #define strncpy_s(a, b, c, d) strncpy(a, c, d) 107 | #define vsprintf_s(a, b, c, d) vsprintf(a, c, d) 108 | #define _strdup strdup 109 | #define _stricmp stricmp 110 | 111 | 112 | #endif 113 | 114 | #endif //LOG_INNER_INCLUDE_H 115 | -------------------------------------------------------------------------------- /src/log_multi_thread.h: -------------------------------------------------------------------------------- 1 | #ifndef LOG_MULTI_THREAD_UTIL_H 2 | #define LOG_MULTI_THREAD_UTIL_H 3 | 4 | #include "log_inner_include.h" 5 | 6 | 7 | //不同操作系统资源相关的工具宏定义 8 | #ifdef WIN32 9 | /// * @brief 临界区资源 10 | typedef LPCRITICAL_SECTION CRITICALSECTION; 11 | #define INVALID_CRITSECT NULL 12 | 13 | /** 14 | ******************************************************************** 15 | * 创建互斥锁 16 | ******************************************************************** 17 | */ 18 | static inline CRITICALSECTION CreateCriticalSection() 19 | { 20 | CRITICALSECTION cs = (CRITICAL_SECTION *)malloc(sizeof(CRITICAL_SECTION)); 21 | InitializeCriticalSection(cs); 22 | return cs; 23 | } 24 | 25 | /** 26 | ******************************************************************** 27 | * 删除互斥锁 28 | ******************************************************************** 29 | */ 30 | static inline void DeleteCriticalSection(CRITICALSECTION cs) { 31 | if (cs != INVALID_CRITSECT) { 32 | DeleteCriticalSection(cs); 33 | free(cs); 34 | } 35 | } 36 | 37 | /// * @brief 加锁 38 | #define CS_ENTER(cs) EnterCriticalSection(cs) 39 | /// * @brief 解锁 40 | #define CS_LEAVE(cs) LeaveCriticalSection(cs) 41 | 42 | /// * @brief 互斥锁 43 | #define MUTEX CRITICAL_SECTION 44 | /// * @brief 加锁 45 | #define MUTEX_LOCK(mutex) EnterCriticalSection(&mutex) 46 | /// * @brief 解锁 47 | #define MUTEX_UNLOCK(mutex) LeaveCriticalSection(&mutex) 48 | /// * @brief 互斥锁初始化 49 | #define MUTEX_INIT(mutex) InitializeCriticalSection(&mutex) 50 | /// * @brief 互斥锁销毁 51 | #define MUTEX_DESTROY(mutex) DeleteCriticalSection(&mutex) 52 | 53 | //信号量资源 54 | /// * @brief 信号量 55 | typedef HANDLE SEMA; 56 | /// * @brief 等待信号量一定时间 57 | #define SEMA_WAIT_TIME(sema, delay) WaitForSingleObject(sema, delay) 58 | /// * @brief 一直阻塞地进行等待信号量 59 | #define SEMA_WAIT(sema) WaitForSingleObject(sema, INFINITE) 60 | /// * @brief 释放信号量 61 | #define SEMA_POST(sema) ReleaseSemaphore(sema, 1, NULL) 62 | /// * @brief 尝试获取一个信号量 63 | #define SEMA_TRYWAIT(sema) WaitForSingleObject(sema, 0) 64 | /// * @brief 销毁信号量 65 | #define SEMA_DESTROY(sema) CloseHandle(sema) 66 | /// * @brief 初始化信号量， 输入的为：信号量的最大值，初始信号量个数 67 | #define SEMA_INIT(sema, initCount, maxCount) sema = CreateSemaphore(NULL, initCount, maxCount, NULL) 68 | /// * @brief 初始一个带有名称的信号量，用于多进程交互 69 | #define SEMA_INIT_NAME(sema, initCount, maxCount, semaName) sema = CreateSemaphore(NULL, initCount, maxCount, semaName) 70 | /// * @brief 信号量等待超时 71 | #define SEMA_WAIT_TIMEOUT WAIT_TIMEOUT 72 | /// * @brief 等待到信号量 73 | #define SEMA_WAIT_OK WAIT_OBJECT_0 74 | 75 | typedef HANDLE THREAD 76 | 77 | #define snprintf sprintf_s 78 | 79 | #define ATOMICINT volatile long 80 | 81 | #define ATOMICINT_INC(pAtopicInt) InterlockedIncrement(pAtopicInt) 82 | #define ATOMICINT_DEC(pAtopicInt) InterlockedDecrement(pAtopicInt) 83 | #define ATOMICINT_ADD(pAtopicInt, addVal) InterlockedAdd(pAtopicInt, addVal) 84 | #define ATOMICINT_EXCHANGEADD(pAtopicInt, addVal) InterlockedExchangeAdd(pAtopicInt, addVal) 85 | #define ATOMICINT_EXCHANGE(pAtopicInt, exchangeVal) InterlockedExchange(pAtopicInt, exchangeVal) 86 | #define ATOMICINT_COMPAREEXCAHNGE(pAtopicInt, exchangeVal, cmpVal) InterlockedCompareExchange(pAtopicInt, exchangeVal, cmpVal) 87 | 88 | #elif defined(_VXWORKS) 89 | 90 | //临界区资源 91 | typedef SEM_ID CRITICALSECTION; 92 | #define INVALID_CRITSECT NULL 93 | static inline CRITICALSECTION CreateCriticalSection(int spinCount = 0) 94 | { 95 | CRITICALSECTION cs = semMCreate(SEM_Q_PRIORITY | SEM_DELETE_SAFE | SEM_INVERSION_SAFE); 96 | if (cs == NULL) 97 | { 98 | perror("vxworks create MUTUAL EXCLUSION SEMAPHORE failed\n"); 99 | } 100 | return cs; 101 | } 102 | 103 | static inline void DeleteCriticalSection(CRITICALSECTION & cs) 104 | { 105 | semDelete(cs); 106 | cs = INVALID_CRITSECT; 107 | } 108 | 109 | #define CS_ENTER(cs) semTake(cs, WAIT_FOREVER) 110 | #define CS_LEAVE(cs) semGive(cs) 111 | 112 | #define MUTEX SEM_ID 113 | #define MUTEX_LOCK(mutex) semTake(mutex, WAIT_FOREVER) 114 | #define MUTEX_UNLOCK(mutex) semGive(mutex) 115 | #define MUTEX_INIT(mutex) mutex = semBCreate(SEM_Q_FIFO,SEM_FULL) 116 | #define MUTEX_DESTROY(mutex) semDelete(mutex) 117 | //信号量资源 118 | #define SEMA SEM_ID 119 | #define SEMA_WAIT_TIME(sema,delay) semTake(sema, delay) 120 | #define SEMA_WAIT(sema) semTake(sema, WAIT_FOREVER) 121 | #define SEMA_POST(sema) semGive(sema) 122 | #define SEMA_DESTROY(sema) semDelete(sema) 123 | #define SEMA_INIT(sema, initCount, maxCount) sema = semCCreate(SEM_Q_FIFO,initCount) 124 | #define SEMA_WAIT_TIMEOUT ERROR 125 | //线程资源 126 | #define THREADID int 127 | #define SOCKET int 128 | #define closesocket(s_) close(s_) 129 | #define SOCKET_ERROR -1 130 | 131 | #else 132 | 133 | //临界区资源 134 | typedef pthread_mutex_t* CRITICALSECTION; 135 | #define INVALID_CRITSECT NULL 136 | 137 | static inline CRITICALSECTION CreateCriticalSection() { 138 | CRITICALSECTION cs = (pthread_mutex_t *)malloc(sizeof(pthread_mutex_t)); 139 | assert(cs != INVALID_CRITSECT); 140 | pthread_mutex_init(cs, NULL); 141 | return cs; 142 | } 143 | 144 | static inline void DeleteCriticalSection(CRITICALSECTION cs) { 145 | if (cs != INVALID_CRITSECT) { 146 | pthread_mutex_destroy(cs); 147 | free(cs); 148 | } 149 | } 150 | 151 | #define CS_ENTER(cs) pthread_mutex_lock(cs) 152 | #define CS_LEAVE(cs) pthread_mutex_unlock(cs) 153 | 154 | typedef pthread_cond_t* COND; 155 | typedef int COND_WAIT_T; 156 | #define COND_WAIT_OK 0 157 | #define COND_WAIT_TIMEOUT ETIMEDOUT 158 | 159 | #define INVALID_COND NULL 160 | 161 | static inline COND CreateCond() { 162 | COND cond = (COND)malloc(sizeof(pthread_cond_t)); 163 | assert(cond != INVALID_CRITSECT); 164 | pthread_cond_init(cond, NULL); 165 | return cond; 166 | } 167 | 168 | static inline void DeleteCond(COND cond) { 169 | if (cond != INVALID_COND) { 170 | pthread_cond_destroy(cond); 171 | free(cond); 172 | } 173 | } 174 | 175 | #define COND_SIGNAL(cond) pthread_cond_signal(cond) 176 | #define COND_SIGNAL_ALL(cond) pthread_cond_broadcast(cond) 177 | 178 | static inline COND_WAIT_T COND_WAIT_TIME(COND cond, CRITICALSECTION cs, int32_t waitMs) { 179 | struct timeval now; 180 | struct timespec outTime; 181 | gettimeofday(&now, NULL); 182 | 183 | now.tv_usec += ((waitMs) % 1000) * 1000; 184 | if (now.tv_usec > 1000000) 185 | { 186 | now.tv_usec -= 1000000; 187 | ++now.tv_sec; 188 | } 189 | outTime.tv_sec = now.tv_sec + (waitMs) / 1000; 190 | outTime.tv_nsec = now.tv_usec * 1000; 191 | return pthread_cond_timedwait(cond, cs, &outTime); 192 | } 193 | 194 | static inline int64_t GET_TIME_US() { 195 | struct timeval now; 196 | gettimeofday(&now, NULL); 197 | return (int64_t)now.tv_sec * 1000000 + now.tv_usec; 198 | } 199 | 200 | 201 | #define MUTEX pthread_mutex_t 202 | #define SEMA sem_t 203 | #define MUTEX_LOCK(mutex) pthread_mutex_lock(&mutex) 204 | #define MUTEX_UNLOCK(mutex) pthread_mutex_unlock(&mutex) 205 | #define MUTEX_INIT(mutex) pthread_mutex_init(&mutex,NULL) 206 | #define MUTEX_DESTROY(mutex) pthread_mutex_destroy(&mutex) 207 | 208 | // not supported in mac os 209 | #ifdef __linux__ 210 | static inline int sema_wait_time_(sem_t* sema, unsigned int delayMs) 211 | { 212 | struct timespec ts; 213 | 214 | struct timeval tv; 215 | 216 | gettimeofday(&tv, NULL); 217 | tv.tv_usec += (delayMs % 1000) * 1000; 218 | tv.tv_sec += delayMs / 1000; 219 | if (tv.tv_usec > 1000000) { 220 | tv.tv_usec -= 1000000; 221 | ++tv.tv_sec; 222 | } 223 | 224 | ts.tv_sec = tv.tv_sec; 225 | ts.tv_nsec = tv.tv_usec * 1000; 226 | 227 | return sem_timedwait(sema, &ts) == 0 ? 0 : ETIMEDOUT; 228 | } 229 | 230 | #define SEMA_WAIT_TIME(sema,delay) sema_wait_time_(&sema,delay) 231 | 232 | #endif 233 | 234 | #define SEMA_WAIT(sema) sem_wait(&sema) 235 | #define SEMA_POST(sema) sem_post(&sema) 236 | #define SEMA_TRYWAIT(sema) sem_trywait(&sema) 237 | #define SEMA_DESTROY(sema) sem_destroy(&sema) 238 | #define SEMA_INIT(sema, initCount, maxCount) sem_init(&sema,0,initCount) 239 | #define SEMA_INIT_NAME(sema, initCount, maxCount, semaName) sema = sem_open(semaName, O_CREAT, 0, initCount) 240 | #define WAIT_OBJECT_0 0 241 | #define WAIT_TIMEOUT ETIMEDOUT 242 | #define SEMA_WAIT_TIMEOUT ETIMEDOUT 243 | #define SEMA_WAIT_OK 0 244 | 245 | typedef pthread_t THREAD; 246 | #define THREAD_INIT(thread, func, param) pthread_create(&(thread), NULL, func, param) 247 | #define THREAD_JOIN(thread) pthread_join(thread, NULL) 248 | 249 | #define ATOMICINT volatile long 250 | 251 | #define ATOMICINT_INC(pAtopicInt) __sync_add_and_fetch(pAtopicInt, 1) 252 | #define ATOMICINT_DEC(pAtopicInt) __sync_add_and_fetch(pAtopicInt, -1) 253 | #define ATOMICINT_ADD(pAtopicInt, addVal) __sync_add_and_fetch(pAtopicInt, addVal) 254 | #define ATOMICINT_EXCHANGEADD(pAtopicInt, addVal) __sync_fetch_and_add(pAtopicInt, addVal) 255 | #define ATOMICINT_EXCHANGE(pAtopicInt, exchangeVal) __sync_val_compare_and_swap(pAtopicInt, *pAtopicInt, exchangeVal) 256 | #define ATOMICINT_COMPAREEXCAHNGE(pAtopicInt, exchangeVal, cmpVal) __sync_val_compare_and_swap(pAtopicInt, cmpVal, exchangeVal) 257 | 258 | typedef struct _FILETIME 259 | { 260 | unsigned long dwLowDateTime; 261 | unsigned long dwHighDateTime; 262 | } FILETIME; 263 | 264 | #endif //WIN32 265 | 266 | #endif //LOG_MULTI_THREAD_UTIL_H 267 | -------------------------------------------------------------------------------- /src/log_producer_client.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 20/11/2017. 3 | // 4 | 5 | #include "log_producer_client.h" 6 | #include "log_producer_manager.h" 7 | #include "inner_log.h" 8 | #include "log_api.h" 9 | #include 10 | #include 11 | 12 | 13 | static uint32_t s_init_flag = 0; 14 | static log_producer_result s_last_result = 0; 15 | 16 | extern log_queue * g_sender_data_queue; 17 | extern THREAD * g_send_threads; 18 | extern int32_t g_send_thread_count; 19 | extern volatile uint8_t g_send_thread_destroy; 20 | 21 | extern void * log_producer_send_thread_global(void * param); 22 | extern void log_producer_send_thread_global_inner(log_producer_send_param * send_param); 23 | extern void destroy_log_producer_manager_tail(log_producer_manager * manager); 24 | 25 | typedef struct _producer_client_private { 26 | 27 | log_producer_manager * producer_manager; 28 | log_producer_config * producer_config; 29 | }producer_client_private ; 30 | 31 | struct _log_producer { 32 | log_producer_client * root_client; 33 | }; 34 | 35 | log_producer_result log_producer_env_init(int32_t log_global_flag) 36 | { 37 | // if already init, just return s_last_result 38 | if (s_init_flag == 1) 39 | { 40 | return s_last_result; 41 | } 42 | s_init_flag = 1; 43 | if (0 != sls_log_init(log_global_flag)) 44 | { 45 | s_last_result = LOG_PRODUCER_INVALID; 46 | } 47 | else 48 | { 49 | s_last_result = LOG_PRODUCER_OK; 50 | } 51 | return s_last_result; 52 | } 53 | 54 | void log_producer_env_destroy() 55 | { 56 | if (s_init_flag == 0) 57 | { 58 | return; 59 | } 60 | s_init_flag = 0; 61 | 62 | // destroy global send threads 63 | if (g_send_threads != NULL) { 64 | g_send_thread_destroy = 1; 65 | aos_info_log("join global sender thread pool begin, thread count : %d", g_send_thread_count); 66 | int32_t threadId = 0; 67 | for (; threadId < g_send_thread_count; ++threadId) 68 | { 69 | THREAD_JOIN(g_send_threads[threadId]); 70 | aos_info_log("join one global sender thread pool done, thread id : %d", threadId); 71 | } 72 | free(g_send_threads); 73 | aos_info_log("flush out global sender queue begin"); 74 | while (log_queue_size(g_sender_data_queue) > 0) 75 | { 76 | // @note : we must process all data in queue, otherwise this will cause memory leak 77 | log_producer_send_param * send_param = (log_producer_send_param *)log_queue_trypop(g_sender_data_queue); 78 | log_producer_send_thread_global_inner(send_param); 79 | } 80 | aos_info_log("flush out global sender queue success"); 81 | log_queue_destroy(g_sender_data_queue); 82 | g_sender_data_queue = NULL; 83 | g_send_thread_destroy = 0; 84 | g_send_thread_count = 0; 85 | g_send_threads = NULL; 86 | aos_info_log("join global sender thread pool success"); 87 | } 88 | sls_log_destroy(); 89 | } 90 | 91 | log_producer * create_log_producer(log_producer_config * config, on_log_producer_send_done_function send_done_function) 92 | { 93 | if (!log_producer_config_is_valid(config)) 94 | { 95 | return NULL; 96 | } 97 | log_producer * producer = (log_producer *)malloc(sizeof(log_producer)); 98 | log_producer_client * producer_client = (log_producer_client *)malloc(sizeof(log_producer_client)); 99 | producer_client_private * client_private = (producer_client_private *)malloc(sizeof(producer_client_private)); 100 | producer_client->private_data = client_private; 101 | client_private->producer_config = config; 102 | client_private->producer_manager = create_log_producer_manager(config); 103 | client_private->producer_manager->send_done_function = send_done_function; 104 | 105 | if(client_private->producer_manager == NULL) 106 | { 107 | // free 108 | free(producer_client); 109 | free(client_private); 110 | free(producer); 111 | return NULL; 112 | } 113 | aos_debug_log("create producer client success, config : %s", config->logstore); 114 | producer_client->valid_flag = 1; 115 | producer->root_client = producer_client; 116 | return producer; 117 | } 118 | 119 | 120 | void destroy_log_producer(log_producer * producer) 121 | { 122 | if (producer == NULL) 123 | { 124 | return; 125 | } 126 | log_producer_client * client = producer->root_client; 127 | client->valid_flag = 0; 128 | producer_client_private * client_private = (producer_client_private *)client->private_data; 129 | destroy_log_producer_manager(client_private->producer_manager); 130 | 131 | free(client_private); 132 | free(client); 133 | free(producer); 134 | } 135 | 136 | extern log_producer_client * get_log_producer_client(log_producer * producer, const char * config_name) 137 | { 138 | if (producer == NULL) 139 | { 140 | return NULL; 141 | } 142 | return producer->root_client; 143 | } 144 | 145 | void log_producer_client_network_recover(log_producer_client * client) 146 | { 147 | if (client == NULL) 148 | { 149 | return; 150 | } 151 | log_producer_manager * manager = ((producer_client_private *)client->private_data)->producer_manager; 152 | manager->networkRecover = 1; 153 | } 154 | 155 | log_producer_result log_producer_client_add_log(log_producer_client * client, int32_t kv_count, ...) 156 | { 157 | if (client == NULL || !client->valid_flag) 158 | { 159 | return LOG_PRODUCER_INVALID; 160 | } 161 | va_list argp; 162 | va_start(argp, kv_count); 163 | 164 | int32_t pairs = kv_count / 2; 165 | 166 | char ** keys = (char **)malloc(pairs * sizeof(char *)); 167 | char ** values = (char **)malloc(pairs * sizeof(char *)); 168 | size_t * key_lens = (size_t *)malloc(pairs * sizeof(size_t)); 169 | size_t * val_lens = (size_t *)malloc(pairs * sizeof(size_t)); 170 | 171 | int32_t i = 0; 172 | for (; i < pairs; ++i) 173 | { 174 | const char * key = va_arg(argp, const char *); 175 | const char * value = va_arg(argp, const char *); 176 | keys[i] = (char *)key; 177 | values[i] = (char *)value; 178 | key_lens[i] = strlen(key); 179 | val_lens[i] = strlen(value); 180 | } 181 | 182 | log_producer_manager * manager = ((producer_client_private *)client->private_data)->producer_manager; 183 | 184 | log_producer_result rst = log_producer_manager_add_log(manager, (uint32_t)time(NULL), pairs, keys, key_lens, values, val_lens); 185 | free(keys); 186 | free(values); 187 | free(key_lens); 188 | free(val_lens); 189 | return rst; 190 | } 191 | 192 | log_producer_result log_producer_client_add_log_with_len(log_producer_client * client, int32_t pair_count, char ** keys, size_t * key_lens, char ** values, size_t * val_lens) 193 | { 194 | return log_producer_client_add_log_with_len_time(client, (uint32_t)time(NULL), pair_count, keys, key_lens, values, val_lens); 195 | } 196 | 197 | log_producer_result log_producer_client_add_log_with_len_time(log_producer_client * client, uint32_t time_sec, int32_t pair_count, char ** keys, size_t * key_lens, char ** values, size_t * value_lens) 198 | { 199 | if (client == NULL || !client->valid_flag) 200 | { 201 | return LOG_PRODUCER_INVALID; 202 | } 203 | 204 | log_producer_manager * manager = ((producer_client_private *)client->private_data)->producer_manager; 205 | 206 | return log_producer_manager_add_log(manager, time_sec, pair_count, keys, key_lens, values, value_lens); 207 | 208 | } 209 | 210 | 211 | 212 | log_producer_result log_producer_client_add_raw_log_buffer(log_producer_client * client, size_t log_bytes, size_t compressed_bytes, const unsigned char * raw_buffer) 213 | { 214 | if (client == NULL || !client->valid_flag || raw_buffer == NULL) 215 | { 216 | return LOG_PRODUCER_INVALID; 217 | } 218 | 219 | log_producer_manager * manager = ((producer_client_private *)client->private_data)->producer_manager; 220 | 221 | return log_producer_manager_send_raw_buffer(manager, log_bytes, compressed_bytes, raw_buffer); 222 | } 223 | 224 | log_producer_result log_producer_global_send_thread_init(int32_t log_global_send_thread_count, int32_t log_global_send_queue_size) 225 | { 226 | if (log_global_send_thread_count <= 0 || log_global_send_queue_size <= 0 || g_send_threads != NULL) { 227 | return LOG_PRODUCER_INVALID; 228 | } 229 | g_send_thread_count = log_global_send_thread_count; 230 | g_send_threads = (THREAD *)malloc(sizeof(THREAD) * g_send_thread_count); 231 | 232 | g_sender_data_queue = log_queue_create(log_global_send_queue_size); 233 | g_send_thread_destroy = 0; 234 | int32_t threadId = 0; 235 | for (; threadId < g_send_thread_count; ++threadId) 236 | { 237 | THREAD_INIT(g_send_threads[threadId], log_producer_send_thread_global, g_sender_data_queue); 238 | } 239 | return LOG_PRODUCER_OK; 240 | } 241 | -------------------------------------------------------------------------------- /src/log_producer_client.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 20/11/2017. 3 | // 4 | 5 | #ifndef LOG_C_SDK_LOG_PRODUCER_CLIENT_H 6 | #define LOG_C_SDK_LOG_PRODUCER_CLIENT_H 7 | 8 | 9 | #include "log_define.h" 10 | #include "log_producer_config.h" 11 | LOG_CPP_START 12 | 13 | 14 | /** 15 | * log producer client 16 | */ 17 | typedef struct _log_producer_client 18 | { 19 | volatile int32_t valid_flag; 20 | int32_t log_level; 21 | void * private_data; 22 | }log_producer_client; 23 | 24 | typedef struct _log_producer log_producer; 25 | 26 | /** 27 | * init log producer environment 28 | * @param log_global_flag global log config flag 29 | * @note should been called before create any log producer client 30 | * @note no multi thread safe 31 | * @return OK if success, others the error code 32 | */ 33 | LOG_EXPORT log_producer_result log_producer_env_init(int32_t log_global_flag); 34 | 35 | /** 36 | * create global send thread pool 37 | * 38 | * @note if producer have no send thread, use global send thread pool to send logs 39 | * @note this thread pool will been destroyed when you call log_producer_env_destroy 40 | * @note not thread safe 41 | * 42 | * @param log_global_send_thread_count 43 | * @param log_global_send_queue_size recommend values : for server apps, set 10000; for client apps, set 1000; for iot devices, set 100 44 | * @return 45 | */ 46 | LOG_EXPORT log_producer_result log_producer_global_send_thread_init(int32_t log_global_send_thread_count, int32_t log_global_send_queue_size); 47 | 48 | /** 49 | * destroy log producer environment 50 | * @note should been called after all log producer clients destroyed 51 | * @note no multi thread safe 52 | */ 53 | LOG_EXPORT void log_producer_env_destroy(); 54 | 55 | /** 56 | * create log producer with a producer config 57 | * @param config log_producer_config 58 | * @param send_done_function this function will be called when send done(can be ok or fail), set to NULL if you don't care about it 59 | * @return producer client ptr, NULL if create fail 60 | */ 61 | LOG_EXPORT log_producer * create_log_producer(log_producer_config * config, on_log_producer_send_done_function send_done_function); 62 | 63 | /** 64 | * destroy log producer 65 | * @param producer 66 | * @note no multi thread safe 67 | */ 68 | LOG_EXPORT void destroy_log_producer(log_producer * producer); 69 | 70 | /** 71 | * get client from producer 72 | * @param producer 73 | * @param config_name useless now, set NULL 74 | * @return the specific producer client, root client if config_name is NULL or no specific config, 75 | */ 76 | LOG_EXPORT log_producer_client * get_log_producer_client(log_producer * producer, const char * config_name); 77 | 78 | 79 | /** 80 | * force send data when network recover 81 | * @param client 82 | */ 83 | LOG_EXPORT void log_producer_client_network_recover(log_producer_client * client); 84 | 85 | /** 86 | * add log to producer, this may return LOG_PRODUCER_DROP_ERROR if buffer is full. 87 | * if you care about this log very much, retry when return LOG_PRODUCER_DROP_ERROR. 88 | * 89 | * @example log_producer_client_add_log(client, 4, "key_1", "value_1", "key_2", "value_2") 90 | * @note log param ... must be const char * or char * with '\0' ended 91 | * @note multi thread safe 92 | * @param client 93 | * @param kv_count key value count 94 | * @param ... log params : key, value pairs, must be const char * or char * 95 | * @return ok if success, LOG_PRODUCER_DROP_ERROR if buffer is full, LOG_PRODUCER_INVALID if client is destroyed. 96 | */ 97 | LOG_EXPORT log_producer_result log_producer_client_add_log(log_producer_client * client, int32_t kv_count, ...); 98 | 99 | /** 100 | * add log to producer, this may return LOG_PRODUCER_DROP_ERROR if buffer is full. 101 | * if you care about this log very much, retry when return LOG_PRODUCER_DROP_ERROR. 102 | * 103 | * @param client 104 | * @param pair_count key value pair count 105 | * @note pair_count not kv_count 106 | * @param keys the key array 107 | * @param key_lens the key len array 108 | * @param values the value array 109 | * @param value_lens the value len array 110 | * @return ok if success, LOG_PRODUCER_DROP_ERROR if buffer is full, LOG_PRODUCER_INVALID if client is destroyed. 111 | */ 112 | LOG_EXPORT log_producer_result log_producer_client_add_log_with_len(log_producer_client * client, int32_t pair_count, char ** keys, size_t * key_lens, char ** values, size_t * value_lens); 113 | 114 | /** 115 | * add log to producer, this may return LOG_PRODUCER_DROP_ERROR if buffer is full. 116 | * if you care about this log very much, retry when return LOG_PRODUCER_DROP_ERROR. 117 | * 118 | * @param client 119 | * @param time_sec log time 120 | * @param pair_count key value pair count 121 | * @note pair_count not kv_count 122 | * @param keys the key array 123 | * @param key_lens the key len array 124 | * @param values the value array 125 | * @param value_lens the value len array 126 | * @param flush if this log info need to send right, 1 mean flush and 0 means NO 127 | * @return ok if success, LOG_PRODUCER_DROP_ERROR if buffer is full, LOG_PRODUCER_INVALID if client is destroyed, LOG_PRODUCER_PERSISTENT_ERROR is save binlog failed. 128 | */ 129 | LOG_EXPORT log_producer_result log_producer_client_add_log_with_len_time(log_producer_client * client, uint32_t time_sec, int32_t pair_count, char ** keys, size_t * key_lens, char ** values, size_t * value_lens); 130 | 131 | 132 | /** 133 | * add raw log buffer to client, this function is used to send buffers which can not send out when producer has destroyed 134 | * @param client 135 | * @param log_bytes 136 | * @param compressed_bytes 137 | * @param raw_buffer 138 | * @return ok if success, LOG_PRODUCER_DROP_ERROR if buffer is full, LOG_PRODUCER_INVALID if client is destroyed. 139 | */ 140 | LOG_EXPORT log_producer_result log_producer_client_add_raw_log_buffer(log_producer_client * client, size_t log_bytes, size_t compressed_bytes, const unsigned char * raw_buffer); 141 | 142 | LOG_CPP_END 143 | 144 | #endif //LOG_C_SDK_LOG_PRODUCER_CLIENT_H 145 | -------------------------------------------------------------------------------- /src/log_producer_common.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 21/11/2017. 3 | // 4 | 5 | #include "log_producer_common.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | log_producer_result LOG_PRODUCER_OK = 0; 12 | log_producer_result LOG_PRODUCER_INVALID = 1; 13 | log_producer_result LOG_PRODUCER_WRITE_ERROR = 2; 14 | log_producer_result LOG_PRODUCER_DROP_ERROR = 3; 15 | log_producer_result LOG_PRODUCER_SEND_NETWORK_ERROR = 4; 16 | log_producer_result LOG_PRODUCER_SEND_QUOTA_ERROR = 5; 17 | log_producer_result LOG_PRODUCER_SEND_UNAUTHORIZED = 6; 18 | log_producer_result LOG_PRODUCER_SEND_SERVER_ERROR = 7; 19 | log_producer_result LOG_PRODUCER_SEND_DISCARD_ERROR = 8; 20 | log_producer_result LOG_PRODUCER_SEND_TIME_ERROR = 9; 21 | log_producer_result LOG_PRODUCER_SEND_EXIT_BUFFERED = 10; 22 | 23 | 24 | int is_log_producer_result_ok(log_producer_result rst) 25 | { 26 | return rst == LOG_PRODUCER_OK; 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/log_producer_common.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 21/11/2017. 3 | // 4 | 5 | #ifndef LOG_C_SDK_LOG_PRODUCER_COMMON_H_H 6 | #define LOG_C_SDK_LOG_PRODUCER_COMMON_H_H 7 | 8 | #include "log_define.h" 9 | #include 10 | #include 11 | LOG_CPP_START 12 | 13 | 14 | /** 15 | * log producer result for all operation 16 | */ 17 | typedef int log_producer_result; 18 | 19 | /** 20 | * callback function for producer client 21 | * @param config_name this is logstore name 22 | * @param result send result 23 | * @param log_bytes log group packaged bytes 24 | * @param compressed_bytes lz4 compressed bytes 25 | * @param req_id request id. Must check NULL when use it 26 | * @param error_message if send result is not ok, error message is set. Must check NULL when use it 27 | * @param raw_buffer lz4 buffer 28 | * @note you can only read raw_buffer, but can't modify or free it 29 | */ 30 | typedef void (*on_log_producer_send_done_function)(const char * config_name, log_producer_result result, size_t log_bytes, size_t compressed_bytes, const char * req_id, const char * error_message, const unsigned char * raw_buffer); 31 | 32 | extern log_producer_result LOG_PRODUCER_OK; 33 | extern log_producer_result LOG_PRODUCER_INVALID; 34 | extern log_producer_result LOG_PRODUCER_WRITE_ERROR; 35 | extern log_producer_result LOG_PRODUCER_DROP_ERROR; 36 | extern log_producer_result LOG_PRODUCER_SEND_NETWORK_ERROR; 37 | extern log_producer_result LOG_PRODUCER_SEND_QUOTA_ERROR; 38 | extern log_producer_result LOG_PRODUCER_SEND_UNAUTHORIZED; 39 | extern log_producer_result LOG_PRODUCER_SEND_SERVER_ERROR; 40 | extern log_producer_result LOG_PRODUCER_SEND_DISCARD_ERROR; 41 | extern log_producer_result LOG_PRODUCER_SEND_TIME_ERROR; 42 | // this error code is passed when producer is being destroyed, you should save this buffer to local 43 | extern log_producer_result LOG_PRODUCER_SEND_EXIT_BUFFERED; 44 | 45 | 46 | /** 47 | * check if rst if ok 48 | * @param rst 49 | * @return 1 if ok, 0 not ok 50 | */ 51 | LOG_EXPORT int is_log_producer_result_ok(log_producer_result rst); 52 | 53 | 54 | LOG_CPP_END 55 | 56 | #endif //LOG_C_SDK_LOG_PRODUCER_COMMON_H_H 57 | -------------------------------------------------------------------------------- /src/log_producer_config.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 20/11/2017. 3 | // 4 | 5 | #include "log_producer_config.h" 6 | #include "sds.h" 7 | #include 8 | #include 9 | #include "inner_log.h" 10 | 11 | static void _set_default_producer_config(log_producer_config * pConfig) 12 | { 13 | pConfig->logBytesPerPackage = 3 * 1024 * 1024; 14 | pConfig->logCountPerPackage = 2048; 15 | pConfig->packageTimeoutInMS = 3000; 16 | pConfig->maxBufferBytes = 64 * 1024 * 1024; 17 | 18 | pConfig->connectTimeoutSec = 10; 19 | pConfig->sendTimeoutSec = 15; 20 | pConfig->destroySenderWaitTimeoutSec = 1; 21 | pConfig->destroyFlusherWaitTimeoutSec = 1; 22 | pConfig->compressType = LOG_COMPRESS_LZ4; 23 | pConfig->using_https = 0; 24 | } 25 | 26 | 27 | static void _copy_config_string(const char * value, sds * src_value) 28 | { 29 | if (value == NULL || src_value == NULL) 30 | { 31 | return; 32 | } 33 | size_t strLen = strlen(value); 34 | if (*src_value == NULL) 35 | { 36 | *src_value = sdsnewEmpty(strLen); 37 | } 38 | *src_value = sdscpylen(*src_value, value, strLen); 39 | } 40 | 41 | 42 | log_producer_config * create_log_producer_config() 43 | { 44 | log_producer_config* pConfig = (log_producer_config*)malloc(sizeof(log_producer_config)); 45 | memset(pConfig, 0, sizeof(log_producer_config)); 46 | _set_default_producer_config(pConfig); 47 | pConfig->authVersion = AUTH_VERSION_1; 48 | pConfig->region = NULL; 49 | return pConfig; 50 | } 51 | 52 | 53 | void destroy_log_producer_config(log_producer_config * pConfig) 54 | { 55 | if (pConfig->project != NULL) 56 | { 57 | sdsfree(pConfig->project); 58 | } 59 | if (pConfig->logstore != NULL) 60 | { 61 | sdsfree(pConfig->logstore); 62 | } 63 | if (pConfig->endpoint != NULL) 64 | { 65 | sdsfree(pConfig->endpoint); 66 | } 67 | if (pConfig->accessKey != NULL) 68 | { 69 | sdsfree(pConfig->accessKey); 70 | } 71 | if (pConfig->accessKeyId != NULL) 72 | { 73 | sdsfree(pConfig->accessKeyId); 74 | } 75 | if (pConfig->topic != NULL) 76 | { 77 | sdsfree(pConfig->topic); 78 | } 79 | if (pConfig->source != NULL) 80 | { 81 | sdsfree(pConfig->source); 82 | } 83 | if (pConfig->netInterface != NULL) 84 | { 85 | sdsfree(pConfig->netInterface); 86 | } 87 | if (pConfig->remote_address != NULL) 88 | { 89 | sdsfree(pConfig->remote_address); 90 | } 91 | if (pConfig->securityToken != NULL) 92 | { 93 | sdsfree(pConfig->securityToken); 94 | } 95 | if (pConfig->securityTokenLock != NULL) 96 | { 97 | DeleteCriticalSection(pConfig->securityTokenLock); 98 | } 99 | if (pConfig->tagCount > 0 && pConfig->tags != NULL) 100 | { 101 | int i = 0; 102 | for (; i < pConfig->tagCount; ++i) 103 | { 104 | sdsfree(pConfig->tags[i].key); 105 | sdsfree(pConfig->tags[i].value); 106 | } 107 | free(pConfig->tags); 108 | } 109 | if (pConfig->region != NULL) 110 | { 111 | sdsfree(pConfig->region); 112 | } 113 | free(pConfig); 114 | } 115 | 116 | #ifdef LOG_PRODUCER_DEBUG 117 | void log_producer_config_print(log_producer_config * pConfig, FILE * file) 118 | { 119 | fprintf(file, "endpoint : %s\n", pConfig->endpoint); 120 | fprintf(file,"project : %s\n", pConfig->project); 121 | fprintf(file,"logstore : %s\n", pConfig->logstore); 122 | fprintf(file,"accessKeyId : %s\n", pConfig->accessKeyId); 123 | fprintf(file,"accessKey : %s\n", pConfig->accessKey); 124 | fprintf(file,"configName : %s\n", pConfig->configName); 125 | fprintf(file,"topic : %s\n", pConfig->topic); 126 | fprintf(file,"logLevel : %d\n", pConfig->logLevel); 127 | 128 | fprintf(file,"packageTimeoutInMS : %d\n", pConfig->packageTimeoutInMS); 129 | fprintf(file, "logCountPerPackage : %d\n", pConfig->logCountPerPackage); 130 | fprintf(file, "logBytesPerPackage : %d\n", pConfig->logBytesPerPackage); 131 | fprintf(file, "maxBufferBytes : %d\n", pConfig->maxBufferBytes); 132 | 133 | 134 | fprintf(file, "tags: \n"); 135 | int32_t i = 0; 136 | for (i = 0; i < pConfig->tagCount; ++i) 137 | { 138 | fprintf(file, "tag key : %s, value : %s \n", pConfig->tags[i].key, pConfig->tags[i].value); 139 | } 140 | 141 | } 142 | #endif 143 | 144 | void log_producer_config_set_packet_timeout(log_producer_config * config, int32_t time_out_ms) 145 | { 146 | if (config == NULL || time_out_ms < 0) 147 | { 148 | return; 149 | } 150 | config->packageTimeoutInMS = time_out_ms; 151 | } 152 | void log_producer_config_set_packet_log_count(log_producer_config * config, int32_t log_count) 153 | { 154 | if (config == NULL || log_count < 0) 155 | { 156 | return; 157 | } 158 | config->logCountPerPackage = log_count; 159 | } 160 | void log_producer_config_set_packet_log_bytes(log_producer_config * config, int32_t log_bytes) 161 | { 162 | if (config == NULL || log_bytes < 0) 163 | { 164 | return; 165 | } 166 | config->logBytesPerPackage = log_bytes; 167 | } 168 | void log_producer_config_set_max_buffer_limit(log_producer_config * config, int64_t max_buffer_bytes) 169 | { 170 | if (config == NULL || max_buffer_bytes < 0) 171 | { 172 | return; 173 | } 174 | config->maxBufferBytes = max_buffer_bytes; 175 | } 176 | 177 | void log_producer_config_set_send_thread_count(log_producer_config * config, int32_t thread_count) 178 | { 179 | if (config == NULL || thread_count < 0) 180 | { 181 | return; 182 | } 183 | config->sendThreadCount = thread_count; 184 | } 185 | 186 | 187 | void log_producer_config_set_net_interface(log_producer_config * config, const char * net_interface) 188 | { 189 | if (config == NULL || net_interface == NULL) 190 | { 191 | return; 192 | } 193 | _copy_config_string(net_interface, &config->netInterface); 194 | } 195 | 196 | void log_producer_config_set_remote_address(log_producer_config * config, const char * remote_address) 197 | { 198 | if (config == NULL || remote_address == NULL) 199 | { 200 | return; 201 | } 202 | _copy_config_string(remote_address, &config->remote_address); 203 | } 204 | 205 | void log_producer_config_set_connect_timeout_sec(log_producer_config * config, int32_t connect_timeout_sec) 206 | { 207 | if (config == NULL || connect_timeout_sec <= 0) 208 | { 209 | return; 210 | } 211 | config->connectTimeoutSec = connect_timeout_sec; 212 | } 213 | 214 | 215 | void log_producer_config_set_send_timeout_sec(log_producer_config * config, int32_t send_timeout_sec) 216 | { 217 | if (config == NULL || send_timeout_sec <= 0) 218 | { 219 | return; 220 | } 221 | config->sendTimeoutSec = send_timeout_sec; 222 | } 223 | 224 | void log_producer_config_set_destroy_flusher_wait_sec(log_producer_config * config, int32_t destroy_flusher_wait_sec) 225 | { 226 | if (config == NULL || destroy_flusher_wait_sec <= 0) 227 | { 228 | return; 229 | } 230 | config->destroyFlusherWaitTimeoutSec = destroy_flusher_wait_sec; 231 | } 232 | 233 | void log_producer_config_set_destroy_sender_wait_sec(log_producer_config * config, int32_t destroy_sender_wait_sec) 234 | { 235 | if (config == NULL || destroy_sender_wait_sec <= 0) 236 | { 237 | return; 238 | } 239 | config->destroySenderWaitTimeoutSec = destroy_sender_wait_sec; 240 | } 241 | 242 | void log_producer_config_set_compress_type(log_producer_config * config, int32_t compress_type) 243 | { 244 | if (config == NULL || 245 | compress_type < (int32_t)LOG_COMPRESS_NONE || 246 | compress_type > (int32_t)LOG_COMPRESS_ZSTD) 247 | { 248 | return; 249 | } 250 | config->compressType = (log_compress_type)compress_type; 251 | } 252 | 253 | void log_producer_config_set_using_http(log_producer_config * config, int32_t using_https) 254 | { 255 | if (config == NULL || using_https < 0) 256 | { 257 | return; 258 | } 259 | config->using_https = using_https; 260 | } 261 | 262 | void log_producer_config_set_log_queue_size(log_producer_config * config, int32_t log_queue_size) 263 | { 264 | if (config == NULL || log_queue_size < 0) 265 | { 266 | return; 267 | } 268 | config->logQueueSize = log_queue_size; 269 | } 270 | 271 | void log_producer_config_add_tag(log_producer_config * pConfig, const char * key, const char * value) 272 | { 273 | if(key == NULL || value == NULL) 274 | { 275 | return; 276 | } 277 | ++pConfig->tagCount; 278 | if (pConfig->tags == NULL || pConfig->tagCount > pConfig->tagAllocSize) 279 | { 280 | if(pConfig->tagAllocSize == 0) 281 | { 282 | pConfig->tagAllocSize = 4; 283 | } 284 | else 285 | { 286 | pConfig->tagAllocSize *= 2; 287 | } 288 | log_producer_config_tag * tagArray = (log_producer_config_tag *)malloc(sizeof(log_producer_config_tag) * pConfig->tagAllocSize); 289 | if (pConfig->tags != NULL) 290 | { 291 | memcpy(tagArray, pConfig->tags, sizeof(log_producer_config_tag) * (pConfig->tagCount - 1)); 292 | free(pConfig->tags); 293 | } 294 | pConfig->tags = tagArray; 295 | } 296 | int32_t tagIndex = pConfig->tagCount - 1; 297 | pConfig->tags[tagIndex].key = sdsnew(key); 298 | pConfig->tags[tagIndex].value = sdsnew(value); 299 | 300 | } 301 | 302 | 303 | void log_producer_config_set_endpoint(log_producer_config * config, const char * endpoint) 304 | { 305 | if (strlen(endpoint) < 8) { 306 | return; 307 | } 308 | if (strncmp(endpoint, "http://", 7) == 0) 309 | { 310 | endpoint += 7; 311 | } 312 | else if (strncmp(endpoint, "https://", 8) == 0) 313 | { 314 | config->using_https = 1; 315 | endpoint += 8; 316 | } 317 | _copy_config_string(endpoint, &config->endpoint); 318 | } 319 | 320 | void log_producer_config_set_project(log_producer_config * config, const char * project) 321 | { 322 | _copy_config_string(project, &config->project); 323 | } 324 | void log_producer_config_set_logstore(log_producer_config * config, const char * logstore) 325 | { 326 | _copy_config_string(logstore, &config->logstore); 327 | } 328 | 329 | void log_producer_config_set_access_id(log_producer_config * config, const char * access_id) 330 | { 331 | _copy_config_string(access_id, &config->accessKeyId); 332 | } 333 | 334 | void log_producer_config_set_access_key(log_producer_config * config, const char * access_key) 335 | { 336 | _copy_config_string(access_key, &config->accessKey); 337 | } 338 | 339 | 340 | void log_producer_config_reset_security_token(log_producer_config * config, const char * access_id, const char * access_secret, const char * security_token) 341 | { 342 | if (config->securityTokenLock == NULL) 343 | { 344 | config->securityTokenLock = CreateCriticalSection(); 345 | } 346 | CS_ENTER(config->securityTokenLock); 347 | _copy_config_string(access_id, &config->accessKeyId); 348 | _copy_config_string(access_secret, &config->accessKey); 349 | _copy_config_string(security_token, &config->securityToken); 350 | CS_LEAVE(config->securityTokenLock); 351 | } 352 | 353 | void log_producer_config_set_auth_version(log_producer_config * config, auth_version auth_version) 354 | { 355 | config->authVersion = auth_version; 356 | } 357 | 358 | void log_producer_config_set_region(log_producer_config * config, const char * region) 359 | { 360 | _copy_config_string(region, &config->region); 361 | } 362 | 363 | void log_producer_config_get_security(log_producer_config * config, char ** access_id, char ** access_secret, char ** security_token) 364 | { 365 | if (config->securityTokenLock == NULL) 366 | { 367 | _copy_config_string(config->accessKeyId, access_id); 368 | _copy_config_string(config->accessKey, access_secret); 369 | } 370 | else 371 | { 372 | CS_ENTER(config->securityTokenLock); 373 | _copy_config_string(config->accessKeyId, access_id); 374 | _copy_config_string(config->accessKey, access_secret); 375 | _copy_config_string(config->securityToken, security_token); 376 | CS_LEAVE(config->securityTokenLock); 377 | } 378 | } 379 | 380 | void log_producer_config_set_topic(log_producer_config * config, const char * topic) 381 | { 382 | _copy_config_string(topic, &config->topic); 383 | } 384 | 385 | void log_producer_config_set_source(log_producer_config * config, const char * source) 386 | { 387 | _copy_config_string(source, &config->source); 388 | } 389 | 390 | int log_producer_config_is_valid(log_producer_config * config) 391 | { 392 | if (config == NULL) 393 | { 394 | aos_error_log("invalid producer config"); 395 | return 0; 396 | } 397 | if (config->endpoint == NULL || config->project == NULL || config->logstore == NULL) 398 | { 399 | aos_error_log("invalid producer config destination params"); 400 | return 0; 401 | } 402 | if (config->accessKey == NULL || config->accessKeyId == NULL) 403 | { 404 | aos_error_log("invalid producer config authority params"); 405 | return 0; 406 | } 407 | if (config->packageTimeoutInMS < 0 || config->maxBufferBytes < 0 || config->logCountPerPackage < 0 || config->logBytesPerPackage < 0) 408 | { 409 | aos_error_log("invalid producer config log merge and buffer params"); 410 | return 0; 411 | } 412 | return 1; 413 | } 414 | -------------------------------------------------------------------------------- /src/log_producer_config.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 20/11/2017. 3 | // 4 | 5 | #ifndef LOG_C_SDK_LOG_PRODUCER_CONFIG_H 6 | #define LOG_C_SDK_LOG_PRODUCER_CONFIG_H 7 | 8 | #include "log_define.h" 9 | #include "log_producer_common.h" 10 | #include 11 | #include 12 | #include 13 | #include "log_multi_thread.h" 14 | LOG_CPP_START 15 | 16 | 17 | typedef struct _log_producer_config_tag 18 | { 19 | char * key; 20 | char * value; 21 | }log_producer_config_tag; 22 | 23 | typedef struct _log_producer_config 24 | { 25 | char * endpoint; 26 | char * project; 27 | char * logstore; 28 | char * accessKeyId; 29 | char * accessKey; 30 | char * securityToken; 31 | char * topic; 32 | char * source; 33 | auth_version authVersion; 34 | char * region; 35 | CRITICALSECTION securityTokenLock; 36 | log_producer_config_tag * tags; 37 | int32_t tagAllocSize; 38 | int32_t tagCount; 39 | 40 | int32_t sendThreadCount; 41 | 42 | int32_t packageTimeoutInMS; 43 | int32_t logCountPerPackage; 44 | int32_t logBytesPerPackage; 45 | int32_t maxBufferBytes; 46 | int32_t logQueueSize; 47 | 48 | char * netInterface; 49 | char * remote_address; 50 | int32_t connectTimeoutSec; 51 | int32_t sendTimeoutSec; 52 | int32_t destroyFlusherWaitTimeoutSec; 53 | int32_t destroySenderWaitTimeoutSec; 54 | 55 | log_compress_type compressType; 56 | int32_t using_https; // default http, 0 http, 1 https 57 | 58 | }log_producer_config; 59 | 60 | 61 | /** 62 | * create a empty producer config, should set config params manually 63 | * @return empty producer config 64 | */ 65 | LOG_EXPORT log_producer_config * create_log_producer_config(); 66 | 67 | 68 | /** 69 | * set producer config endpoint 70 | * @note if endpoint start with "https", then set using_https 1 71 | * @note strlen(endpoint) must >= 8 72 | * @param config 73 | * @param endpoint 74 | */ 75 | LOG_EXPORT void log_producer_config_set_endpoint(log_producer_config * config, const char * endpoint); 76 | 77 | /** 78 | * set producer config project 79 | * @param config 80 | * @param project 81 | */ 82 | LOG_EXPORT void log_producer_config_set_project(log_producer_config * config, const char * project); 83 | 84 | /** 85 | * set producer config logstore 86 | * @param config 87 | * @param logstore 88 | */ 89 | LOG_EXPORT void log_producer_config_set_logstore(log_producer_config * config, const char * logstore); 90 | 91 | /** 92 | * set producer config access id 93 | * @param config 94 | * @param access_id 95 | */ 96 | LOG_EXPORT void log_producer_config_set_access_id(log_producer_config * config, const char * access_id); 97 | 98 | 99 | /** 100 | * set producer config access key 101 | * @param config 102 | * @param access_id 103 | */ 104 | LOG_EXPORT void log_producer_config_set_access_key(log_producer_config * config, const char * access_id); 105 | 106 | /** 107 | * set producer config auth_version 108 | * @param config 109 | * @param auth_version 110 | */ 111 | LOG_EXPORT void log_producer_config_set_auth_version(log_producer_config * config, auth_version auth_version); 112 | 113 | 114 | /** 115 | * set producer config region 116 | * @param config 117 | * @param region 118 | */ 119 | LOG_EXPORT void log_producer_config_set_region(log_producer_config * config, const char * region); 120 | 121 | 122 | /** 123 | * reset producer config security token (thread safe) 124 | * @note if you want to use security token to send logs, you must call this function when create config and reset token before expired. 125 | * if token has expired, producer will drop logs after 6 hours 126 | * @param config 127 | * @param access_id 128 | */ 129 | LOG_EXPORT void log_producer_config_reset_security_token(log_producer_config * config, const char * access_id, const char * access_secret, const char * security_token); 130 | 131 | 132 | /** 133 | * inner api 134 | */ 135 | LOG_EXPORT void log_producer_config_get_security(log_producer_config * config, char ** access_id, char ** access_secret, char ** security_token); 136 | 137 | /** 138 | * set producer config topic 139 | * @param config 140 | * @param topic 141 | */ 142 | LOG_EXPORT void log_producer_config_set_topic(log_producer_config * config, const char * topic); 143 | 144 | /** 145 | * set producer source, this source will been set to every loggroup's "source" field 146 | * @param config 147 | * @param source 148 | */ 149 | LOG_EXPORT void log_producer_config_set_source(log_producer_config * config, const char * source); 150 | 151 | /** 152 | * add tag, this tag will been set to every loggroup's "tag" field 153 | * @param config 154 | * @param key 155 | * @param value 156 | */ 157 | LOG_EXPORT void log_producer_config_add_tag(log_producer_config * config, const char * key, const char * value); 158 | 159 | /** 160 | * set loggroup timeout, if delta time from first log's time >= time_out_ms, current loggroup will been flushed out 161 | * @param config 162 | * @param time_out_ms 163 | */ 164 | LOG_EXPORT void log_producer_config_set_packet_timeout(log_producer_config * config, int32_t time_out_ms); 165 | 166 | /** 167 | * set loggroup max log count, if loggoup's logs count >= log_count, current loggroup will been flushed out 168 | * @param config 169 | * @param log_count 170 | */ 171 | LOG_EXPORT void log_producer_config_set_packet_log_count(log_producer_config * config, int32_t log_count); 172 | 173 | /** 174 | * set loggroup max log bytes, if loggoup's log bytes >= log_bytes, current loggroup will been flushed out 175 | * @param config 176 | * @param log_bytes 177 | */ 178 | LOG_EXPORT void log_producer_config_set_packet_log_bytes(log_producer_config * config, int32_t log_bytes); 179 | 180 | /** 181 | * set max buffer size, if total buffer size > max_buffer_bytes, all send will fail immediately 182 | * @param config 183 | * @param max_buffer_bytes 184 | */ 185 | LOG_EXPORT void log_producer_config_set_max_buffer_limit(log_producer_config * config, int64_t max_buffer_bytes); 186 | 187 | /** 188 | * set send thread count, default is 0. 189 | * @note if thread count is 0, flusher thread is in the charge of send logs. 190 | * @note if thread count > 1, then producer will create $(thread_count) threads to send logs. 191 | * @param config 192 | * @param thread_count 193 | */ 194 | LOG_EXPORT void log_producer_config_set_send_thread_count(log_producer_config * config, int32_t thread_count); 195 | 196 | /** 197 | * set interface to send log out 198 | * @param config 199 | * @param net_interface 200 | */ 201 | LOG_EXPORT void log_producer_config_set_net_interface(log_producer_config * config, const char * net_interface); 202 | 203 | /** 204 | * set remote_address to send log out 205 | * @param config 206 | * @param remote_address 207 | */ 208 | LOG_EXPORT void log_producer_config_set_remote_address(log_producer_config * config, const char * remote_address); 209 | 210 | /** 211 | * set connect timeout seconds 212 | * @param config 213 | * @param connect_timeout_sec 214 | */ 215 | LOG_EXPORT void log_producer_config_set_connect_timeout_sec(log_producer_config * config, int32_t connect_timeout_sec); 216 | 217 | /** 218 | * set send timeout seconds 219 | * @param config 220 | * @param send_timeout_sec 221 | */ 222 | LOG_EXPORT void log_producer_config_set_send_timeout_sec(log_producer_config * config, int32_t send_timeout_sec); 223 | 224 | /** 225 | * set wait seconds when destroy flusher 226 | * @param config 227 | * @param destroy_flusher_wait_sec 228 | */ 229 | LOG_EXPORT void log_producer_config_set_destroy_flusher_wait_sec(log_producer_config * config, int32_t destroy_flusher_wait_sec); 230 | 231 | /** 232 | * set wait seconds when destroy sender 233 | * @param config 234 | * @param destroy_sender_wait_sec 235 | */ 236 | LOG_EXPORT void log_producer_config_set_destroy_sender_wait_sec(log_producer_config * config, int32_t destroy_sender_wait_sec); 237 | 238 | /** 239 | * set compress type, default 1 (lz4) 240 | * @param config 241 | * @param compress_type only support 1 or 0. 1 -> lz4, 0 -> no compress 242 | */ 243 | LOG_EXPORT void log_producer_config_set_compress_type(log_producer_config * config, int32_t compress_type); 244 | 245 | /** 246 | * default http, 0 http, 1 https 247 | * @param config 248 | * @param using_https 249 | */ 250 | LOG_EXPORT void log_producer_config_set_using_http(log_producer_config * config, int32_t using_https); 251 | 252 | 253 | /** 254 | * default http, 0 http, 1 https 255 | * @param config 256 | * @param log_queue_size 257 | */ 258 | LOG_EXPORT void log_producer_config_set_log_queue_size(log_producer_config * config, int32_t log_queue_size); 259 | 260 | 261 | 262 | /** 263 | * destroy config, this will free all memory allocated by this config 264 | * @param config 265 | */ 266 | LOG_EXPORT void destroy_log_producer_config(log_producer_config * config); 267 | 268 | #ifdef LOG_PRODUCER_DEBUG 269 | /** 270 | * print this config 271 | * @param config 272 | */ 273 | void log_producer_config_print(log_producer_config * config, FILE * pFile); 274 | 275 | #endif 276 | 277 | /** 278 | * check if given config is valid 279 | * @param config 280 | * @return 1 valid, 0 invalid 281 | */ 282 | LOG_EXPORT int log_producer_config_is_valid(log_producer_config * config); 283 | 284 | 285 | 286 | LOG_CPP_END 287 | 288 | #endif //LOG_C_SDK_LOG_PRODUCER_CONFIG_H 289 | -------------------------------------------------------------------------------- /src/log_producer_manager.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 20/11/2017. 3 | // 4 | 5 | #ifndef LOG_C_SDK_LOG_PRODUCER_MANAGER_H 6 | #define LOG_C_SDK_LOG_PRODUCER_MANAGER_H 7 | 8 | #include "log_define.h" 9 | LOG_CPP_START 10 | 11 | #include "log_producer_config.h" 12 | #include "log_producer_sender.h" 13 | #include "log_builder.h" 14 | #include "log_queue.h" 15 | #include "log_multi_thread.h" 16 | 17 | typedef struct _log_producer_manager 18 | { 19 | log_producer_config * producer_config; 20 | volatile uint32_t shutdown; 21 | volatile uint32_t networkRecover; 22 | volatile uint32_t totalBufferSize; 23 | log_queue * loggroup_queue; 24 | log_queue * sender_data_queue; 25 | THREAD * send_threads; 26 | THREAD flush_thread; 27 | CRITICALSECTION lock; 28 | COND triger_cond; 29 | log_group_builder * builder; 30 | int32_t firstLogTime; 31 | char * source; 32 | char * pack_prefix; 33 | volatile uint32_t pack_index; 34 | on_log_producer_send_done_function send_done_function; 35 | log_producer_send_param ** send_param_queue; 36 | uint64_t send_param_queue_size; 37 | volatile uint64_t send_param_queue_read; 38 | volatile uint64_t send_param_queue_write; 39 | ATOMICINT ref_count; // only used when global send thread works 40 | }log_producer_manager; 41 | 42 | extern log_producer_manager * create_log_producer_manager(log_producer_config * producer_config); 43 | extern void destroy_log_producer_manager(log_producer_manager * manager); 44 | 45 | extern log_producer_result log_producer_manager_add_log(log_producer_manager * producer_manager, uint32_t log_time, int32_t pair_count, char ** keys, size_t * key_lens, char ** values, size_t * val_lens); 46 | 47 | extern log_producer_result log_producer_manager_send_raw_buffer(log_producer_manager * producer_manager, size_t log_bytes, size_t compressed_bytes, const unsigned char * raw_buffer); 48 | 49 | LOG_CPP_END 50 | 51 | #endif //LOG_C_SDK_LOG_PRODUCER_MANAGER_H 52 | -------------------------------------------------------------------------------- /src/log_producer_sender.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 20/11/2017. 3 | // 4 | 5 | #include "log_producer_sender.h" 6 | #include "log_api.h" 7 | #include "log_producer_manager.h" 8 | #include "inner_log.h" 9 | #include "lz4.h" 10 | #include "sds.h" 11 | #include 12 | #include 13 | #include 14 | 15 | const char* LOGE_SERVER_BUSY = "ServerBusy"; 16 | const char* LOGE_INTERNAL_SERVER_ERROR = "InternalServerError"; 17 | const char* LOGE_UNAUTHORIZED = "Unauthorized"; 18 | const char* LOGE_WRITE_QUOTA_EXCEED = "WriteQuotaExceed"; 19 | const char* LOGE_SHARD_WRITE_QUOTA_EXCEED = "ShardWriteQuotaExceed"; 20 | const char* LOGE_TIME_EXPIRED = "RequestTimeExpired"; 21 | 22 | #define SEND_SLEEP_INTERVAL_MS 100 23 | #define MAX_NETWORK_ERROR_SLEEP_MS 3600000 24 | #define BASE_NETWORK_ERROR_SLEEP_MS 1000 25 | #define MAX_QUOTA_ERROR_SLEEP_MS 60000 26 | #define BASE_QUOTA_ERROR_SLEEP_MS 3000 27 | #define INVALID_TIME_TRY_INTERVAL 3000 28 | 29 | #define DROP_FAIL_DATA_TIME_SECOND (3600 * 6) 30 | 31 | #define SEND_TIME_INVALID_FIX 32 | 33 | extern volatile uint8_t g_send_thread_destroy; 34 | 35 | extern void destroy_log_producer_manager_tail(log_producer_manager * manager); 36 | 37 | typedef struct _send_error_info 38 | { 39 | log_producer_send_result last_send_error; 40 | int32_t last_sleep_ms; 41 | int32_t first_error_time; 42 | }send_error_info; 43 | 44 | int32_t log_producer_on_send_done(log_producer_send_param * send_param, post_log_result * result, send_error_info * error_info); 45 | 46 | #ifdef SEND_TIME_INVALID_FIX 47 | 48 | void _rebuild_time(lz4_log_buf * lz4_buf, lz4_log_buf ** new_lz4_buf) 49 | { 50 | aos_debug_log("rebuild log."); 51 | char * buf = (char *)malloc(lz4_buf->raw_length); 52 | if (LZ4_decompress_safe((const char* )lz4_buf->data, buf, lz4_buf->length, lz4_buf->raw_length) <= 0) 53 | { 54 | free(buf); 55 | aos_fatal_log("LZ4_decompress_safe error"); 56 | return; 57 | } 58 | uint32_t nowTime = time(NULL); 59 | fix_log_group_time(buf, lz4_buf->raw_length, nowTime); 60 | 61 | int compress_bound = LZ4_compressBound(lz4_buf->raw_length); 62 | char *compress_data = (char *)malloc(compress_bound); 63 | int compressed_size = LZ4_compress_default((char *)buf, compress_data, lz4_buf->raw_length, compress_bound); 64 | if(compressed_size <= 0) 65 | { 66 | aos_fatal_log("LZ4_compress_default error"); 67 | free(buf); 68 | free(compress_data); 69 | return; 70 | } 71 | *new_lz4_buf = (lz4_log_buf*)malloc(sizeof(lz4_log_buf) + compressed_size); 72 | (*new_lz4_buf)->length = compressed_size; 73 | (*new_lz4_buf)->raw_length = lz4_buf->raw_length; 74 | memcpy((*new_lz4_buf)->data, compress_data, compressed_size); 75 | free(buf); 76 | free(compress_data); 77 | return; 78 | } 79 | 80 | #endif 81 | 82 | void * log_producer_send_thread(void * param) 83 | { 84 | log_producer_manager * producer_manager = (log_producer_manager *)param; 85 | 86 | if (producer_manager->sender_data_queue == NULL) 87 | { 88 | return NULL; 89 | } 90 | 91 | while (!producer_manager->shutdown) 92 | { 93 | void * send_param = log_queue_pop(producer_manager->sender_data_queue, 30); 94 | if (send_param != NULL) 95 | { 96 | log_producer_send_fun(send_param); 97 | } 98 | } 99 | 100 | return NULL; 101 | } 102 | 103 | void log_producer_send_thread_global_inner(log_producer_send_param * send_param) 104 | { 105 | if (send_param != NULL) 106 | { 107 | log_producer_manager * producer_manager = (log_producer_manager *)send_param->producer_manager; 108 | log_producer_send_fun(send_param); 109 | // @note after log_producer_send_fun, send_param has been destroyed 110 | if (ATOMICINT_DEC(&producer_manager->ref_count) == 0) 111 | { 112 | aos_info_log("producer's ref count is 0, destroy this producer, project : %s, logstore : %s", 113 | producer_manager->producer_config->project, producer_manager->producer_config->logstore); 114 | destroy_log_producer_manager_tail(producer_manager); 115 | } 116 | } 117 | } 118 | 119 | void * log_producer_send_thread_global(void * param) 120 | { 121 | log_queue * send_log_queue = (log_queue * )param; 122 | 123 | while (!g_send_thread_destroy) 124 | { 125 | log_producer_send_param * send_param = (log_producer_send_param *)log_queue_pop(send_log_queue, 300); 126 | log_producer_send_thread_global_inner(send_param); 127 | } 128 | 129 | return NULL; 130 | } 131 | 132 | void * log_producer_send_fun(void * param) 133 | { 134 | log_producer_send_param * send_param = (log_producer_send_param *)param; 135 | if (send_param->magic_num != LOG_PRODUCER_SEND_MAGIC_NUM) 136 | { 137 | aos_fatal_log("invalid send param, magic num not found, num 0x%x", send_param->magic_num); 138 | return NULL; 139 | } 140 | 141 | if (send_param->log_buf == NULL) 142 | { 143 | aos_info_log("receive producer destroy event, project : %s, logstore : %s", send_param->producer_config->project, send_param->producer_config->logstore); 144 | free(send_param); 145 | return NULL; 146 | } 147 | 148 | log_producer_config * config = send_param->producer_config; 149 | 150 | send_error_info error_info; 151 | memset(&error_info, 0, sizeof(error_info)); 152 | 153 | log_producer_manager * producer_manager = (log_producer_manager *)send_param->producer_manager; 154 | 155 | do 156 | { 157 | if (producer_manager->shutdown) 158 | { 159 | aos_info_log("send fail but shutdown signal received, force exit"); 160 | if (producer_manager->send_done_function != NULL) 161 | { 162 | producer_manager->send_done_function(producer_manager->producer_config->logstore, LOG_PRODUCER_SEND_EXIT_BUFFERED, send_param->log_buf->raw_length, send_param->log_buf->length, 163 | NULL, "producer is being destroyed, producer has no time to send this buffer out", send_param->log_buf->data); 164 | } 165 | break; 166 | } 167 | lz4_log_buf * send_buf = send_param->log_buf; 168 | #ifdef SEND_TIME_INVALID_FIX 169 | uint32_t nowTime = time(NULL); 170 | if (nowTime - send_param->builder_time > 600 || send_param->builder_time > nowTime || error_info.last_send_error == LOG_SEND_TIME_ERROR) 171 | { 172 | _rebuild_time(send_param->log_buf, &send_buf); 173 | send_param->builder_time = nowTime; 174 | } 175 | #endif 176 | log_post_option option; 177 | memset(&option, 0, sizeof(log_post_option)); 178 | option.connect_timeout = config->connectTimeoutSec; 179 | option.operation_timeout = config->sendTimeoutSec; 180 | option.interface = config->netInterface; 181 | option.compress_type = config->compressType; 182 | option.remote_address = config->remote_address; 183 | option.using_https = config->using_https; 184 | 185 | sds accessKeyId = NULL; 186 | sds accessKey = NULL; 187 | sds stsToken = NULL; 188 | log_producer_config_get_security(config, &accessKeyId, &accessKey, &stsToken); 189 | 190 | post_log_result * rst = post_logs_from_lz4buf(config->endpoint, accessKeyId, 191 | accessKey, stsToken, 192 | config->project, config->logstore, 193 | send_buf, &option, 194 | config->authVersion, config->region); 195 | sdsfree(accessKeyId); 196 | sdsfree(accessKey); 197 | sdsfree(stsToken); 198 | 199 | int32_t sleepMs = log_producer_on_send_done(send_param, rst, &error_info); 200 | 201 | post_log_result_destroy(rst); 202 | 203 | // tmp buffer, free 204 | if (send_buf != send_param->log_buf) 205 | { 206 | free(send_buf); 207 | } 208 | 209 | if (sleepMs <= 0) 210 | { 211 | break; 212 | } 213 | int i =0; 214 | for (i = 0; i < sleepMs; i += SEND_SLEEP_INTERVAL_MS) 215 | { 216 | usleep(SEND_SLEEP_INTERVAL_MS * 1000); 217 | if (producer_manager->shutdown || producer_manager->networkRecover) 218 | { 219 | break; 220 | } 221 | } 222 | 223 | if (producer_manager->networkRecover) 224 | { 225 | producer_manager->networkRecover = 0; 226 | } 227 | 228 | }while(1); 229 | 230 | // at last, free all buffer 231 | free_lz4_log_buf(send_param->log_buf); 232 | free(send_param); 233 | 234 | return NULL; 235 | 236 | } 237 | 238 | int32_t log_producer_on_send_done(log_producer_send_param * send_param, post_log_result * result, send_error_info * error_info) 239 | { 240 | log_producer_send_result send_result = AosStatusToResult(result); 241 | log_producer_manager * producer_manager = (log_producer_manager *)send_param->producer_manager; 242 | if (producer_manager->send_done_function != NULL) 243 | { 244 | log_producer_result callback_result = send_result == LOG_SEND_OK ? 245 | LOG_PRODUCER_OK : 246 | (LOG_PRODUCER_SEND_NETWORK_ERROR + send_result - LOG_SEND_NETWORK_ERROR); 247 | producer_manager->send_done_function(producer_manager->producer_config->logstore, 248 | callback_result, 249 | send_param->log_buf->raw_length, 250 | send_param->log_buf->length, 251 | result->requestID, 252 | result->errorMessage, 253 | send_param->log_buf->data); 254 | } 255 | switch (send_result) 256 | { 257 | case LOG_SEND_OK: 258 | break; 259 | case LOG_SEND_TIME_ERROR: 260 | // if no this marco, drop data 261 | #ifdef SEND_TIME_INVALID_FIX 262 | error_info->last_send_error = LOG_SEND_TIME_ERROR; 263 | error_info->last_sleep_ms = INVALID_TIME_TRY_INTERVAL; 264 | return error_info->last_sleep_ms; 265 | #else 266 | break; 267 | #endif 268 | case LOG_SEND_QUOTA_EXCEED: 269 | if (error_info->last_send_error != LOG_SEND_QUOTA_EXCEED) 270 | { 271 | error_info->last_send_error = LOG_SEND_QUOTA_EXCEED; 272 | error_info->last_sleep_ms = BASE_QUOTA_ERROR_SLEEP_MS; 273 | error_info->first_error_time = time(NULL); 274 | } 275 | else 276 | { 277 | if (error_info->last_sleep_ms < MAX_QUOTA_ERROR_SLEEP_MS) 278 | { 279 | error_info->last_sleep_ms *= 2; 280 | } 281 | 282 | #ifndef SEND_TIME_INVALID_FIX 283 | // only drop data when SEND_TIME_INVALID_FIX not defined 284 | if (time(NULL) - error_info->first_error_time > DROP_FAIL_DATA_TIME_SECOND) 285 | { 286 | break; 287 | } 288 | #endif 289 | } 290 | aos_warn_log("send quota error, project : %s, logstore : %s, buffer len : %d, raw len : %d, code : %d, error msg : %s", 291 | send_param->producer_config->project, 292 | send_param->producer_config->logstore, 293 | (int)send_param->log_buf->length, 294 | (int)send_param->log_buf->raw_length, 295 | result->statusCode, 296 | result->errorMessage); 297 | return error_info->last_sleep_ms; 298 | case LOG_SEND_SERVER_ERROR : 299 | case LOG_SEND_NETWORK_ERROR: 300 | if (error_info->last_send_error != LOG_SEND_NETWORK_ERROR) 301 | { 302 | error_info->last_send_error = LOG_SEND_NETWORK_ERROR; 303 | error_info->last_sleep_ms = BASE_NETWORK_ERROR_SLEEP_MS; 304 | error_info->first_error_time = time(NULL); 305 | } 306 | else 307 | { 308 | if (error_info->last_sleep_ms < MAX_NETWORK_ERROR_SLEEP_MS) 309 | { 310 | error_info->last_sleep_ms *= 2; 311 | } 312 | #ifndef SEND_TIME_INVALID_FIX 313 | // only drop data when SEND_TIME_INVALID_FIX not defined 314 | if (time(NULL) - error_info->first_error_time > DROP_FAIL_DATA_TIME_SECOND) 315 | { 316 | break; 317 | } 318 | #endif 319 | } 320 | aos_warn_log("send network error, project : %s, logstore : %s, buffer len : %d, raw len : %d, code : %d, error msg : %s", 321 | send_param->producer_config->project, 322 | send_param->producer_config->logstore, 323 | (int)send_param->log_buf->length, 324 | (int)send_param->log_buf->raw_length, 325 | result->statusCode, 326 | result->errorMessage == NULL ? "" : result->errorMessage); 327 | return error_info->last_sleep_ms; 328 | default: 329 | // discard data 330 | break; 331 | 332 | } 333 | 334 | CS_ENTER(producer_manager->lock); 335 | producer_manager->totalBufferSize -= send_param->log_buf->length; 336 | CS_LEAVE(producer_manager->lock); 337 | if (send_result == LOG_SEND_OK) 338 | { 339 | aos_debug_log("send success, project : %s, logstore : %s, buffer len : %d, raw len : %d, total buffer : %d, code : %d, error msg : %s", 340 | send_param->producer_config->project, 341 | send_param->producer_config->logstore, 342 | (int)send_param->log_buf->length, 343 | (int)send_param->log_buf->raw_length, 344 | (int)producer_manager->totalBufferSize, 345 | result->statusCode, 346 | result->errorMessage); 347 | } 348 | else 349 | { 350 | aos_warn_log("send fail, discard data, project : %s, logstore : %s, buffer len : %d, raw len : %d, total buffer : %d, code : %d, error msg : %s", 351 | send_param->producer_config->project, 352 | send_param->producer_config->logstore, 353 | (int)send_param->log_buf->length, 354 | (int)send_param->log_buf->raw_length, 355 | (int)producer_manager->totalBufferSize, 356 | result->statusCode, 357 | result->errorMessage); 358 | } 359 | 360 | return 0; 361 | } 362 | 363 | log_producer_result log_producer_send_data(log_producer_send_param * send_param) 364 | { 365 | log_producer_send_fun(send_param); 366 | return LOG_PRODUCER_OK; 367 | } 368 | 369 | log_producer_send_result AosStatusToResult(post_log_result * result) 370 | { 371 | if (result->statusCode / 100 == 2) 372 | { 373 | return LOG_SEND_OK; 374 | } 375 | if (result->statusCode <= 0) 376 | { 377 | return LOG_SEND_NETWORK_ERROR; 378 | } 379 | if (result->statusCode >= 500 || result->requestID == NULL) 380 | { 381 | return LOG_SEND_SERVER_ERROR; 382 | } 383 | if (result->statusCode == 403) 384 | { 385 | return LOG_SEND_QUOTA_EXCEED; 386 | } 387 | if (result->statusCode == 401 || result->statusCode == 404) 388 | { 389 | return LOG_SEND_UNAUTHORIZED; 390 | } 391 | if (result->errorMessage != NULL && strstr(result->errorMessage, LOGE_TIME_EXPIRED) != NULL) 392 | { 393 | return LOG_SEND_TIME_ERROR; 394 | } 395 | return LOG_SEND_DISCARD_ERROR; 396 | 397 | } 398 | 399 | log_producer_send_param * create_log_producer_send_param(log_producer_config * producer_config, 400 | void * producer_manager, 401 | lz4_log_buf * log_buf, 402 | uint32_t builder_time) 403 | { 404 | log_producer_send_param * param = (log_producer_send_param *)malloc(sizeof(log_producer_send_param)); 405 | param->producer_config = producer_config; 406 | param->producer_manager = producer_manager; 407 | param->log_buf = log_buf; 408 | param->magic_num = LOG_PRODUCER_SEND_MAGIC_NUM; 409 | param->builder_time = builder_time; 410 | return param; 411 | } 412 | 413 | log_producer_send_param * create_log_producer_destroy_param(log_producer_config * producer_config, void *producer_manager) 414 | { 415 | log_producer_send_param * param = (log_producer_send_param *)malloc(sizeof(log_producer_send_param)); 416 | param->producer_config = producer_config; 417 | param->producer_manager = producer_manager; 418 | param->log_buf = NULL; 419 | param->magic_num = LOG_PRODUCER_SEND_MAGIC_NUM; 420 | param->builder_time = 0; 421 | return param; 422 | } 423 | 424 | 425 | -------------------------------------------------------------------------------- /src/log_producer_sender.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 20/11/2017. 3 | // 4 | 5 | #ifndef LOG_C_SDK_LOG_PRODUCER_SENDER_H 6 | #define LOG_C_SDK_LOG_PRODUCER_SENDER_H 7 | 8 | 9 | #include "log_define.h" 10 | #include "log_producer_config.h" 11 | #include "log_builder.h" 12 | LOG_CPP_START 13 | 14 | #define LOG_PRODUCER_SEND_MAGIC_NUM 0x1B35487A 15 | 16 | typedef int32_t log_producer_send_result; 17 | 18 | #define LOG_SEND_OK 0 19 | #define LOG_SEND_NETWORK_ERROR 1 20 | #define LOG_SEND_QUOTA_EXCEED 2 21 | #define LOG_SEND_UNAUTHORIZED 3 22 | #define LOG_SEND_SERVER_ERROR 4 23 | #define LOG_SEND_DISCARD_ERROR 5 24 | #define LOG_SEND_TIME_ERROR 6 25 | 26 | extern const char* LOGE_SERVER_BUSY;//= "ServerBusy"; 27 | extern const char* LOGE_INTERNAL_SERVER_ERROR;//= "InternalServerError"; 28 | extern const char* LOGE_UNAUTHORIZED;//= "Unauthorized"; 29 | extern const char* LOGE_WRITE_QUOTA_EXCEED;//="WriteQuotaExceed"; 30 | extern const char* LOGE_SHARD_WRITE_QUOTA_EXCEED;//= "ShardWriteQuotaExceed"; 31 | extern const char* LOGE_TIME_EXPIRED;//= "RequestTimeExpired"; 32 | 33 | 34 | typedef struct _log_producer_send_param 35 | { 36 | log_producer_config * producer_config; 37 | void * producer_manager; 38 | lz4_log_buf * log_buf; 39 | uint32_t magic_num; 40 | uint32_t builder_time; 41 | }log_producer_send_param; 42 | 43 | extern void * log_producer_send_fun(void * send_param); 44 | 45 | extern log_producer_result log_producer_send_data(log_producer_send_param * send_param); 46 | 47 | extern log_producer_send_result AosStatusToResult(post_log_result * result); 48 | 49 | extern log_producer_send_param * create_log_producer_send_param(log_producer_config * producer_config, 50 | void * producer_manager, 51 | lz4_log_buf * log_buf, 52 | uint32_t builder_time); 53 | 54 | extern log_producer_send_param * create_log_producer_destroy_param(log_producer_config * producer_config, void * producer_manager); 55 | 56 | LOG_CPP_END 57 | 58 | #endif //LOG_C_SDK_LOG_PRODUCER_SENDER_H 59 | -------------------------------------------------------------------------------- /src/log_queue.c: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 29/12/2017. 3 | // 4 | #include "log_queue.h" 5 | #include "log_multi_thread.h" 6 | 7 | 8 | struct _log_queue{ 9 | void ** data; 10 | int64_t head; 11 | int64_t tail; 12 | int32_t size; 13 | CRITICALSECTION mutex; 14 | COND notempty; 15 | }; 16 | 17 | log_queue * log_queue_create(int32_t size) 18 | { 19 | void * buffer = malloc(sizeof(void *) * size + sizeof(log_queue)); 20 | memset(buffer, 0, sizeof(void *) * size + sizeof(log_queue)); 21 | log_queue * queue = (log_queue *)buffer; 22 | queue->data = (void **)(buffer + sizeof(log_queue)); 23 | queue->size = size; 24 | queue->mutex = CreateCriticalSection(); 25 | queue->notempty = CreateCond(); 26 | return queue; 27 | } 28 | 29 | void log_queue_destroy(log_queue * queue) 30 | { 31 | DeleteCriticalSection(queue->mutex); 32 | DeleteCond(queue->notempty); 33 | free(queue); 34 | } 35 | 36 | int32_t log_queue_size(log_queue * queue) 37 | { 38 | CS_ENTER(queue->mutex); 39 | int32_t len = queue->tail - queue->head; 40 | CS_LEAVE(queue->mutex); 41 | return len; 42 | } 43 | 44 | int32_t log_queue_isfull(log_queue * queue) 45 | { 46 | CS_ENTER(queue->mutex); 47 | int32_t rst = (int32_t)((queue->tail - queue->head) == queue->size); 48 | CS_LEAVE(queue->mutex); 49 | return rst; 50 | } 51 | 52 | int32_t log_queue_push(log_queue * queue, void * data) 53 | { 54 | CS_ENTER(queue->mutex); 55 | if (queue->tail - queue->head == queue->size) 56 | { 57 | CS_LEAVE(queue->mutex); 58 | return -1; 59 | } 60 | queue->data[queue->tail++ % queue->size] = data; 61 | CS_LEAVE(queue->mutex); 62 | COND_SIGNAL(queue->notempty); 63 | return 0; 64 | } 65 | 66 | void * log_queue_pop(log_queue * queue, int32_t waitMs) { 67 | CS_ENTER(queue->mutex); 68 | if (queue->tail == queue->head) { 69 | COND_WAIT_TIME(queue->notempty, queue->mutex, waitMs); 70 | } 71 | void * result = NULL; 72 | if (queue->tail > queue->head) 73 | { 74 | result = queue->data[queue->head++ % queue->size]; 75 | } 76 | CS_LEAVE(queue->mutex); 77 | return result; 78 | } 79 | 80 | void * log_queue_trypop(log_queue * queue) 81 | { 82 | CS_ENTER(queue->mutex); 83 | void * result = NULL; 84 | if (queue->tail > queue->head) 85 | { 86 | result = queue->data[queue->head++ % queue->size]; 87 | } 88 | CS_LEAVE(queue->mutex); 89 | return result; 90 | } 91 | 92 | -------------------------------------------------------------------------------- /src/log_queue.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ZhangCheng on 29/12/2017. 3 | // 4 | 5 | #ifndef LOG_C_SDK_LOG_QUEUE_H 6 | #define LOG_C_SDK_LOG_QUEUE_H 7 | 8 | #include 9 | 10 | typedef struct _log_queue log_queue; 11 | 12 | log_queue * log_queue_create(int32_t max_size); 13 | 14 | void log_queue_destroy(log_queue * queue); 15 | 16 | int32_t log_queue_size(log_queue * queue); 17 | 18 | int32_t log_queue_isfull(log_queue * queue); 19 | 20 | 21 | int32_t log_queue_push(log_queue * queue, void * data); 22 | 23 | void * log_queue_pop(log_queue * queue, int32_t waitMs); 24 | 25 | void * log_queue_trypop(log_queue * queue); 26 | 27 | #endif //LOG_C_SDK_LOG_QUEUE_H 28 | -------------------------------------------------------------------------------- /src/log_util.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "log_util.h" 4 | #include "md5.h" 5 | #include "hmac-sha.h" 6 | #include 7 | 8 | static const char *g_hex_hash = "0123456789ABCDEF"; 9 | 10 | void md5_to_string(const char * buffer, int bufLen, char * md5) 11 | { 12 | unsigned char md5Buf[16]; 13 | mbedtls_md5((const unsigned char *)buffer, bufLen, md5Buf); 14 | int i = 0; 15 | for(; i < 32; i+=2) 16 | { 17 | md5[i] = g_hex_hash[md5Buf[i >> 1] >> 4]; 18 | md5[i+1] = g_hex_hash[md5Buf[i >> 1] & 0xF]; 19 | } 20 | } 21 | 22 | int aos_base64_encode(const unsigned char *in, int inLen, char *out) 23 | { 24 | static const char *ENC = 25 | "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 26 | 27 | char *original_out = out; 28 | 29 | while (inLen) { 30 | // first 6 bits of char 1 31 | *out++ = ENC[*in >> 2]; 32 | if (!--inLen) { 33 | // last 2 bits of char 1, 4 bits of 0 34 | *out++ = ENC[(*in & 0x3) << 4]; 35 | *out++ = '='; 36 | *out++ = '='; 37 | break; 38 | } 39 | // last 2 bits of char 1, first 4 bits of char 2 40 | *out++ = ENC[((*in & 0x3) << 4) | (*(in + 1) >> 4)]; 41 | in++; 42 | if (!--inLen) { 43 | // last 4 bits of char 2, 2 bits of 0 44 | *out++ = ENC[(*in & 0xF) << 2]; 45 | *out++ = '='; 46 | break; 47 | } 48 | // last 4 bits of char 2, first 2 bits of char 3 49 | *out++ = ENC[((*in & 0xF) << 2) | (*(in + 1) >> 6)]; 50 | in++; 51 | // last 6 bits of char 3 52 | *out++ = ENC[*in & 0x3F]; 53 | in++, inLen--; 54 | } 55 | 56 | return (int)(out - original_out); 57 | } 58 | 59 | int signature_to_base64(const char * sig, int sigLen, const char * key, int keyLen, char * base64) 60 | { 61 | unsigned char sha1Buf[20]; 62 | hmac_sha1(sha1Buf, key, keyLen << 3, sig, sigLen << 3); 63 | return aos_base64_encode((const unsigned char *)sha1Buf, 20, base64); 64 | } 65 | 66 | void sha256_to_hex_string(uint8_t hash[32], char hex_str[65]) { 67 | const char hex_digits[] = "0123456789abcdef"; 68 | int i; 69 | for (i = 0; i < 32; ++i) { 70 | hex_str[i * 2] = hex_digits[(hash[i] >> 4) & 0x0F]; 71 | hex_str[i * 2 + 1] = hex_digits[hash[i] & 0x0F]; 72 | } 73 | hex_str[64] = '\0'; // Null-terminate the string 74 | } 75 | 76 | char *sha256_hash_to_hex(const char *input, uint32 length) { 77 | if (!input) { 78 | return NULL; 79 | } 80 | 81 | sha256_context ctx; 82 | uint8_t hash[32]; 83 | char *output = (char*)malloc(65); // 64 chars for hex + 1 for '\0' 84 | if (output == NULL) { 85 | return NULL; // Allocation failed 86 | } 87 | 88 | sha256_starts(&ctx); 89 | sha256_update(&ctx, (uint8_t *)input, length); 90 | sha256_finish(&ctx, hash); 91 | 92 | sha256_to_hex_string(hash, output); 93 | 94 | return output; 95 | } -------------------------------------------------------------------------------- /src/log_util.h: -------------------------------------------------------------------------------- 1 | #ifndef LIBLOG_UTIL_H 2 | #define LIBLOG_UTIL_H 3 | 4 | #include "log_define.h" 5 | #include "sha256.h" 6 | #include 7 | LOG_CPP_START 8 | 9 | void md5_to_string(const char * buffer, int bufLen, char * md5); 10 | 11 | int signature_to_base64(const char * sig, int sigLen, const char * key, int keyLen, char * base64); 12 | 13 | void sha256_to_hex_string(uint8_t hash[32], char hex_str[65]); 14 | char *sha256_hash_to_hex(const char *input, uint32 length); 15 | 16 | LOG_CPP_END 17 | #endif 18 | -------------------------------------------------------------------------------- /src/md5.c: -------------------------------------------------------------------------------- 1 | #include "md5.h" 2 | #include 3 | 4 | /* 5 | * 32-bit integer manipulation macros (little endian) 6 | */ 7 | #ifndef GET_UINT32_LE 8 | #define GET_UINT32_LE(n,b,i) \ 9 | { \ 10 | (n) = ( (uint32_t) (b)[(i) ] ) \ 11 | | ( (uint32_t) (b)[(i) + 1] << 8 ) \ 12 | | ( (uint32_t) (b)[(i) + 2] << 16 ) \ 13 | | ( (uint32_t) (b)[(i) + 3] << 24 ); \ 14 | } 15 | #endif 16 | 17 | #ifndef PUT_UINT32_LE 18 | #define PUT_UINT32_LE(n,b,i) \ 19 | { \ 20 | (b)[(i) ] = (unsigned char) ( ( (n) ) & 0xFF ); \ 21 | (b)[(i) + 1] = (unsigned char) ( ( (n) >> 8 ) & 0xFF ); \ 22 | (b)[(i) + 2] = (unsigned char) ( ( (n) >> 16 ) & 0xFF ); \ 23 | (b)[(i) + 3] = (unsigned char) ( ( (n) >> 24 ) & 0xFF ); \ 24 | } 25 | #endif 26 | 27 | void mbedtls_md5_init( mbedtls_md5_context *ctx ) 28 | { 29 | memset( ctx, 0, sizeof( mbedtls_md5_context ) ); 30 | } 31 | 32 | void mbedtls_md5_clone( mbedtls_md5_context *dst, 33 | const mbedtls_md5_context *src ) 34 | { 35 | *dst = *src; 36 | } 37 | 38 | /* 39 | * MD5 context setup 40 | */ 41 | void mbedtls_md5_starts( mbedtls_md5_context *ctx ) 42 | { 43 | ctx->total[0] = 0; 44 | ctx->total[1] = 0; 45 | 46 | ctx->state[0] = 0x67452301; 47 | ctx->state[1] = 0xEFCDAB89; 48 | ctx->state[2] = 0x98BADCFE; 49 | ctx->state[3] = 0x10325476; 50 | } 51 | 52 | #if !defined(MBEDTLS_MD5_PROCESS_ALT) 53 | void mbedtls_md5_process( mbedtls_md5_context *ctx, const unsigned char data[64] ) 54 | { 55 | uint32_t X[16], A, B, C, D; 56 | 57 | GET_UINT32_LE( X[ 0], data, 0 ); 58 | GET_UINT32_LE( X[ 1], data, 4 ); 59 | GET_UINT32_LE( X[ 2], data, 8 ); 60 | GET_UINT32_LE( X[ 3], data, 12 ); 61 | GET_UINT32_LE( X[ 4], data, 16 ); 62 | GET_UINT32_LE( X[ 5], data, 20 ); 63 | GET_UINT32_LE( X[ 6], data, 24 ); 64 | GET_UINT32_LE( X[ 7], data, 28 ); 65 | GET_UINT32_LE( X[ 8], data, 32 ); 66 | GET_UINT32_LE( X[ 9], data, 36 ); 67 | GET_UINT32_LE( X[10], data, 40 ); 68 | GET_UINT32_LE( X[11], data, 44 ); 69 | GET_UINT32_LE( X[12], data, 48 ); 70 | GET_UINT32_LE( X[13], data, 52 ); 71 | GET_UINT32_LE( X[14], data, 56 ); 72 | GET_UINT32_LE( X[15], data, 60 ); 73 | 74 | #define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n))) 75 | 76 | #define P(a,b,c,d,k,s,t) \ 77 | { \ 78 | a += F(b,c,d) + X[k] + t; a = S(a,s) + b; \ 79 | } 80 | 81 | A = ctx->state[0]; 82 | B = ctx->state[1]; 83 | C = ctx->state[2]; 84 | D = ctx->state[3]; 85 | 86 | #define F(x,y,z) (z ^ (x & (y ^ z))) 87 | 88 | P( A, B, C, D, 0, 7, 0xD76AA478 ); 89 | P( D, A, B, C, 1, 12, 0xE8C7B756 ); 90 | P( C, D, A, B, 2, 17, 0x242070DB ); 91 | P( B, C, D, A, 3, 22, 0xC1BDCEEE ); 92 | P( A, B, C, D, 4, 7, 0xF57C0FAF ); 93 | P( D, A, B, C, 5, 12, 0x4787C62A ); 94 | P( C, D, A, B, 6, 17, 0xA8304613 ); 95 | P( B, C, D, A, 7, 22, 0xFD469501 ); 96 | P( A, B, C, D, 8, 7, 0x698098D8 ); 97 | P( D, A, B, C, 9, 12, 0x8B44F7AF ); 98 | P( C, D, A, B, 10, 17, 0xFFFF5BB1 ); 99 | P( B, C, D, A, 11, 22, 0x895CD7BE ); 100 | P( A, B, C, D, 12, 7, 0x6B901122 ); 101 | P( D, A, B, C, 13, 12, 0xFD987193 ); 102 | P( C, D, A, B, 14, 17, 0xA679438E ); 103 | P( B, C, D, A, 15, 22, 0x49B40821 ); 104 | 105 | #undef F 106 | 107 | #define F(x,y,z) (y ^ (z & (x ^ y))) 108 | 109 | P( A, B, C, D, 1, 5, 0xF61E2562 ); 110 | P( D, A, B, C, 6, 9, 0xC040B340 ); 111 | P( C, D, A, B, 11, 14, 0x265E5A51 ); 112 | P( B, C, D, A, 0, 20, 0xE9B6C7AA ); 113 | P( A, B, C, D, 5, 5, 0xD62F105D ); 114 | P( D, A, B, C, 10, 9, 0x02441453 ); 115 | P( C, D, A, B, 15, 14, 0xD8A1E681 ); 116 | P( B, C, D, A, 4, 20, 0xE7D3FBC8 ); 117 | P( A, B, C, D, 9, 5, 0x21E1CDE6 ); 118 | P( D, A, B, C, 14, 9, 0xC33707D6 ); 119 | P( C, D, A, B, 3, 14, 0xF4D50D87 ); 120 | P( B, C, D, A, 8, 20, 0x455A14ED ); 121 | P( A, B, C, D, 13, 5, 0xA9E3E905 ); 122 | P( D, A, B, C, 2, 9, 0xFCEFA3F8 ); 123 | P( C, D, A, B, 7, 14, 0x676F02D9 ); 124 | P( B, C, D, A, 12, 20, 0x8D2A4C8A ); 125 | 126 | #undef F 127 | 128 | #define F(x,y,z) (x ^ y ^ z) 129 | 130 | P( A, B, C, D, 5, 4, 0xFFFA3942 ); 131 | P( D, A, B, C, 8, 11, 0x8771F681 ); 132 | P( C, D, A, B, 11, 16, 0x6D9D6122 ); 133 | P( B, C, D, A, 14, 23, 0xFDE5380C ); 134 | P( A, B, C, D, 1, 4, 0xA4BEEA44 ); 135 | P( D, A, B, C, 4, 11, 0x4BDECFA9 ); 136 | P( C, D, A, B, 7, 16, 0xF6BB4B60 ); 137 | P( B, C, D, A, 10, 23, 0xBEBFBC70 ); 138 | P( A, B, C, D, 13, 4, 0x289B7EC6 ); 139 | P( D, A, B, C, 0, 11, 0xEAA127FA ); 140 | P( C, D, A, B, 3, 16, 0xD4EF3085 ); 141 | P( B, C, D, A, 6, 23, 0x04881D05 ); 142 | P( A, B, C, D, 9, 4, 0xD9D4D039 ); 143 | P( D, A, B, C, 12, 11, 0xE6DB99E5 ); 144 | P( C, D, A, B, 15, 16, 0x1FA27CF8 ); 145 | P( B, C, D, A, 2, 23, 0xC4AC5665 ); 146 | 147 | #undef F 148 | 149 | #define F(x,y,z) (y ^ (x | ~z)) 150 | 151 | P( A, B, C, D, 0, 6, 0xF4292244 ); 152 | P( D, A, B, C, 7, 10, 0x432AFF97 ); 153 | P( C, D, A, B, 14, 15, 0xAB9423A7 ); 154 | P( B, C, D, A, 5, 21, 0xFC93A039 ); 155 | P( A, B, C, D, 12, 6, 0x655B59C3 ); 156 | P( D, A, B, C, 3, 10, 0x8F0CCC92 ); 157 | P( C, D, A, B, 10, 15, 0xFFEFF47D ); 158 | P( B, C, D, A, 1, 21, 0x85845DD1 ); 159 | P( A, B, C, D, 8, 6, 0x6FA87E4F ); 160 | P( D, A, B, C, 15, 10, 0xFE2CE6E0 ); 161 | P( C, D, A, B, 6, 15, 0xA3014314 ); 162 | P( B, C, D, A, 13, 21, 0x4E0811A1 ); 163 | P( A, B, C, D, 4, 6, 0xF7537E82 ); 164 | P( D, A, B, C, 11, 10, 0xBD3AF235 ); 165 | P( C, D, A, B, 2, 15, 0x2AD7D2BB ); 166 | P( B, C, D, A, 9, 21, 0xEB86D391 ); 167 | 168 | #undef F 169 | 170 | ctx->state[0] += A; 171 | ctx->state[1] += B; 172 | ctx->state[2] += C; 173 | ctx->state[3] += D; 174 | } 175 | #endif /* !MBEDTLS_MD5_PROCESS_ALT */ 176 | 177 | /* 178 | * MD5 process buffer 179 | */ 180 | void mbedtls_md5_update( mbedtls_md5_context *ctx, const unsigned char *input, size_t ilen ) 181 | { 182 | size_t fill; 183 | uint32_t left; 184 | 185 | if( ilen == 0 ) 186 | return; 187 | 188 | left = ctx->total[0] & 0x3F; 189 | fill = 64 - left; 190 | 191 | ctx->total[0] += (uint32_t) ilen; 192 | ctx->total[0] &= 0xFFFFFFFF; 193 | 194 | if( ctx->total[0] < (uint32_t) ilen ) 195 | ctx->total[1]++; 196 | 197 | if( left && ilen >= fill ) 198 | { 199 | memcpy( (void *) (ctx->buffer + left), input, fill ); 200 | mbedtls_md5_process( ctx, ctx->buffer ); 201 | input += fill; 202 | ilen -= fill; 203 | left = 0; 204 | } 205 | 206 | while( ilen >= 64 ) 207 | { 208 | mbedtls_md5_process( ctx, input ); 209 | input += 64; 210 | ilen -= 64; 211 | } 212 | 213 | if( ilen > 0 ) 214 | { 215 | memcpy( (void *) (ctx->buffer + left), input, ilen ); 216 | } 217 | } 218 | 219 | static const unsigned char md5_padding[64] = 220 | { 221 | 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 225 | }; 226 | 227 | /* 228 | * MD5 final digest 229 | */ 230 | void mbedtls_md5_finish( mbedtls_md5_context *ctx, unsigned char output[16] ) 231 | { 232 | uint32_t last, padn; 233 | uint32_t high, low; 234 | unsigned char msglen[8]; 235 | 236 | high = ( ctx->total[0] >> 29 ) 237 | | ( ctx->total[1] << 3 ); 238 | low = ( ctx->total[0] << 3 ); 239 | 240 | PUT_UINT32_LE( low, msglen, 0 ); 241 | PUT_UINT32_LE( high, msglen, 4 ); 242 | 243 | last = ctx->total[0] & 0x3F; 244 | padn = ( last < 56 ) ? ( 56 - last ) : ( 120 - last ); 245 | 246 | mbedtls_md5_update( ctx, md5_padding, padn ); 247 | mbedtls_md5_update( ctx, msglen, 8 ); 248 | 249 | PUT_UINT32_LE( ctx->state[0], output, 0 ); 250 | PUT_UINT32_LE( ctx->state[1], output, 4 ); 251 | PUT_UINT32_LE( ctx->state[2], output, 8 ); 252 | PUT_UINT32_LE( ctx->state[3], output, 12 ); 253 | } 254 | 255 | 256 | /* 257 | * output = MD5( input buffer ) 258 | */ 259 | void mbedtls_md5( const unsigned char *input, size_t ilen, unsigned char output[16] ) 260 | { 261 | mbedtls_md5_context ctx; 262 | 263 | mbedtls_md5_init( &ctx ); 264 | mbedtls_md5_starts( &ctx ); 265 | mbedtls_md5_update( &ctx, input, ilen ); 266 | mbedtls_md5_finish( &ctx, output ); 267 | } 268 | -------------------------------------------------------------------------------- /src/md5.h: -------------------------------------------------------------------------------- 1 | #ifndef MD5_H_ 2 | #define MD5_H_ 3 | 4 | #include 5 | #include 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | /** 12 | * \brief MD5 context structure 13 | */ 14 | typedef struct 15 | { 16 | uint32_t total[2]; /*!< number of bytes processed */ 17 | uint32_t state[4]; /*!< intermediate digest state */ 18 | unsigned char buffer[64]; /*!< data block being processed */ 19 | } mbedtls_md5_context; 20 | 21 | /** 22 | * \brief Initialize MD5 context 23 | * 24 | * \param ctx MD5 context to be initialized 25 | */ 26 | void mbedtls_md5_init( mbedtls_md5_context *ctx ); 27 | 28 | 29 | /** 30 | * \brief Clone (the state of) an MD5 context 31 | * 32 | * \param dst The destination context 33 | * \param src The context to be cloned 34 | */ 35 | void mbedtls_md5_clone( mbedtls_md5_context *dst, 36 | const mbedtls_md5_context *src ); 37 | 38 | /** 39 | * \brief MD5 context setup 40 | * 41 | * \param ctx context to be initialized 42 | */ 43 | void mbedtls_md5_starts( mbedtls_md5_context *ctx ); 44 | 45 | /** 46 | * \brief MD5 process buffer 47 | * 48 | * \param ctx MD5 context 49 | * \param input buffer holding the data 50 | * \param ilen length of the input data 51 | */ 52 | void mbedtls_md5_update( mbedtls_md5_context *ctx, const unsigned char *input, size_t ilen ); 53 | 54 | /** 55 | * \brief MD5 final digest 56 | * 57 | * \param ctx MD5 context 58 | * \param output MD5 checksum result 59 | */ 60 | void mbedtls_md5_finish( mbedtls_md5_context *ctx, unsigned char output[16] ); 61 | 62 | /* Internal use */ 63 | void mbedtls_md5_process( mbedtls_md5_context *ctx, const unsigned char data[64] ); 64 | 65 | 66 | /** 67 | * \brief Output = MD5( input buffer ) 68 | * 69 | * \param input buffer holding the data 70 | * \param ilen length of the input data 71 | * \param output MD5 checksum result 72 | */ 73 | void mbedtls_md5( const unsigned char *input, size_t ilen, unsigned char output[16] ); 74 | 75 | 76 | #ifdef __cplusplus 77 | } 78 | #endif 79 | 80 | 81 | #endif /*MD5_H_*/ 82 | -------------------------------------------------------------------------------- /src/sds.c: -------------------------------------------------------------------------------- 1 | /* SDSLib, A C dynamic strings library 2 | * 3 | * Copyright (c) 2006-2012, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include "sds.h" 37 | 38 | size_t sdslen(const sds s) { 39 | struct sdshdr *sh = (struct sdshdr *) (s - (sizeof(struct sdshdr))); 40 | return sh->len; 41 | } 42 | 43 | size_t sdsavail(const sds s) { 44 | struct sdshdr *sh = (struct sdshdr *) (s - (sizeof(struct sdshdr))); 45 | return sh->free; 46 | } 47 | 48 | /* Create a new sds string with the content specified by the 'init' pointer 49 | * and 'initlen'. 50 | * If NULL is used for 'init' the string is initialized with zero bytes. 51 | * 52 | * The string is always null-termined (all the sds strings are, always) so 53 | * even if you create an sds string with: 54 | * 55 | * mystring = sdsnewlen("abc",3); 56 | * 57 | * You can print the string with printf() as there is an implicit \0 at the 58 | * end of the string. However the string is binary safe and can contain 59 | * \0 characters in the middle, as the length is stored in the sds header. */ 60 | sds sdsnewlen(const void *init, size_t initlen) { 61 | struct sdshdr *sh; 62 | 63 | if (init) { 64 | sh = malloc(sizeof(struct sdshdr) + initlen + 1); 65 | } else { 66 | sh = calloc(sizeof(struct sdshdr) + initlen + 1, 1); 67 | } 68 | if (sh == NULL) return NULL; 69 | sh->len = initlen; 70 | sh->free = 0; 71 | if (initlen && init) 72 | memcpy(sh->buf, init, initlen); 73 | sh->buf[initlen] = '\0'; 74 | return (char *) sh->buf; 75 | } 76 | 77 | 78 | sds sdsnewEmpty(size_t preAlloclen) { 79 | struct sdshdr *sh; 80 | 81 | sh = malloc(sizeof(struct sdshdr) + preAlloclen + 1); 82 | if (sh == NULL) return NULL; 83 | sh->len = 0; 84 | sh->free = preAlloclen; 85 | sh->buf[0] = '\0'; 86 | return (char *) sh->buf; 87 | } 88 | 89 | 90 | /* Create an empty (zero length) sds string. Even in this case the string 91 | * always has an implicit null term. */ 92 | sds sdsempty(void) { 93 | return sdsnewlen("", 0); 94 | } 95 | 96 | /* Create a new sds string starting from a null terminated C string. */ 97 | sds sdsnew(const char *init) { 98 | size_t initlen = (init == NULL) ? 0 : strlen(init); 99 | return sdsnewlen(init, initlen); 100 | } 101 | 102 | /* Duplicate an sds string. */ 103 | sds sdsdup(const sds s) { 104 | if (s == NULL) return NULL; 105 | return sdsnewlen(s, sdslen(s)); 106 | } 107 | 108 | /* Free an sds string. No operation is performed if 's' is NULL. */ 109 | void sdsfree(sds s) { 110 | if (s == NULL) return; 111 | free(s - sizeof(struct sdshdr)); 112 | } 113 | 114 | /* Set the sds string length to the length as obtained with strlen(), so 115 | * considering as content only up to the first null term character. 116 | * 117 | * This function is useful when the sds string is hacked manually in some 118 | * way, like in the following example: 119 | * 120 | * s = sdsnew("foobar"); 121 | * s[2] = '\0'; 122 | * sdsupdatelen(s); 123 | * printf("%d\n", sdslen(s)); 124 | * 125 | * The output will be "2", but if we comment out the call to sdsupdatelen() 126 | * the output will be "6" as the string was modified but the logical length 127 | * remains 6 bytes. */ 128 | void sdsupdatelen(sds s) { 129 | struct sdshdr *sh = (void *) (s - (sizeof(struct sdshdr))); 130 | int reallen = strlen(s); 131 | sh->free += (sh->len - reallen); 132 | sh->len = reallen; 133 | } 134 | 135 | /* Modify an sds string in-place to make it empty (zero length). 136 | * However all the existing buffer is not discarded but set as free space 137 | * so that next append operations will not require allocations up to the 138 | * number of bytes previously available. */ 139 | void sdsclear(sds s) { 140 | struct sdshdr *sh = (void *) (s - (sizeof(struct sdshdr))); 141 | sh->free += sh->len; 142 | sh->len = 0; 143 | sh->buf[0] = '\0'; 144 | } 145 | 146 | /* Enlarge the free space at the end of the sds string so that the caller 147 | * is sure that after calling this function can overwrite up to addlen 148 | * bytes after the end of the string, plus one more byte for nul term. 149 | * 150 | * Note: this does not change the *length* of the sds string as returned 151 | * by sdslen(), but only the free buffer space we have. */ 152 | sds sdsMakeRoomFor(sds s, size_t addlen) { 153 | struct sdshdr *sh, *newsh; 154 | size_t free = sdsavail(s); 155 | size_t len, newlen; 156 | 157 | if (free >= addlen) return s; 158 | len = sdslen(s); 159 | sh = (void *) (s - (sizeof(struct sdshdr))); 160 | newlen = (len + addlen); 161 | if (newlen < SDS_MAX_PREALLOC) 162 | newlen *= 2; 163 | else 164 | newlen += SDS_MAX_PREALLOC; 165 | newsh = realloc(sh, sizeof(struct sdshdr) + newlen + 1); 166 | if (newsh == NULL) return NULL; 167 | 168 | newsh->free = newlen - len; 169 | return newsh->buf; 170 | } 171 | 172 | /* Reallocate the sds string so that it has no free space at the end. The 173 | * contained string remains not altered, but next concatenation operations 174 | * will require a reallocation. 175 | * 176 | * After the call, the passed sds string is no longer valid and all the 177 | * references must be substituted with the new pointer returned by the call. */ 178 | sds sdsRemoveFreeSpace(sds s) { 179 | struct sdshdr *sh; 180 | 181 | sh = (void *) (s - (sizeof(struct sdshdr))); 182 | sh = realloc(sh, sizeof(struct sdshdr) + sh->len + 1); 183 | sh->free = 0; 184 | return sh->buf; 185 | } 186 | 187 | /* Return the total size of the allocation of the specifed sds string, 188 | * including: 189 | * 1) The sds header before the pointer. 190 | * 2) The string. 191 | * 3) The free buffer at the end if any. 192 | * 4) The implicit null term. 193 | */ 194 | size_t sdsAllocSize(sds s) { 195 | struct sdshdr *sh = (void *) (s - (sizeof(struct sdshdr))); 196 | 197 | return sizeof(*sh) + sh->len + sh->free + 1; 198 | } 199 | 200 | /* Increment the sds length and decrements the left free space at the 201 | * end of the string according to 'incr'. Also set the null term 202 | * in the new end of the string. 203 | * 204 | * This function is used in order to fix the string length after the 205 | * user calls sdsMakeRoomFor(), writes something after the end of 206 | * the current string, and finally needs to set the new length. 207 | * 208 | * Note: it is possible to use a negative increment in order to 209 | * right-trim the string. 210 | * 211 | * Usage example: 212 | * 213 | * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the 214 | * following schema, to cat bytes coming from the kernel to the end of an 215 | * sds string without copying into an intermediate buffer: 216 | * 217 | * oldlen = sdslen(s); 218 | * s = sdsMakeRoomFor(s, BUFFER_SIZE); 219 | * nread = read(fd, s+oldlen, BUFFER_SIZE); 220 | * ... check for nread <= 0 and handle it ... 221 | * sdsIncrLen(s, nread); 222 | */ 223 | void sdsIncrLen(sds s, int incr) { 224 | struct sdshdr *sh = (void *) (s - (sizeof(struct sdshdr))); 225 | 226 | if (incr >= 0) 227 | assert(sh->free >= (unsigned int) incr); 228 | else 229 | assert(sh->len >= (unsigned int) (-incr)); 230 | sh->len += incr; 231 | sh->free -= incr; 232 | s[sh->len] = '\0'; 233 | } 234 | 235 | /* Grow the sds to have the specified length. Bytes that were not part of 236 | * the original length of the sds will be set to zero. 237 | * 238 | * if the specified length is smaller than the current length, no operation 239 | * is performed. */ 240 | sds sdsgrowzero(sds s, size_t len) { 241 | struct sdshdr *sh = (void *) (s - (sizeof(struct sdshdr))); 242 | size_t totlen, curlen = sh->len; 243 | 244 | if (len <= curlen) return s; 245 | s = sdsMakeRoomFor(s, len - curlen); 246 | if (s == NULL) return NULL; 247 | 248 | /* Make sure added region doesn't contain garbage */ 249 | sh = (void *) (s - (sizeof(struct sdshdr))); 250 | memset(s + curlen, 0, (len - curlen + 1)); /* also set trailing \0 byte */ 251 | totlen = sh->len + sh->free; 252 | sh->len = len; 253 | sh->free = totlen - sh->len; 254 | return s; 255 | } 256 | 257 | /* Append the specified binary-safe string pointed by 't' of 'len' bytes to the 258 | * end of the specified sds string 's'. 259 | * 260 | * After the call, the passed sds string is no longer valid and all the 261 | * references must be substituted with the new pointer returned by the call. */ 262 | sds sdscatlen(sds s, const void *t, size_t len) { 263 | struct sdshdr *sh; 264 | size_t curlen = sdslen(s); 265 | 266 | s = sdsMakeRoomFor(s, len); 267 | if (s == NULL) return NULL; 268 | sh = (void *) (s - (sizeof(struct sdshdr))); 269 | memcpy(s + curlen, t, len); 270 | sh->len = curlen + len; 271 | sh->free = sh->free - len; 272 | s[curlen + len] = '\0'; 273 | return s; 274 | } 275 | 276 | 277 | sds sdscatchar(sds s, char c) { 278 | struct sdshdr *sh; 279 | size_t curlen = sdslen(s); 280 | 281 | s = sdsMakeRoomFor(s, 1); 282 | if (s == NULL) return NULL; 283 | sh = (void *) (s - (sizeof(struct sdshdr))); 284 | s[curlen] = c; 285 | s[curlen + 1] = '\0'; 286 | ++sh->len; 287 | --sh->free; 288 | return s; 289 | } 290 | 291 | 292 | /* Append the specified null termianted C string to the sds string 's'. 293 | * 294 | * After the call, the passed sds string is no longer valid and all the 295 | * references must be substituted with the new pointer returned by the call. */ 296 | sds sdscat(sds s, const char *t) { 297 | if (s == NULL || t == NULL) { 298 | return s; 299 | } 300 | return sdscatlen(s, t, strlen(t)); 301 | } 302 | 303 | /* Append the specified sds 't' to the existing sds 's'. 304 | * 305 | * After the call, the modified sds string is no longer valid and all the 306 | * references must be substituted with the new pointer returned by the call. */ 307 | sds sdscatsds(sds s, const sds t) { 308 | return sdscatlen(s, t, sdslen(t)); 309 | } 310 | 311 | /* Destructively modify the sds string 's' to hold the specified binary 312 | * safe string pointed by 't' of length 'len' bytes. */ 313 | sds sdscpylen(sds s, const char *t, size_t len) { 314 | struct sdshdr *sh = (void *) (s - (sizeof(struct sdshdr))); 315 | size_t totlen = sh->free + sh->len; 316 | 317 | if (totlen < len) { 318 | s = sdsMakeRoomFor(s, len - sh->len); 319 | if (s == NULL) return NULL; 320 | sh = (void *) (s - (sizeof(struct sdshdr))); 321 | totlen = sh->free + sh->len; 322 | } 323 | memcpy(s, t, len); 324 | s[len] = '\0'; 325 | sh->len = len; 326 | sh->free = totlen - len; 327 | return s; 328 | } 329 | 330 | /* Like sdscpylen() but 't' must be a null-termined string so that the length 331 | * of the string is obtained with strlen(). */ 332 | sds sdscpy(sds s, const char *t) { 333 | return sdscpylen(s, t, strlen(t)); 334 | } 335 | 336 | 337 | 338 | /* Like sdscatprintf() but gets va_list instead of being variadic. */ 339 | sds sdscatvprintf(sds s, const char *fmt, va_list ap) { 340 | va_list cpy; 341 | char staticbuf[1024], *buf = staticbuf, *t; 342 | size_t buflen = strlen(fmt) * 2; 343 | 344 | /* We try to start using a static buffer for speed. 345 | * If not possible we revert to heap allocation. */ 346 | if (buflen > sizeof(staticbuf)) { 347 | buf = malloc(buflen); 348 | if (buf == NULL) return NULL; 349 | } else { 350 | buflen = sizeof(staticbuf); 351 | } 352 | 353 | /* Try with buffers two times bigger every time we fail to 354 | * fit the string in the current buffer size. */ 355 | while (1) { 356 | buf[buflen - 2] = '\0'; 357 | va_copy(cpy, ap); 358 | vsnprintf(buf, buflen, fmt, cpy); 359 | va_end(cpy); 360 | if (buf[buflen - 2] != '\0') { 361 | if (buf != staticbuf) free(buf); 362 | buflen *= 2; 363 | buf = malloc(buflen); 364 | if (buf == NULL) return NULL; 365 | continue; 366 | } 367 | break; 368 | } 369 | 370 | /* Finally concat the obtained string to the SDS string and return it. */ 371 | t = sdscat(s, buf); 372 | if (buf != staticbuf) free(buf); 373 | return t; 374 | } 375 | 376 | /* Append to the sds string 's' a string obtained using printf-alike format 377 | * specifier. 378 | * 379 | * After the call, the modified sds string is no longer valid and all the 380 | * references must be substituted with the new pointer returned by the call. 381 | * 382 | * Example: 383 | * 384 | * s = sdsnew("Sum is: "); 385 | * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b). 386 | * 387 | * Often you need to create a string from scratch with the printf-alike 388 | * format. When this is the need, just use sdsempty() as the target string: 389 | * 390 | * s = sdscatprintf(sdsempty(), "... your format ...", args); 391 | */ 392 | sds sdscatprintf(sds s, const char *fmt, ...) { 393 | va_list ap; 394 | char *t; 395 | va_start(ap, fmt); 396 | t = sdscatvprintf(s, fmt, ap); 397 | va_end(ap); 398 | return t; 399 | } 400 | 401 | 402 | -------------------------------------------------------------------------------- /src/sds.h: -------------------------------------------------------------------------------- 1 | /* SDSLib, A C dynamic strings library 2 | * 3 | * Copyright (c) 2006-2010, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #ifndef __SDS_H 32 | #define __SDS_H 33 | 34 | #define SDS_MAX_PREALLOC (1024*1024) 35 | 36 | #include 37 | #include 38 | 39 | #ifdef WIN32 40 | #define inline __inline 41 | #endif 42 | 43 | typedef char *sds; 44 | 45 | struct sdshdr { 46 | unsigned int len; 47 | unsigned int free; 48 | char buf[]; 49 | }; 50 | 51 | size_t sdslen(const sds s); 52 | 53 | size_t sdsavail(const sds s); 54 | 55 | sds sdsnewlen(const void *init, size_t initlen); 56 | 57 | sds sdsnewEmpty(size_t preAlloclen); 58 | 59 | sds sdsnew(const char *init); 60 | 61 | sds sdsempty(void); 62 | 63 | size_t sdslen(const sds s); 64 | 65 | sds sdsdup(const sds s); 66 | 67 | void sdsfree(sds s); 68 | 69 | size_t sdsavail(const sds s); 70 | 71 | sds sdsgrowzero(sds s, size_t len); 72 | 73 | sds sdscatlen(sds s, const void *t, size_t len); 74 | 75 | sds sdscat(sds s, const char *t); 76 | 77 | sds sdscatchar(sds s, char c); 78 | 79 | sds sdscatsds(sds s, const sds t); 80 | 81 | sds sdscpylen(sds s, const char *t, size_t len); 82 | 83 | sds sdscpy(sds s, const char *t); 84 | 85 | sds sdscatvprintf(sds s, const char *fmt, va_list ap); 86 | 87 | #ifdef __GNUC__ 88 | 89 | sds sdscatprintf(sds s, const char *fmt, ...) 90 | __attribute__((format(printf, 2, 3))); 91 | 92 | #else 93 | sds sdscatprintf(sds s, const char *fmt, ...); 94 | #endif 95 | 96 | 97 | #endif 98 | -------------------------------------------------------------------------------- /src/sha1.c: -------------------------------------------------------------------------------- 1 | 2 | #include /* memcpy & co */ 3 | #include 4 | #include "sha1.h" 5 | 6 | #ifndef LITTLE_ENDIAN 7 | #define LITTLE_ENDIAN 8 | #endif 9 | 10 | /********************************************************************************************************/ 11 | 12 | /** 13 | * \brief initialises given SHA-1 context 14 | * 15 | */ 16 | void sha1_init(sha1_ctx_t *state){ 17 | state->h[0] = 0x67452301; 18 | state->h[1] = 0xefcdab89; 19 | state->h[2] = 0x98badcfe; 20 | state->h[3] = 0x10325476; 21 | state->h[4] = 0xc3d2e1f0; 22 | state->length = 0; 23 | } 24 | 25 | /********************************************************************************************************/ 26 | /* some helping functions */ 27 | uint32_t rotl32(uint32_t n, uint8_t bits){ 28 | return ((n<>(32-bits))); 29 | } 30 | 31 | uint32_t change_endian32(uint32_t x){ 32 | return (((x)<<24) | ((x)>>24) | (((x)& 0x0000ff00)<<8) | (((x)& 0x00ff0000)>>8)); 33 | } 34 | 35 | 36 | /* three SHA-1 inner functions */ 37 | uint32_t ch(uint32_t x, uint32_t y, uint32_t z){ 38 | return ((x&y)^((~x)&z)); 39 | } 40 | 41 | uint32_t maj(uint32_t x, uint32_t y, uint32_t z){ 42 | return ((x&y)^(x&z)^(y&z)); 43 | } 44 | 45 | uint32_t parity(uint32_t x, uint32_t y, uint32_t z){ 46 | return ((x^y)^z); 47 | } 48 | 49 | /********************************************************************************************************/ 50 | /** 51 | * \brief "add" a block to the hash 52 | * This is the core function of the hash algorithm. To understand how it's working 53 | * and what thoese variables do, take a look at FIPS-182. This is an "alternativ" implementation 54 | */ 55 | 56 | #define MASK 0x0000000f 57 | 58 | typedef uint32_t (*pf_t)(uint32_t x, uint32_t y, uint32_t z); 59 | 60 | void sha1_nextBlock (sha1_ctx_t *state, const void *block){ 61 | uint32_t a[5]; 62 | uint32_t w[16]; 63 | uint32_t temp; 64 | uint8_t t,s,fi, fib; 65 | pf_t f[] = {ch,parity,maj,parity}; 66 | uint32_t k[4]={ 0x5a827999, 67 | 0x6ed9eba1, 68 | 0x8f1bbcdc, 69 | 0xca62c1d6}; 70 | 71 | /* load the w array (changing the endian and so) */ 72 | for(t=0; t<16; ++t){ 73 | w[t] = change_endian32(((uint32_t*)block)[t]); 74 | } 75 | 76 | 77 | /* load the state */ 78 | memcpy(a, state->h, 5*sizeof(uint32_t)); 79 | 80 | 81 | /* the fun stuff */ 82 | for(fi=0,fib=0,t=0; t<=79; ++t){ 83 | s = t & MASK; 84 | if(t>=16){ 85 | w[s] = rotl32( w[(s+13)&MASK] ^ w[(s+8)&MASK] ^ 86 | w[(s+ 2)&MASK] ^ w[s] ,1); 87 | } 88 | 89 | uint32_t dtemp; 90 | temp = rotl32(a[0],5) + (dtemp=f[fi](a[1],a[2],a[3])) + a[4] + k[fi] + w[s]; 91 | memmove(&(a[1]), &(a[0]), 4*sizeof(uint32_t)); /* e=d; d=c; c=b; b=a; */ 92 | a[0] = temp; 93 | a[2] = rotl32(a[2],30); /* we might also do rotr32(c,2) */ 94 | fib++; 95 | if(fib==20){ 96 | fib=0; 97 | fi = (fi+1)%4; 98 | } 99 | } 100 | 101 | /* update the state */ 102 | for(t=0; t<5; ++t){ 103 | state->h[t] += a[t]; 104 | } 105 | state->length += 512; 106 | } 107 | 108 | /********************************************************************************************************/ 109 | 110 | void sha1_lastBlock(sha1_ctx_t *state, const void *block, uint16_t length){ 111 | uint8_t lb[SHA1_BLOCK_BYTES]; /* local block */ 112 | while(length>=SHA1_BLOCK_BITS){ 113 | sha1_nextBlock(state, block); 114 | length -= SHA1_BLOCK_BITS; 115 | block = (uint8_t*)block + SHA1_BLOCK_BYTES; 116 | } 117 | state->length += length; 118 | memset(lb, 0, SHA1_BLOCK_BYTES); 119 | memcpy (lb, block, (length+7)>>3); 120 | 121 | /* set the final one bit */ 122 | lb[length>>3] |= 0x80>>(length & 0x07); 123 | 124 | if (length>512-64-1){ /* not enouth space for 64bit length value */ 125 | sha1_nextBlock(state, lb); 126 | state->length -= 512; 127 | memset(lb, 0, SHA1_BLOCK_BYTES); 128 | } 129 | /* store the 64bit length value */ 130 | #if defined LITTLE_ENDIAN 131 | /* this is now rolled up */ 132 | uint8_t i; 133 | for (i=0; i<8; ++i){ 134 | lb[56+i] = ((uint8_t*)&(state->length))[7-i]; 135 | } 136 | #elif defined BIG_ENDIAN 137 | *((uint64_t)&(lb[56])) = state->length; 138 | #endif 139 | sha1_nextBlock(state, lb); 140 | } 141 | 142 | /********************************************************************************************************/ 143 | 144 | void sha1_ctx2hash (void *dest, sha1_ctx_t *state){ 145 | #if defined LITTLE_ENDIAN 146 | uint8_t i; 147 | for(i=0; i<5; ++i){ 148 | ((uint32_t*)dest)[i] = change_endian32(state->h[i]); 149 | } 150 | #elif BIG_ENDIAN 151 | if (dest != state->h) 152 | memcpy(dest, state->h, SHA1_HASH_BITS/8); 153 | #else 154 | # error unsupported endian type! 155 | #endif 156 | } 157 | 158 | /********************************************************************************************************/ 159 | /** 160 | * 161 | * 162 | */ 163 | void sha1 (void *dest, const void *msg, uint32_t length){ 164 | sha1_ctx_t s; 165 | sha1_init(&s); 166 | while(length & (~0x0001ff)){ /* length>=512 */ 167 | sha1_nextBlock(&s, msg); 168 | msg = (uint8_t*)msg + SHA1_BLOCK_BITS/8; /* increment pointer to next block */ 169 | length -= SHA1_BLOCK_BITS; 170 | } 171 | sha1_lastBlock(&s, msg, length); 172 | sha1_ctx2hash(dest, &s); 173 | } 174 | -------------------------------------------------------------------------------- /src/sha1.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef SHA1_H_ 3 | #define SHA1_H_ 4 | 5 | #include 6 | /** \def SHA1_HASH_BITS 7 | * definees the size of a SHA-1 hash in bits 8 | */ 9 | 10 | /** \def SHA1_HASH_BYTES 11 | * definees the size of a SHA-1 hash in bytes 12 | */ 13 | 14 | /** \def SHA1_BLOCK_BITS 15 | * definees the size of a SHA-1 input block in bits 16 | */ 17 | 18 | /** \def SHA1_BLOCK_BYTES 19 | * definees the size of a SHA-1 input block in bytes 20 | */ 21 | #define SHA1_HASH_BITS 160 22 | #define SHA1_HASH_BYTES (SHA1_HASH_BITS/8) 23 | #define SHA1_BLOCK_BITS 512 24 | #define SHA1_BLOCK_BYTES (SHA1_BLOCK_BITS/8) 25 | 26 | /** \typedef sha1_ctx_t 27 | * \brief SHA-1 context type 28 | * 29 | * A vatiable of this type may hold the state of a SHA-1 hashing process 30 | */ 31 | typedef struct { 32 | uint32_t h[5]; 33 | uint64_t length; 34 | } sha1_ctx_t; 35 | 36 | /** \typedef sha1_hash_t 37 | * \brief hash value type 38 | * A variable of this type may hold a SHA-1 hash value 39 | */ 40 | /* 41 | typedef uint8_t sha1_hash_t[SHA1_HASH_BITS/8]; 42 | */ 43 | 44 | /** \fn sha1_init(sha1_ctx_t *state) 45 | * \brief initializes a SHA-1 context 46 | * This function sets a ::sha1_ctx_t variable to the initialization vector 47 | * for SHA-1 hashing. 48 | * \param state pointer to the SHA-1 context variable 49 | */ 50 | void sha1_init(sha1_ctx_t *state); 51 | 52 | /** \fn sha1_nextBlock(sha1_ctx_t *state, const void *block) 53 | * \brief process one input block 54 | * This function processes one input block and updates the hash context 55 | * accordingly 56 | * \param state pointer to the state variable to update 57 | * \param block pointer to the message block to process 58 | */ 59 | void sha1_nextBlock (sha1_ctx_t *state, const void *block); 60 | 61 | /** \fn sha1_lastBlock(sha1_ctx_t *state, const void *block, uint16_t length_b) 62 | * \brief processes the given block and finalizes the context 63 | * This function processes the last block in a SHA-1 hashing process. 64 | * The block should have a maximum length of a single input block. 65 | * \param state pointer to the state variable to update and finalize 66 | * \param block pointer to themessage block to process 67 | * \param length_b length of the message block in bits 68 | */ 69 | void sha1_lastBlock (sha1_ctx_t *state, const void *block, uint16_t length_b); 70 | 71 | /** \fn sha1_ctx2hash(sha1_hash_t *dest, sha1_ctx_t *state) 72 | * \brief convert a state variable into an actual hash value 73 | * Writes the hash value corresponding to the state to the memory pointed by dest. 74 | * \param dest pointer to the hash value destination 75 | * \param state pointer to the hash context 76 | */ 77 | void sha1_ctx2hash (void *dest, sha1_ctx_t *state); 78 | 79 | /** \fn sha1(sha1_hash_t *dest, const void *msg, uint32_t length_b) 80 | * \brief hashing a message which in located entirely in RAM 81 | * This function automatically hashes a message which is entirely in RAM with 82 | * the SHA-1 hashing algorithm. 83 | * \param dest pointer to the hash value destination 84 | * \param msg pointer to the message which should be hashed 85 | * \param length_b length of the message in bits 86 | */ 87 | void sha1(void *dest, const void *msg, uint32_t length_b); 88 | 89 | 90 | 91 | #endif /*SHA1_H_*/ 92 | -------------------------------------------------------------------------------- /src/sha256.c: -------------------------------------------------------------------------------- 1 | /* 2 | * FIPS-180-2 compliant SHA-256 implementation 3 | * 4 | * Copyright (C) 2001-2003 Christophe Devine 5 | * 6 | * This program is free software; you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation; either version 2 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program; if not, write to the Free Software 18 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 19 | */ 20 | 21 | #include 22 | 23 | #include "sha256.h" 24 | 25 | #define GET_UINT32(n,b,i) \ 26 | { \ 27 | (n) = ( (uint32) (b)[(i) ] << 24 ) \ 28 | | ( (uint32) (b)[(i) + 1] << 16 ) \ 29 | | ( (uint32) (b)[(i) + 2] << 8 ) \ 30 | | ( (uint32) (b)[(i) + 3] ); \ 31 | } 32 | 33 | #define PUT_UINT32(n,b,i) \ 34 | { \ 35 | (b)[(i) ] = (uint8) ( (n) >> 24 ); \ 36 | (b)[(i) + 1] = (uint8) ( (n) >> 16 ); \ 37 | (b)[(i) + 2] = (uint8) ( (n) >> 8 ); \ 38 | (b)[(i) + 3] = (uint8) ( (n) ); \ 39 | } 40 | 41 | void sha256_starts( sha256_context *ctx ) 42 | { 43 | ctx->total[0] = 0; 44 | ctx->total[1] = 0; 45 | 46 | ctx->state[0] = 0x6A09E667; 47 | ctx->state[1] = 0xBB67AE85; 48 | ctx->state[2] = 0x3C6EF372; 49 | ctx->state[3] = 0xA54FF53A; 50 | ctx->state[4] = 0x510E527F; 51 | ctx->state[5] = 0x9B05688C; 52 | ctx->state[6] = 0x1F83D9AB; 53 | ctx->state[7] = 0x5BE0CD19; 54 | } 55 | 56 | void sha256_process( sha256_context *ctx, uint8 data[64] ) 57 | { 58 | uint32 temp1, temp2, W[64]; 59 | uint32 A, B, C, D, E, F, G, H; 60 | 61 | GET_UINT32( W[0], data, 0 ); 62 | GET_UINT32( W[1], data, 4 ); 63 | GET_UINT32( W[2], data, 8 ); 64 | GET_UINT32( W[3], data, 12 ); 65 | GET_UINT32( W[4], data, 16 ); 66 | GET_UINT32( W[5], data, 20 ); 67 | GET_UINT32( W[6], data, 24 ); 68 | GET_UINT32( W[7], data, 28 ); 69 | GET_UINT32( W[8], data, 32 ); 70 | GET_UINT32( W[9], data, 36 ); 71 | GET_UINT32( W[10], data, 40 ); 72 | GET_UINT32( W[11], data, 44 ); 73 | GET_UINT32( W[12], data, 48 ); 74 | GET_UINT32( W[13], data, 52 ); 75 | GET_UINT32( W[14], data, 56 ); 76 | GET_UINT32( W[15], data, 60 ); 77 | 78 | #define SHR(x,n) ((x & 0xFFFFFFFF) >> n) 79 | #define ROTR(x,n) (SHR(x,n) | (x << (32 - n))) 80 | 81 | #define S0(x) (ROTR(x, 7) ^ ROTR(x,18) ^ SHR(x, 3)) 82 | #define S1(x) (ROTR(x,17) ^ ROTR(x,19) ^ SHR(x,10)) 83 | 84 | #define S2(x) (ROTR(x, 2) ^ ROTR(x,13) ^ ROTR(x,22)) 85 | #define S3(x) (ROTR(x, 6) ^ ROTR(x,11) ^ ROTR(x,25)) 86 | 87 | #define F0(x,y,z) ((x & y) | (z & (x | y))) 88 | #define F1(x,y,z) (z ^ (x & (y ^ z))) 89 | 90 | #define R(t) \ 91 | ( \ 92 | W[t] = S1(W[t - 2]) + W[t - 7] + \ 93 | S0(W[t - 15]) + W[t - 16] \ 94 | ) 95 | 96 | #define P(a,b,c,d,e,f,g,h,x,K) \ 97 | { \ 98 | temp1 = h + S3(e) + F1(e,f,g) + K + x; \ 99 | temp2 = S2(a) + F0(a,b,c); \ 100 | d += temp1; h = temp1 + temp2; \ 101 | } 102 | 103 | A = ctx->state[0]; 104 | B = ctx->state[1]; 105 | C = ctx->state[2]; 106 | D = ctx->state[3]; 107 | E = ctx->state[4]; 108 | F = ctx->state[5]; 109 | G = ctx->state[6]; 110 | H = ctx->state[7]; 111 | 112 | P( A, B, C, D, E, F, G, H, W[ 0], 0x428A2F98 ); 113 | P( H, A, B, C, D, E, F, G, W[ 1], 0x71374491 ); 114 | P( G, H, A, B, C, D, E, F, W[ 2], 0xB5C0FBCF ); 115 | P( F, G, H, A, B, C, D, E, W[ 3], 0xE9B5DBA5 ); 116 | P( E, F, G, H, A, B, C, D, W[ 4], 0x3956C25B ); 117 | P( D, E, F, G, H, A, B, C, W[ 5], 0x59F111F1 ); 118 | P( C, D, E, F, G, H, A, B, W[ 6], 0x923F82A4 ); 119 | P( B, C, D, E, F, G, H, A, W[ 7], 0xAB1C5ED5 ); 120 | P( A, B, C, D, E, F, G, H, W[ 8], 0xD807AA98 ); 121 | P( H, A, B, C, D, E, F, G, W[ 9], 0x12835B01 ); 122 | P( G, H, A, B, C, D, E, F, W[10], 0x243185BE ); 123 | P( F, G, H, A, B, C, D, E, W[11], 0x550C7DC3 ); 124 | P( E, F, G, H, A, B, C, D, W[12], 0x72BE5D74 ); 125 | P( D, E, F, G, H, A, B, C, W[13], 0x80DEB1FE ); 126 | P( C, D, E, F, G, H, A, B, W[14], 0x9BDC06A7 ); 127 | P( B, C, D, E, F, G, H, A, W[15], 0xC19BF174 ); 128 | P( A, B, C, D, E, F, G, H, R(16), 0xE49B69C1 ); 129 | P( H, A, B, C, D, E, F, G, R(17), 0xEFBE4786 ); 130 | P( G, H, A, B, C, D, E, F, R(18), 0x0FC19DC6 ); 131 | P( F, G, H, A, B, C, D, E, R(19), 0x240CA1CC ); 132 | P( E, F, G, H, A, B, C, D, R(20), 0x2DE92C6F ); 133 | P( D, E, F, G, H, A, B, C, R(21), 0x4A7484AA ); 134 | P( C, D, E, F, G, H, A, B, R(22), 0x5CB0A9DC ); 135 | P( B, C, D, E, F, G, H, A, R(23), 0x76F988DA ); 136 | P( A, B, C, D, E, F, G, H, R(24), 0x983E5152 ); 137 | P( H, A, B, C, D, E, F, G, R(25), 0xA831C66D ); 138 | P( G, H, A, B, C, D, E, F, R(26), 0xB00327C8 ); 139 | P( F, G, H, A, B, C, D, E, R(27), 0xBF597FC7 ); 140 | P( E, F, G, H, A, B, C, D, R(28), 0xC6E00BF3 ); 141 | P( D, E, F, G, H, A, B, C, R(29), 0xD5A79147 ); 142 | P( C, D, E, F, G, H, A, B, R(30), 0x06CA6351 ); 143 | P( B, C, D, E, F, G, H, A, R(31), 0x14292967 ); 144 | P( A, B, C, D, E, F, G, H, R(32), 0x27B70A85 ); 145 | P( H, A, B, C, D, E, F, G, R(33), 0x2E1B2138 ); 146 | P( G, H, A, B, C, D, E, F, R(34), 0x4D2C6DFC ); 147 | P( F, G, H, A, B, C, D, E, R(35), 0x53380D13 ); 148 | P( E, F, G, H, A, B, C, D, R(36), 0x650A7354 ); 149 | P( D, E, F, G, H, A, B, C, R(37), 0x766A0ABB ); 150 | P( C, D, E, F, G, H, A, B, R(38), 0x81C2C92E ); 151 | P( B, C, D, E, F, G, H, A, R(39), 0x92722C85 ); 152 | P( A, B, C, D, E, F, G, H, R(40), 0xA2BFE8A1 ); 153 | P( H, A, B, C, D, E, F, G, R(41), 0xA81A664B ); 154 | P( G, H, A, B, C, D, E, F, R(42), 0xC24B8B70 ); 155 | P( F, G, H, A, B, C, D, E, R(43), 0xC76C51A3 ); 156 | P( E, F, G, H, A, B, C, D, R(44), 0xD192E819 ); 157 | P( D, E, F, G, H, A, B, C, R(45), 0xD6990624 ); 158 | P( C, D, E, F, G, H, A, B, R(46), 0xF40E3585 ); 159 | P( B, C, D, E, F, G, H, A, R(47), 0x106AA070 ); 160 | P( A, B, C, D, E, F, G, H, R(48), 0x19A4C116 ); 161 | P( H, A, B, C, D, E, F, G, R(49), 0x1E376C08 ); 162 | P( G, H, A, B, C, D, E, F, R(50), 0x2748774C ); 163 | P( F, G, H, A, B, C, D, E, R(51), 0x34B0BCB5 ); 164 | P( E, F, G, H, A, B, C, D, R(52), 0x391C0CB3 ); 165 | P( D, E, F, G, H, A, B, C, R(53), 0x4ED8AA4A ); 166 | P( C, D, E, F, G, H, A, B, R(54), 0x5B9CCA4F ); 167 | P( B, C, D, E, F, G, H, A, R(55), 0x682E6FF3 ); 168 | P( A, B, C, D, E, F, G, H, R(56), 0x748F82EE ); 169 | P( H, A, B, C, D, E, F, G, R(57), 0x78A5636F ); 170 | P( G, H, A, B, C, D, E, F, R(58), 0x84C87814 ); 171 | P( F, G, H, A, B, C, D, E, R(59), 0x8CC70208 ); 172 | P( E, F, G, H, A, B, C, D, R(60), 0x90BEFFFA ); 173 | P( D, E, F, G, H, A, B, C, R(61), 0xA4506CEB ); 174 | P( C, D, E, F, G, H, A, B, R(62), 0xBEF9A3F7 ); 175 | P( B, C, D, E, F, G, H, A, R(63), 0xC67178F2 ); 176 | 177 | ctx->state[0] += A; 178 | ctx->state[1] += B; 179 | ctx->state[2] += C; 180 | ctx->state[3] += D; 181 | ctx->state[4] += E; 182 | ctx->state[5] += F; 183 | ctx->state[6] += G; 184 | ctx->state[7] += H; 185 | } 186 | 187 | void sha256_update( sha256_context *ctx, uint8 *input, uint32 length ) 188 | { 189 | uint32 left, fill; 190 | 191 | if( ! length ) return; 192 | 193 | left = ctx->total[0] & 0x3F; 194 | fill = 64 - left; 195 | 196 | ctx->total[0] += length; 197 | ctx->total[0] &= 0xFFFFFFFF; 198 | 199 | if( ctx->total[0] < length ) 200 | ctx->total[1]++; /* #nocov */ 201 | 202 | if( left && length >= fill ) 203 | { 204 | memcpy( (void *) (ctx->buffer + left), 205 | (void *) input, fill ); 206 | sha256_process( ctx, ctx->buffer ); 207 | length -= fill; 208 | input += fill; 209 | left = 0; 210 | } 211 | 212 | while( length >= 64 ) 213 | { 214 | sha256_process( ctx, input ); 215 | length -= 64; 216 | input += 64; 217 | } 218 | 219 | if( length ) 220 | { 221 | memcpy( (void *) (ctx->buffer + left), 222 | (void *) input, length ); 223 | } 224 | } 225 | 226 | static uint8 sha256_padding[64] = 227 | { 228 | 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 232 | }; 233 | 234 | void sha256_finish( sha256_context *ctx, uint8 digest[32] ) 235 | { 236 | uint32 last, padn; 237 | uint32 high, low; 238 | uint8 msglen[8]; 239 | 240 | high = ( ctx->total[0] >> 29 ) 241 | | ( ctx->total[1] << 3 ); 242 | low = ( ctx->total[0] << 3 ); 243 | 244 | PUT_UINT32( high, msglen, 0 ); 245 | PUT_UINT32( low, msglen, 4 ); 246 | 247 | last = ctx->total[0] & 0x3F; 248 | padn = ( last < 56 ) ? ( 56 - last ) : ( 120 - last ); 249 | 250 | sha256_update( ctx, sha256_padding, padn ); 251 | sha256_update( ctx, msglen, 8 ); 252 | 253 | PUT_UINT32( ctx->state[0], digest, 0 ); 254 | PUT_UINT32( ctx->state[1], digest, 4 ); 255 | PUT_UINT32( ctx->state[2], digest, 8 ); 256 | PUT_UINT32( ctx->state[3], digest, 12 ); 257 | PUT_UINT32( ctx->state[4], digest, 16 ); 258 | PUT_UINT32( ctx->state[5], digest, 20 ); 259 | PUT_UINT32( ctx->state[6], digest, 24 ); 260 | PUT_UINT32( ctx->state[7], digest, 28 ); 261 | } -------------------------------------------------------------------------------- /src/sha256.h: -------------------------------------------------------------------------------- 1 | #ifndef _SHA256_H 2 | #define _SHA256_H 3 | 4 | #ifndef uint8 5 | #define uint8 unsigned char 6 | #endif 7 | 8 | #ifndef uint32 9 | #define uint32 unsigned long int 10 | #endif 11 | 12 | typedef struct 13 | { 14 | uint32 total[2]; 15 | uint32 state[8]; 16 | uint8 buffer[64]; 17 | } 18 | sha256_context; 19 | 20 | void sha256_starts( sha256_context *ctx ); 21 | void sha256_update( sha256_context *ctx, uint8 *input, uint32 length ); 22 | void sha256_finish( sha256_context *ctx, uint8 digest[32] ); 23 | 24 | #endif /* sha256.h */ --------------------------------------------------------------------------------