├── AUTHORS
├── LICENSE.TXT
├── Makefile
├── README.md
├── TODO
├── freq
    ├── appid.h
    ├── appid_manager.cpp
    ├── appid_manager.h
    ├── appname.h
    ├── appname_manager.cpp
    ├── appname_manager.h
    ├── expression.h
    ├── freq_item.h
    ├── freq_manager.cpp
    ├── freq_manager.h
    ├── freqlib.cpp
    ├── libwxfreq.h
    ├── plain_expression.cpp
    ├── plain_expression.h
    ├── rule_item.h
    ├── rule_manager.cpp
    └── rule_manager.h
├── main.cpp
├── net
    ├── acceptor.cpp
    ├── acceptor.h
    ├── iothread.cpp
    ├── iothread.h
    ├── server.cpp
    ├── server.h
    ├── server_conf.cpp
    ├── server_conf.h
    ├── server_inner.cpp
    ├── server_inner.h
    ├── util.cpp
    ├── util.h
    ├── worker.cpp
    ├── worker.h
    ├── workerpool.cpp
    └── workerpool.h
├── server.conf
├── test
    ├── demo.cpp
    ├── performance_test.cpp
    ├── test.conf
    └── tools.cpp
└── util
    ├── config.cpp
    ├── config.h
    ├── config_hash.cpp
    ├── config_hash.h
    ├── freq_stat.h
    ├── log.cpp
    ├── log.h
    ├── map_freq_stat.cpp
    ├── map_freq_stat.h
    ├── multi_hash_base.h
    ├── multi_hash_table.cpp
    ├── multi_hash_table.h
    ├── options.cpp
    ├── options.h
    ├── reload.cpp
    ├── reload.h
    ├── shm_freq_stat.cpp
    ├── shm_freq_stat.h
    ├── slice.cpp
    └── slice.h


/AUTHORS:
--------------------------------------------------------------------------------
 1 | # Names should be added to this file like so:
 2 | # Name or Organization <email address>
 3 | 
 4 | Tencent Inc. 2016
 5 | 
 6 | # Initial version authors:
 7 | Jack Liang	<jackliang@tencent.com>
 8 | Robin Tang	<robintang@tencent.com>
 9 | Arthur Zou	<arthurzou@tencent.com>
10 | 
11 | # Partial list of contributors:
12 | 


--------------------------------------------------------------------------------
/LICENSE.TXT:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Tencent/libwxfreq/8281a2d530588f44661b42ea9be9474ba9316014/LICENSE.TXT


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | NAME = libwxfreq
 2 | SERVERNAME = WxFreq
 3 | LIBNAME = libwxfreq.a
 4 | OUTPUTDIR = ./output
 5 | INCLUDE = $(OUTPUTDIR)/include/$(NAME)
 6 | LIB64 = $(OUTPUTDIR)/lib64
 7 | LIBINSTALLDIR = /usr/local/lib64/
 8 | HEADERINSTALLDIR = /usr/local/include/
 9 | DIRLIBEVENT=/data1/mm64/arthurzou/archive
10 | 
11 | CC = g++
12 | LDFLAGS = -lwxfreq -pthread -L$(LIB64) -I $(dir $(INCLUDE))
13 | CPPFLAGS = $(CFLAGS)
14 | CPPFLAGS += -g -O2 -Wall -std=c++11
15 | INC = -I ./util -I ./freq -I ./net --std=c++11
16 | 
17 | CPPFILE += $(wildcard freq/*.cpp)
18 | CPPFILE += $(wildcard util/*.cpp)
19 | OBJ = $(patsubst %.cpp,  %.o, $(CPPFILE))
20 | DFILE = $(patsubst %.o,  %.d, $(OBJ))
21 | 
22 | NETCPPFILE += main.cpp
23 | NETCPPFILE += $(wildcard net/*.cpp)
24 | NETOBJ = $(patsubst %.cpp,  %.o, $(NETCPPFILE))
25 | NETDFILE = $(patsubst %.o,  %.d, $(NETOBJ))
26 | 
27 | 
28 | .PHONY : all clean test
29 | 
30 | $(SERVERNAME) : $(OBJ) $(NETOBJ) main.o
31 | 	@rm -rf $(DFILE)
32 | 	@rm -rf $(NETDFILE)
33 | 	$(CC) $(CPPFLAGS)  $^ -o $@ -pthread -levent -lrt -static -L $(DIRLIBEVENT)
34 | 
35 | $(LIBNAME) : $(OBJ)
36 | 	@rm -rf $(OUTPUTDIR)
37 | 	@rm -rf $(DFILE)
38 | 	@mkdir -p $(LIB64)
39 | 	@mkdir -p $(INCLUDE)
40 | 	ar scr $@ $(OBJ)
41 | 	mv $@  $(LIB64)
42 | 	cp freq/libwxfreq.h $(INCLUDE)
43 | 	@$(CC) -MM freq/libwxfreq.h $(INC) | \
44 | 		tr " " "\n" | grep ".h" | xargs -I{} cp {} $(INCLUDE)
45 | 	@find $(INCLUDE) -name "*.h" | xargs sed -i 's;#include ";#include "$(NAME)/;'
46 | 
47 | %: test/%.cpp $(LIBNAME)
48 | 	$(CC) $(CPPFLAGS)  $< -o $@.out $(LDFLAGS)
49 | 
50 | 
51 | -include $(DFILE)
52 | -include $(NETDFILE)
53 | 
54 | %.d: %.cpp
55 | 	@$(CC) $(INC) -MT $*.o -MM $< > $@ && \
56 | 		echo -e '\t$$(CC) $$(CPPFLAGS) $(INC) -c $$< -o $$@' >> $@
57 | 
58 | clean:
59 | 	@echo -n "clean workspace ..."
60 | 	@rm  -rf $(OUTPUTDIR) $(OBJ) $(NETOBJ) $(NETDFILE) $(DFILE) *.out $(SERVERNAME)
61 | 	@echo -e "\t\t\033[31m[done]\033[0m"
62 | 
63 | install: $(LIBNAME)
64 | 	cp -r $(INCLUDE)  $(HEADERINSTALLDIR)
65 | 	cp -r $(LIB64)/$(LIBNAME) $(LIBINSTALLDIR)
66 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # libwxfreq
  2 | 
  3 | ## 编译
  4 |   1. 编译lib: make libwxfreq.a
  5 |   2. 编译server: 依赖libevent库，所以要修改DIRLIBEVENT变量为libevent.a所在目录，并确定libevent相关头文件在头文件搜索路径中， 运行make
  6 | 
  7 | ## 安装
  8 |   1. 修改makefile文件中的HEADERINSTALLDIR和LIBINSTALLDIR变量
  9 |   2. make install
 10 | 
 11 | ## 单元测试
 12 |   1. 以测试文件为目标make. 例如: make demo
 13 | 
 14 | ## 简介
 15 |   libwxfreq是一个高性能频率限制库。通过简洁的几个接口便可以实现通用的可配置的频率
 16 | 限制功能。配合其他网络框架，便可以实现一个通用的高性能频率控制服务。
 17 | 
 18 | ## 特点：
 19 |   1. 高度可配置。统计时长可配，规则可配。
 20 |   2. 支持任意key类型。
 21 |  
 22 | ## 配置文件
 23 |   1. 格式
 24 |    
 25 |    appid的元信息，指明apppid的统计时长。默认第一个统计时长是60s，第二个统计时  
 26 |    时长是3600s， 第三个是86400s。可以手动修改任何一个统计时长为[0, 86400]  
 27 | 
 28 |   [appid]
 29 |   
 30 |   5 = 30, 60, 90 // appid为5的统计时长分别是30s 60s 90s  
 31 |   25 = 10, 60   // appid为25的统计时长分别是10s 60s 86400s  
 32 |  
 33 |   频率规则， 用于说明某个appid的阈值， 拦截等级等  
 34 |   [match_rule_name] // 规则名字，用户指明中了哪条规则  
 35 |   match_appid = 5   // 匹配的appid                              
 36 |   block_level = 1  // 拦截等级， 大于0  
 37 |   rule_type = user // 规则对应的key类型  
 38 |   item = min_interval >= 4 // 具体的阈值表达式， 支持大于和等于
 39 |   ```
 40 |   min_interval 表示第一个统计时长的统计值，                  
 41 |   mid_interval 表示第二个统计时长的统计值，                        
 42 |   max_interval 表示第三个统计时长的统计值.
 43 |   ```
 44 | 
 45 |   2. 例子
 46 |   ```
 47 |   [appid]
 48 |   100 = 30, 60, 90
 49 | 
 50 |   [rule1]
 51 |   match_appid = 100
 52 |   item =  min_interval >= 10
 53 |   block_level = 1
 54 |   rule_type = user
 55 | 
 56 |   [rule2]
 57 |   match_appid = 100
 58 |   item =  mid_interval >= 15 
 59 |   block_level = 1
 60 |   rule_type = user
 61 | 
 62 |   [rule3]
 63 |   match_appid = 100
 64 |   item =  max_interval >= 20
 65 |   block_level = 1
 66 |   rule_type = user
 67 | 
 68 |   [rule4]
 69 |   match_appid = 100
 70 |   item =  min_interval >= 100
 71 |   block_level = 1
 72 |   rule_type = ip 
 73 | 
 74 |   [rule5]
 75 |   match_appid = 100
 76 |   item =  mid_interval >= 150
 77 |   block_level = 1
 78 |   rule_type = ip 
 79 | 
 80 |   [rule6]
 81 |   match_appid = 100
 82 |   item =  max_interval >= 200 
 83 |   block_level = 1
 84 |   rule_type = ip 
 85 |   ```
 86 | 
 87 | ## API接口
 88 | 
 89 |   设置规则配置文件
 90 |   > void SetRuleConfFile(const char *filename);
 91 | 
 92 |   注册一个新的统计维度。type_name 指明类型名称， zero_init标志初始化时是否需要
 93 |   初始化为0， key 是共享内存key，item_cnt 表示共享内存大小，最大能统计的item个
 94 |   数。
 95 |   > int RegisterNewShmStat(const char* type_name, bool zero_init, key_t key,
 96 |   >                      unsigned int item_cnt);
 97 | 
 98 |   系统初始化
 99 |   > bool InitFreq();
100 | 
101 |   上报一次频率到系统中。type_name 是统计类型，说明key的含义。 key是主体标识，
102 |   appid 是业务标识， cnt 表示本次需要累加的次数。 例如，统计QQ号为3402393864的
103 |   用户在appid为5的服务上的频率，使用ReportAndCheck("user", 3402393864, 5, 1);
104 |   同样统计ip为202.204.105.8访问appid的5的频率，使用ReportAndCheck( "IP", 
105 |   3402393864, 5, 1); 因为202.204.105.8的整形表示也是3402393864。type_name指明了
106 |   key的含义。
107 |   > struct BlockResult ReportAndCheck(const char* type_name, const char *key,
108 |   >                                   const uint32_t appid, const uint32_t cnt);
109 | 
110 |   只检查当前是否中频率
111 |   > struct BlockResult OnlyCheck(const char* type_name, const char *key,
112 |   >                             const uint32_t appid);
113 | 
114 |   只上报，不检查
115 |   > int OnlyReport(const char* type_name, const char *key, const uint32_t appid,
116 |   >              const uint32_t cnt);
117 | 
118 |   获取频率统计值, cnt为0获取当前统计值，不为零表示先累计再获取
119 |   > struct FreqCache GetCache(const char* type_name, const char *key,
120 |   >                          const uint32_t appid, const uint32_t cnt = 0);
121 | 
122 |   加白操作，linger_time表示加白有效期
123 |   > int AddWhite(const char* type_name, const char *key, const uint32_t appid,
124 |   >            const uint32_t linger_time);
125 | 
126 |   删除白名单
127 |   > int DeleteWhite(const char* type_name, const char *key, const uint32_t appid);
128 | 
129 |   手动加拦截
130 |   > int AddBlock(const char* type_name, const char *key, const uint32_t appid,
131 |   >            const uint32_t linger_time, const uint32_t block_level);
132 | 
133 |   手动删除拦截
134 |   > int DeleteBlock(const char* type_name, const char *key, const uint32_t appid);
135 | 
136 |   设置写日志方法，默认写本地文件/tmp/libwxfreq.log
137 |   > void SetLogFunc(LogFunction logfunc);
138 | 
139 |   设置重启时恢复白名单/拦截方法，默认从本地文件/tmp/libwxfreq_db.$type_name读取
140 |   > void SetLoadFunc(LoadFunc func);
141 | 
142 |   设置白名单/加拦截数据持久化方法。默认写本地文件/tmp/libwxfreq_db.$type_name
143 |   > void SetDumpFunc(DumpFunc func);
144 | 


--------------------------------------------------------------------------------
/TODO:
--------------------------------------------------------------------------------
1 | 1. 表达式支持逻辑运算
2 | 


--------------------------------------------------------------------------------
/freq/appid.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_APPID_H_
18 | #define FREQLIB_FREQ_APPID_H_
19 | 
20 | #include <stdint.h>
21 | #include <stdio.h>
22 | #include <string>
23 | 
24 | namespace libwxfreq {
25 | 
26 | class AppidMeta {
27 |  public:
28 |   AppidMeta()
29 |     : min_interval_(60),
30 |       mid_interval_(3600),
31 |       max_interval_(86400) {
32 |   }
33 | 
34 |   inline void set_min_interval(const uint32_t& min_interval) {
35 |     min_interval_ = min_interval;
36 |   }
37 | 
38 |   inline void set_mid_interval(const uint32_t& mid_interval) {
39 |     mid_interval_ = mid_interval;
40 |   }
41 | 
42 |   inline void set_max_interval(const uint32_t& max_interval) {
43 |     max_interval_ = max_interval;
44 |   }
45 | 
46 |   inline uint16_t min_interval() const {
47 |     return min_interval_;
48 |   }
49 | 
50 |   inline uint16_t mid_interval() const {
51 |     return mid_interval_;
52 |   }
53 |   inline uint16_t max_interval() const {
54 |     return max_interval_;
55 |   }
56 | 
57 |   inline std::string DebugString() const {
58 |     char buf[128];
59 |     snprintf(buf, sizeof(buf), "min_interval=%d, mid_interval=%d,"
60 |             " max_interval=%d", min_interval_, mid_interval_, max_interval_);
61 |     return buf;
62 |   }
63 | 
64 |  private:
65 |   uint16_t min_interval_;
66 |   uint16_t mid_interval_;
67 |   uint32_t max_interval_;
68 | };
69 | 
70 | }  // namespace libwxfreq
71 | 
72 | #endif  // FREQLIB_FREQ_APPID_H_
73 | 


--------------------------------------------------------------------------------
/freq/appid_manager.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "appid_manager.h"
 18 | #include <unistd.h>
 19 | #include <string.h>
 20 | #include <stdio.h>
 21 | #include <vector>
 22 | #include "config.h"
 23 | #include "log.h"
 24 | 
 25 | 
 26 | namespace libwxfreq {
 27 | 
 28 | static AppidMeta  dummymeta;
 29 | AppidMapVecAppidMeta  AppidManager::appid_map_[2];
 30 | unsigned char AppidManager::index_ = 0;
 31 | 
 32 | void AppidManager::UpdateAppidMeta(const Config& config) {
 33 |   int ret = 0;
 34 |   unsigned int appid = 0; 
 35 |   std::string interval_value; //60, 3600, 86400
 36 | 
 37 |   unsigned char new_index = (index_ + 1) % 2;
 38 |   AppidMapVecAppidMeta& tmp_appid  = appid_map_[new_index];
 39 |   tmp_appid.clear();
 40 | 
 41 |   std::vector<std::string> keys;
 42 |   config.GetKeysBySection("appid", keys);
 43 | 
 44 |   for (std::vector<std::string>::iterator it = keys.begin();
 45 |       it != keys.end(); ++it) {
 46 |     appid = atoi(it->c_str());
 47 |     ret = config.ReadItem("appid", *it, "", interval_value);
 48 |     if (ret != 0 || appid == 0) {
 49 |       gLog("[%s][%d]: read %s appid %d failed %d\n", __FILE__, __LINE__,
 50 |           it->c_str(), appid, ret);
 51 |       continue;
 52 |     }
 53 | 
 54 |     int32_t min = 0, mid = 0, max = 0;
 55 |     char *save_ptr = NULL;
 56 |     char *p = NULL;
 57 |     char buf[128];
 58 |     snprintf(buf, sizeof(buf), "%s", interval_value.c_str());
 59 |     p = strtok_r(buf, ",", &save_ptr);
 60 |     if (p != NULL) {
 61 |       min = atoi(p);
 62 |       if (min > 0 && min <= 86400) {
 63 |         tmp_appid[appid].set_min_interval(min);
 64 |       }
 65 |     }
 66 | 
 67 |     p = strtok_r(NULL, ",", &save_ptr);
 68 |     if (p != NULL) {
 69 |       mid = atoi(p);
 70 |       if (mid > 0 && mid <= 86400) {
 71 |         tmp_appid[appid].set_mid_interval(mid);
 72 |       }
 73 |     }
 74 | 
 75 |     p = strtok_r(NULL, ",", &save_ptr);
 76 |     if (p != NULL) {
 77 |       max = atoi(p);
 78 |       if (max > 0 && max <= 86400) {
 79 |         tmp_appid[appid].set_max_interval(max);
 80 |       }
 81 |     }
 82 |   }
 83 | 
 84 |   index_ = new_index;
 85 |   gLog("[%s][%d]: index = %d\n", __FILE__, __LINE__, index_);
 86 |   for (AppidMapVecAppidMeta::iterator it = appid_map_[index_].begin();
 87 |       it != appid_map_[index_].end(); ++it) {
 88 |     gLog("[%s][%d]: appid = %u, meta = %s\n", __FILE__, __LINE__,
 89 |         it->first, it->second.DebugString().c_str());
 90 |   }
 91 | }
 92 | 
 93 | const AppidMeta* AppidManager::GetAppidMeta(const uint32_t appid) {
 94 |   AppidMapVecAppidMeta::iterator it = appid_map_[index_].find(appid);
 95 |   if (it == appid_map_[index_].end()) {
 96 |     gLog("[%s][%d]: can't find appid %d meta\n", __FILE__, __LINE__, appid);
 97 |     return NULL;
 98 |   }
 99 |   return &(it->second);
100 | }
101 | 
102 | }  // idspace libwxfreq
103 | 


--------------------------------------------------------------------------------
/freq/appid_manager.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_APPID_MANAGER_H_
18 | #define FREQLIB_FREQ_APPID_MANAGER_H_
19 | #include <vector>
20 | #include <map>
21 | #include <string>
22 | #include "appid.h"
23 | 
24 | namespace libwxfreq {
25 | class Config;
26 | 
27 | typedef std::map<uint32_t, AppidMeta> AppidMapVecAppidMeta;
28 | 
29 | class AppidManager {
30 |  public:
31 |   static const AppidMeta* GetAppidMeta(const uint32_t appid);
32 |   static void UpdateAppidMeta(const Config& config);
33 | 
34 |  private:
35 |   static AppidMapVecAppidMeta appid_map_[2];
36 |   static uint8_t index_;
37 | };
38 | 
39 | }  // namespace libwxfreq
40 | #endif  // FREQLIB_FREQ_APPID_MANAGER_H_
41 | 


--------------------------------------------------------------------------------
/freq/appname.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_APPNAME_H_
18 | #define FREQLIB_FREQ_APPNAME_H_
19 | 
20 | #include <stdint.h>
21 | #include <stdio.h>
22 | #include <string>
23 | 
24 | namespace libwxfreq {
25 | 
26 | class AppNameMeta {
27 |  public:
28 |   AppNameMeta()
29 |     : appname_(),
30 |       appid_(0),
31 |       min_interval_(60),
32 |       mid_interval_(3600),
33 |       max_interval_(86400) {
34 |   }
35 | 
36 |   inline void set_min_interval(const uint32_t& min_interval) {
37 |     min_interval_ = min_interval;
38 |   }
39 | 
40 |   inline void set_mid_interval(const uint32_t& mid_interval) {
41 |     mid_interval_ = mid_interval;
42 |   }
43 | 
44 |   inline void set_max_interval(const uint32_t& max_interval) {
45 |     max_interval_ = max_interval;
46 |   }
47 | 
48 |   inline void set_appid(const uint32_t& appid) {
49 |     appid_ = appid;
50 |   }
51 | 
52 |   inline void set_appname(const std::string& appname) {
53 |     appname_ = appname;
54 |   }
55 | 
56 |   inline std::string appname() const {
57 |     return appname_;
58 |   }
59 | 
60 |   inline uint16_t min_interval() const {
61 |     return min_interval_;
62 |   }
63 | 
64 |   inline uint16_t mid_interval() const {
65 |     return mid_interval_;
66 |   }
67 |   inline uint16_t max_interval() const {
68 |     return max_interval_;
69 |   }
70 | 
71 |   inline uint32_t appid() const {
72 |     return appid_;
73 |   }
74 | 
75 |   inline std::string DebugString() const {
76 |     char buf[128];
77 |     snprintf(buf, sizeof(buf), "appname=%s, appid=%d, min_interval=%d, "
78 |         "mid_interval=%d, max_interval=%d", appname_.c_str(), appid_,
79 |         min_interval_, mid_interval_, max_interval_);
80 |     return buf;
81 |   }
82 | 
83 |  private:
84 |   std::string appname_;
85 |   uint32_t appid_;
86 |   uint16_t min_interval_;
87 |   uint16_t mid_interval_;
88 |   uint32_t max_interval_;
89 | };
90 | 
91 | }  // namespace libwxfreq
92 | 
93 | #endif  // FREQLIB_FREQ_APPNAME_H_
94 | 


--------------------------------------------------------------------------------
/freq/appname_manager.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "appname_manager.h"
18 | #include <unistd.h>
19 | #include <stdio.h>
20 | #include <vector>
21 | #include "config.h"
22 | #include "log.h"
23 | 
24 | 
25 | namespace libwxfreq {
26 | 
27 | static AppNameMeta  dummymeta;
28 | AppanmeMapVecAppnameMeta  AppnameManager::appname_map_[2];
29 | unsigned char AppnameManager::index_ = 0;
30 | 
31 | void AppnameManager::UpdateAppNameMeta(const Config& config) {
32 |   int ret = 0;
33 |   unsigned int appid = 0, interval = 0;
34 |   std::string appname, interval_key;
35 | 
36 |   unsigned char new_index = (index_ + 1) % 2;
37 |   AppanmeMapVecAppnameMeta& tmp_appname  = appname_map_[new_index];
38 |   tmp_appname.clear();
39 | 
40 |   std::vector<std::string> keys;
41 |   config.GetKeysBySection("appname", keys);
42 | 
43 |   for (std::vector<std::string>::iterator it = keys.begin();
44 |       it != keys.end(); ++it) {
45 |     appid = 0;
46 |     std::size_t pos = it->find(":");
47 |     if (pos == std::string::npos) {
48 |       ret = config.ReadItem("appname", *it, 0, appid);
49 |       if (ret != 0 || appid == 0) {
50 |         gLog("[%s][%d]: read %s appid %d failed\n", __FILE__, __LINE__,
51 |             it->c_str(), appid);
52 |         continue;
53 |       }
54 |       tmp_appname[*it].set_appname(*it);
55 |       tmp_appname[*it].set_appid(appid);
56 |     } else {
57 |       appname = it->substr(0, pos);
58 |       interval_key = it->substr(pos + 1);
59 |       AppNameMeta& meta = tmp_appname[appname];
60 | 
61 |       ret = config.ReadItem("appname", *it, 0, interval);
62 |       if (ret != 0) {
63 |         gLog("[%s][%d]: read %s %s failed\n", __FILE__, __LINE__,
64 |             appname.c_str(), interval_key.c_str());
65 |         continue;
66 |       }
67 |       if (interval_key == "min_interval") {
68 |         meta.set_min_interval(interval);
69 |       } else if (interval_key == "mid_interval") {
70 |         meta.set_mid_interval(interval);
71 |       } else if (interval_key == "max_interval") {
72 |         meta.set_max_interval(interval);
73 |       }
74 |     }
75 |   }
76 |   index_ = new_index;
77 |   gLog("[%s][%d]: index = %d\n", __FILE__, __LINE__, index_);
78 |   for (AppanmeMapVecAppnameMeta::iterator it = appname_map_[index_].begin();
79 |       it != appname_map_[index_].end(); ++it) {
80 |     gLog("[%s][%d]: appname = %s, meta = %s\n", __FILE__, __LINE__,
81 |         it->first.c_str(), it->second.DebugString().c_str());
82 |   }
83 | }
84 | 
85 | const AppNameMeta* AppnameManager::GetAppNameMeta(const std::string& appname) {
86 |   AppanmeMapVecAppnameMeta::iterator it = appname_map_[index_].find(appname);
87 |   if (it == appname_map_[index_].end()) {
88 |     gLog("[%s][%d]: can't find appname %s meta\n", __FILE__, __LINE__,
89 |         appname.c_str());
90 |     return NULL;
91 |   }
92 |   return &(it->second);
93 | }
94 | 
95 | }  // namespace libwxfreq
96 | 


--------------------------------------------------------------------------------
/freq/appname_manager.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_APPNAME_MANAGER_H_
18 | #define FREQLIB_FREQ_APPNAME_MANAGER_H_
19 | #include <vector>
20 | #include <map>
21 | #include <string>
22 | #include "appname.h"
23 | 
24 | namespace libwxfreq {
25 | class Config;
26 | 
27 | typedef std::map<std::string, AppNameMeta> AppanmeMapVecAppnameMeta;
28 | 
29 | class AppnameManager {
30 |  public:
31 |   static const AppNameMeta* GetAppNameMeta(const std::string& appname);
32 |   static void UpdateAppNameMeta(const Config& config);
33 | 
34 |  private:
35 |   static AppanmeMapVecAppnameMeta appname_map_[2];
36 |   static uint8_t index_;
37 | };
38 | 
39 | }  // namespace libwxfreq
40 | #endif  // FREQLIB_FREQ_APPNAME_MANAGER_H_
41 | 


--------------------------------------------------------------------------------
/freq/expression.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_EXPRESSION_H_
18 | #define FREQLIB_FREQ_EXPRESSION_H_
19 | #include <string>
20 | #include "freq_item.h"
21 | 
22 | namespace libwxfreq {
23 | 
24 | class Expression {
25 |  public:
26 |   virtual ~Expression() {}
27 |   virtual bool ParseItem(const std::string& item) = 0;
28 |   virtual bool IsMatch(const FreqItem& freqitem) const = 0;
29 |   virtual std::string DebugString() const = 0;
30 |   virtual Expression* Clone() const = 0;
31 | };
32 | 
33 | 
34 | }  // namespace libwxfreq
35 | 
36 | 
37 | #endif  // FREQLIB_FREQ_EXPRESSION_H_
38 | 


--------------------------------------------------------------------------------
/freq/freq_item.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_FREQ_ITEM_H_
18 | #define FREQLIB_FREQ_FREQ_ITEM_H_
19 | #include <stdint.h>
20 | #include <string>
21 | 
22 | #pragma pack(1)
23 | namespace libwxfreq {
24 | 
25 | struct FreqKey {
26 |   char key[32];
27 |   uint32_t appid;
28 | };
29 | 
30 | struct FreqItem {
31 |   unsigned int level1_cnt;
32 |   unsigned int level2_cnt;
33 |   unsigned int level3_cnt;
34 | };
35 | 
36 | }  // namespace libwxfreq
37 | #pragma pack()
38 | #endif  // FREQLIB_FREQ_FREQ_ITEM_H_
39 | 


--------------------------------------------------------------------------------
/freq/freq_manager.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "freq_manager.h"
 18 | #include "freq_item.h"
 19 | #include "appid.h"
 20 | #include "freq_stat.h"
 21 | #include "log.h"
 22 | #include "appid_manager.h"
 23 | 
 24 | namespace libwxfreq {
 25 | 
 26 | TypeName2FreqStatMap FreqManager::type_name_map_;
 27 | const uint32_t FreqManager::kWhiteFlag = 0xffffffff;
 28 | const uint32_t FreqManager::kAddBlockFlag = 0xfffffffe;
 29 | static AppidMeta dummy_appid_meta;
 30 | 
 31 | inline uint32_t FreqManager::TimeSplit(time_t timestamp, uint32_t scale) {
 32 |   return timestamp/scale;
 33 | }
 34 | 
 35 | bool FreqManager::Init(const TypeName2FreqStatMap& type_name_map) {
 36 |   TypeName2FreqStatMap::iterator it = type_name_map_.begin();
 37 |   while (it != type_name_map_.end()) {
 38 |     delete it->second;
 39 |     it->second = NULL;
 40 |     ++it;
 41 |   }
 42 |   type_name_map_ = type_name_map;
 43 |   it = type_name_map_.begin();
 44 | 
 45 |   while (it != type_name_map_.end()) {
 46 |     if (it->second->TryReload() == false ) {
 47 |       gLog("[%s][%d]: ERROR type name %s TryReload failed\n",
 48 |           __FILE__, __LINE__, it->first.c_str());
 49 |     }
 50 |     ++it;
 51 |   }
 52 |   return true;
 53 | }
 54 | 
 55 | const FreqItem* FreqManager::CommAdd(FreqStat *freq_stat,
 56 |                                     const std::string& key,
 57 |                                     const AppidMeta& appid_meta,
 58 |                                     const uint32_t appid,
 59 |                                     const uint32_t cnt) {
 60 |   FreqItem *freq_item = NULL;
 61 |   uint32_t& last_time = freq_stat->GetItem(key, appid, cnt, freq_item);
 62 |   if (freq_item == NULL)  {
 63 |     gLog("[%s][%d]: appid %u key %s GetItem empty\n", __FILE__, __LINE__,
 64 |         appid, key.c_str());
 65 |     return NULL;
 66 |   }
 67 | 
 68 |   if (freq_item->level1_cnt == kWhiteFlag) {
 69 |     gLog("[%s][%d]: appid %u key %s white\n", __FILE__, __LINE__,
 70 |         appid, key.c_str());
 71 |     return freq_item;
 72 |   } else if (freq_item->level1_cnt == kAddBlockFlag) {
 73 |     gLog("[%s][%d]: appid %u key %s blocked\n", __FILE__, __LINE__,
 74 |         appid, key.c_str());
 75 |     return freq_item;
 76 |   }
 77 | 
 78 |   time_t current_time = time(NULL);
 79 |   uint32_t current_interval_level3 = TimeSplit(current_time,
 80 |                                                 appid_meta.max_interval());
 81 |   uint32_t current_interval_level2 = TimeSplit(current_time,
 82 |                                                 appid_meta.mid_interval());
 83 |   uint32_t current_interval_level1 = TimeSplit(current_time,
 84 |                                               appid_meta.min_interval());
 85 | 
 86 |   uint32_t last_interval_level3 = TimeSplit(last_time,
 87 |                                             appid_meta.max_interval());
 88 |   uint32_t last_interval_level2 = TimeSplit(last_time,
 89 |                                             appid_meta.mid_interval());
 90 |   uint32_t last_interval_level1 = TimeSplit(last_time,
 91 |                                             appid_meta.min_interval());
 92 | 
 93 |   if (last_interval_level1 != current_interval_level1) {
 94 |     freq_item->level1_cnt = cnt;
 95 |   } else {
 96 |     __sync_fetch_and_add(&freq_item->level1_cnt, cnt);
 97 |   }
 98 | 
 99 |   if (last_interval_level2 != current_interval_level2) {
100 |     freq_item->level2_cnt = cnt;
101 |   } else {
102 |     __sync_fetch_and_add(&freq_item->level2_cnt, cnt);
103 |   }
104 | 
105 |   if (last_interval_level3 != current_interval_level3) {
106 |     freq_item->level3_cnt = cnt;
107 |   } else {
108 |     __sync_fetch_and_add(&freq_item->level3_cnt, cnt);
109 |   }
110 |   last_time = current_time;
111 |   return freq_item;
112 | }
113 | 
114 | const FreqItem* FreqManager::Add(const char* type_name,
115 |                                 const std::string& key,
116 |                                 const uint32_t appid,
117 |                                 const uint32_t cnt) {
118 |   if (type_name == NULL) {
119 |     gLog("[%s][%d]: type name empty\n", __FILE__, __LINE__);
120 |     return NULL;
121 |   }
122 |   if (appid == 0) {
123 |     gLog("[%s][%d]: appid = 0\n", __FILE__, __LINE__);
124 |     return NULL;
125 |   }
126 |   TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);
127 |   if (it == type_name_map_.end()) {
128 |     gLog("[%s][%d]: type name %s not exists\n", __FILE__, __LINE__, type_name);
129 |     return NULL;
130 |   }
131 | 
132 |   const AppidMeta *meta = AppidManager::GetAppidMeta(appid);
133 |   if (meta == NULL) {
134 |     return CommAdd(it->second, key, dummy_appid_meta, appid, cnt);
135 |   } else {
136 |     return CommAdd(it->second, key, *meta, appid, cnt);
137 |   }
138 | }
139 | 
140 | bool FreqManager::AddWhite(const char* type_name, const std::string& key,
141 |                           const uint32_t appid, const uint32_t linger_time) {
142 |   if (type_name == NULL) {
143 |     gLog("[%s][%d]: type name empty\n", __FILE__, __LINE__);
144 |     return false;
145 |   }
146 |   if (appid == 0) {
147 |     gLog("[%s][%d]: appid = 0\n", __FILE__, __LINE__);
148 |     return false;
149 |   }
150 |   TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);
151 |   if (it == type_name_map_.end()) {
152 |     gLog("[%s][%d]: type name %s not exists\n", __FILE__, __LINE__, type_name);
153 |     return false;
154 |   }
155 |   FreqStat *freq_stat = it->second;
156 | 
157 |   if (freq_stat == NULL) {
158 |     gLog("[%s][%d]: type name %s freq_stat NULL\n",
159 |         __FILE__, __LINE__, type_name);
160 |     return false;
161 |   }
162 | 
163 |   FreqItem *freq_item = NULL;
164 |   uint32_t& last_time = freq_stat->GetItem(key, appid, 1, freq_item);
165 |   if (freq_item == NULL)  {
166 |     gLog("[%s][%d]: appid %u key %s GetItem empty\n",
167 |         __FILE__, __LINE__, appid, key.c_str());
168 |     return false;
169 |   }
170 | 
171 |   unsigned int expired_time = time(NULL) + linger_time;
172 |   freq_item->level1_cnt = kWhiteFlag;
173 |   last_time = expired_time - 90000;
174 |   return true;
175 | }
176 | 
177 | bool FreqManager::DeleteWhite(const char* type_name, const std::string& key,
178 |                               const uint32_t appid) {
179 |   if (type_name == NULL) {
180 |     gLog("[%s][%d]: type name empty\n", __FILE__, __LINE__);
181 |     return false;
182 |   }
183 | 
184 |   TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);
185 |   if (it == type_name_map_.end()) {
186 |     gLog("[%s][%d]: type name %s not exists\n", __FILE__, __LINE__, type_name);
187 |     return false;
188 |   }
189 |   FreqStat *freq_stat = it->second;
190 | 
191 |   if (freq_stat == NULL) {
192 |     gLog("[%s][%d]: type name %s freq_stat NULL\n",
193 |         __FILE__, __LINE__, type_name);
194 |     return false;
195 |   }
196 | 
197 |   FreqItem *freq_item = NULL;
198 |   uint32_t& last_time = freq_stat->GetItem(key, appid, 0, freq_item);
199 |   if (freq_item == NULL)  {
200 |     return true;
201 |   }
202 | 
203 |   if (GetWhiteLevel(freq_item) > 0) {
204 |     last_time = 1; //expired
205 |   }
206 |   return true;
207 | }
208 | 
209 | bool FreqManager::AddBlock(const char* type_name, const std::string& key,
210 |                           const uint32_t appid, const uint32_t linger_time,
211 |                           const uint32_t block_level) {
212 |   if (type_name == NULL) {
213 |     gLog("[%s][%d]: type name empty\n", __FILE__, __LINE__);
214 |     return false;
215 |   }
216 | 
217 |   TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);
218 |   if (it == type_name_map_.end()) {
219 |     gLog("[%s][%d]: type name %s not exists\n", __FILE__, __LINE__, type_name);
220 |     return false;
221 |   }
222 |   FreqStat *freq_stat = it->second;
223 | 
224 | 
225 |   FreqItem *freq_item = NULL;
226 |   uint32_t& last_time = freq_stat->GetItem(key, appid, 1, freq_item);
227 |   if (freq_item == NULL)  {
228 |     gLog("[%s][%d]: appid %d key %s GetItem empty\n",
229 |         __FILE__, __LINE__, appid, key.c_str());
230 |     return false;
231 |   }
232 | 
233 |   if (freq_stat == NULL) {
234 |     gLog("[%s][%d]: type name %s freq_stat NULL\n",
235 |         __FILE__, __LINE__, type_name);
236 |     return false;
237 |   }
238 | 
239 |   if (GetWhiteLevel(freq_item) > 0) {
240 |     return true;
241 |   }
242 | 
243 |   unsigned int expired_time = time(NULL) + linger_time;
244 |   freq_item->level1_cnt = kAddBlockFlag;
245 |   freq_item->level2_cnt = block_level;
246 |   last_time = expired_time - 90000;
247 |   return true;
248 | }
249 | 
250 | bool FreqManager::DeleteBlock(const char* type_name, const std::string& key,
251 |                               const uint32_t appid) {
252 |   if (type_name == NULL) {
253 |     gLog("[%s][%d]: type name empty\n", __FILE__, __LINE__);
254 |     return false;
255 |   }
256 | 
257 |   TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);
258 |   if (it == type_name_map_.end()) {
259 |     gLog("[%s][%d]: type name %s not exists\n", __FILE__, __LINE__, type_name);
260 |     return false;
261 |   }
262 |   FreqStat *freq_stat = it->second;
263 | 
264 |   if (freq_stat == NULL) {
265 |     gLog("[%s][%d]: type name %s freq_stat NULL\n",
266 |         __FILE__, __LINE__, type_name);
267 |     return false;
268 |   }
269 | 
270 | 
271 |   FreqItem *freq_item = NULL;
272 |   uint32_t& last_time = freq_stat->GetItem(key, appid, 0, freq_item);
273 |   if (freq_item == NULL)  {
274 |     return true;
275 |   }
276 | 
277 |   if (GetWhiteLevel(freq_item) > 0) {
278 |     return true;
279 |   }
280 | 
281 |   last_time = 1;
282 |   return true;
283 | }
284 | 
285 | uint32_t FreqManager::GetWhiteLevel(const FreqItem* item) {
286 |   return item->level1_cnt == kWhiteFlag ? 1 : 0;
287 | }
288 | 
289 | uint32_t FreqManager::GetBlackLevel(const FreqItem* item) {
290 |   if (item->level1_cnt == kAddBlockFlag) {
291 |     return item->level2_cnt;
292 |   }
293 |   return 0;
294 | }
295 | 
296 | }  // namespace libwxfreq
297 | 


--------------------------------------------------------------------------------
/freq/freq_manager.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_FREQ_MANAGER_H_
18 | #define FREQLIB_FREQ_FREQ_MANAGER_H_
19 | #include <stdint.h>
20 | #include <string>
21 | #include <map>
22 | 
23 | namespace libwxfreq {
24 | 
25 | class FreqStat;
26 | class AppidMeta;
27 | class FreqItem;
28 | typedef std::map<std::string, FreqStat *> TypeName2FreqStatMap;
29 | 
30 | class FreqManager {
31 |  public:
32 |   static bool Init(const TypeName2FreqStatMap& type_name_map);
33 |   static const FreqItem* Add(const char* type_name, const std::string& key,
34 |                             const uint32_t appid, const uint32_t cnt);
35 |   static bool AddWhite(const char* type_name, const std::string& key,
36 |                       const uint32_t appid, const uint32_t linger_time);
37 |   static bool DeleteWhite(const char* type_name, const std::string& key,
38 |                         const uint32_t appid);
39 |   static bool AddBlock(const char* type_name, const std::string& key,
40 |                       const uint32_t appid, const uint32_t linger_time,
41 |                       const uint32_t block_level);
42 |   static bool DeleteBlock(const char* type_name, const std::string& key,
43 |                           const uint32_t appid);
44 |   static uint32_t GetWhiteLevel(const FreqItem* item);
45 |   static uint32_t GetBlackLevel(const FreqItem* item);
46 | 
47 |  private:
48 |   static const uint32_t kWhiteFlag;
49 |   static const uint32_t kAddBlockFlag;
50 |   static const FreqItem* CommAdd(FreqStat *freq_stat, const std::string& key,
51 |                                 const AppidMeta& appid_meta,
52 |                                 const uint32_t appid, const uint32_t cnt);
53 |   static uint32_t TimeSplit(time_t timestamp, uint32_t scale);
54 |   static TypeName2FreqStatMap type_name_map_;
55 | };
56 | 
57 | }  // namespace libwxfreq
58 | #endif  // FREQLIB_FREQ_FREQ_MANAGER_H_
59 | 


--------------------------------------------------------------------------------
/freq/freqlib.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "libwxfreq.h"
 18 | #include <string.h>
 19 | #include <stdio.h>
 20 | #include "appid_manager.h"
 21 | #include "freq_manager.h"
 22 | #include "rule_manager.h"
 23 | #include "map_freq_stat.h"
 24 | #include "shm_freq_stat.h"
 25 | #include "options.h"
 26 | #include "freq_stat.h"
 27 | #include "log.h"
 28 | #include "reload.h"
 29 | 
 30 | using namespace libwxfreq;
 31 | 
 32 | const static uint32_t kDeleteTime = 86400;
 33 | TypeName2FreqStatMap type_name_map;
 34 | 
 35 | int RegisterNewMapStat(const char* type_name, bool /*zero_init*/,
 36 |                       key_t /*key*/, unsigned int /*item_cnt*/) {
 37 |   if (type_name == NULL) {
 38 |     gLog("[%s][%d]: type name empty\n", __FILE__, __LINE__);
 39 |     return kErrorInput;
 40 |   }
 41 |   TypeName2FreqStatMap::iterator it = type_name_map.find(type_name);
 42 |   if (it != type_name_map.end()) delete it->second;
 43 |   FreqStat * tmp_map = new  MapFreqStat();
 44 |   if (tmp_map == NULL) {
 45 |     gLog("[%s][%d]: alloc stat memory error, type name:%s\n",
 46 |         __FILE__, __LINE__, type_name);
 47 |     return kErrorSystem;
 48 |   }
 49 |   tmp_map->set_type_name(type_name);
 50 |   type_name_map[type_name] = tmp_map;
 51 |   return kOK;
 52 | }
 53 | 
 54 | int RegisterNewShmStat(const char* type_name, bool zero_init,
 55 |                       key_t key, unsigned int item_cnt) {
 56 |   if (type_name == NULL) {
 57 |     gLog("[%s][%d]: type name empty\n", __FILE__, __LINE__);
 58 |     return kErrorInput;
 59 |   }
 60 |   TypeName2FreqStatMap::iterator it = type_name_map.find(type_name);
 61 |   if (it != type_name_map.end()) delete it->second;
 62 |   FreqStat * tmp_map = new  ShmFreqStat(zero_init, key, item_cnt);
 63 |   if (tmp_map == NULL) {
 64 |     gLog("[%s][%d]: alloc stat memory error, type name:%s\n",
 65 |         __FILE__, __LINE__, type_name);
 66 |     return kErrorSystem;
 67 |   }
 68 |   tmp_map->set_type_name(type_name);
 69 |   type_name_map[type_name] = tmp_map;
 70 |   return kOK;
 71 | }
 72 | 
 73 | void SetRuleConfFile(const char *filename) {
 74 |   RuleManager::set_rule_conf_file(filename);
 75 | }
 76 | 
 77 | bool InitFreq() {
 78 |   RuleManager::Init();
 79 |   return FreqManager::Init(type_name_map);
 80 | }
 81 | 
 82 | static BlockResult MakeBlockResult(unsigned int block_level,
 83 |                                   const char* match_rule) {
 84 |   BlockResult result;
 85 |   result.block_level = block_level;
 86 |   strncpy(result.match_rule, match_rule, sizeof(result.match_rule));
 87 |   return result;
 88 | }
 89 | 
 90 | BlockResult ReportAndCheck(const char* type_name, const char *key,
 91 |                           const uint32_t appid, const uint32_t cnt) {
 92 |   const FreqItem* p =  NULL;
 93 |   p = FreqManager::Add(type_name, key, appid, cnt);
 94 |   if (p == NULL) return MakeBlockResult(kErrorOutOfMemory, "");
 95 | 
 96 |   RuleManager::Result res = RuleManager::IsMatchRule(type_name, appid, *p);
 97 |   return MakeBlockResult(res.block_level, res.match_rule.c_str());
 98 | }
 99 | 
100 | struct BlockResult OnlyCheck(const char* type_name, const char *key,
101 |                             const uint32_t appid) {
102 |   return ReportAndCheck(type_name, key, appid, 0);
103 | }
104 | 
105 | int OnlyReport(const char* type_name, const char *key, const uint32_t appid,
106 |               const uint32_t cnt) {
107 |   if (FreqManager::Add(type_name, key, appid, cnt) == NULL)
108 |     return kErrorOutOfMemory;
109 |   return 0;
110 | }
111 | 
112 | 
113 | static FreqCache MakeFreqCache(const uint32_t level1, const uint32_t level2,
114 |                               const uint32_t level3) {
115 |   FreqCache freqcache = {level1, level2, level3};
116 |   return freqcache;
117 | }
118 | 
119 | 
120 | FreqCache GetCache(const char* type_name,  const char *key,
121 |                   const uint32_t appid, const uint32_t cnt) {
122 |   const FreqItem* p =  NULL;
123 |   p = FreqManager::Add(type_name, key, appid, cnt);
124 |   if (p == NULL) return MakeFreqCache(0, 0, 0);
125 |   return MakeFreqCache(p->level1_cnt, p->level2_cnt, p->level3_cnt);
126 | }
127 | 
128 | void SetLogFunc(LogFunction logfunc) {
129 |   SetLog(logfunc);
130 | }
131 | 
132 | int AddWhite(const char* type_name, const char *key, const uint32_t appid,
133 |             const uint32_t linger_time) {
134 |   if (gDumpFunc("AddWhite", type_name, key, appid, linger_time, 1) != 0)
135 |     return kErrorSystem;
136 |   if (FreqManager::AddWhite(type_name, key, appid, linger_time) == false)
137 |     return kErrorOutOfMemory;
138 |   return 0;
139 | }
140 | 
141 | int DeleteWhite(const char* type_name, const char *key, const uint32_t appid) {
142 |   if (gDumpFunc("DeleteWhite", type_name, key, appid, kDeleteTime, 1) != 0)
143 |     return kErrorSystem;
144 |   if (FreqManager::DeleteWhite(type_name, key, appid) == false)
145 |     return kErrorOutOfMemory;
146 |   return 0;
147 | }
148 | 
149 | int AddBlock(const char* type_name, const char *key, const uint32_t appid,
150 |             const uint32_t linger_time, const uint32_t block_level) {
151 |   if (gDumpFunc("AddBlock", type_name, key, appid,
152 |                 linger_time, block_level) != 0) return kErrorSystem;
153 |   if (FreqManager::AddBlock(type_name, key, appid, linger_time,
154 |                             block_level) == false) return kErrorOutOfMemory;
155 |   return 0;
156 | }
157 | 
158 | int DeleteBlock(const char* type_name, const char *key, const uint32_t appid) {
159 |   if (gDumpFunc("DeleteBlock", type_name, key, appid, kDeleteTime, 1) != 0)
160 |     return kErrorSystem;
161 |   if (FreqManager::DeleteBlock(type_name, key, appid) == false)
162 |     return kErrorOutOfMemory;
163 |   return 0;
164 | }
165 | 
166 | void SetLoadFunc(LoadFunc func) {
167 |   libwxfreq::SetLoadFunc(func);
168 | }
169 | 
170 | void SetDumpFunc(DumpFunc func) {
171 |   libwxfreq::SetDumpFunc(func);
172 | }
173 | 


--------------------------------------------------------------------------------
/freq/libwxfreq.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_FREQLIB_H_
18 | #define FREQLIB_FREQ_FREQLIB_H_
19 | #include <sys/shm.h>
20 | #include <stdint.h>
21 | #ifdef __cplusplus
22 | extern "C" {
23 | #endif
24 | 
25 | enum {
26 |   kOK = 0,
27 |   kErrorOutOfMemory = -1,
28 |   kErrorNoAppName = -2,
29 |   kErrorSystem = -3,
30 |   kErrorInput = -4,
31 | };
32 | 
33 | struct BlockResult {
34 |     int block_level;
35 |     char match_rule[32];
36 | };
37 | 
38 | struct FreqCache {
39 |   uint32_t level1_cnt;
40 |   uint32_t level2_cnt;
41 |   uint32_t level3_cnt;
42 | };
43 | 
44 | int RegisterNewShmStat(const char* type_name, bool zero_init, key_t key,
45 |                       unsigned int item_cnt);
46 | int RegisterNewMapStat(const char* type_name, bool zero_init, key_t key,
47 |                       unsigned int item_cnt);
48 | 
49 | void SetRuleConfFile(const char *filename);
50 | 
51 | bool InitFreq();
52 | 
53 | struct BlockResult ReportAndCheck(const char* type_name, const char *key,
54 |                                   const uint32_t appid, const uint32_t cnt);
55 | struct BlockResult OnlyCheck(const char* type_name, const char *key,
56 |                             const uint32_t appid);
57 | int OnlyReport(const char* type_name, const char *key, const uint32_t appid,
58 |               const uint32_t cnt);
59 | struct FreqCache GetCache(const char* type_name, const char *key,
60 |                           const uint32_t appid, const uint32_t cnt = 0);
61 | 
62 | int AddWhite(const char* type_name, const char *key, const uint32_t appid,
63 |             const uint32_t linger_time);
64 | int DeleteWhite(const char* type_name, const char *key, const uint32_t appid);
65 | int AddBlock(const char* type_name, const char *key, const uint32_t appid,
66 |             const uint32_t linger_time, const uint32_t block_level);
67 | int DeleteBlock(const char* type_name, const char *key, const uint32_t appid);
68 | 
69 | typedef int (*LogFunction)(const char* format, ...);
70 | void SetLogFunc(LogFunction logfunc);
71 | 
72 | typedef int (*LoadFunc)(const char* type_name);
73 | void SetLoadFunc(LoadFunc func);
74 | 
75 | typedef int (*DumpFunc)(const char* opname, const char* type_name,
76 |                         const char *key, const uint32_t appid,
77 |                         const unsigned int linger_time,
78 |                         const uint32_t block_level);
79 | void SetDumpFunc(DumpFunc func);
80 | 
81 | #ifdef __cplusplus
82 | }
83 | #endif
84 | 
85 | #endif  // FREQLIB_FREQ_FREQLIB_H_
86 | 


--------------------------------------------------------------------------------
/freq/plain_expression.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "plain_expression.h"
18 | #include <string.h>
19 | #include <stdio.h>
20 | #include <stdlib.h>
21 | #include "slice.h"
22 | #include "log.h"
23 | 
24 | namespace libwxfreq {
25 | 
26 | static bool Greater(const FreqItem& freqitem, const unsigned int index,
27 |                     const unsigned int threshold) {
28 |   if (index >= 3) return false;
29 |   const unsigned int *p = (const unsigned int *)&freqitem;
30 |   return p[index] > threshold;
31 | }
32 | 
33 | static bool GreaterOrEqual(const FreqItem& freqitem, const unsigned int index,
34 |                           const unsigned int threshold) {
35 |   if (index >= 3) return false;
36 |   const unsigned int *p = (const unsigned int *)&freqitem;
37 |   return p[index] >= threshold;
38 | }
39 | 
40 | bool PlainExpression::ParseItem(const std::string& item) {
41 |   const char *ptr = NULL;
42 |   ptr = strstr(item.c_str(), ">");
43 |   if (ptr == NULL) return false;
44 |   Slice keyword(item.c_str(), ptr);
45 |   keyword.StrTrim(" ");
46 |   if (keyword == "min_interval") {
47 |     key_index_ = 0;
48 |   } else if (keyword == "mid_interval") {
49 |     key_index_ = 1;
50 |   } else if (keyword == "max_interval") {
51 |     key_index_ = 2;
52 |   } else {
53 |     gLog("[%s][%d]: wrong keyword %s, item %s\n",
54 |         __FILE__, __LINE__, keyword.ToStr().c_str(), item.c_str());
55 |     return false;
56 |   }
57 |   ptr++;
58 |   if (*ptr == '=') {
59 |     ptr++;
60 |     cmp_ = GreaterOrEqual;
61 |   } else {
62 |     cmp_ = Greater;
63 |   }
64 | 
65 |   if (*ptr == '\0') {
66 |     gLog("[%s][%d]: wrong syntax item %s\n", __FILE__, __LINE__, item.c_str());
67 |     return false;
68 |   }
69 |   threshold_ = strtoul(ptr, NULL, 10);
70 |   return threshold_ > 0;
71 | }
72 | 
73 | bool PlainExpression::IsMatch(const FreqItem& freqitem) const {
74 |     return cmp_(freqitem, key_index_, threshold_);
75 | }
76 | 
77 | std::string PlainExpression::DebugString() const {
78 |   std::string op = "NULL";
79 |   if (cmp_ == GreaterOrEqual) op = "GreaterOrEqual";
80 |   else if (cmp_ == Greater) op = "Greater";
81 | 
82 |   char buf[128];
83 |   snprintf(buf, sizeof(buf), "key_index:%u\tthreshold:%u\tcmp:%s\taddr:%lx",
84 |           key_index_, threshold_, op.c_str(), (unsigned long int)cmp_);
85 | 
86 |   return std::string(buf);
87 | }
88 | 
89 | Expression* PlainExpression::Clone() const {
90 |   return new PlainExpression(cmp_, threshold_, key_index_);
91 | }
92 | 
93 | }  // namespace libwxfreq
94 | 


--------------------------------------------------------------------------------
/freq/plain_expression.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_PLAIN_EXPRESSION_H_
18 | #define FREQLIB_FREQ_PLAIN_EXPRESSION_H_
19 | 
20 | #include <string>
21 | #include "expression.h"
22 | #include "log.h"
23 | 
24 | namespace libwxfreq {
25 | 
26 | class PlainExpression: public Expression {
27 |  public:
28 |   PlainExpression() { }
29 | 
30 |   bool ParseItem(const std::string& item);
31 |   bool IsMatch(const FreqItem& freqitem) const;
32 |   std::string DebugString() const;
33 |   Expression* Clone() const;
34 | 
35 |  private:
36 |   typedef bool (*cmp)(const FreqItem&, const unsigned int, const unsigned int);
37 | 
38 |   PlainExpression(const cmp func, const unsigned int threshold,
39 |                   const unsigned char key_index)
40 |     : cmp_(func),
41 |       threshold_(threshold),
42 |       key_index_(key_index) { }
43 | 
44 |   // parse item
45 |   cmp cmp_;
46 |   unsigned int threshold_;
47 |   uint8_t key_index_;
48 | };
49 | 
50 | }  // namespace libwxfreq
51 | 
52 | #endif  // FREQLIB_FREQ_PLAIN_EXPRESSION_H_
53 | 


--------------------------------------------------------------------------------
/freq/rule_item.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_RULE_ITEM_H_
18 | #define FREQLIB_FREQ_RULE_ITEM_H_
19 | 
20 | #include <stdio.h>
21 | #include <string>
22 | #include "expression.h"
23 | 
24 | namespace libwxfreq {
25 | class FreqItem;
26 | 
27 | class RuleItem {
28 |  public:
29 |   RuleItem(const std::string& rulename, uint16_t blocklevel)
30 |     : match_rulename_(rulename),
31 |     block_level_(blocklevel),
32 |     expression_(NULL) { }
33 | 
34 |   RuleItem(const RuleItem& other) {
35 |     *this = other;
36 |     expression_ = other.expression_->Clone();
37 |   }
38 | 
39 |   ~RuleItem() {
40 |     if (expression_ != NULL) delete expression_;
41 |   }
42 | 
43 |   inline uint16_t block_level() const {
44 |     return block_level_;
45 |   }
46 | 
47 |   inline const std::string& match_rulename() const {
48 |     return match_rulename_;
49 |   }
50 | 
51 |   inline void set_match_rulename(const std::string& match_rulename) {
52 |     match_rulename_ = match_rulename;
53 |   }
54 | 
55 |   inline void set_block_level(uint16_t block_level) {
56 |     block_level_ = block_level;
57 |   }
58 | 
59 |   inline void set_expression(Expression* p) {
60 |     if (expression_ != NULL) delete expression_;
61 |     expression_ = p;
62 |   }
63 | 
64 |   inline Expression* expression() {
65 |     return expression_;
66 |   }
67 | 
68 |   inline bool ParseItem(const std::string& item) {
69 |     return expression_->ParseItem(item);
70 |   }
71 | 
72 |   inline bool IsMatch(const FreqItem& freqitem) const {
73 |     return expression_->IsMatch(freqitem);
74 |   }
75 | 
76 |   inline std::string DebugString() {
77 |     std::string expression = expression_->DebugString();
78 |     char buf[512];
79 |     snprintf(buf, sizeof(buf), "match_rulename:%s\tblock_level:%u\t"
80 |             "expression:%s", match_rulename_.c_str(), block_level_,
81 |             expression.c_str());
82 |     return std::string(buf);
83 |   }
84 | 
85 |  private:
86 |   std::string match_rulename_;
87 |   uint16_t block_level_;
88 |   Expression* expression_;
89 | };
90 | 
91 | }  // namespace libwxfreq
92 | #endif  // FREQLIB_FREQ_RULE_ITEM_H_
93 | 


--------------------------------------------------------------------------------
/freq/rule_manager.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "rule_manager.h"
 18 | #include <sys/stat.h>
 19 | #include <unistd.h>
 20 | #include <vector>
 21 | #include "plain_expression.h"
 22 | #include "config.h"
 23 | #include "log.h"
 24 | #include "appid_manager.h"
 25 | #include "freq_item.h"
 26 | #include "rule_item.h"
 27 | #include "freq_manager.h"
 28 | 
 29 | namespace libwxfreq {
 30 | 
 31 | std::string RuleManager::rule_conf_file_;
 32 | bool RuleManager::is_loading_ = false;
 33 | time_t RuleManager::last_modify_time_ = 0;
 34 | unsigned char RuleManager::index_ = 0;
 35 | TypeNameMap RuleManager::map_[2];
 36 | 
 37 | int RuleManager::Init() {
 38 |   return TryLoadFile();
 39 | }
 40 | 
 41 | int RuleManager::TryLoadFile() {
 42 |   if (rule_conf_file_.empty()) {
 43 |     gLog("[%s][%d]: rule confile empty\n", __FILE__, __LINE__);
 44 |     return -1;
 45 |   }
 46 | 
 47 |   struct stat _filestat;
 48 |   if (stat(rule_conf_file_.c_str(), &_filestat) < 0) {
 49 |     gLog("[%s][%d]: stat confile %s failed\n",
 50 |         __FILE__, __LINE__, rule_conf_file_.c_str());
 51 |     return -2;
 52 |   }
 53 |   if (_filestat.st_mtime > last_modify_time_ &&
 54 |       time(NULL) - _filestat.st_mtime >= 2) {
 55 |     if (__sync_bool_compare_and_swap(&is_loading_, false, true)) {
 56 |       last_modify_time_ = _filestat.st_mtime;
 57 |       LoadFile();
 58 |       is_loading_ = false;
 59 |     }
 60 |   }
 61 |   return 0;
 62 | }
 63 | 
 64 | int RuleManager::LoadFile() {
 65 |   gLog("[%s][%d]: Load confile %s\n",
 66 |       __FILE__, __LINE__, rule_conf_file_.c_str());
 67 |   Config cfg(rule_conf_file_);
 68 |   if (cfg.Init() != 0) {
 69 |     gLog("[%s][%d]: int confile %s failed\n",
 70 |         __FILE__, __LINE__, rule_conf_file_.c_str());
 71 |     return -1;
 72 |   }
 73 | 
 74 |   unsigned char new_index = (index_ + 1) % 2;
 75 |   map_[new_index].clear();
 76 | 
 77 |   std::vector<std::string> sectionList;
 78 |   cfg.section_list(sectionList);
 79 |   for (std::vector<std::string>::iterator it = sectionList.begin();
 80 |       it != sectionList.end(); ++it) {
 81 |       if (*it != "appid") InsertRule(cfg, it->c_str());
 82 |   }
 83 | 
 84 |   AppidManager::UpdateAppidMeta(cfg);
 85 | 
 86 |   index_ = new_index;
 87 |   gLog("[%s][%d]: new index %d\n", __FILE__, __LINE__, index_);
 88 |   return 0;
 89 | }
 90 | 
 91 | int RuleManager::InsertRule(const Config& config, const std::string& section) {
 92 |   std::string item;
 93 |   std::string rule_type;
 94 |   unsigned int block_level = 0;
 95 |   unsigned int match_appid;
 96 |   unsigned int new_index = (index_ + 1) % 2;
 97 | 
 98 |   int ret1 = config.ReadItem(section, "item", "", item);
 99 |   int ret2 = config.ReadItem(section, "match_appid", 0, match_appid);
100 |   int ret3 = config.ReadItem(section, "block_level", 0, block_level);
101 |   int ret4 = config.ReadItem(section, "rule_type", "user", rule_type);
102 | 
103 |   if (ret1 != 0 || ret2 != 0 || ret3 != 0 || ret4 < 0) {
104 |     gLog("[%s][%d]: insert rule section %s failed %d|%d|%d|%d\n",
105 |         __FILE__, __LINE__, section.c_str(), ret1, ret2, ret3, ret4);
106 |     return -1;
107 |   }
108 | 
109 |   if (block_level <= 0) {
110 |     gLog("[%s][%d]: insert rule section %s block_level %d, skip\n",
111 |         __FILE__, __LINE__, section.c_str(), block_level);
112 |     return -2;
113 |   }
114 | 
115 |   RuleItem rule_item(section, block_level);
116 |   rule_item.set_expression(new PlainExpression());
117 |   if (rule_item.ParseItem(item) == false) {
118 |     gLog("[%s][%d]: insert rule section %s parse failed skip\n",
119 |         __FILE__, __LINE__, section.c_str());
120 |     return -3;
121 |   }
122 | 
123 |   map_[new_index][rule_type][match_appid].push_back(rule_item);
124 | 
125 |   gLog("[%s][%d]: insert rule section %s match_appid %u block_level %d"
126 |       " rule_type %s item %s rule_item %s\n", __FILE__, __LINE__,
127 |       section.c_str(), match_appid, block_level, rule_type.c_str(),
128 |       item.c_str(), rule_item.DebugString().c_str());
129 |   return 0;
130 | }
131 | 
132 | std::string RuleManager::DebugString() {
133 |   TryLoadFile();
134 | 
135 |   TypeNameMap& current_map = map_[index_];
136 |   std::string rule_string;
137 |   for (TypeNameMap::iterator cit = current_map.begin();
138 |       cit != current_map.end(); ++cit) {
139 |     for (AppidMapVecRuleItem::iterator it = cit->second.begin();
140 |         it != cit->second.end(); ++it) {
141 |       for (std::vector<RuleItem>::iterator vit = it->second.begin();
142 |           vit != it->second.end(); ++vit) {
143 |         rule_string.append("match_appid:\t");
144 |         rule_string.append(std::to_string(it->first));
145 |         rule_string.append("\n");
146 |         rule_string.append(vit->DebugString());
147 |       }
148 |     }
149 |   }
150 |   return rule_string;
151 | }
152 | 
153 | RuleManager::Result RuleManager::IsMatch(const uint32_t appid,
154 |                                         const FreqItem& freq_item,
155 |                                         const AppidMapVecRuleItem& map) {
156 |   TryLoadFile();
157 | 
158 |   AppidMapVecRuleItem::const_iterator it = map.find(appid);
159 |   if (it == map.end()) {
160 |     return RuleManager::Result(0, "");
161 |   }
162 | 
163 |   for (std::vector<RuleItem>::const_iterator vit = it->second.begin();
164 |       vit != it->second.end(); ++vit) {
165 |     if (vit->IsMatch(freq_item)) {
166 |       RuleManager::Result res(vit->block_level(), vit->match_rulename());
167 |       return res;
168 |     }
169 |   }
170 |   return RuleManager::Result(0, "");
171 | }
172 | 
173 | RuleManager::Result RuleManager::IsMatchRule(const char * type_name,
174 |                                             const uint32_t appid,
175 |                                             const FreqItem& freq_item) {
176 |   if (type_name == NULL || FreqManager::GetWhiteLevel(&freq_item) > 0) {
177 |     return RuleManager::Result(0, "");
178 |   }
179 |   if (FreqManager::GetBlackLevel(&freq_item) > 0) {
180 |     return RuleManager::Result(FreqManager::GetBlackLevel(&freq_item), "black");
181 |   }
182 |   TypeNameMap::iterator it = map_[index_].find(type_name);
183 |   if (it == map_[index_].end()) {
184 |     return RuleManager::Result(0, "");
185 |   }
186 |   return IsMatch(appid, freq_item, it->second);
187 | }
188 | 
189 | }  // namespace libwxfreq
190 | 


--------------------------------------------------------------------------------
/freq/rule_manager.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_FREQ_RULE_MANAGER_H_
18 | #define FREQLIB_FREQ_RULE_MANAGER_H_
19 | #include <pthread.h>
20 | #include <vector>
21 | #include <map>
22 | #include <string>
23 | 
24 | namespace libwxfreq {
25 | 
26 | class Config;
27 | class FreqItem;
28 | class RuleItem;
29 | 
30 | typedef std::map<unsigned int, std::vector<RuleItem> > AppidMapVecRuleItem;
31 | typedef std::map<std::string, AppidMapVecRuleItem> TypeNameMap;
32 | 
33 | class RuleManager {
34 |  public:
35 |   struct Result {
36 |     int block_level;
37 |     std::string match_rule;
38 | 
39 |     Result(): block_level(0), match_rule() { }
40 |     Result(int b, const std::string& matchrule)
41 |       : block_level(b),
42 |         match_rule(matchrule) { }
43 |   };
44 | 
45 |  public:
46 |   static inline void set_rule_conf_file(const std::string& filename) {
47 |     rule_conf_file_ = filename;
48 |   }
49 |   static int Init();
50 | 
51 |   static std::string DebugString();
52 |   static Result IsMatchRule(const char * type_name, const uint32_t appid,
53 |                             const FreqItem& freq_item);
54 | 
55 |  private:
56 |   static int LoadFile();
57 |   static int TryLoadFile();
58 |   static int InsertRule(const Config& config, const std::string& section);
59 | 
60 |   static Result IsMatch(const uint32_t appid, const FreqItem& freq_item,
61 |                         const AppidMapVecRuleItem& map);
62 | 
63 |  private:
64 |   static std::string rule_conf_file_;
65 |   static bool is_loading_;
66 |   static time_t last_modify_time_;
67 | 
68 |   static TypeNameMap map_[2];
69 |   static unsigned char index_;
70 | };
71 | 
72 | }  // namespace libwxfreq
73 | #endif  // FREQLIB_FREQ_RULE_MANAGER_H_
74 | 


--------------------------------------------------------------------------------
/main.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include  <string.h>
18 | #include  <signal.h>
19 | #include  <event.h>
20 | #include  <iostream>
21 | #include  "server.h"
22 | #include  "worker.h"
23 | #include  "libwxfreq.h"
24 | 
25 | using std::cout;
26 | using std::endl;
27 | 
28 | static inline int Log(const char* format, ...) {
29 |   return 0;
30 | }
31 | 
32 | int main(int argc, char* argv[]) {
33 |   libwxfreq::ServerConf& conf = libwxfreq::ServerConf::GetInstance();
34 |   conf.LoadFile("./server.conf");
35 |   SetLogFunc(Log);
36 | 
37 |   SetRuleConfFile(conf.libwxfreq_conf_path().c_str());
38 |   const std::vector<libwxfreq::StatType>& type = conf.stat_vec();
39 |   for (std::vector<libwxfreq::StatType>::const_iterator it = type.cbegin();
40 |         it != type.cend(); ++it) {
41 |     int ret = RegisterNewShmStat(it->GetTypeName().c_str(), it->init() > 0,
42 |             it->shm_key(), it->item_count());
43 |     if (ret != 0) {
44 |       cout << "RegisterNewShmStat failed, ret = " << ret << endl;
45 |       return -1;
46 |     }
47 |   }
48 | 
49 |   if (InitFreq() == false) {
50 |     cout << "InitFreq failed" << endl;
51 |     return -2;
52 |   }
53 | 
54 |   libwxfreq::Server server(libwxfreq::TxtFreqReq);
55 |   // libwxfreq::Server server(ip, port, libwxfreq::Echo, worker_num);
56 |   server.Run();
57 |   return 0;
58 | }
59 | 


--------------------------------------------------------------------------------
/net/acceptor.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include <sys/socket.h>
18 | #include <sys/un.h>
19 | #include <arpa/inet.h>
20 | #include <unistd.h>
21 | #include <stdlib.h>
22 | 
23 | #include "acceptor.h"
24 | #include "server.h"
25 | namespace libwxfreq {
26 | Acceptor::Acceptor(Server* s)
27 |     :s_(s) {
28 | }
29 | 
30 | void Acceptor::LoopAccept(const char *ip, const int port) {
31 |   int listen_fd = socket(AF_INET, SOCK_STREAM, 0);
32 |   struct sockaddr_in addr;
33 |   memset(&addr, 0, sizeof(addr));
34 |   addr.sin_family = AF_INET;
35 |   addr.sin_port = htons(port);
36 |   addr.sin_addr.s_addr = inet_addr(ip);
37 |   if (bind(listen_fd, (struct sockaddr*) &addr, sizeof(addr)) < 0) {
38 |     close(listen_fd);
39 |     exit(-1);
40 |   }
41 |   if (listen(listen_fd, 1024) < 0) {
42 |     close(listen_fd);
43 |     exit(-1);
44 |   }
45 |   while (true) {
46 |     struct sockaddr_in addr_in;
47 |     socklen_t socklen = sizeof(addr_in);
48 |     int accepted_fd = accept(listen_fd, (struct sockaddr*) &addr_in, &socklen);
49 |     if (accepted_fd > 0) {
50 |       s_->pool().AddNewConn(accepted_fd);
51 |     }
52 |   }
53 |   close(listen_fd);
54 | }
55 | 
56 | }  // namespace libwxfreq
57 | 


--------------------------------------------------------------------------------
/net/acceptor.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef LIBFEQ_NET_ACCEPTOR_H_
18 | #define LIBFEQ_NET_ACCEPTOR_H_
19 | 
20 | namespace libwxfreq {
21 | class Server;
22 | class Acceptor {
23 |  public:
24 |   explicit Acceptor(Server *s);
25 |   void LoopAccept(const char *ip, const int port);
26 |  private:
27 |   Server* s_;
28 | };
29 | 
30 | }  // namespace libwxfreq
31 | 
32 | #endif  // LIBFEQ_NET_ACCEPTOR_H_
33 | 


--------------------------------------------------------------------------------
/net/iothread.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "iothread.h"
 18 | #include <event.h>
 19 | #include <sys/socket.h>
 20 | #include <string.h>
 21 | #include <arpa/inet.h>
 22 | #include <netinet/in.h>
 23 | #include <errno.h>
 24 | #include <stdio.h>
 25 | #include <algorithm>
 26 | #include <iostream>
 27 | #include "util.h"
 28 | #include "server_conf.h"
 29 | 
 30 | namespace libwxfreq {
 31 | using std::string;
 32 | using std::max;
 33 | 
 34 | static NotifySessionStat notify_stat;
 35 | static CloseSessionStat  close_stat;
 36 | static ReadSessionStat   read_stat;
 37 | static CheckSessionStat   check_stat;
 38 | static WriteSessionStat  write_stat;
 39 | static ProcessSessionStat process_stat;
 40 | 
 41 | static size_t ValidatePkg(const char* buf, size_t len) {
 42 |   const char *pn = static_cast<const char*>(memchr(buf, '\n', len));
 43 |   const char *pr = static_cast<const char*>(memchr(buf, '\r', len));
 44 |   if (pn == NULL && pr == NULL)  {
 45 |     return -1;
 46 |   }
 47 |   const char *p = pn != NULL ? pn : pr;
 48 |   if (pr != NULL && pr < p) p = pr;
 49 |   while ((*p == '\0' || *p == '\r' || *p == '\n' ||
 50 |         *p == ' ' || *p == '\t') && p < buf + len)  p++;
 51 |   return p - buf;
 52 | }
 53 | 
 54 | 
 55 | SKBuff::SKBuff():buffer_(array_), write_pos_(0), read_pos_(0),
 56 |           capacity_(LIBFREQ_BUFFER_SIZE) {
 57 |             buffer_[0] ='\0';
 58 | }
 59 | 
 60 | SKBuff::~SKBuff() {
 61 |   if (buffer_ != array_) {
 62 |     delete[] buffer_;
 63 |   }
 64 | }
 65 | 
 66 | char* SKBuff::GetWritePos() {
 67 |   return buffer_ + write_pos_;
 68 | }
 69 | 
 70 | char* SKBuff::GetReadPos() {
 71 |   return buffer_ + read_pos_;
 72 | }
 73 | 
 74 | size_t SKBuff::AddWritePos(size_t size) {
 75 |   write_pos_ += size;
 76 |   return write_pos_;
 77 | }
 78 | 
 79 | size_t SKBuff::AddReadPos(size_t size) {
 80 |   read_pos_ += size;
 81 |   return read_pos_;
 82 | }
 83 | 
 84 | size_t SKBuff::GetUseableSize() {
 85 |   if (capacity_ - write_pos_ < 32) {
 86 |     ExpandSize(capacity_);
 87 |   }
 88 |   return capacity_ - write_pos_;
 89 | }
 90 | 
 91 | void SKBuff::Attach(SKBuff& buff) {
 92 |   Reset();
 93 |   write_pos_ = buff.write_pos_;
 94 |   read_pos_ = buff.read_pos_;
 95 |   capacity_ = buff.capacity_;
 96 |   if (buff.array_ == buff.buffer_) {
 97 |     buffer_ = array_;
 98 |     memcpy(buffer_, buff.GetReadPos(), buff.GetUnReadSize());
 99 |     read_pos_ = 0;
100 |     write_pos_ = buff.GetUnReadSize();
101 |   } else {
102 |     buffer_ = buff.buffer_;
103 |     buff.buffer_ = NULL;
104 |   }
105 | }
106 | void SKBuff::Attach(const char* buff, size_t len) {
107 |   Reset();
108 |   ExpandSize(len);
109 |   memcpy(buffer_, buff, len);
110 |   write_pos_ = len;
111 | }
112 | 
113 | char* SKBuff::GetBuffer() {
114 |   return buffer_;
115 | }
116 | 
117 | size_t SKBuff::GetBufferLen() {
118 |   return write_pos_;
119 | }
120 | 
121 | size_t SKBuff::GetUnReadSize() {
122 |   return write_pos_ - read_pos_;
123 | }
124 | 
125 | void SKBuff::Reset() {
126 |   if (buffer_ != array_) delete[] buffer_;
127 |   buffer_ = array_;
128 |   write_pos_ = 0;
129 |   read_pos_ = 0;
130 |   capacity_ = LIBFREQ_BUFFER_SIZE;
131 |   buffer_[0] = '\0';
132 | }
133 | 
134 | void SKBuff::Shrink() {
135 |   memmove(buffer_, buffer_ + read_pos_, GetUnReadSize());
136 |   write_pos_ -= read_pos_;
137 |   read_pos_ = 0;
138 | }
139 | 
140 | void SKBuff::ExpandSize(size_t len) {
141 |   if (len > capacity_ - write_pos_) {
142 |     capacity_ = max(capacity_ * 2, len + write_pos_);
143 |     char * tmp = new char[capacity_];
144 |     memcpy(tmp, buffer_, write_pos_);
145 |     if (buffer_ != array_) delete[] buffer_;
146 |     buffer_ = tmp;
147 |   }
148 | }
149 | 
150 | void IOThread::run() {
151 |   thread_->base = event_base_new();
152 |   ConnSession *session = new ConnSession();
153 |   session->fd = thread_->receive_fd;
154 |   session->iothread = this;
155 |   session->stat = &notify_stat;
156 |   SetSockNonBlock(session->fd);
157 |   event_set(&session->event, thread_->receive_fd,
158 |       EV_READ | EV_PERSIST, ConnSession::Handler, session);
159 | 
160 |   event_base_set(thread_->base, &session->event);
161 | 
162 |   if (event_add(&session->event, 0) != 0) {
163 |     exit(-1);
164 |   }
165 | 
166 |   // run thread
167 |   if (pthread_create(&thread_->tid, NULL, IOThread::ThreadEntry, thread_) != 0) {
168 |     exit(-1);
169 |   }
170 | }
171 | 
172 | IOThread::IOThread(Thread * t)
173 |       :thread_(t) {
174 | }
175 | 
176 | Thread& IOThread::thread() {
177 |   return *thread_;
178 | }
179 | 
180 | void* IOThread::ThreadEntry(void *args) {
181 |   Thread *thread = static_cast<Thread *>(args);
182 |   event_base_loop(thread->base, 0);
183 |   event_base_free(thread->base);
184 |   return NULL;
185 | }
186 | 
187 | void ConnSession::Handler(int fd, int16_t event, void* args) {
188 |   ConnSession *session = static_cast<ConnSession *>(args);
189 |   if (event & EV_TIMEOUT) {
190 |     session->stat = &close_stat;
191 |   }
192 |   while (session->stat->Handler(event, session) > 0) {
193 |     continue;
194 |   }
195 | }
196 | 
197 | ConnSession::~ConnSession() {
198 |   event_del(&event);
199 |   event_del(&timeout_event);
200 |   close(fd);
201 | }
202 | 
203 | const std::string& NotifySessionStat::GetName() const  {
204 |   static std::string name = "notify";
205 |   return name;
206 | }
207 | 
208 | int NotifySessionStat::Handler(int16_t event, ConnSession* session) {
209 |   char buf[1];
210 |   int count = read(session->fd, buf, sizeof(buf));
211 |   if (count <= 0) return kDone;
212 | 
213 |   ConnItem *item = session->iothread->thread().queue.DeQueue();
214 |   if (item == NULL)  return kDone;
215 | 
216 |   ConnSession *new_conn_session = new ConnSession();
217 |   new_conn_session->fd = item->fd;
218 |   SetSockNonBlock(item->fd);
219 |   new_conn_session->iothread = session->iothread;
220 |   new_conn_session->stat = &read_stat;
221 |   new_conn_session->keep_alive = false;
222 | 
223 |   event_set(&new_conn_session->event, new_conn_session->fd,
224 |       EV_READ | EV_PERSIST, ConnSession::Handler, new_conn_session);
225 |   event_base_set(session->iothread->thread().base, &new_conn_session->event);
226 | 
227 |   event_set(&new_conn_session->timeout_event, -1, 0,
228 |             ConnSession::Handler, new_conn_session);
229 |   event_base_set(session->iothread->thread().base, &new_conn_session->timeout_event);
230 | 
231 |   ServerConf& conf = ServerConf::GetInstance();
232 |   struct timeval tv = {conf.timeout(), 0};
233 |   if (event_add(&new_conn_session->event, 0) != 0 ||
234 |       event_add(&new_conn_session->timeout_event, &tv) != 0) {
235 |     new_conn_session->stat = &close_stat;
236 |     return kContinue;
237 |   }
238 |   return kDone;
239 | }
240 | 
241 | const std::string& CloseSessionStat::GetName() const  {
242 |   static std::string name = "close";
243 |   return name;
244 | }
245 | 
246 | int CloseSessionStat::Handler(int16_t event, ConnSession* session) {
247 |   delete session;
248 |   return kDone;
249 | }
250 | 
251 | const std::string& ReadSessionStat::GetName() const  {
252 |   static std::string name = "read";
253 |   return name;
254 | }
255 | 
256 | int ReadSessionStat::Handler(int16_t event, ConnSession* session) {
257 |   int read_size = read(session->fd, session->recv.GetWritePos(), session->recv.GetUseableSize());
258 |   if (read_size <= 0) {
259 |     session->stat = &close_stat;
260 |     return kContinue;
261 |   }
262 |   session->recv.AddWritePos(read_size);
263 |   session->stat = &check_stat;
264 |   return kContinue;
265 | }
266 | 
267 | const std::string& CheckSessionStat::GetName() const  {
268 |   static std::string name = "check";
269 |   return name;
270 | }
271 | 
272 | int CheckSessionStat::Handler(int16_t event, ConnSession* session) {
273 |   int pkglen = ValidatePkg(session->recv.GetBuffer(), session->recv.GetBufferLen());
274 |   if (pkglen <= 0) {
275 |     session->stat = &read_stat;
276 |     return kDone;  // continue read more data
277 |   }
278 |   session->in.Attach(session->recv.GetReadPos(), pkglen);
279 |   session->recv.AddReadPos(pkglen);
280 |   session->recv.Shrink();
281 |   session->stat = &process_stat;
282 |   return kContinue;
283 | }
284 | 
285 | const std::string& WriteSessionStat::GetName() const  {
286 |   static std::string name = "write";
287 |   return name;
288 | }
289 | 
290 | int WriteSessionStat::Handler(int16_t event, ConnSession* session) {
291 |   size_t size = write(session->fd, session->out.GetReadPos(), session->out.GetUnReadSize());
292 |   if (size < 0) {
293 |     session->stat = &close_stat;
294 |     return kContinue;
295 |   }
296 |   if (size < session->out.GetUnReadSize()) {
297 |     session->out.AddReadPos(size);
298 |     return kDone;  // continue to write;
299 |   }
300 |   if (session->keep_alive) {
301 |     event_del(&session->event);
302 |     event_set(&session->event, session->fd,
303 |         EV_READ | EV_PERSIST, ConnSession::Handler, session);
304 | 
305 |     event_base_set(session->iothread->thread().base, &session->event);
306 |     if (event_add(&session->event, 0) != 0) {
307 |       session->stat = &close_stat;
308 |       return kContinue;
309 |     }
310 |     session->stat = &check_stat;
311 |     session->out.Reset();
312 |     return kContinue;
313 |   } else {
314 |     session->stat = &close_stat;
315 |     return kContinue;
316 |   }
317 |   return kDone;
318 | }
319 | 
320 | const std::string& ProcessSessionStat::GetName() const  {
321 |   static std::string name = "process";
322 |   return name;
323 | }
324 | 
325 | int ProcessSessionStat::Handler(int16_t event, ConnSession* session) {
326 |   if (strncmp(session->in.GetBuffer(), "keep_alive", 10) == 0) {
327 |     session->keep_alive = true;
328 |     session->in.Reset();
329 |     event_del(&session->timeout_event);
330 |     session->stat = &check_stat;
331 |     return kContinue;
332 |   }
333 | 
334 |   string req(session->in.GetBuffer(), session->in.GetBufferLen());
335 |   string resp;
336 |   session->iothread->thread().dispatch(req, resp);
337 |   session->out.Attach(resp.data(), resp.size());
338 | 
339 |   event_del(&session->event);
340 |   event_set(&session->event, session->fd,
341 |       EV_WRITE | EV_PERSIST, ConnSession::Handler, session);
342 |   event_base_set(session->iothread->thread().base, &session->event);
343 |   if (event_add(&session->event, 0) != 0) {
344 |     session->stat = &close_stat;
345 |     return kContinue;
346 |   }
347 |   session->stat = &write_stat;
348 |   return kDone;
349 | }
350 | 
351 | }  // namespace libwxfreq
352 | 


--------------------------------------------------------------------------------
/net/iothread.h:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #ifndef LIBFREQ_NET_IOTHREAD_H_
 18 | #define LIBFREQ_NET_IOTHREAD_H_
 19 | #include <string>
 20 | #include <cstdint>
 21 | #include "server_inner.h"
 22 | 
 23 | #define LIBFREQ_BUFFER_SIZE 128
 24 | namespace libwxfreq {
 25 | class IOThread {
 26 |  public:
 27 |   explicit IOThread(Thread * t);
 28 |   void run();
 29 |   Thread& thread();
 30 |  private:
 31 |   static void* ThreadEntry(void *);
 32 |   Thread *thread_;
 33 | };
 34 | 
 35 | class ConnSession;
 36 | class SessionStat {
 37 |  public:
 38 |   virtual const std::string& GetName() const = 0;
 39 |   virtual int Handler(int16_t event, ConnSession* session) = 0;
 40 |  protected:
 41 |   enum {
 42 |     kDone = 0,
 43 |     kContinue = 1,
 44 |   };
 45 | };
 46 | 
 47 | class NotifySessionStat:public SessionStat {
 48 |  public:
 49 |   const std::string& GetName() const;
 50 |   int Handler(int16_t event, ConnSession* session);
 51 | };
 52 | 
 53 | class CloseSessionStat:public SessionStat {
 54 |  public:
 55 |   const std::string& GetName() const;
 56 |   int Handler(int16_t event, ConnSession* session);
 57 | };
 58 | 
 59 | class ReadSessionStat:public SessionStat {
 60 |  public:
 61 |   const std::string& GetName() const;
 62 |   int Handler(int16_t event, ConnSession* session);
 63 | };
 64 | 
 65 | class CheckSessionStat:public SessionStat {
 66 |  public:
 67 |   const std::string& GetName() const;
 68 |   int Handler(int16_t event, ConnSession* session);
 69 | };
 70 | 
 71 | class WriteSessionStat:public SessionStat {
 72 |  public:
 73 |   const std::string& GetName() const;
 74 |   int Handler(int16_t event, ConnSession* session);
 75 | };
 76 | 
 77 | class ProcessSessionStat:public SessionStat {
 78 |  public:
 79 |   const std::string& GetName() const;
 80 |   int Handler(int16_t event, ConnSession* session);
 81 | };
 82 | 
 83 | class SKBuff {
 84 |  public:
 85 |   SKBuff();
 86 |   ~SKBuff();
 87 |   char* GetWritePos();
 88 |   char* GetReadPos();
 89 |   size_t GetUseableSize();
 90 |   void Attach(SKBuff& buff);
 91 |   void Attach(const char* buff, size_t len);
 92 |   char* GetBuffer();
 93 |   size_t GetBufferLen();
 94 |   size_t GetUnReadSize();
 95 |   void Reset();
 96 |   void Shrink();
 97 |   size_t AddWritePos(size_t size);
 98 |   size_t AddReadPos(size_t size);
 99 |  private:
100 |   void ExpandSize(size_t len);
101 |  private:
102 |   char array_[LIBFREQ_BUFFER_SIZE];
103 |   char* buffer_;
104 |   size_t write_pos_;
105 |   size_t read_pos_;
106 |   size_t capacity_;
107 | };
108 | 
109 | struct ConnSession {
110 |   int fd;
111 |   bool    keep_alive;
112 |   SessionStat  *stat;
113 |   SKBuff  recv;
114 |   SKBuff  in;
115 |   SKBuff  out;
116 |   IOThread*   iothread;
117 |   struct event event;
118 |   struct event timeout_event;
119 |   static void Handler(int, int16_t, void *);
120 |   ~ConnSession();
121 | };
122 | }  // namespace libwxfreq
123 | 
124 | #endif  // LIBFREQ_NET_IOTHREAD_H_
125 | 


--------------------------------------------------------------------------------
/net/server.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "server.h"
18 | #include "util.h"
19 | 
20 | namespace libwxfreq {
21 | Server::Server(const char *ip, const int port, Dispatch func,
22 |                 const int worker_num)
23 |             :server_ip_(ip), listent_port_(port), pool_(func, worker_num),
24 |              acceptor_(this) {
25 | }
26 | 
27 | Server::Server(Dispatch func)
28 |             :pool_(func), acceptor_(this) {
29 |   const ServerConf& conf = ServerConf::GetInstance();
30 |   server_ip_ = conf.listen_ip();
31 |   listent_port_ = conf.listen_port();
32 |   pool_.SetWorkerNum(conf.worker_num());
33 | }
34 | 
35 | void Server::Run() {
36 |   if (Daemonize() != 0) exit(-1);
37 |   if (!pool_.Run()) exit(-1);
38 |   acceptor_.LoopAccept(server_ip_.c_str(), listent_port_);
39 | }
40 | 
41 | WorkerPool& Server::pool() {
42 |   return pool_;
43 | }
44 | 
45 | }  // namespace libwxfreq
46 | 


--------------------------------------------------------------------------------
/net/server.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef LIBFREQ_NET_SERVER_H_
18 | #define LIBFREQ_NET_SERVER_H_
19 | 
20 | #include <functional>
21 | #include <string>
22 | #include "acceptor.h"
23 | #include "workerpool.h"
24 | #include "server_conf.h"
25 | 
26 | namespace libwxfreq {
27 | 
28 | class Server {
29 |  public:
30 |   Server(const char *ip, const int port, Dispatch func,
31 |          const int worker_num = 4);
32 |   explicit Server(Dispatch func);
33 |   void Run();
34 |   WorkerPool& pool();
35 |  private:
36 |   int worker_num_;
37 |   std::string server_ip_;
38 |   int listent_port_;
39 |   WorkerPool pool_;
40 |   Acceptor acceptor_;
41 | };
42 | 
43 | }  // namespace libwxfreq
44 | 
45 | #endif  // LIBFREQ_NET_SERVER_H_
46 | 


--------------------------------------------------------------------------------
/net/server_conf.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "server_conf.h"
 18 | #include "config.h"
 19 | 
 20 | namespace libwxfreq {
 21 | using std::string;
 22 | using std::to_string;
 23 | 
 24 | StatType::StatType(const StatType& left):typename_(left.typename_),
 25 |     init_(left.init_), shm_key_(left.shm_key_), item_count_(left.item_count_) {
 26 | }
 27 | 
 28 | StatType::StatType(const std::string& tyname, uint32_t init,
 29 |               uint32_t shm_key, uint32_t item_count)
 30 |               :typename_(tyname), init_(init), shm_key_(shm_key),
 31 |               item_count_(item_count) {
 32 | }
 33 | 
 34 | const std::string& StatType::GetTypeName() const {
 35 |   return typename_;
 36 | }
 37 | uint32_t StatType::init() const {
 38 |   return init_;
 39 | }
 40 | uint32_t StatType::shm_key() const {
 41 |   return shm_key_;
 42 | }
 43 | 
 44 | uint32_t StatType::item_count() const {
 45 |   return item_count_;
 46 | }
 47 | 
 48 | ServerConf& ServerConf::GetInstance() {
 49 |     static ServerConf conf_;
 50 |     return conf_;
 51 | }
 52 | 
 53 | ServerConf::ServerConf() {
 54 |   has_init_ = false;
 55 | }
 56 | 
 57 | bool ServerConf::LoadFile(const std::string& file) {
 58 |   if (has_init_) return true;
 59 | 
 60 |   Config config(file);
 61 |   if (config.Init() != 0) return false;
 62 | 
 63 |   int32_t stat_num = 0;
 64 |   config.ReadItem("General", "libwxfreq_conf_path", "", libwxfreq_conf_path_);
 65 |   config.ReadItem("General", "stat_type_num", 0, stat_num);
 66 | 
 67 |   config.ReadItem("Server", "listen_ip", "", listen_ip_);
 68 |   config.ReadItem("Server", "listen_port", 0, listen_port_);
 69 |   config.ReadItem("Server", "worke_num", 0, worke_num_);
 70 |   config.ReadItem("Server", "timeout", 1, timeout_);
 71 |   stat_vec_.clear();
 72 |   for (int i = 0; i < stat_num; i++) {
 73 |     string typename_, section_name = "Stat_";
 74 |     uint32_t init = 0, shm_key = 0, item_count;
 75 |     section_name += to_string(i);
 76 | 
 77 |     config.ReadItem(section_name, "typename", "", typename_);
 78 |     config.ReadItem(section_name, "init", 0, init);
 79 |     config.ReadItem(section_name, "shm_key", 0, shm_key);
 80 |     config.ReadItem(section_name, "item_count", 0, item_count);
 81 | 
 82 |     StatType type(typename_, init, shm_key, item_count);
 83 |     stat_vec_.push_back(type);
 84 |   }
 85 | 
 86 |   if (stat_num > 0) {
 87 |     has_init_ = true;
 88 |   }
 89 |   return has_init_;
 90 | }
 91 | 
 92 | const std::string ServerConf::listen_ip() const {
 93 |   return listen_ip_;
 94 | }
 95 | const std::string ServerConf::libwxfreq_conf_path() const {
 96 |   return libwxfreq_conf_path_;
 97 | }
 98 | uint32_t ServerConf::listen_port() const {
 99 |   return listen_port_;
100 | }
101 | uint32_t ServerConf::worker_num() const {
102 |   return worke_num_;
103 | }
104 | const std::vector<StatType>& ServerConf::stat_vec() const {
105 |   return stat_vec_;
106 | }
107 | uint32_t ServerConf::timeout() const {
108 |   return timeout_;
109 | }
110 | 
111 | }  // namespace libwxfreq
112 | 
113 | 


--------------------------------------------------------------------------------
/net/server_conf.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef LIBFREQ_NET_SERVER_CONF_H_
18 | #define LIBFREQ_NET_SERVER_CONF_H_
19 | #include <stdint.h>
20 | #include <string>
21 | #include <vector>
22 | 
23 | 
24 | namespace libwxfreq {
25 | class StatType {
26 |  public:
27 |   StatType(const StatType& left);
28 |   StatType(const std::string& tyname, uint32_t init,
29 |               uint32_t shm_key, uint32_t item_count);
30 |   const std::string& GetTypeName() const;
31 |   uint32_t init() const;
32 |   uint32_t shm_key() const;
33 |   uint32_t item_count() const;
34 | 
35 |  private:
36 |   std::string typename_;
37 |   uint32_t init_;
38 |   uint32_t shm_key_;
39 |   uint32_t item_count_;
40 | };
41 | 
42 | class ServerConf {
43 |  public:
44 |   static ServerConf& GetInstance();
45 |   ServerConf(const ServerConf& a) = delete;
46 |   bool LoadFile(const std::string& file);
47 |   const std::string listen_ip() const;
48 |   const std::string libwxfreq_conf_path() const;
49 |   uint32_t listen_port() const;
50 |   uint32_t worker_num() const;
51 |   const std::vector<StatType>& stat_vec() const;
52 |   uint32_t timeout() const;
53 | 
54 |  private:
55 |   ServerConf();
56 | 
57 |  private:
58 |   std::string libwxfreq_conf_path_;
59 |   uint32_t stat_type_num_;
60 |   std::string listen_ip_;
61 |   uint32_t listen_port_;
62 |   uint32_t worke_num_;
63 |   uint32_t timeout_;
64 |   std::vector<StatType> stat_vec_;
65 |   bool has_init_;
66 | };
67 | 
68 | }  // namespace libwxfreq
69 | #endif  // LIBFREQ_NET_SERVER_CONF_H_
70 | 


--------------------------------------------------------------------------------
/net/server_inner.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "server_inner.h"
18 | 
19 | namespace libwxfreq {
20 | void ConnQueue::EnQueue(int fd) {
21 |   if ((head_ + 1) % 1024 != tail_) {
22 |     array_[head_++].fd = fd;
23 |     head_ %= 1024;
24 |   }
25 | }
26 | 
27 | ConnItem* ConnQueue::DeQueue() {
28 |   ConnItem* item = NULL;
29 |   if (tail_ != head_) {
30 |     item = &array_[tail_++];
31 |     tail_ %= 1024;
32 |   }
33 |   return item;
34 | }
35 | 
36 | }  // namespace libwxfreq
37 | 


--------------------------------------------------------------------------------
/net/server_inner.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_NET_SERVER_INNER_H_
18 | #define FREQLIB_NET_SERVER_INNER_H_
19 | 
20 | #include <sys/types.h>
21 | #include <sys/socket.h>
22 | #include <sys/time.h>
23 | #include <netinet/in.h>
24 | #include <event.h>
25 | #include <netdb.h>
26 | #include <pthread.h>
27 | #include <unistd.h>
28 | #include <assert.h>
29 | #include <string>
30 | #include <functional>
31 | 
32 | namespace libwxfreq {
33 | typedef std::function< void(const std::string&, std::string& resp) > Dispatch;
34 | 
35 | struct ConnItem {
36 |   int fd;
37 | };
38 | 
39 | class ConnQueue {
40 |  public:
41 |   ConnQueue() :head_(0), tail_(0) {
42 |   }
43 |   void EnQueue(int fd);
44 |   ConnItem* DeQueue();
45 |  private:
46 |   struct ConnItem array_[1024];
47 |   int head_;
48 |   int tail_;
49 | };
50 | 
51 | class IOThread;
52 | 
53 | struct Thread {
54 |   pthread_t  tid;
55 |   IOThread *iothread;
56 |   struct event_base *base;
57 |   int receive_fd;  // worker end
58 |   int send_fd;  // control end
59 |   struct ConnQueue queue;
60 |   Dispatch dispatch;
61 | };
62 | 
63 | }  // namespace libwxfreq
64 | #endif  // FREQLIB_NET_SERVER_INNER_H_
65 | 


--------------------------------------------------------------------------------
/net/util.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include <signal.h>
18 | #include <fcntl.h>
19 | #include <stdio.h>
20 | #include <stdlib.h>
21 | #include <unistd.h>
22 | 
23 | namespace libwxfreq {
24 | int Daemonize() {
25 |   int fd = 0;
26 |   if (fork() != 0) exit(0);
27 |   // child process, create a new session
28 |   if (setsid() == -1) return -1;
29 | 
30 |   umask(0);
31 |   chdir("/");
32 |   if ((fd = open("/dev/null", O_RDWR, 0)) != -1) {
33 |     if (dup2(fd, STDIN_FILENO) < 0) return -1;
34 |     if (dup2(fd, STDOUT_FILENO) < 0) return -1;
35 |     if (dup2(fd, STDERR_FILENO) < 0) return -1;
36 |   }
37 | 
38 |   signal(SIGPIPE, SIG_IGN);
39 |   return 0;
40 | }
41 | 
42 | 
43 | void SetSockNonBlock(int sock) {
44 |   int flags;
45 |   flags = fcntl(sock, F_GETFL, 0);
46 |   if (flags >= 0) {
47 |     fcntl(sock, F_SETFL, flags | O_NONBLOCK);
48 |   }
49 | }
50 | 
51 | }  // namespace libwxfreq
52 | 


--------------------------------------------------------------------------------
/net/util.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef LIBFREQ_NET_UTIL_H_
18 | #define LIBFREQ_NET_UTIL_H_
19 | namespace libwxfreq {
20 | int Daemonize();
21 | void SetSockNonBlock(int sock);
22 | }  // namespace libwxfreq
23 | 
24 | #endif  // LIBFREQ_NET_UTIL_H_
25 | 


--------------------------------------------------------------------------------
/net/worker.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include <string.h>
 18 | #include <cstring>
 19 | #include <string>
 20 | #include <sstream>
 21 | #include <cstdio>
 22 | #include <vector>
 23 | 
 24 | #include "libwxfreq.h"
 25 | namespace libwxfreq {
 26 | using std::string;
 27 | using std::vector;
 28 | using std::ostringstream;
 29 | 
 30 | void Echo(const string& req, string& resp) {
 31 |   resp = req;
 32 | }
 33 | 
 34 | void SplitReq(const char* req, vector<string>& cmd) {
 35 |   const char* end = req;
 36 |   const char* start = NULL;
 37 | 
 38 |   while(*end != '\n' && *end != '\r' && *end != '\0') {
 39 |     if (*end == ' ' || *end == '\t') {
 40 |        if (start != NULL) {
 41 |          cmd.push_back(string(start, end - start));
 42 |          start = NULL;
 43 |        }
 44 |     } else {
 45 |       if (start == NULL) {
 46 |         start = end;
 47 |       }
 48 |     }
 49 |     end++;
 50 |   }
 51 |   if (start != NULL) {
 52 |     cmd.push_back(string(start, end - start));
 53 |   }
 54 | }
 55 | 
 56 | 
 57 | void TxtFreqReq(const string& req, string& resp) {
 58 |   vector<string> parsed_req;
 59 |   SplitReq(req.c_str(), parsed_req);
 60 | 
 61 |   if (parsed_req.size() == 0) return;
 62 |   string cmd = parsed_req[0];
 63 |   ostringstream ostr;
 64 | 
 65 |   if (strcasecmp(cmd.c_str(), "ReportAndCheck") == 0) {
 66 |     if (parsed_req.size() != 5) {
 67 |       ostr << "{\"code\": -1, \"msg\":\"wrong cmd, [ReportAndCheck type user key appid cnt]\"}\n\n";
 68 |     } else {
 69 |       uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);
 70 |       uint32_t cnt = strtoul(parsed_req[4].c_str(), NULL, 10);
 71 |       BlockResult bs = ReportAndCheck(parsed_req[1].c_str(),
 72 |                    parsed_req[2].c_str(), appid, cnt);
 73 |       ostr << "{\"code\": 0, \"block_level\": " << bs.block_level << ", \"match_rule\": \"" << bs.match_rule <<"\"}\n\n";
 74 |     }
 75 |   } else if (strcasecmp(cmd.c_str(), "OnlyCheck") == 0) {
 76 |     if (parsed_req.size() != 4) {
 77 |       ostr << "{\"code\": -1, \"msg\":\"wrong cmd, [OnlyCheck type user key appid]\"}\n\n";
 78 |     } else {
 79 |       uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);
 80 |       BlockResult bs = OnlyCheck(parsed_req[1].c_str(),
 81 |                  parsed_req[2].c_str(), appid);
 82 |       ostr << "{\"code\": 0, \"block_level\": " << bs.block_level << ", \"match_rule\": \"" << bs.match_rule << "\"}\n\n";
 83 |     }
 84 |   } else if (strcasecmp(cmd.c_str(), "OnlyReport") == 0) {
 85 |     if (parsed_req.size() != 5) {
 86 |       ostr << "{\"code\": -1, \"msg\":\"wrong cmd, [OnlyReport type user key appid cnt]\"}\n\n";
 87 |     } else {
 88 |       uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);
 89 |       uint32_t cnt = strtoul(parsed_req[4].c_str(), NULL, 10);
 90 |       int ret = OnlyReport(parsed_req[1].c_str(),
 91 |                  parsed_req[2].c_str(), appid, cnt);
 92 |       ostr << "{\"code\": " << ret << "}\n\n";
 93 |     }
 94 |   } else if (strcasecmp(cmd.c_str(), "GetCache") == 0) {
 95 |     if (parsed_req.size() != 4) {
 96 |       ostr << "{\"code\": -1, \"msg\":\"wrong cmd, [GetCache type user key appid]\"}\n\n";
 97 |     } else {
 98 |       uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);
 99 |       FreqCache cache = GetCache(parsed_req[1].c_str(),
100 |                    parsed_req[2].c_str(), appid);
101 |       ostr << "{\"code\": 0, \"cnt1\": " << cache.level1_cnt <<", \"cnt2\": "
102 |           << cache.level2_cnt <<", \"cnt3\": " << cache.level3_cnt << "}\n\n";
103 |     }
104 |   } else if (strcasecmp(cmd.c_str(), "AddWhite") == 0) {
105 |     if (parsed_req.size() != 5) {
106 |       ostr << "{\"code\": -1, \"msg\":\"wrong cmd, [AddWhite type user key appid linger_time]\"}\n\n";
107 |     } else {
108 |       uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);
109 |       uint32_t linger_time = strtoul(parsed_req[4].c_str(), NULL, 10);
110 |       int ret = AddWhite(parsed_req[1].c_str(), parsed_req[2].c_str(), appid, linger_time);
111 |       ostr << "{\"code\": " << ret << "}\n\n";
112 |     }
113 |   } else if (strcasecmp(cmd.c_str(), "DeleteWhite") == 0) {
114 |     if (parsed_req.size() != 4) {
115 |       ostr << "{\"code\": -1, \"msg\":\"wrong cmd, [DeleteWhite type user key appid]\"}\n\n";
116 |     } else {
117 |       uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);
118 |       int ret = DeleteWhite(parsed_req[1].c_str(), parsed_req[2].c_str(), appid);
119 |       ostr << "{\"code\": " << ret << "}\n\n";
120 |     }
121 |   } else if (strcasecmp(cmd.c_str(), "AddBlock") == 0) {
122 |     if (parsed_req.size() != 6) {
123 |       ostr << "{\"code\": -1, \"msg\":\"wrong cmd, [AddBlock type user key appid linger_time block_level]\"}\n\n";
124 |     } else {
125 |       uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);
126 |       uint32_t linger_time = strtoul(parsed_req[4].c_str(), NULL, 10);
127 |       uint32_t block_level = strtoul(parsed_req[5].c_str(), NULL, 10);
128 |       int ret = AddBlock(parsed_req[1].c_str(), parsed_req[2].c_str(), appid, linger_time, block_level);
129 |       ostr << "{\"code\": " << ret << "}\n\n";
130 |     }
131 |   } else if (strcasecmp(cmd.c_str(), "DeleteBlock") == 0) {
132 |     if (parsed_req.size() != 4) {
133 |       ostr << "{\"code\": -1, \"msg\":\"wrong cmd, [DeleteBlock type user key appid]\"}\n\n";
134 |     } else {
135 |       uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);
136 |       int ret = DeleteBlock(parsed_req[1].c_str(), parsed_req[2].c_str(), appid);
137 |       ostr << "{\"code\": " << ret << "}\n\n";
138 |     }
139 |   } else if (strcasecmp(cmd.c_str(), "Help") == 0) {
140 |       ostr << "cmdlist:\n1: ReportAndCheck\n2: OnlyCheck\n3: OnlyReport\n"
141 |                     "4: GetCache\n5: AddWhite\n6: DeleteWhite\n7: AddBlock\n"
142 |                     "8: DeleteBlock\n\n";
143 |   } else {
144 |       ostr << "{\"code\": -1, \"msg\":\"unkown cmd " << cmd << ", try [help]\"}\n\n";
145 |   }
146 | 
147 |   resp = ostr.str();
148 | }
149 | 
150 | }  // namespace libwxfreq
151 | 


--------------------------------------------------------------------------------
/net/worker.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef LIBFREQ_NET_WORKER_H_
18 | #define LIBFREQ_NET_WORKER_H_
19 | #include <string>
20 | namespace libwxfreq {
21 | void Echo(const std::string& req, std::string& resp);
22 | void TxtFreqReq(const std::string& req, std::string&  resp);
23 | }  // namespace libwxfreq
24 | 
25 | #endif  // LIBFREQ_NET_WORKER_H_
26 | 
27 | 
28 | 


--------------------------------------------------------------------------------
/net/workerpool.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "workerpool.h"
18 | #include <unistd.h>
19 | #include <stdlib.h>
20 | #include <string.h>
21 | #include "iothread.h"
22 | 
23 | namespace libwxfreq {
24 | WorkerPool::WorkerPool(Dispatch func, int num)
25 |           :pool_(NULL), pool_size_(num),
26 |           last_accept_worker_index_(0), func_(func) {
27 | }
28 | 
29 | void WorkerPool::SetWorkerNum(int num) {
30 |   pool_size_ = num;
31 | }
32 | 
33 | bool WorkerPool::Run() {
34 |     pool_ = static_cast<Thread*>(malloc(pool_size_ * sizeof(Thread)));
35 |     memset(reinterpret_cast<void*>(pool_), 0, pool_size_ * sizeof(Thread));
36 |     for (int i = 0; i < pool_size_; i++) {
37 |         int fds[2] = {0};
38 |         if (pipe(fds) != 0) {
39 |           return false;
40 |         }
41 |         pool_[i].receive_fd = fds[0];
42 |         pool_[i].send_fd = fds[1];
43 |         pool_[i].dispatch = func_;
44 | 
45 |         pool_[i].iothread = new IOThread(&pool_[i]);
46 |         pool_[i].iothread->run();
47 |     }
48 |     return true;
49 | }
50 | 
51 | WorkerPool::~WorkerPool() {
52 |     delete[] pool_;
53 | }
54 | 
55 | void WorkerPool::AddNewConn(int fd) {
56 |     last_accept_worker_index_ = (last_accept_worker_index_ + 1) % pool_size_;
57 |     Thread &cur = pool_[last_accept_worker_index_];
58 |     cur.queue.EnQueue(fd);
59 |     Notify(cur);
60 | }
61 | 
62 | void WorkerPool::Notify(Thread& thread) {
63 |     int fd = thread.send_fd;
64 |     char cmd[1] = {'c'};
65 |     write(fd, cmd, 1);
66 | }
67 | 
68 | }  // namespace libwxfreq
69 | 


--------------------------------------------------------------------------------
/net/workerpool.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef LIBFREQ_NET_WORKERPOOL_H_
18 | #define LIBFREQ_NET_WORKERPOOL_H_
19 | #include "server_inner.h"
20 | namespace libwxfreq {
21 | class WorkerPool {
22 |  public:
23 |   explicit WorkerPool(Dispatch func, int num = 0);
24 |   void SetWorkerNum(int num);
25 |   ~WorkerPool();
26 |   void AddNewConn(int fd);
27 |   bool Run();
28 |  private:
29 |   void Notify(Thread& thread);
30 |  private:
31 |   Thread* pool_;
32 |   int pool_size_;
33 |   size_t last_accept_worker_index_;
34 |   Dispatch func_;
35 | };
36 | 
37 | }  // namespace libwxfreq
38 | #endif  // LIBFREQ_NET_WORKERPOOL_H_
39 | 


--------------------------------------------------------------------------------
/server.conf:
--------------------------------------------------------------------------------
 1 | [General]
 2 | libwxfreq_conf_path = ./test.conf
 3 | stat_type_num = 2
 4 | 
 5 | [Server]
 6 | listen_ip = 10.0.0.1
 7 | listen_port = 5678 
 8 | worke_num = 5
 9 | timeout = 10
10 | 
11 | [Stat_0]
12 | typename = user
13 | init = 0
14 | shm_key = 20180313
15 | item_count = 100000000
16 | 
17 | [Stat_1]
18 | typename = ip 
19 | init = 0
20 | shm_key = 20180314
21 | item_count = 100000000
22 | 
23 | 


--------------------------------------------------------------------------------
/test/demo.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "libwxfreq/libwxfreq.h"
18 | #include <stdlib.h>
19 | #include <unistd.h>
20 | #include <stdio.h>
21 | #include <time.h>
22 | 
23 | int main(int argc, char* argv[]) {
24 |   if (argc != 7){
25 |     printf("%s conf appid key keytype shm_key sleep_time(ms)\n", argv[0]);
26 |     return 0;
27 |   }
28 |   SetRuleConfFile(argv[1]); 
29 |   int appid = strtoul(argv[2], NULL, 10);
30 |   const char* key = argv[3];
31 |   const char* keytype = argv[4];
32 |   int shm_key = strtoul(argv[5], NULL, 10);
33 |   int sleep_time = strtoul(argv[6], NULL, 10);
34 |   sleep_time *= 1000;
35 |   
36 |   int ret = RegisterNewShmStat(keytype, false, shm_key,  1000000);
37 |   if (ret != 0) {
38 |     printf("RegisterNewShmStat failed, ret = %d\n", ret);
39 |     return -1;
40 |   }
41 | 
42 |   if (InitFreq() == false) {
43 |     printf("InitFreq failed\n");
44 |     return -2;
45 |   }
46 | 
47 |   while (true) {
48 |     struct BlockResult res;
49 |     res = ReportAndCheck(keytype, key, appid, 1);
50 |     printf("time %lu blocklevel %u matchrule %s\n", time(NULL),
51 |           res.block_level, res.match_rule);
52 |     usleep(sleep_time);
53 |   }
54 | }
55 | 


--------------------------------------------------------------------------------
/test/performance_test.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "libwxfreq/libwxfreq.h"
18 | #include <stdlib.h>
19 | #include <unistd.h>
20 | #include <stdio.h>
21 | #include <time.h>
22 | #include <stdlib.h>
23 | #include <string>
24 | #include <sys/time.h>
25 | 
26 | static inline int Log(const char* format, ...) { }
27 | int main(int argc, char* argv[]) {
28 |   if (argc != 3){
29 |     printf("%s conf item_count\n", argv[0]);
30 |     return 0;
31 |   }
32 |   SetRuleConfFile(argv[1]); 
33 |   SetLogFunc(Log);
34 |   int item_count = strtoul(argv[2], NULL, 10);
35 |   
36 |   int ret = RegisterNewShmStat("user", false, rand(),  item_count);
37 |   if (ret != 0) {
38 |     printf("RegisterNewShmStat failed, ret = %d\n", ret);
39 |     return -1;
40 |   }
41 | 
42 |   if (InitFreq() == false) {
43 |     printf("InitFreq failed\n");
44 |     return -2;
45 |   }
46 | 
47 |   int oom_num = 0, first_time = 0;
48 |   struct timeval start, end;
49 |   gettimeofday(&start, NULL);
50 |   printf("start time %u %u\n", start.tv_sec, start.tv_usec);
51 |   for (int i = 0; i < item_count; i++) {
52 |     struct BlockResult res;
53 |     res = ReportAndCheck("user", "test", rand(), 1);
54 |     if (res.block_level == kErrorOutOfMemory) {
55 |       oom_num ++;
56 |       if (first_time == 0) first_time = i + 1;
57 |     }
58 |   }
59 |   gettimeofday(&end, NULL);
60 |   printf("end time %u %u\n", end.tv_sec, end.tv_usec);
61 |   printf("average time %u per second\n", (unsigned int)(item_count/(end.tv_sec-start.tv_sec)));
62 |   printf("oom count %u first oom %u memory usage %.3f\n", oom_num, first_time, 100 - 100.0 * oom_num/item_count);
63 |   return 0;
64 | }
65 | 
66 | 


--------------------------------------------------------------------------------
/test/test.conf:
--------------------------------------------------------------------------------
 1 | [appid]
 2 | 100 = 30, 60, 90
 3 | 
 4 | [rule1]
 5 | match_appid = 100
 6 | item =  min_interval >= 10
 7 | block_level = 1
 8 | rule_type = user
 9 | 
10 | [rule2]
11 | match_appid = 100
12 | item =  mid_interval >= 15 
13 | block_level = 1
14 | rule_type = user
15 | 
16 | [rule3]
17 | match_appid = 100
18 | item =  max_interval >= 20
19 | block_level = 1
20 | rule_type = user
21 | 
22 | [rule4]
23 | match_appid = 100
24 | item =  min_interval >= 100
25 | block_level = 1
26 | rule_type = ip 
27 | 
28 | [rule5]
29 | match_appid = 100
30 | item =  mid_interval >= 150
31 | block_level = 1
32 | rule_type = ip 
33 | 
34 | [rule6]
35 | match_appid = 100
36 | item =  max_interval >= 200 
37 | block_level = 1
38 | rule_type = ip 
39 | 


--------------------------------------------------------------------------------
/test/tools.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "libwxfreq/libwxfreq.h"
18 | #include <stdlib.h>
19 | #include <unistd.h>
20 | #include <stdio.h>
21 | #include <time.h>
22 | #include <string.h>
23 | 
24 | int main(int argc, char* argv[]) {
25 |   if (argc != 9){
26 |     printf("%s conf appid key keytype shm_key func level time\n", argv[0]);
27 |     return 0;
28 |   }
29 |   SetRuleConfFile(argv[1]); 
30 |   int appid = strtoul(argv[2], NULL, 10);
31 |   const char* key = argv[3];
32 |   const char* keytype = argv[4];
33 |   int shm_key = strtoul(argv[5], NULL, 10);
34 |   const char* func = argv[6];
35 |   int block_level = strtoul(argv[7], NULL, 10);
36 |   int linger_time = strtoul(argv[8], NULL, 10);
37 |   
38 |   int ret = RegisterNewShmStat(keytype, false, shm_key,  1000000);
39 |   if (ret != 0) {
40 |     printf("RegisterNewShmStat failed, ret = %d\n", ret);
41 |     return -1;
42 |   }
43 | 
44 |   if (InitFreq() == false) {
45 |     printf("InitFreq failed\n");
46 |     return -2;
47 |   }
48 | 
49 |   if (strcmp(func, "addblock") == 0) {
50 |     ret = AddBlock(keytype, key, appid,  linger_time, block_level);
51 |   } else if (strcmp(func, "deleteblock") == 0) { 
52 |     ret = DeleteBlock(keytype, key, appid);
53 |   } else if (strcmp(func, "addwhite") == 0) { 
54 |     ret = AddWhite(keytype, key, appid, linger_time);
55 |   } else if (strcmp(func, "deletewhite") == 0) { 
56 |     ret = DeleteWhite(keytype, key, appid);
57 |   } else {
58 |     printf("unknown func\n");
59 |     return -3;
60 |   }
61 |   printf("%s return ret %d\n", func, ret);
62 |   return ret;
63 | }
64 | 


--------------------------------------------------------------------------------
/util/config.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "config.h"
 18 | #include <sys/types.h>
 19 | #include <sys/stat.h>
 20 | #include <stdio.h>
 21 | #include <string.h>
 22 | #include <malloc.h>
 23 | #include <stdlib.h>
 24 | #include <unistd.h>
 25 | 
 26 | #include "log.h"
 27 | #include "slice.h"
 28 | 
 29 | namespace libwxfreq {
 30 | 
 31 | 
 32 | Config::Config(const std::string& conf_filename) {
 33 |   conf_filename_ = conf_filename;
 34 | }
 35 | 
 36 | const std::string & Config::conf_filename() {
 37 |   return conf_filename_;
 38 | }
 39 | 
 40 | void Config::set_conf_filename(const std::string &conf_filename) {
 41 |   conf_filename_ = conf_filename;
 42 | }
 43 | 
 44 | int Config::Init(void) {
 45 |   int ret = LoadFile();
 46 |   if (0 != ret) {
 47 |     gLog("[%s][%d]: init config %s error return %d\n", __FILE__, __LINE__,
 48 |         conf_filename_.c_str(), ret);
 49 |     Reset();
 50 |   }
 51 |   return ret;
 52 | }
 53 | 
 54 | int Config::Reset(void) {
 55 |   conf_filename_buf_.clear();
 56 |   section_list_.clear();
 57 |   table_.Clear();
 58 | }
 59 | 
 60 | 
 61 | int Config::ParserConfig() {
 62 |   Slice section, key, value;
 63 |   const char *eol = NULL, *tmp = NULL;
 64 |   const char *ptr = conf_filename_buf_.c_str();
 65 |   const char *endptr = conf_filename_buf_.c_str() + conf_filename_buf_.size();
 66 | 
 67 |   while (ptr < endptr) {
 68 |     eol = strchr(ptr, '\n');  // getline
 69 |     if (eol == NULL) eol = endptr;  // last line
 70 |     while (*ptr == ' ' || *ptr == '\t') ptr++;  // ltrim
 71 | 
 72 |     if (*ptr == '[') {  // section
 73 |       ptr += 1;
 74 |       tmp = ptr;
 75 |       while (*ptr != ']' && ptr < eol) ptr++;
 76 |       section = Slice(tmp, ptr);
 77 |       section.StrTrim("\t ");
 78 |       section_list_.push_back(section);
 79 |     } else if (NULL == strchr("#;\n", *ptr)) {  // item
 80 |       tmp = ptr;
 81 |       while (*ptr != '=' && ptr < eol) ptr++;
 82 |       key = Slice(tmp, ptr);
 83 |       key.StrTrim("\t ");
 84 | 
 85 |       if (*ptr == '=' && section.start() != NULL) {
 86 |         ptr += 1;
 87 |         tmp = ptr;
 88 |         while (ptr < eol) ptr++;
 89 |         value = Slice(tmp, ptr);
 90 |         value.StrTrim("\t ");
 91 |         table_.Add(section, key, value);
 92 |       }
 93 |     }
 94 |     ptr = eol + 1;  // next line
 95 |   }
 96 |   return section_list_.size() > 0 ? 0 : -1;
 97 | }
 98 | 
 99 | int Config::LoadFile(void) {
100 |   FILE * fp = fopen(conf_filename_.c_str(), "r");
101 |   if (NULL != fp) {
102 |     struct stat fileStat;
103 |     if (0 == fstat(fileno(fp), &fileStat)) {
104 |       Reset();
105 |       if (fileStat.st_size == 0) return 0;
106 |       char *tmp = reinterpret_cast<char*>(malloc(fileStat.st_size + 64));
107 |       fread(tmp, fileStat.st_size, 1, fp);
108 |       tmp[fileStat.st_size] = '\0';  // append '\0' make strchr happy
109 |       conf_filename_buf_ = tmp;
110 |       free(tmp);
111 |       ParserConfig();
112 |     } else {
113 |       gLog("[%s][%d]: open confile %s error\n",
114 |           __FILE__, __LINE__, conf_filename_.c_str());
115 |     }
116 |     fclose(fp);
117 |     fp = NULL;
118 |   }
119 |   return section_list_.size() == 0 ? -1 : 0;
120 | }
121 | 
122 | 
123 | void Config::section_list(std::vector<std::string>& sectionlist) {
124 |   for (size_t i = 0; i < section_list_.size(); i++) {
125 |     sectionlist.push_back(std::string(section_list_[i].start(),
126 |                         section_list_[i].end() - section_list_[i].start()));
127 |   }
128 | }
129 | 
130 | void Config::GetKeysBySection(const std::string &section,
131 |                               std::vector<std::string>& keys) const {
132 |   std::vector<Slice> slice_keys;
133 |   table_.GetKeysBySection(section, slice_keys);
134 |   for (std::vector<Slice>::iterator it = slice_keys.begin();
135 |       it != slice_keys.end(); ++it) {
136 |     keys.push_back(it->ToStr());
137 |   }
138 | }
139 | 
140 | 
141 | int Config::ReadItem(const std::string& section, const std::string& key,
142 |                     const char* defaultvalue, std::string& itemvalue) const {
143 |   if (section.size() <= 0 || key.size() <= 0 ) return -1;
144 |   std::string value;
145 |   Slice idxsec = section;
146 |   Slice idxkey = key;
147 |   Slice val = table_.Get(idxsec, idxkey);
148 | 
149 |   if (val.start() != NULL) {
150 |     value.assign(val.start(), val.end() - val.start());
151 |     std::stringstream ss(value, std::ios_base::in);
152 |     itemvalue = ss.str();
153 |     return 0;
154 |   } else {
155 |     std::stringstream ss;
156 |     ss << defaultvalue;
157 |     itemvalue = ss.str();
158 |     return 1;
159 |   }
160 | }
161 | 
162 | }  // namespace libwxfreq
163 | 
164 | 


--------------------------------------------------------------------------------
/util/config.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_CONFIG_H_
18 | #define FREQLIB_UTIL_CONFIG_H_
19 | 
20 | #include <string>
21 | #include <vector>
22 | #include <sstream>
23 | #include "config_hash.h"
24 | 
25 | namespace libwxfreq {
26 | 
27 | class Config {
28 |  public:
29 |   Config() { }
30 |   ~Config() { }
31 |   explicit Config(const std::string &configfile);
32 | 
33 |   int Init();
34 |   const std::string & conf_filename();
35 |   void set_conf_filename(const std::string &conf_filename);
36 | 
37 |   void section_list(std::vector<std::string>& sectionlist);
38 |   void GetKeysBySection(const std::string &section,
39 |                         std::vector<std::string>& keys) const;
40 | 
41 |   template <typename T1, typename T2>
42 |   int ReadItem(const std::string& section, const std::string& key,
43 |               const T1& defaultvalue, T2& itemvalue) const;
44 | 
45 |   int ReadItem(const std::string& section, const std::string& key,
46 |               const char* defaultvalue, std::string& itemvalue) const;
47 | 
48 |  private:
49 |   int Reset();
50 |   int ParserConfig();
51 |   int LoadFile(void);
52 | 
53 |   std::string conf_filename_;
54 |   std::string conf_filename_buf_;
55 |   std::vector<Slice> section_list_;
56 |   ConfigHashTable table_;
57 | };
58 | 
59 | template <typename T1, typename T2>
60 | int Config::ReadItem(const std::string& section, const std::string& key,
61 |                     const T1& defaultvalue, T2& itemvalue) const {
62 |   if (section.size() <= 0 || key.size() <= 0 ) return -1;
63 | 
64 |   std::string value;
65 |   Slice idxsec = section;
66 |   Slice idxkey = key;
67 |   Slice val = table_.Get(idxsec, idxkey);
68 | 
69 |   if (val.start() != NULL) {
70 |     value.assign(val.start(), val.end() - val.start());
71 |     std::stringstream ss(value, std::ios_base::in);
72 |     ss >> itemvalue;
73 |     return 0;
74 |   } else {
75 |     std::stringstream ss;
76 |     ss << defaultvalue;
77 |     ss >> itemvalue;
78 |     return 1;
79 |   }
80 | }
81 | }  // namespace libwxfreq
82 | 
83 | #endif  // FREQLIB_UTIL_CONFIG_H_
84 | 


--------------------------------------------------------------------------------
/util/config_hash.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include <stdlib.h>
 18 | #include "config_hash.h"
 19 | 
 20 | namespace libwxfreq {
 21 | 
 22 | // ConfigHashNode
 23 | ConfigHashNode::ConfigHashNode():hash_num_(0) { }
 24 | 
 25 | ConfigHashNode::ConfigHashNode(const unsigned int hash_num, const Slice &sec,
 26 |                  const Slice &key, const Slice &val) {
 27 |   hash_num_ = hash_num;
 28 |   section_ = sec;
 29 |   key_ = key;
 30 |   value_ = val;
 31 | }
 32 | 
 33 | inline unsigned ConfigHashNode::hash_num() const {
 34 |   return hash_num_;
 35 | }
 36 | 
 37 | inline const Slice& ConfigHashNode::section() const {
 38 |   return section_;
 39 | }
 40 | 
 41 | inline const Slice& ConfigHashNode::key() const {
 42 |   return key_;
 43 | }
 44 | 
 45 | inline const Slice & ConfigHashNode::value() const {
 46 |   return value_;
 47 | }
 48 | 
 49 | // ConfigHashTable
 50 | ConfigHashTable::ConfigHashTable() {
 51 | }
 52 | 
 53 | ConfigHashTable::ConfigHashTable(const ConfigHashTable &other) {
 54 |   table_ = other.table_;
 55 | }
 56 | 
 57 | ConfigHashTable::~ConfigHashTable() {
 58 | }
 59 | 
 60 | void ConfigHashTable::Clear() {
 61 |   table_.clear();
 62 | }
 63 | 
 64 | // using FNV1 hash function
 65 | inline unsigned int ConfigHashTable::HashFun(const Slice &sec,
 66 |                                             const Slice &key) const {
 67 |   static const unsigned int FNV1_BASIS = 2166136261lu;
 68 |   static const unsigned int FNV1_PRIME = 16777619;
 69 |   unsigned int ret = FNV1_BASIS;
 70 |   const char *ptr, *endptr;
 71 | 
 72 |   endptr = sec.end();
 73 |   for (ptr = sec.start(); ptr < endptr; ptr++) {
 74 |     ret *= FNV1_PRIME;
 75 |     ret ^= tolower(*ptr);
 76 |   }
 77 | 
 78 |   endptr = key.end();
 79 |   for (ptr = key.start(); ptr < endptr; ptr++) {
 80 |     ret *= FNV1_PRIME;
 81 |     ret ^= tolower(*ptr);
 82 |   }
 83 |   return ret;
 84 | }
 85 | 
 86 | void ConfigHashTable::Add(const Slice &sec, const Slice &key,
 87 |                           const Slice &val) {
 88 |   unsigned int hashnum = HashFun(sec, key);
 89 |   table_[hashnum].push_back(ConfigHashNode(hashnum, sec, key, val));
 90 | 
 91 |   std::vector<ConfigHashNode>::iterator it = table_[hashnum].end() - 1;
 92 |   unsigned int sec_hashnum = HashFun(sec, Slice());
 93 |   keys_table_[sec_hashnum].push_back(it);
 94 | }
 95 | 
 96 | Slice ConfigHashTable::Get(const Slice &sec, const Slice &key) const {
 97 |   unsigned int hashnum = HashFun(sec, key);
 98 | 
 99 |   HashTable::const_iterator tit = table_.find(hashnum);
100 | 
101 |   if (tit == table_.end()) return Slice();
102 | 
103 |   for (std::vector<ConfigHashNode>::const_iterator it = tit->second.begin();
104 |       it != tit->second.end(); ++it) {
105 |     if (it->hash_num() == hashnum && it->section() == sec && it->key() == key) {
106 |       return it->value();
107 |     }
108 |   }
109 | 
110 |   return Slice();
111 | }
112 | 
113 | void ConfigHashTable::GetKeysBySection(const Slice &sec,
114 |                                       std::vector<Slice> &keys) const {
115 |   unsigned int hashnum = HashFun(sec, Slice());
116 |   HashIteratorTable::const_iterator tit = keys_table_.find(hashnum);
117 | 
118 |   if (tit == keys_table_.end()) return;
119 | 
120 |   for (std::vector<std::vector<ConfigHashNode>::iterator>::const_iterator
121 |       it = tit->second.begin(); it != tit->second.end(); ++it) {
122 |     if ((*it)->section() == sec) {
123 |       keys.push_back((*it)->key());
124 |     }
125 |   }
126 | }
127 | 
128 | bool ConfigHashTable::Empty() {
129 |   return table_.empty();
130 | }
131 | 
132 | }  // namespace libwxfreq
133 | 
134 | 


--------------------------------------------------------------------------------
/util/config_hash.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_CONFIG_HASH_H_
18 | #define FREQLIB_UTIL_CONFIG_HASH_H_
19 | 
20 | #include<vector>
21 | #include<string>
22 | #include<map>
23 | #include "slice.h"
24 | 
25 | namespace libwxfreq {
26 | 
27 | class ConfigHashNode {
28 |  public:
29 |   ConfigHashNode();
30 |   ConfigHashNode(const unsigned int hash_num, const Slice &sec,
31 |                  const Slice &key, const Slice &val);
32 |   unsigned hash_num() const;
33 |   const Slice & section() const;
34 |   const Slice & key() const;
35 |   const Slice & value() const;
36 | 
37 |  private:
38 |   unsigned int hash_num_;
39 |   Slice section_, key_, value_;
40 | };
41 | 
42 | 
43 | 
44 | class ConfigHashTable {
45 |  public:
46 |   ConfigHashTable();
47 |   ConfigHashTable(const ConfigHashTable &other);
48 |   ~ConfigHashTable();
49 | 
50 | 
51 |   void Clear();
52 |   void Add(const Slice &sec, const Slice &key, const Slice &val);
53 |   Slice Get(const Slice &sec, const Slice &key) const;
54 |   void GetKeysBySection(const Slice &sec, std::vector<Slice> &keys) const;
55 |   bool Empty();
56 | 
57 |  private:
58 |   typedef std::map<unsigned int, std::vector<ConfigHashNode> > HashTable;
59 |   typedef std::map<unsigned int, std::vector<std::vector<ConfigHashNode>::iterator> > HashIteratorTable;
60 |   // using FNV1 hash function
61 |   inline unsigned int HashFun(const Slice &sec, const Slice &key) const;
62 |   HashTable table_;
63 |   HashIteratorTable keys_table_;
64 | };
65 | 
66 | }  // namespace libwxfreq
67 | 
68 | #endif  // FREQLIB_UTIL_CONFIG_HASH_H_
69 | 


--------------------------------------------------------------------------------
/util/freq_stat.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_FREQ_STAT_H_
18 | #define FREQLIB_UTIL_FREQ_STAT_H_
19 | #include <stdint.h>
20 | #include <string>
21 | 
22 | namespace libwxfreq {
23 | class FreqItem;
24 | 
25 | class FreqStat {
26 |  public:
27 |   virtual uint32_t& GetItem(const std::string& key, const uint32_t appid,
28 |                             const uint32_t cnt, FreqItem* &freq_item) = 0;
29 |   virtual bool TryReload() = 0;
30 |   void set_type_name(const std::string& type_name) {
31 |     type_name_ = type_name;
32 |   }
33 |   
34 |   const std::string& type_name() const {
35 |     return type_name_;
36 |   }
37 | 
38 |   virtual ~FreqStat() { }
39 |  protected:
40 |   std::string type_name_;
41 | };
42 | 
43 | }  // namespace libwxfreq
44 | 
45 | #endif  // FREQLIB_UTIL_FREQ_STAT_H_
46 | 


--------------------------------------------------------------------------------
/util/log.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "log.h"
 18 | #include <stdint.h>
 19 | #include <sys/time.h>
 20 | #include <sys/types.h>
 21 | #include <sys/syscall.h>
 22 | #include <time.h>
 23 | #include <stdio.h>
 24 | #include <assert.h>
 25 | #include <unistd.h>
 26 | #include <errno.h>
 27 | #include <stdarg.h>
 28 | 
 29 | namespace libwxfreq {
 30 | 
 31 | static const char* kDefaultLogFileName = "/tmp/libwxfreq.log";
 32 | 
 33 | static inline pid_t gettid() {
 34 |   return syscall(SYS_gettid);
 35 | }
 36 | 
 37 | static void Logv(FILE *file, const char* format, va_list ap) {
 38 |   const uint64_t thread_id = gettid();
 39 | 
 40 |   char buffer[500];
 41 |   for (int iter = 0; iter < 2; iter++) {
 42 |     char* base;
 43 |     int bufsize;
 44 |     if (iter == 0) {
 45 |       bufsize = sizeof(buffer);
 46 |       base = buffer;
 47 |     } else {
 48 |       bufsize = 30000;
 49 |       base = new char[bufsize];
 50 |     }
 51 |     char* p = base;
 52 |     char* limit = base + bufsize;
 53 | 
 54 |     struct timeval now_tv;
 55 |     gettimeofday(&now_tv, NULL);
 56 |     const time_t seconds = now_tv.tv_sec;
 57 |     struct tm t;
 58 |     localtime_r(&seconds, &t);
 59 |     p += snprintf(p, limit - p,
 60 |                   "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ",
 61 |                   t.tm_year + 1900,
 62 |                   t.tm_mon + 1,
 63 |                   t.tm_mday,
 64 |                   t.tm_hour,
 65 |                   t.tm_min,
 66 |                   t.tm_sec,
 67 |                   static_cast<int>(now_tv.tv_usec),
 68 |                   static_cast<uint64_t>(thread_id));
 69 | 
 70 |     // Print the message
 71 |     if (p < limit) {
 72 |       va_list backup_ap;
 73 |       va_copy(backup_ap, ap);
 74 |       p += vsnprintf(p, limit - p, format, backup_ap);
 75 |       va_end(backup_ap);
 76 |     }
 77 | 
 78 |     // Truncate to available space if necessary
 79 |     if (p >= limit) {
 80 |       if (iter == 0) {
 81 |         continue;       // Try again with larger buffer
 82 |       } else {
 83 |         p = limit - 1;
 84 |       }
 85 |     }
 86 | 
 87 |     // Add newline if necessary
 88 |     if (p == base || p[-1] != '\n') {
 89 |       *p++ = '\n';
 90 |     }
 91 | 
 92 |     assert(p <= limit);
 93 |     fwrite(base, 1, p - base, file);
 94 |     fflush(file);
 95 |     if (base != buffer) {
 96 |       delete[] base;
 97 |     }
 98 |     break;
 99 |   }
100 | }
101 | 
102 | static int Log(const char* format, ...) {
103 |   static FILE* file = fopen(kDefaultLogFileName, "a");
104 |   if (file == NULL) return -1;
105 |   va_list ap;
106 |   va_start(ap, format);
107 |   Logv(file, format, ap);
108 |   va_end(ap);
109 |   return 0;
110 | }
111 | 
112 | LogFunc gLog = Log;
113 | 
114 | }  //  namespace libwxfreq
115 | 
116 | 


--------------------------------------------------------------------------------
/util/log.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_LOG_H_
18 | #define FREQLIB_UTIL_LOG_H_
19 | 
20 | namespace libwxfreq {
21 | 
22 | typedef int (*LogFunc)(const char* format, ...);
23 | 
24 | extern LogFunc gLog;
25 | 
26 | }  // namespace libwxfreq
27 | #endif  // FREQLIB_UTIL_LOG_H_
28 | 


--------------------------------------------------------------------------------
/util/map_freq_stat.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "map_freq_stat.h"
18 | #include <cstring>
19 | #include "reload.h"
20 | 
21 | namespace libwxfreq {
22 | static uint32_t dummy_timestamp = 0;
23 | 
24 | bool FreqKeyCmp(const FreqKey&a, const FreqKey&b) {
25 |   if (a.appid < b.appid) {
26 |     return true;
27 |   } else if (a.appid == b.appid) {
28 |     return strncmp(a.key, b.key, sizeof(a.key)) < 0;
29 |   }
30 |   return false;
31 | }
32 | 
33 | uint32_t& MapFreqStat::GetItem(const std::string& key, const uint32_t appid,
34 |                                const uint32_t cnt, FreqItem* &freq_item) {
35 |   FreqKey freqkey;
36 |   freqkey.appid = appid;
37 |   strncpy(freqkey.key, key.c_str(), sizeof(freqkey.key));
38 |   if (cnt != 0) {
39 |     FreqItemForMap& freqitemformap = item_map[freqkey];
40 |     freq_item = reinterpret_cast<FreqItem*>(&freqitemformap);
41 |     return freqitemformap.timestamp;
42 |   } else {
43 |     ItemMap::iterator it = item_map.find(freqkey);
44 |     if (it == item_map.end()) {
45 |       freq_item = NULL;
46 |       return dummy_timestamp;
47 |     } else {
48 |       freq_item = reinterpret_cast<FreqItem*>(&(it->second));
49 |       return it->second.timestamp;
50 |     }
51 |   }
52 | }
53 | 
54 | bool MapFreqStat::TryReload() {
55 |   return gLoadFunc(type_name_.c_str()) == 0;
56 | }
57 | 
58 | }  // namespace libwxfreq
59 | 


--------------------------------------------------------------------------------
/util/map_freq_stat.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_MAP_FREQ_STAT_H_
18 | #define FREQLIB_UTIL_MAP_FREQ_STAT_H_
19 | #include <map>
20 | #include <string>
21 | #include "freq_item.h"
22 | #include "freq_stat.h"
23 | 
24 | namespace libwxfreq {
25 | #pragma pack(1)
26 | struct FreqItemForMap : public FreqItem {
27 |   uint32_t timestamp;
28 | };
29 | #pragma pack()
30 | bool FreqKeyCmp(const FreqKey&a, const FreqKey&b);
31 | 
32 | class MapFreqStat : public FreqStat {
33 |  public:
34 |   MapFreqStat() : item_map(FreqKeyCmp) { }
35 |   virtual uint32_t& GetItem(const std::string& key, const uint32_t appid,
36 |                             const uint32_t cnt, FreqItem* &freq_item);
37 |   virtual bool TryReload();
38 | 
39 |  private:
40 |   typedef bool (*cmp) (const FreqKey&a, const FreqKey&b);
41 |   typedef std::map<FreqKey, FreqItemForMap, cmp> ItemMap;
42 |   ItemMap item_map;
43 | };
44 | 
45 | }  // namespace libwxfreq
46 | 
47 | #endif  // FREQLIB_UTIL_MAP_FREQ_STAT_H_
48 | 


--------------------------------------------------------------------------------
/util/multi_hash_base.h:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #ifndef FREQLIB_UTIL_MULTI_HASH_BASE_H_
 18 | #define FREQLIB_UTIL_MULTI_HASH_BASE_H_
 19 | 
 20 | #include <stdio.h>
 21 | #include <stdint.h>
 22 | #include <stdlib.h>
 23 | #include <unistd.h>
 24 | #include <string.h>
 25 | #include <math.h>
 26 | #include <vector>
 27 | #include <iostream>
 28 | #include "log.h"
 29 | 
 30 | namespace libwxfreq {
 31 | 
 32 | static const unsigned int   MAX_LEVEL_CNT       = 30;
 33 | static const unsigned int   MAIN_LEVEL_CNT      = 5;
 34 | static const float    MAIN_LEVEL_ITEM_RATE  = 0.9;
 35 | 
 36 | 
 37 | template <typename Key, typename Value>
 38 | class MultiHashBase {
 39 | 
 40 |  enum MultiHashRet {
 41 |   enMultiHashKeyNotFound    = -3,
 42 |   enMultiHashBucketFull   = -2,
 43 |   enMultiHashNotInit      = -1,
 44 |   enMultiHashOK         = 0,
 45 |   enMultiHashFoundEmptyItem   = 1,
 46 |   enMultiHashFoundExpiredItem = 2,
 47 | };
 48 | 
 49 |  public:
 50 |   MultiHashBase();
 51 |   virtual ~MultiHashBase();
 52 |   typedef Key KeyType;
 53 |   typedef Value ValueType;
 54 | 
 55 |  public:
 56 |   int MultiHashInit(char *base, size_t len,
 57 |                     unsigned int max_level = MAX_LEVEL_CNT,
 58 |                     unsigned int main_level = MAIN_LEVEL_CNT);
 59 | 
 60 |   int GetValueForRead(const KeyType &input_key, ValueType *&value);
 61 |   int GetValueForWrite(const KeyType &input_key, ValueType *&value);
 62 | 
 63 |  protected:
 64 |   virtual int InitLevelHash();
 65 | 
 66 |   unsigned int item_size() { return item_size_ ;}
 67 |   virtual size_t HashKey(const KeyType &input_key) = 0;
 68 |   virtual bool KeyCmp(const KeyType &input_key, const KeyType &key_in_mem) = 0;
 69 |   virtual bool IsExpired(const KeyType &key, const ValueType &value,
 70 |                         void *old_value) = 0;
 71 |   virtual bool ExpiredOccupy(const KeyType &input_key, KeyType &key_in_mem,
 72 |                             ValueType &value_in_mem, void *old_value) = 0;
 73 |   virtual bool IsEmpty(const KeyType &key, const ValueType &value) = 0;
 74 |   virtual bool EmptyOccupy(const KeyType &input_key, KeyType &key_in_mem) = 0;
 75 |   virtual void AfterFindReadItem(ValueType &value) = 0;
 76 |   virtual void AfterFindWriteItem(ValueType &value) = 0;
 77 | 
 78 |  private:
 79 |   char  *shm_mem_base_;
 80 |   unsigned int  max_level_;
 81 |   unsigned int  main_level_;
 82 |   unsigned int  item_size_;
 83 |   size_t shm_mem_len_;
 84 |   size_t max_item_cnt_;
 85 |   std::vector<size_t> size_meta_;
 86 | 
 87 |  private:
 88 |   static size_t GetBiggestPrimer(size_t ulluppervalue);
 89 |   static bool IsPrimer(size_t ullvalue);
 90 | };
 91 | 
 92 | 
 93 | template <typename Key, typename Value>
 94 | MultiHashBase<Key, Value>::MultiHashBase()
 95 |     : shm_mem_base_(NULL),
 96 |       shm_mem_len_(0),
 97 |       max_level_(MAX_LEVEL_CNT),
 98 |       main_level_(MAIN_LEVEL_CNT) {
 99 |   item_size_ = sizeof(KeyType) + sizeof(ValueType);
100 | }
101 | 
102 | template <typename Key, typename Value>
103 | MultiHashBase<Key, Value>::~MultiHashBase() { }
104 | 
105 | 
106 | template <typename Key, typename Value>
107 | int MultiHashBase<Key, Value>::MultiHashInit(char *base, size_t len,
108 |                                              unsigned int max_level,
109 |                                              unsigned int main_level) {
110 |   shm_mem_base_ = base;
111 |   shm_mem_len_ = len;
112 |   max_level_ = max_level >= 2 * MAX_LEVEL_CNT ? 2 * MAX_LEVEL_CNT : max_level;
113 |   max_level_ = max_level_ <= MAX_LEVEL_CNT / 2 ? MAX_LEVEL_CNT/ 2 : max_level_;
114 |   main_level_ = main_level >= MAIN_LEVEL_CNT ? MAIN_LEVEL_CNT : main_level;
115 |   main_level_ = main_level_<= MAIN_LEVEL_CNT / 2 ? MAIN_LEVEL_CNT / 2 : main_level_;
116 |   main_level_ = main_level_ <= max_level_ ? main_level_ : max_level_ - 1;
117 | 
118 |   max_item_cnt_ = len / item_size_;
119 |   size_meta_.reserve(max_level_);
120 |   int ret = InitLevelHash();
121 |   if (0 != ret) {
122 |     return -__LINE__;
123 |   }
124 |   return enMultiHashOK;
125 | }
126 | 
127 | template <typename Key, typename Value>
128 | int MultiHashBase<Key, Value>::InitLevelHash() {
129 |   unsigned int conflict_level = max_level_ - main_level_;
130 |   size_t level_size =
131 |     (size_t)(MAIN_LEVEL_ITEM_RATE * max_item_cnt_ / main_level_);
132 |     gLog("[%s][%d]: level_size %llu rate %f cnt %llu\n", __FILE__, __LINE__, 
133 |           level_size, MAIN_LEVEL_ITEM_RATE, max_item_cnt_);
134 |   
135 |   size_t current_level_size = 0;
136 |   size_t had_push_item_cnt = 0;
137 | 
138 |   for (int i = 0; i < max_level_; i++) {
139 |     if (i == max_level_ -1) {
140 |       size_meta_[i] = max_item_cnt_ - had_push_item_cnt;
141 |       break;
142 |     }
143 | 
144 |     if (level_size  <= 2) {
145 |       max_level_ = i + 1;
146 |       size_meta_[i] = max_item_cnt_ - had_push_item_cnt;
147 |       return enMultiHashOK;
148 |     }
149 | 
150 |     current_level_size = GetBiggestPrimer(level_size);
151 |     size_meta_[i] = current_level_size;
152 |     had_push_item_cnt += current_level_size;
153 |     if (i + 1 == main_level_) {
154 |       level_size = (max_item_cnt_ - had_push_item_cnt) / conflict_level;
155 |     } else {
156 |       level_size = current_level_size - 1;
157 |     }
158 |   }
159 | 
160 |   for (int i = 0; i < max_level_; i++) {
161 |     gLog("[%s][%d]: level %d size %llu\n", __FILE__, __LINE__, 
162 |           i, size_meta_[i]);
163 |   }
164 |   return enMultiHashOK;
165 | }
166 | 
167 | template <typename Key, typename Value>
168 | int MultiHashBase<Key, Value>::GetValueForRead(const KeyType &input_key,
169 |                                               ValueType *&value) {
170 |   int     ret = enMultiHashKeyNotFound;
171 |   size_t offset = 0;
172 |   unsigned int  find_level = 0;
173 |   size_t hash_key = HashKey(input_key);
174 |   for (find_level = 0; find_level < max_level_; find_level++) {
175 |     size_t index = hash_key % size_meta_[find_level];
176 | 
177 |     KeyType* key_in_mem =
178 |       reinterpret_cast<KeyType *>(shm_mem_base_ + offset + (item_size_ * index));
179 |     ValueType* value_in_mem = reinterpret_cast<ValueType *>(
180 |         reinterpret_cast<char *>(key_in_mem) + sizeof(KeyType));
181 | 
182 |     if (KeyCmp(input_key, *key_in_mem) == true) {
183 |       uint32_t timestamp = 0;
184 |       if (IsExpired(*key_in_mem, *value_in_mem, &timestamp)) {
185 |         ret = enMultiHashKeyNotFound;
186 |         break;
187 |       }
188 |       value = value_in_mem;
189 |       ret = enMultiHashOK;
190 |       AfterFindReadItem(*value_in_mem);
191 |       break;
192 |     } else if (IsEmpty(*key_in_mem, *value_in_mem) == true) {
193 |       ret = enMultiHashKeyNotFound;
194 |       break;
195 |     }
196 |     offset += size_meta_[find_level] * item_size_;
197 |   }
198 | 
199 |   return ret;
200 | }
201 | 
202 | template <typename Key, typename Value>
203 | int MultiHashBase<Key, Value>::GetValueForWrite(const KeyType &input_key,
204 |                                                 ValueType *&value) {
205 |   int ret = enMultiHashBucketFull;
206 |   ValueType * read_value_ptr;
207 |   ret = GetValueForRead(input_key, read_value_ptr);
208 |   if (ret == enMultiHashOK) {
209 |     value = read_value_ptr;
210 |     AfterFindWriteItem(*value);
211 |     return ret;
212 |   }
213 | 
214 |   size_t offset  = 0;
215 |   unsigned int  find_level    = 0;
216 |   ret = enMultiHashBucketFull;
217 |   size_t hash_key = HashKey(input_key);
218 |   for (find_level = 0; find_level < max_level_; find_level++) {
219 |     size_t index = hash_key % size_meta_[find_level];
220 |     KeyType* key_in_mem =
221 |       reinterpret_cast<KeyType *>(shm_mem_base_ + offset + (item_size_ * index));
222 |     ValueType* value_in_mem = reinterpret_cast<ValueType *>(
223 |         reinterpret_cast<char *>(key_in_mem) + sizeof(KeyType));
224 | 
225 |     uint32_t timestamp = 0;
226 |     if (IsEmpty(*key_in_mem, *value_in_mem) == true) {
227 |       if (EmptyOccupy(input_key, *key_in_mem) == true) {
228 |         memset(reinterpret_cast<void *>(value_in_mem), 0 , sizeof(ValueType));
229 |         ret = enMultiHashOK;
230 |         value = value_in_mem;
231 |         break;
232 |       }
233 |     } else if (IsExpired(*key_in_mem, *value_in_mem, &timestamp) == true) {
234 |       if (ExpiredOccupy(input_key, *key_in_mem, *value_in_mem, &timestamp) == true) {
235 |         memset(reinterpret_cast<void *>(value_in_mem), 0 , sizeof(ValueType));
236 |         ret = enMultiHashOK;
237 |         value = value_in_mem;
238 |         break;
239 |       }
240 |     }
241 |     offset += size_meta_[find_level] * item_size_;
242 |   }
243 |   if (ret == enMultiHashOK) {
244 |     gLog("[%s][%d]: level %d hash %llu\n", __FILE__, __LINE__, 
245 |           find_level, hash_key);
246 |   } else {
247 |     gLog("[%s][%d]: not found hash %llu\n", __FILE__, __LINE__, hash_key);
248 |   }
249 |   return ret;
250 | }
251 | 
252 | template <typename Key, typename Value>
253 | bool MultiHashBase<Key, Value>::IsPrimer(size_t ullValue) {
254 |   if (0 == (ullValue % 2)) {
255 |     return false;
256 |   }
257 | 
258 |   size_t ullEnd = (size_t)sqrt(ullValue) + 1;
259 |   if (ullEnd > (ullValue / 2)) {
260 |     ullEnd = ullValue / 2;
261 |   }
262 | 
263 |   for (size_t i = 3; i <= ullEnd; i++) {
264 |     if (0 == (ullValue % i)) {
265 |         return false;
266 |     }
267 |   }
268 | 
269 |   return true;
270 | }
271 | 
272 | template <typename Key, typename Value>
273 | size_t MultiHashBase<Key, Value>::GetBiggestPrimer(
274 |     size_t ullUpperValue) {
275 |   for (size_t i = ullUpperValue; i > 1; i--) {
276 |     if (IsPrimer(i)) {
277 |         return i;
278 |     }
279 |   }
280 |   return 1;
281 | }
282 | 
283 | }  // namespace libwxfreq
284 | 
285 | #endif  // FREQLIB_UTIL_MULTI_HASH_BASE_H_
286 | 
287 | 


--------------------------------------------------------------------------------
/util/multi_hash_table.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "multi_hash_table.h"
 18 | #include <errno.h>
 19 | #include <string.h>
 20 | #include <stdlib.h>
 21 | #include <stdint.h>
 22 | #include <stdio.h>
 23 | namespace libwxfreq {
 24 | 
 25 | static size_t BKDRHash(const char *str) {
 26 |   unsigned int seed = 131;  // 31 131 1313 13131 131313 etc..
 27 |   unsigned int hash = 0;
 28 |   while (*str) {
 29 |     hash = hash * seed + (*str++);
 30 |   }
 31 |   return hash;
 32 | }
 33 | 
 34 | static size_t IntHash(uint32_t appid) {
 35 |   register size_t key = appid;
 36 |   return key * 2654435761;
 37 | }
 38 | 
 39 | MultiHashTable::MultiHashTable():init_flag_(NULL) { }
 40 | MultiHashTable::~MultiHashTable() { }
 41 | 
 42 | int MultiHashTable::MultiHashTableInit(bool zero_init, key_t shmkey,
 43 |                                        int shmflag, uint64_t item_cnt) {
 44 |   size_t shm_size = (size_t)(item_size() * item_cnt + 1);
 45 | 
 46 |   int iShmID = 0;
 47 |   iShmID = ::shmget(shmkey, shm_size, shmflag | IPC_CREAT);
 48 |   if (iShmID < 0 && errno == EINVAL) {
 49 |     // 共享内存大小发生变化
 50 |     iShmID = ::shmget(shmkey, 0, 0);
 51 |     if (iShmID == -1) return -__LINE__;
 52 |     if (::shmctl(iShmID, IPC_RMID, NULL) == -1) return -__LINE__;
 53 |     iShmID = ::shmget(shmkey, shm_size, shmflag | IPC_CREAT);
 54 |   }
 55 |   if (iShmID <0) return -__LINE__;
 56 | 
 57 |   char *base = NULL;
 58 |   base = reinterpret_cast<char *>(::shmat(iShmID, NULL, 0));
 59 |   if (reinterpret_cast<char *>(-1) == base) {
 60 |     return -__LINE__;
 61 |   }
 62 | 
 63 |   init_flag_ = base + shm_size - 1;
 64 |   if (zero_init) {
 65 |     memset(base, 0, shm_size);
 66 |   }
 67 | 
 68 |   return MultiHashInit(base, shm_size);
 69 | }
 70 | 
 71 | size_t MultiHashTable::HashKey(const KeyType &input_key) {
 72 |   return BKDRHash(input_key.key) * IntHash(input_key.appid);
 73 | }
 74 | 
 75 | bool MultiHashTable::KeyCmp(const KeyType &input_key,
 76 |                             const KeyType &key_in_mem) {
 77 |   size_t size = sizeof(input_key.key);
 78 |   uint32_t input_hash = *reinterpret_cast<const uint32_t*>(
 79 |       input_key.key + size - sizeof(uint32_t));
 80 |   uint32_t mem_hash = *reinterpret_cast<const uint32_t*>(
 81 |       key_in_mem.key + size - sizeof(uint32_t));
 82 |   return input_key.appid == key_in_mem.appid &&
 83 |          input_hash == mem_hash &&
 84 |          strncmp(key_in_mem.key, input_key.key, size) == 0;
 85 | }
 86 | 
 87 | bool MultiHashTable::IsExpired(const KeyType &input_key,
 88 |                               const ValueType &value, void *old_value) {
 89 |   *reinterpret_cast<uint32_t*>(old_value)  = value.timestamp;
 90 |   return value.timestamp != 0 && time(NULL) - value.timestamp >= 90000;
 91 | }
 92 | 
 93 | bool MultiHashTable::IsEmpty(const KeyType &input_key,
 94 |                             const ValueType &/*value*/) {
 95 |   return input_key.appid == 0;
 96 | }
 97 | void MultiHashTable::AfterFindReadItem(ValueType &value) { }
 98 | void MultiHashTable::AfterFindWriteItem(ValueType &value) { }
 99 | 
100 | bool MultiHashTable::ExpiredOccupy(const KeyType &input_key,
101 |                                   KeyType &key_in_mem, ValueType &value_in_mem,
102 |                                   void *old_value) {
103 |   if (__sync_bool_compare_and_swap(&value_in_mem.timestamp,
104 |                                   *reinterpret_cast<uint32_t*>(old_value), 0)) {
105 |     memcpy(key_in_mem.key, input_key.key, sizeof(input_key.key));
106 |     key_in_mem.appid = input_key.appid;
107 |     return true;
108 |   }
109 |   return false;
110 | }
111 | 
112 | bool MultiHashTable::EmptyOccupy(const KeyType &input_key,
113 |                                 KeyType &key_in_mem) {
114 |   if (__sync_bool_compare_and_swap(&key_in_mem.appid, 0, input_key.appid)) {
115 |     memcpy(key_in_mem.key, input_key.key, sizeof(input_key.key));
116 |     return true;
117 |   }
118 |   return false;
119 | }
120 | 
121 | bool MultiHashTable::IsValid() {
122 |   return init_flag_!= NULL && *init_flag_ == 17;
123 | }
124 | 
125 | void MultiHashTable::MarkAsValid() {
126 |   if (init_flag_ != NULL) *init_flag_ = 17;
127 | }
128 | 
129 | void MultiHashTable::MarkAsInValid() {
130 |   if (init_flag_ != NULL) *init_flag_ = 0;
131 | }
132 | 
133 | }  //  namespace libwxfreq
134 | 
135 | 


--------------------------------------------------------------------------------
/util/multi_hash_table.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_MULTI_HASH_TABLE_H_
18 | #define FREQLIB_UTIL_MULTI_HASH_TABLE_H_
19 | 
20 | #include <sys/shm.h>
21 | #include <sys/stat.h>
22 | #include "multi_hash_base.h"
23 | #include "freq_item.h"
24 | 
25 | namespace libwxfreq {
26 | 
27 | #pragma pack(1)
28 | struct FreqItemForShm:public libwxfreq::FreqItem {
29 |   uint32_t timestamp;
30 | };
31 | #pragma pack()
32 | 
33 | class MultiHashTable : public MultiHashBase<libwxfreq::FreqKey, FreqItemForShm> {
34 |  public:
35 |   MultiHashTable();
36 |   virtual ~MultiHashTable();
37 | 
38 |  public:
39 |   int MultiHashTableInit(bool zero_init, key_t shmkey, int shmflag,
40 |                         uint64_t item_cnt);
41 |   bool IsValid();
42 |   void MarkAsValid();
43 |   void MarkAsInValid();
44 | 
45 |  protected:
46 |   size_t HashKey(const KeyType &input_key);
47 |   bool KeyCmp(const KeyType &input_key, const KeyType &key_in_mem);
48 |   bool IsExpired(const KeyType &input_key, const ValueType &value,
49 |                 void *old_value);
50 |   bool IsEmpty(const KeyType &input_key, const ValueType &value);
51 |   void AfterFindReadItem(ValueType &value);
52 |   void AfterFindWriteItem(ValueType &value);
53 |   bool ExpiredOccupy(const KeyType &input_key, KeyType &key_in_mem,
54 |                     ValueType &value_in_mem, void *old_value);
55 |   bool EmptyOccupy(const KeyType &input_key, KeyType &key_in_mem);
56 | 
57 |  private:
58 |   char * init_flag_;
59 | };
60 | 
61 | }  // namespace libwxfreq
62 | #endif  // FREQLIB_UTIL_MULTI_HASH_TABLE_H_
63 | 


--------------------------------------------------------------------------------
/util/options.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "options.h"
18 | 
19 | namespace libwxfreq {
20 | 
21 | extern LogFunc gLog;
22 | extern LoadFunc gLoadFunc;
23 | extern DumpFunc gDumpFunc;
24 | 
25 | void SetLog(LogFunc log) {
26 |   gLog = log;
27 | }
28 | 
29 | void SetLoadFunc(LoadFunc func) {
30 |   gLoadFunc = func;
31 | }
32 | 
33 | void SetDumpFunc(DumpFunc func) {
34 |   gDumpFunc = func;
35 | }
36 | 
37 | }  // namespace libwxfreq
38 | 


--------------------------------------------------------------------------------
/util/options.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_OPTIONS_H_
18 | #define FREQLIB_UTIL_OPTIONS_H_
19 | #include "log.h"
20 | #include "reload.h"
21 | 
22 | namespace libwxfreq {
23 | 
24 | void SetLog(LogFunc log);
25 | 
26 | void SetLoadFunc(LoadFunc func);
27 | void SetDumpFunc(DumpFunc func);
28 | 
29 | }  // namespace libwxfreq
30 | 
31 | #endif  // FREQLIB_UTIL_OPTIONS_H_
32 | 


--------------------------------------------------------------------------------
/util/reload.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "reload.h"
 18 | #include <sys/time.h>
 19 | #include <time.h>
 20 | #include <stdarg.h>
 21 | #include <string>
 22 | #include <sstream>
 23 | #include <fstream>
 24 | #include <cstdio>
 25 | 
 26 | #include "freq_manager.h"
 27 | #include "appid_manager.h"
 28 | #include "log.h"
 29 | 
 30 | namespace libwxfreq {
 31 | 
 32 | static const char* kDefaultWhiteFileNamePrefix = "/tmp/libwxfreq_db.";
 33 | 
 34 | static int LoadFromLocalFile(const char* type_name) {
 35 |   char filename[64];
 36 |   snprintf(filename, sizeof(filename), "%s%s",
 37 |           kDefaultWhiteFileNamePrefix, type_name);
 38 |   std::ifstream ifs;
 39 |   ifs.open(filename, std::fstream::in);
 40 | 
 41 |   if (!ifs.is_open()) {
 42 |     gLog("[%s][%d]: can not open %s\n", __FILE__, __LINE__, filename);
 43 |     return -1;
 44 |   }
 45 | 
 46 |   char line[128];
 47 |   std::string opname, key;
 48 |   uint32_t appid = 0;
 49 |   unsigned int expire_time = 0, block_level = 0;
 50 |   unsigned int now = time(NULL);
 51 |   while (ifs.getline(line, sizeof(line))) {
 52 |     std::stringstream ss(line);
 53 |     ss >> opname >> key >> appid >> expire_time >> block_level;
 54 | 
 55 |     if (opname == "AddWhite") {
 56 |       if (expire_time > now) {
 57 |         if (FreqManager::AddWhite(type_name, key, appid,
 58 |                                   expire_time - now) == false) {
 59 |           gLog("[%s][%d]: reload whitelist failed, typename %s key %s"
 60 |               " appid %u expiretime %u\n", __FILE__, __LINE__, type_name,
 61 |               key.c_str(), appid, expire_time);
 62 |           ifs.close();
 63 |           return -1;
 64 |         }
 65 |       }
 66 |     } else if (opname == "DeleteWhite") {
 67 |       if (FreqManager::DeleteWhite(type_name, key, appid) == false) {
 68 |         gLog("[%s][%d]: reload deletewhite failed, typename %s key %s"
 69 |             " appid %u\n", __FILE__, __LINE__, type_name, key.c_str(), appid);
 70 |         ifs.close();
 71 |         return -1;
 72 |       }
 73 |     } else if (opname == "AddBlock") {
 74 |       if (expire_time > now) {
 75 |         if (FreqManager::AddBlock(type_name, key, appid,
 76 |                                   expire_time - now, block_level) == false) {
 77 |           gLog("[%s][%d]: reload block failed, typename %s key %s appid %u"
 78 |               " expiretime %u blocklevel %u\n", __FILE__, __LINE__, type_name,
 79 |               key.c_str(), appid, expire_time, block_level);
 80 |           ifs.close();
 81 |           return -1;
 82 |         }
 83 |       }
 84 |     } else if (opname == "DeleteBlock") {
 85 |       if (FreqManager::DeleteBlock(type_name, key, appid) == false) {
 86 |         gLog("[%s][%d]: reload deleteblock failed, typename %s key %s"
 87 |             " appid %u\n", __FILE__, __LINE__, type_name, key.c_str(), appid);
 88 |         ifs.close();
 89 |         return -1;
 90 |       }
 91 |     } else {
 92 |       gLog("[%s][%d]: reload unknow typename %s\n", type_name);
 93 |     }
 94 |   }
 95 | 
 96 |   ifs.close();
 97 |   return 0;
 98 | }
 99 | 
100 | static int DumpToLocalFile(const char* opname, const char* type_name,
101 |                           const char *key, const uint32_t appid,
102 |                           const uint32_t linger_time,
103 |                           const uint32_t block_level) {
104 |   char filename[64];
105 |   snprintf(filename, sizeof(filename), "%s%s",
106 |           kDefaultWhiteFileNamePrefix, type_name);
107 |   static std::ofstream ofs(filename, std::fstream::out | std::fstream::app);
108 |   if (!ofs.is_open()) {
109 |     gLog("[%s][%d]: can not open %s\n", __FILE__, __LINE__, filename);
110 |     return -1;
111 |   }
112 |   if (linger_time <= 600) return 0;
113 |   ofs << opname << "\t" << key << "\t" <<  appid << "\t" <<
114 |     time(NULL) + linger_time << "\t" << block_level << std::endl;
115 |   return 0;
116 | }
117 | 
118 | LoadFunc gLoadFunc = LoadFromLocalFile;
119 | DumpFunc gDumpFunc = DumpToLocalFile;
120 | 
121 | }  // namespace libwxfreq
122 | 
123 | 


--------------------------------------------------------------------------------
/util/reload.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_RELOAD_H_
18 | #define FREQLIB_UTIL_RELOAD_H_
19 | #include <stdint.h>
20 | namespace libwxfreq {
21 | 
22 | typedef int (*LoadFunc)(const char* type_name);
23 | typedef int (*DumpFunc)(const char* opname, const char* type_name,
24 |                         const char *key, const uint32_t appid,
25 |                         const uint32_t linger_time,
26 |                         const uint32_t block_level);
27 | 
28 | extern LoadFunc gLoadFunc;
29 | extern DumpFunc gDumpFunc;
30 | 
31 | }  // namespace libwxfreq
32 | #endif  // FREQLIB_UTIL_RELOAD_H_
33 | 


--------------------------------------------------------------------------------
/util/shm_freq_stat.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | * Tencent is pleased to support the open source community by making libwxfreq available.
  3 | *
  4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
  5 | *
  6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
  7 | * except in compliance with the License. You may obtain a copy of the License at
  8 | *
  9 | *               https://opensource.org/licenses/BSD-3-Clause
 10 | *
 11 | * Unless required by applicable law or agreed to in writing, software distributed
 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 14 | * language governing permissions and limitations under the License.
 15 | */
 16 | 
 17 | #include "shm_freq_stat.h"
 18 | #include <stdio.h>
 19 | #include <stdlib.h>
 20 | #include <stdint.h>
 21 | #include <cstring>
 22 | #include "freq_item.h"
 23 | #include "log.h"
 24 | #include "reload.h"
 25 | 
 26 | namespace libwxfreq {
 27 | static uint32_t dummy_timestamp = 0;
 28 | 
 29 | bool ShmKeyLessCmp(const FreqKey& left, const FreqKey& right) {
 30 |   size_t size = sizeof(left.key);
 31 |   uint32_t lef_hash = *reinterpret_cast<const uint32_t*>(
 32 |       left.key + size - sizeof(uint32_t));
 33 |   uint32_t right_hash = *reinterpret_cast<const uint32_t*>(
 34 |       right.key + size - sizeof(uint32_t));
 35 |   if (lef_hash < right_hash) return true;
 36 |   if (lef_hash > right_hash) return false;
 37 |   if (left.appid < right.appid) return true;
 38 |   if (left.appid > right.appid) return false;
 39 |   return strncmp(right.key, left.key, size) < 0;
 40 | }
 41 | 
 42 | uint32_t APHash(const char *str) {
 43 |   uint32_t hash = 0;
 44 |   int i;
 45 | 
 46 |   for (i=0; *str; i++) {
 47 |     if ((i & 1) == 0) {
 48 |       hash ^= ((hash << 7) ^ (*str++) ^ (hash >> 3));
 49 |     } else {
 50 |       hash ^= (~((hash << 11) ^ (*str++) ^ (hash >> 5)));
 51 |     }
 52 |   }
 53 |   return (hash & 0x7FFFFFFF);
 54 | }
 55 | 
 56 | ShmFreqStat::ShmFreqStat(bool zero_init, key_t key, unsigned int item_cnt) :
 57 |   shrink_(false), last_shrink_time_(0), item_map_(ShmKeyLessCmp) {
 58 |   int ret = table_.MultiHashTableInit(zero_init, key,
 59 |                                       S_IRUSR | S_IWUSR, item_cnt);
 60 |   if (ret != 0) {
 61 |     gLog("[%s][%d]: ShmFreqStat constuct failed, key %u ret %d exit %d\n",
 62 |         __FILE__, __LINE__, key, ret, ret);
 63 |     exit(ret);
 64 |   }
 65 |   count_= 0;
 66 |   pthread_rwlock_init(&lock_, NULL);
 67 | }
 68 | 
 69 | void ShmFreqStat::ShrinkMap(time_t now) {
 70 |   if (now - last_shrink_time_ > 3600 &&
 71 |       __sync_bool_compare_and_swap(&shrink_, false, true)) {
 72 |     for (ItemMap::iterator it = item_map_.begin();
 73 |         it != item_map_.end(); ++it) {
 74 |       ItemMap::iterator dit = it;
 75 |       if (static_cast<int>(now - it->second.timestamp) < 90000) {
 76 |         FreqItemForShm* freqitemforshm = NULL;
 77 |         table_.GetValueForWrite(it->first, freqitemforshm);
 78 |         if (freqitemforshm != NULL) {
 79 |           memcpy(freqitemforshm, &it->second, sizeof(FreqItemForShm));
 80 |           pthread_rwlock_wrlock(&lock_);
 81 |           item_map_.erase(dit);
 82 |           pthread_rwlock_unlock(&lock_);
 83 |         } else {  // TODO(arthurzou)
 84 |         }
 85 |       } else {
 86 |           pthread_rwlock_wrlock(&lock_);
 87 |           item_map_.erase(dit);
 88 |           pthread_rwlock_unlock(&lock_);
 89 |         }
 90 |     }
 91 |     last_shrink_time_ = now;
 92 |     shrink_ = false;
 93 |     gLog("[%s][%d]: ShmFreqStat shrinkmap type_name %s size = %u\n",
 94 |         __FILE__, __LINE__, type_name().c_str(), item_map_.size());
 95 |   }
 96 | }
 97 | 
 98 | uint32_t& ShmFreqStat::GetItem(const std::string& key, const uint32_t appid,
 99 |                               const uint32_t cnt, FreqItem* &freq_item) {
100 |   FreqKey freqkey;
101 |   freqkey.appid = appid;
102 |   uint32_t* key_hash_ptr = reinterpret_cast<uint32_t*>(
103 |       freqkey.key + sizeof(freqkey.key) - sizeof(uint32_t));
104 |   *key_hash_ptr = APHash(key.c_str());
105 |   strncpy(freqkey.key, key.c_str(), sizeof(freqkey.key));
106 | 
107 |   time_t now = time(NULL);
108 | 
109 |   FreqItemForShm* freqitemforshm = NULL;
110 |   if (cnt != 0) {
111 |     table_.GetValueForWrite(freqkey, freqitemforshm);
112 |   } else {
113 |     table_.GetValueForRead(freqkey, freqitemforshm);
114 |   }
115 | #ifdef USING_MAP
116 |   ShrinkMap(now);
117 |   if (freqitemforshm == NULL) {  // sad face, find in map
118 |     if (cnt != 0) {
119 |       pthread_rwlock_wrlock(&lock_);
120 |       freqitemforshm = &item_map_[freqkey];
121 |       table_.MarkAsInValid();
122 |       gLog("[%s][%d]: item_map_ type_name %s size %u\n", __FILE__, __LINE__, 
123 |             type_name().c_str(), item_map_.size());
124 |       pthread_rwlock_unlock(&lock_);
125 |       freq_item = reinterpret_cast<FreqItem*>(freqitemforshm);
126 |       return freqitemforshm->timestamp;
127 |     } else {
128 |       pthread_rwlock_rdlock(&lock_);
129 |       ItemMap::iterator it = item_map_.find(freqkey);
130 |       gLog("[%s][%d]: item_map_ type_name %s size %u\n", __FILE__, __LINE__, 
131 |             type_name().c_str(), item_map_.size());
132 |       if (item_map_.size() > 0) {
133 |         table_.MarkAsInValid();
134 |       } else {
135 |         table_.MarkAsValid();
136 |       }
137 |       if (it == item_map_.end() ||
138 |           static_cast<int>(now - it->second.timestamp) >= 90000) {
139 |         freq_item = NULL;
140 |         pthread_rwlock_unlock(&lock_);
141 |         return dummy_timestamp;
142 |       } else {
143 |         freq_item = reinterpret_cast<FreqItem*>(&(it->second));
144 |         pthread_rwlock_unlock(&lock_);
145 |         return it->second.timestamp;
146 |       }
147 |     }
148 |   } else {
149 |     freq_item = reinterpret_cast<FreqItem*>(freqitemforshm);
150 |     if (freq_item->level3_cnt == 0) {
151 |       count_++;
152 |       gLog("[%s][%d]: using shm type_name %s count %u\n", __FILE__, __LINE__, 
153 |           type_name().c_str(), count_);
154 |     }
155 |     return freqitemforshm->timestamp;
156 |   }
157 | #else
158 |   if (freqitemforshm == NULL) {
159 |     freq_item = NULL;
160 |     return dummy_timestamp;
161 |   } else {
162 |     freq_item = reinterpret_cast<FreqItem*>(freqitemforshm);
163 |     return freqitemforshm->timestamp;
164 |   }
165 | #endif
166 | }
167 | 
168 | ShmFreqStat::~ShmFreqStat() {
169 | }
170 | 
171 | bool ShmFreqStat::TryReload() {
172 |   if (!table_.IsValid()) {
173 |     if (gLoadFunc(type_name_.c_str()) == 0) {
174 |       table_.MarkAsValid();
175 |       return true;
176 |     } else {
177 |       return false;
178 |     }
179 |   }
180 |   return true;
181 | }
182 | 
183 | }  // namespace libwxfreq
184 | 


--------------------------------------------------------------------------------
/util/shm_freq_stat.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_SHM_FREQ_STAT_H_
18 | #define FREQLIB_UTIL_SHM_FREQ_STAT_H_
19 | #include <pthread.h>
20 | #include <map>
21 | #include <string>
22 | #include "freq_stat.h"
23 | #include "multi_hash_table.h"
24 | 
25 | namespace libwxfreq {
26 | class FreqItem;
27 | 
28 | class ShmFreqStat : public FreqStat {
29 |  public:
30 |   ShmFreqStat(bool zero_init, key_t key, unsigned int item_cnt);
31 |   virtual uint32_t& GetItem(const std::string& key, const uint32_t appid,
32 |                             const uint32_t cnt, FreqItem* &freq_item);
33 |   virtual bool TryReload();
34 |   virtual ~ShmFreqStat();
35 | 
36 |  private:
37 |   typedef bool (*cmp) (const FreqKey&a, const FreqKey&b);
38 |   typedef std::map<FreqKey, FreqItemForShm, cmp> ItemMap;
39 |   void ShrinkMap(time_t now);
40 |   ItemMap item_map_;
41 |   pthread_rwlock_t lock_;
42 |   unsigned int shrink_;
43 |   unsigned int last_shrink_time_;
44 |   unsigned int count_;
45 |   MultiHashTable table_;
46 | };
47 | 
48 | }  // namespace libwxfreq
49 | 
50 | #endif  // FREQLIB_UTIL_SHM_FREQ_STAT_H_
51 | 


--------------------------------------------------------------------------------
/util/slice.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #include "slice.h"
18 | #include <stdlib.h>
19 | #include <string.h>
20 | 
21 | 
22 | using namespace std;
23 | 
24 | namespace libwxfreq {
25 | 
26 | Slice::Slice() {
27 |   start_ = end_ = NULL;
28 | }
29 | 
30 | Slice::Slice(const char *start, const char *end) {
31 |   start_ = start;
32 |   end_ = end;
33 | }
34 | 
35 | Slice::Slice(const std::string &other_slice) {
36 |   start_ = other_slice.c_str();
37 |   end_ = other_slice.c_str() + other_slice.size();
38 | }
39 | 
40 | std::string Slice::ToStr() const {
41 |   std::string ret(start_, end_- start_);
42 |   return ret;
43 | }
44 | 
45 | void Slice::StrTrim(const char *delimiter) {
46 |   while (start_ < end_ && strchr(delimiter, *start_) != 0) start_++;
47 |   while (start_ < end_ && strchr(delimiter, *(end_-1)) != 0) end_--;
48 | }
49 | 
50 | bool Slice::operator!=(const Slice &other_slice) const {
51 |   return !(*this == other_slice);
52 | }
53 | 
54 | bool Slice::operator==(const Slice &other_slice) const {
55 |   if (end_ - start_ != other_slice.end_ - other_slice.start_) return false;
56 |   return strncasecmp(start_, other_slice.start_, end_ - start_) == 0;
57 | }
58 | 
59 | bool Slice::operator==(const std::string &other_slice)  const {
60 |   if (end_ - start_ != static_cast<int>(other_slice.size())) return false;
61 |   return strncasecmp(start_, other_slice.c_str(), end_ - start_) == 0;
62 | }
63 | 
64 | bool Slice::operator<(const Slice &other_slice) const {
65 |   if (start_ == other_slice.start_) return false;
66 | 
67 |   int minlen = end_ - start_, res = 0;
68 |   if (other_slice.end_ - other_slice.start_ < minlen )
69 |     minlen = other_slice.end_ - other_slice.start_;
70 |   if ((res = strncasecmp( start_, other_slice.start_, minlen )) < 0)
71 |     return true;
72 |   else if (res > 0)
73 |     return false;
74 |   else
75 |     return end_ - start_ < other_slice.end_ - other_slice.start_;
76 | }
77 | 
78 | }  //  namespace libwxfreq
79 | 


--------------------------------------------------------------------------------
/util/slice.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | * Tencent is pleased to support the open source community by making libwxfreq available.
 3 | *
 4 | * Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
 5 | *
 6 | * Licensed under the BSD 3-Clause License (the "License"); you may not use this file
 7 | * except in compliance with the License. You may obtain a copy of the License at
 8 | *
 9 | *               https://opensource.org/licenses/BSD-3-Clause
10 | *
11 | * Unless required by applicable law or agreed to in writing, software distributed
12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
14 | * language governing permissions and limitations under the License.
15 | */
16 | 
17 | #ifndef FREQLIB_UTIL_SLICE_H_
18 | #define FREQLIB_UTIL_SLICE_H_
19 | 
20 | #include<string>
21 | 
22 | namespace libwxfreq {
23 | 
24 | class Slice {
25 |  public:
26 |   Slice();
27 |   Slice(const char *start, const char *end);
28 |   Slice(const std::string &other_str);
29 | 
30 |   std::string ToStr() const;
31 |   void StrTrim(const char *delimiter);
32 | 
33 |   inline const char *start() const { return start_; }
34 |   inline const char *end() const { return end_; }
35 | 
36 |   bool operator!=(const Slice &other_slice) const;
37 |   bool operator==(const Slice &other_slice) const;
38 |   bool operator==(const std::string &other_str) const;
39 |   bool operator<(const Slice &other_slice) const;
40 | 
41 |  private:
42 |   const char *start_, *end_;
43 | };
44 | 
45 | 
46 | }  //  namespace libwxfreq
47 | #endif  // FREQLIB_UTIL_SLICE_H_
48 | 


--------------------------------------------------------------------------------