├── .gitignore ├── config ├── config.toml.example └── config.go ├── controller └── check.go ├── README.md ├── main.go ├── tool └── sensitive.go └── dictionary └── dictionary.txt /.gitignore: -------------------------------------------------------------------------------- 1 | /config/config.toml 2 | /log/pid.lock 3 | /.idea 4 | SensitiveWords -------------------------------------------------------------------------------- /config/config.toml.example: -------------------------------------------------------------------------------- 1 | DictionaryPath = "/dictionary/dictionary.txt" 2 | Port = "9981" 3 | PidFilePath = "/log/pid.lock" 4 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 3 | * time: 6/24/18 3:22 PM 4 | * author: linhuanchao 5 | * e-mail: 873085747@qq.com 6 | */ 7 | 8 | package config 9 | 10 | import ( 11 | "path/filepath" 12 | "os" 13 | "github.com/BurntSushi/toml" 14 | "sync" 15 | ) 16 | 17 | type Config struct { 18 | DictionaryPath string 19 | Port string 20 | PidFilePath string 21 | } 22 | 23 | var once = sync.Once{} 24 | var config *Config 25 | 26 | func GetConfig() *Config{ 27 | once.Do(func() { 28 | currentPath, _ := filepath.Abs(filepath.Dir(os.Args[0])) 29 | toml.DecodeFile(currentPath+"/config/config.toml", &config) 30 | }) 31 | return config 32 | } -------------------------------------------------------------------------------- /controller/check.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 3 | * time: 6/24/18 3:22 PM 4 | * author: linhuanchao 5 | * e-mail: 873085747@qq.com 6 | */ 7 | 8 | package controller 9 | 10 | import ( 11 | "github.com/gin-gonic/gin" 12 | "SensitiveWords/tool" 13 | "net/http" 14 | ) 15 | 16 | func Check(context *gin.Context) { 17 | content := context.Query("content") 18 | sensitiveMap := tool.GetMap() 19 | target, result := sensitiveMap.CheckSensitive(content) 20 | context.JSON(http.StatusOK, gin.H{ 21 | "target" : target, 22 | "result" : result, 23 | }) 24 | } 25 | 26 | func All(context *gin.Context) { 27 | content := context.Query("content") 28 | sensitiveMap := tool.GetMap() 29 | target := sensitiveMap.FindAllSensitive(content) 30 | 31 | type Target struct { 32 | Word string `json:"word"` 33 | WordIndexes []int `json:"word_indexes"` 34 | WordLength int `json:"word_length"` 35 | } 36 | 37 | targetArray := []Target{} 38 | 39 | for key, value := range target{ 40 | t := Target{ 41 | Word: key, 42 | WordIndexes:value.Indexes, 43 | WordLength:value.Len, 44 | } 45 | targetArray = append(targetArray, t) 46 | } 47 | 48 | context.JSON(http.StatusOK,gin.H{ 49 | "target": targetArray, 50 | }) 51 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SensitiveWords 2 | 3 | ## Get 4 | github地址: https://github.com/TomatoMr/SensitiveWords.git 5 | 或者 6 | go get gopkg.in/TomatoMr/SensitiveWords.v1 7 | 8 | ## Introduction 9 | SensitiveWords是基于DFA算法编写的敏感词汇检测插件,可独立部署,也可以集成到项目中. 10 | 11 | ## Usage 12 | 13 | ### 独立部署 14 | 15 | #### 1. 复制配置文件 16 | >cd config 17 | cp config.toml.example config.toml 18 | >> 19 | 20 | #### 2. 构建二进制包 21 | >go build 22 | >> 23 | 24 | #### 3. 使用方法 25 | >-restart 26 | :restart your http server, just like this: -restart or -restart=true|false. 27 | -start [-d] 28 | :up your http server, just like this: -start or -start=true|false [-d or -d=true|false]. 29 | -stop 30 | :down your http server, just like this: -stop or -stop=true|false. 31 | >> 32 | 33 | #### 4. Api 34 | 4.1 /check?content=xxx 35 | >作用:返回目标文本中,第一个敏感词汇 36 | 返回值:target:"", //第一个敏感词 37 | result:"", //是否含有敏感词 38 | >> 39 | 4.2 /all?content=xxx 40 | >作用:返回目标文本中,第一个敏感词汇 41 | 返回值:target:[ 42 | word //敏感词 43 | word_indexes //相同的敏感词在原文本中的索引的数组 44 | word_length //该敏感词的长度 45 | ] 46 | >> 47 | 48 | ### 插件方法 49 | 1. GetMap() 50 | >获取SensitiveMap实例 51 | >> 52 | 2. InitDictionary() 53 | >初始化敏感词典,并获得实例 54 | >> 55 | 3. CheckSensitive(text string) 56 | >接受检测文本,并返回是否含有敏感词和第一个敏感词 57 | >> 58 | 4. FindAllSensitive(text string) 59 | >接受检测文本,并返回所有敏感词 60 | >> 61 | 5. GetConfig() 62 | >返回配置实例 63 | >> 64 | 65 | ### 配置文件说明 66 | >DictionaryPath //敏感词典地址,根目录是本项目地址 67 | Port //http server监听的web端口 68 | PidFilePath //pid文件位置,用于命令行结束程序和重启程序,根目录是本项目地址 69 | >> 70 | 71 | ### 帮助 72 | >Q:重载词典? 73 | A:修改config.toml->修改DictionaryPath->./SensitiveWords -restart 74 | >> 75 | 76 | ### 示例 77 | >1.http://localhost:9981/check?content="脏话" 78 | {"result":true,"target":"脏话"} 79 | 2.http://localhost:9981/all?content="脏话" 80 | {"target":[{"word":"脏话","word_indexes":[0],"word_length":2}]} 81 | >> 82 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 3 | * time: 6/24/18 3:22 PM 4 | * author: linhuanchao 5 | * e-mail: 873085747@qq.com 6 | */ 7 | 8 | package main 9 | import ( 10 | "github.com/gin-gonic/gin" 11 | "SensitiveWords/controller" 12 | "sync" 13 | "flag" 14 | "SensitiveWords/config" 15 | "os" 16 | "path/filepath" 17 | "fmt" 18 | "io/ioutil" 19 | "os/exec" 20 | ) 21 | 22 | var configure *config.Config 23 | var appPath string 24 | var pidPath string 25 | func init() { 26 | appPath, _ = filepath.Abs(filepath.Dir(os.Args[0])) 27 | configure = config.GetConfig() 28 | pidPath = appPath + configure.PidFilePath 29 | gin.SetMode(gin.ReleaseMode) 30 | } 31 | 32 | var wg sync.WaitGroup 33 | func main(){ 34 | var start bool 35 | var stop bool 36 | var daemon bool 37 | var restart bool 38 | flag.BoolVar(&start,"start",false,"up your http server, just like this: -start or -start=true|false.") 39 | flag.BoolVar(&stop,"stop",false,"down your http server, just like this: -stop or -stop=true|false.") 40 | flag.BoolVar(&daemon,"d",false,"daemon, just like this: -start -d or -d=true|false.") 41 | flag.BoolVar(&restart,"restart",false,"restart your http server, just like this: -restart or -restart=true|false.") 42 | flag.Parse() 43 | 44 | if start { 45 | if daemon { 46 | cmd := exec.Command("./SensitiveWords", "-start") 47 | cmd.Start() 48 | os.Exit(0) 49 | } 50 | wg.Add(1) 51 | fmt.Println("http server start.") 52 | Start() 53 | wg.Wait() 54 | } 55 | 56 | if stop { 57 | Stop() 58 | } 59 | 60 | if restart { 61 | Restart() 62 | } 63 | } 64 | 65 | func Start() { 66 | defer wg.Done() 67 | 68 | ioutil.WriteFile(pidPath, []byte(fmt.Sprintf("%d", os.Getpid())), 0666)//记录pid 69 | router := gin.Default() 70 | router.GET("/all", controller.All) 71 | router.GET("/check", controller.Check) 72 | router.Run(":"+configure.Port) 73 | } 74 | 75 | func Stop() { 76 | pid, _ := ioutil.ReadFile(pidPath) 77 | cmd := exec.Command("kill","-9", string(pid)) 78 | cmd.Start() 79 | ioutil.WriteFile(pidPath, nil, 0666)//清除pid 80 | fmt.Println("bye~") 81 | } 82 | 83 | func Restart() { 84 | fmt.Println("restarting...") 85 | pid, _ := ioutil.ReadFile(pidPath) 86 | stop := exec.Command("kill","-9", string(pid)) 87 | stop.Start() 88 | start := exec.Command("./SensitiveWords", "-start", "-d") 89 | start.Start() 90 | } -------------------------------------------------------------------------------- /tool/sensitive.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 3 | * time: 6/24/18 3:22 PM 4 | * author: linhuanchao 5 | * e-mail: 873085747@qq.com 6 | */ 7 | 8 | package tool 9 | 10 | import ( 11 | "os" 12 | "io/ioutil" 13 | "strings" 14 | "path/filepath" 15 | "SensitiveWords/config" 16 | ) 17 | 18 | type SensitiveMap struct { 19 | sensitiveNode map[string]interface{} 20 | isEnd bool 21 | } 22 | 23 | var s *SensitiveMap 24 | 25 | func GetMap() *SensitiveMap { 26 | if s == nil { 27 | currentPath, _ := filepath.Abs(filepath.Dir(os.Args[0])) 28 | configure := config.GetConfig() 29 | dictionaryPath := currentPath + configure.DictionaryPath 30 | s = InitDictionary(s, dictionaryPath) 31 | } 32 | return s 33 | } 34 | 35 | /* 36 | 初始化敏感词词典结构体 37 | */ 38 | func initSensitiveMap() *SensitiveMap { 39 | return &SensitiveMap{ 40 | sensitiveNode:make(map[string]interface{}), 41 | isEnd:false, 42 | } 43 | } 44 | 45 | /* 46 | 读取词典文件 47 | */ 48 | func readDictionary(path string) []string { 49 | file, err := os.Open(path) 50 | if err != nil { 51 | panic(err) 52 | } 53 | defer file.Close() 54 | str, err := ioutil.ReadAll(file) 55 | dictionary := strings.Fields(string(str)) 56 | return dictionary 57 | } 58 | 59 | /* 60 | 初始化敏感词词典，根据DFA算法构建trie 61 | */ 62 | func InitDictionary(s *SensitiveMap, dictionaryPath string) *SensitiveMap{ 63 | s = initSensitiveMap() 64 | dictionary := readDictionary(dictionaryPath) 65 | for _, words := range dictionary { 66 | sMapTmp := s 67 | w := []rune(words) 68 | wordsLength := len(w) 69 | for i:=0; i