├── README-zh_cn.md ├── README.md ├── dict └── default_dict.txt ├── filter ├── ahocorasick.go ├── ahocorasick_test.go ├── dfa.go └── filter.go ├── go.mod ├── go.sum ├── manager.go ├── manager_test.go ├── options.go └── store ├── memory.go ├── mongo.go ├── mysql.go └── store.go /README-zh_cn.md: -------------------------------------------------------------------------------- 1 | # 🚫 go-sensitive 2 | 3 | [![build](https://img.shields.io/badge/build-1.01-brightgreen)](https://github.com/StellarisW/go-sensitive)[![go-version](https://img.shields.io/badge/go-~%3D1.19-30dff3?logo=go)](https://github.com/StellarisW/go-sensitive) 4 | 5 | [English](README.md) | 中文 6 | 7 | > 敏感词过滤, 支持多种数据源加载, 多种过滤算法, 多种操作功能 8 | 9 | ## 🌟 Feature 10 | 11 | - 支持多种操作功能 12 | - `Filter()` 返回过滤后的文本 13 | - `Replace()` 返回替换了敏感词后的文本 14 | - `IsSensitive()` 返回文本是否含有敏感词 15 | - `FindOne()` 返回匹配到的第一个敏感词 16 | - `FindAll()` 返回匹配到的所有敏感词 17 | - `FindAllCount()` 返回匹配到的所有敏感词及出现次数 18 | - 支持多种数据源加载, 动态修改数据源 19 | - 支持内存存储 20 | - 支持mysql存储 21 | - 支持mongo存储 22 | - 支持多种字典加载方式 23 | - 支持运行过程中动态修改数据源 24 | - 支持多种过滤算法 25 | - **DFA** 使用 `trie tree` 数据结构匹配敏感词 26 | - **AC 自动机** 27 | 28 | ## ⚙ Usage 29 | 30 | ```go 31 | package main 32 | 33 | import ( 34 | "fmt" 35 | "github.com/StellarisW/go-sensitive" 36 | ) 37 | 38 | func main() { 39 | filterManager := sensitive.NewFilter( 40 | sensitive.StoreOption{ 41 | Type: sensitive.StoreMemory 42 | }, 43 | sensitive.FilterOption{ 44 | Type: sensitive.FilterDfa 45 | } 46 | ) 47 | 48 | // 加载字典 49 | 50 | err:=filterManager.GetStore().LoadDictPath("path-to-dict") 51 | if err != nil { 52 | fmt.Println(err) 53 | return 54 | } 55 | 56 | // 动态增加词汇 57 | 58 | err=filterManager.GetStore().AddWord("这是敏感词1", "这是敏感词2", "这是敏感词3") 59 | if err != nil { 60 | fmt.Println(err) 61 | return 62 | } 63 | 64 | fmt.Println(filterManager.GetFilter().IsSensitive("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 65 | 66 | fmt.Println(filterManager.GetFilter().Filter("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 67 | 68 | fmt.Println(filterManager.GetFilter().Replace("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词", '*')) 69 | 70 | fmt.Println(filterManager.GetFilter().FindOne("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 71 | 72 | fmt.Println(filterManager.GetFilter().FindAll("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 73 | 74 | fmt.Println(filterManager.GetFilter().FindAllCount("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 75 | } 76 | ``` 77 | 78 | ## ✔ Get 79 | 80 | ``` 81 | $ go get -u github.com/StellarisW/go-sensitive 82 | ``` 83 | 84 | ## 📂 Import 85 | 86 | ```go 87 | import "github.com/StellarisW/go-sensitive" 88 | ``` 89 | 90 | ## 91 | 92 | ## 📌 TODO 93 | 94 | - [ ] add mongo data source support 95 | - [ ] add bloom algorithm -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🚫 go-sensitive 2 | 3 | [![build](https://img.shields.io/badge/build-1.01-brightgreen)](https://github.com/sgoware/go-sensitive)[![go-version](https://img.shields.io/badge/go-~%3D1.19-30dff3?logo=go)](https://github.com/sgoware/go-sensitive) 4 | 5 | English | [中文](README-zh_cn.md) 6 | 7 | > Filter sensitive words, support multiple data sources, filter algorithms and functions 8 | 9 | ## 🌟 Feature 10 | 11 | - support multiple functions 12 | - `Filter()` return filtered text 13 | - `Replace()` return text which sensitive words that is been replaced 14 | - `IsSensitive()` Check whether the text has sensitive word 15 | - `FindOne()` return first sensitive word that has been found in the text 16 | - `FindAll()` return all sensitive word that has been found in the text 17 | - `FindAllCount()` return all sensitive[README-zh_cn.md](README-zh_cn.md) word with its count that has been found in the text 18 | - support multiple data sources with dynamic modification 19 | - support memory storage 20 | - support mysql storage 21 | - support mongo storage 22 | - support multiple ways of add dict 23 | - support dynamic add/del sensitive word while running 24 | - support multiple filter algorithms 25 | - **DFA** use `trie tree` to filter sensitive words 26 | - **Aho–Corasick algorithm** 27 | ## ⚙ Usage 28 | 29 | ```go 30 | package main 31 | 32 | import ( 33 | "fmt" 34 | "github.com/sgoware/go-sensitive" 35 | ) 36 | 37 | func main() { 38 | filterManager := sensitive.NewFilter( 39 | sensitive.StoreOption{ 40 | Type: sensitive.StoreMemory 41 | }, 42 | sensitive.FilterOption{ 43 | Type: sensitive.FilterDfa 44 | } 45 | ) 46 | 47 | // load dict 48 | 49 | err:=filterManager.GetStore().LoadDictPath("path-to-dict") 50 | if err != nil { 51 | fmt.Println(err) 52 | return 53 | } 54 | 55 | // dynamic add sensitive words 56 | 57 | err=filterManager.GetStore().AddWord("这是敏感词1", "这是敏感词2", "这是敏感词3") 58 | if err != nil { 59 | fmt.Println(err) 60 | return 61 | } 62 | 63 | fmt.Println(filterManager.GetFilter().IsSensitive("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 64 | 65 | fmt.Println(filterManager.GetFilter().Filter("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 66 | 67 | fmt.Println(filterManager.GetFilter().Replace("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词", '*')) 68 | 69 | fmt.Println(filterManager.GetFilter().FindOne("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 70 | 71 | fmt.Println(filterManager.GetFilter().FindAll("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 72 | 73 | fmt.Println(filterManager.GetFilter().FindAllCount("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词")) 74 | } 75 | ``` 76 | 77 | ## ✔ Get 78 | 79 | ``` 80 | $ go get -u github.com/sgoware/go-sensitive 81 | ``` 82 | 83 | ## 📂 Import 84 | 85 | ```go 86 | import "github.com/sgoware/go-sensitive" 87 | ``` 88 | 89 | ## 90 | 91 | ## 📌 TODO 92 | 93 | - [ ] add redis data source support 94 | - [ ] add bloom algorithm 95 | -------------------------------------------------------------------------------- /filter/ahocorasick.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "github.com/sgoware/ds/queue/arrayqueue" 5 | ) 6 | 7 | type acNode struct { 8 | value rune 9 | children map[rune]*acNode 10 | word *string 11 | fail *acNode 12 | } 13 | 14 | func newAcNode(r rune) *acNode { 15 | return &acNode{ 16 | value: r, 17 | children: make(map[rune]*acNode), 18 | word: nil, 19 | } 20 | } 21 | 22 | type AcModel struct { 23 | root *acNode 24 | } 25 | 26 | func NewAcModel() *AcModel { 27 | return &AcModel{ 28 | root: newAcNode(0), 29 | } 30 | } 31 | 32 | func (m *AcModel) AddWords(words ...string) { 33 | for _, word := range words { 34 | m.AddWord(word) 35 | } 36 | 37 | m.buildFailPointers() 38 | } 39 | 40 | func (m *AcModel) AddWord(word string) { 41 | now := m.root 42 | runes := []rune(word) 43 | 44 | for _, r := range runes { 45 | if next, ok := now.children[r]; ok { 46 | now = next 47 | } else { 48 | next = newAcNode(r) 49 | now.children[r] = next 50 | now = next 51 | } 52 | } 53 | 54 | now.word = new(string) 55 | *now.word = word 56 | } 57 | 58 | func (m *AcModel) DelWords(words ...string) { 59 | for _, word := range words { 60 | m.DelWord(word) 61 | } 62 | 63 | m.buildFailPointers() 64 | } 65 | 66 | func (m *AcModel) DelWord(word string) { 67 | var lastLeaf *acNode 68 | var lastLeafNextRune rune 69 | now := m.root 70 | runes := []rune(word) 71 | 72 | for _, r := range runes { 73 | if next, ok := now.children[r]; !ok { 74 | return 75 | } else { 76 | if now.word != nil { 77 | lastLeaf = now 78 | lastLeafNextRune = r 79 | } 80 | now = next 81 | } 82 | } 83 | 84 | delete(lastLeaf.children, lastLeafNextRune) 85 | } 86 | 87 | func (m *AcModel) buildFailPointers() { 88 | q := arrayqueue.New(m.root) 89 | 90 | for q.Len() > 0 { 91 | temp, _ := q.Top() 92 | q.Pop() 93 | for _, node := range temp.(*acNode).children { 94 | if temp.(*acNode) == m.root { 95 | node.fail = m.root 96 | } else { 97 | p := temp.(*acNode).fail 98 | for p != nil { 99 | if next, found := p.children[node.value]; found { 100 | node.fail = next 101 | break 102 | } 103 | p = p.fail 104 | } 105 | if p == nil { 106 | node.fail = m.root 107 | } 108 | } 109 | 110 | q.Push(node) 111 | } 112 | } 113 | } 114 | 115 | func (m *AcModel) Listen(addChan, delChan <-chan string) { 116 | go func() { 117 | var words []string 118 | 119 | for word := range addChan { 120 | words = append(words, word) 121 | if len(addChan) == 0 { 122 | m.AddWords(words...) 123 | word = word[:0] 124 | } 125 | } 126 | }() 127 | 128 | go func() { 129 | var words []string 130 | 131 | for word := range delChan { 132 | words = append(words, word) 133 | if len(delChan) == 0 { 134 | m.DelWords(words...) 135 | word = word[:0] 136 | } 137 | } 138 | }() 139 | } 140 | 141 | func (m *AcModel) FindAll(text string) []string { 142 | var matches []string 143 | var found bool 144 | 145 | now := m.root 146 | var temp *acNode 147 | runes := []rune(text) 148 | 149 | for pos := 0; pos < len(runes); pos++ { 150 | _, found = now.children[runes[pos]] 151 | if !found && now != m.root { 152 | now = now.fail 153 | for ; !found && now != m.root; now, found = now.children[runes[pos]] { 154 | now = now.fail 155 | } 156 | } 157 | 158 | // 若找到匹配成功的字符串结点, 则指向那个结点, 否则指向根结点 159 | if next, ok := now.children[runes[pos]]; ok { 160 | now = next 161 | } else { 162 | now = m.root 163 | } 164 | 165 | temp = now 166 | 167 | for temp != m.root { 168 | if temp.word != nil { 169 | matches = append(matches, *temp.word) 170 | } 171 | temp = temp.fail 172 | } 173 | } 174 | 175 | var res []string 176 | set := make(map[string]struct{}) 177 | 178 | for _, word := range matches { 179 | if _, ok := set[word]; !ok { 180 | set[word] = struct{}{} 181 | res = append(res, word) 182 | } 183 | } 184 | 185 | return res 186 | } 187 | 188 | func (m *AcModel) FindAllCount(text string) map[string]int { 189 | res := make(map[string]int) 190 | var found bool 191 | var temp *acNode 192 | 193 | now := m.root 194 | runes := []rune(text) 195 | 196 | for pos := 0; pos < len(runes); pos++ { 197 | _, found = now.children[runes[pos]] 198 | if !found && now != m.root { 199 | now = now.fail 200 | for ; !found && now != m.root; now, found = now.children[runes[pos]] { 201 | now = now.fail 202 | } 203 | } 204 | 205 | // 若找到匹配成功的字符串结点, 则指向那个结点, 否则指向根结点 206 | if next, ok := now.children[runes[pos]]; ok { 207 | now = next 208 | } else { 209 | now = m.root 210 | } 211 | 212 | temp = now 213 | 214 | for temp != m.root { 215 | if temp.word != nil { 216 | res[*temp.word]++ 217 | } 218 | temp = temp.fail 219 | } 220 | } 221 | 222 | return res 223 | } 224 | 225 | func (m *AcModel) FindOne(text string) string { 226 | var found bool 227 | var temp *acNode 228 | 229 | now := m.root 230 | runes := []rune(text) 231 | 232 | for pos := 0; pos < len(runes); pos++ { 233 | _, found = now.children[runes[pos]] 234 | if !found && now != m.root { 235 | now = now.fail 236 | for ; !found && now != m.root; now, found = now.children[runes[pos]] { 237 | now = now.fail 238 | } 239 | } 240 | 241 | // 若找到匹配成功的字符串结点, 则指向那个结点, 否则指向根结点 242 | if next, ok := now.children[runes[pos]]; ok { 243 | now = next 244 | } else { 245 | now = m.root 246 | } 247 | 248 | temp = now 249 | 250 | for temp != m.root { 251 | if temp.word != nil { 252 | return *temp.word 253 | } 254 | temp = temp.fail 255 | } 256 | } 257 | 258 | return "" 259 | } 260 | 261 | func (m *AcModel) IsSensitive(text string) bool { 262 | return m.FindOne(text) != "" 263 | } 264 | 265 | func (m *AcModel) Replace(text string, repl rune) string { 266 | var found bool 267 | var temp *acNode 268 | 269 | now := m.root 270 | runes := []rune(text) 271 | 272 | for pos := 0; pos < len(runes); pos++ { 273 | _, found = now.children[runes[pos]] 274 | if !found && now != m.root { 275 | now = now.fail 276 | for ; !found && now != m.root; now, found = now.children[runes[pos]] { 277 | now = now.fail 278 | } 279 | } 280 | 281 | // 若找到匹配成功的字符串结点, 则指向那个结点, 否则指向根结点 282 | if next, ok := now.children[runes[pos]]; ok { 283 | now = next 284 | } else { 285 | now = m.root 286 | } 287 | 288 | temp = now 289 | 290 | for temp != m.root { 291 | if temp.word != nil { 292 | for i := pos - len([]rune(*temp.word)) + 1; i <= pos; i++ { 293 | runes[i] = repl 294 | } 295 | } 296 | temp = temp.fail 297 | } 298 | } 299 | 300 | return string(runes) 301 | } 302 | 303 | func (m *AcModel) Remove(text string) string { 304 | var found bool 305 | var temp *acNode 306 | 307 | now := m.root 308 | runes := []rune(text) 309 | 310 | for pos := 0; pos < len(runes); pos++ { 311 | _, found = now.children[runes[pos]] 312 | if !found && now != m.root { 313 | now = now.fail 314 | for ; !found && now != m.root; now, found = now.children[runes[pos]] { 315 | now = now.fail 316 | } 317 | } 318 | 319 | // 若找到匹配成功的字符串结点, 则指向那个结点, 否则指向根结点 320 | if next, ok := now.children[runes[pos]]; ok { 321 | now = next 322 | } else { 323 | now = m.root 324 | } 325 | 326 | temp = now 327 | 328 | for temp != m.root { 329 | if temp.word != nil { 330 | runes = append(runes[:pos-len([]rune(*temp.word))+1], runes[pos+1:]...) 331 | pos -= len([]rune(*temp.word)) 332 | } 333 | temp = temp.fail 334 | } 335 | } 336 | 337 | return string(runes) 338 | } 339 | -------------------------------------------------------------------------------- /filter/ahocorasick_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | var ( 9 | words1 = []string{"敏感词1", "敏感词2", "敏感词3"} 10 | 11 | text1 = "敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词" 12 | ) 13 | 14 | func Test_FindAll(t *testing.T) { 15 | type args struct { 16 | words []string 17 | text string 18 | } 19 | 20 | tests := []struct { 21 | name string 22 | args args 23 | result []string 24 | }{ 25 | { 26 | name: "1", 27 | args: args{ 28 | words: words1, 29 | text: text1, 30 | }, 31 | result: []string{"敏感词1", "敏感词2", "敏感词3"}, 32 | }, 33 | } 34 | 35 | for _, tt := range tests { 36 | t.Run(tt.name, func(t *testing.T) { 37 | filter := NewAcModel() 38 | 39 | filter.AddWords(tt.args.words...) 40 | 41 | matchAll := filter.FindAll(tt.args.text) 42 | if !reflect.DeepEqual(matchAll, tt.result) { 43 | t.Errorf("FindAll() = %v, want %v", matchAll, tt.result) 44 | } 45 | }) 46 | } 47 | } 48 | 49 | func Test_FindAllCount(t *testing.T) { 50 | type args struct { 51 | words []string 52 | text string 53 | } 54 | 55 | tests := []struct { 56 | name string 57 | args args 58 | result map[string]int 59 | }{ 60 | { 61 | name: "1", 62 | args: args{ 63 | words: words1, 64 | text: text1, 65 | }, 66 | result: map[string]int{ 67 | "敏感词1": 2, 68 | "敏感词2": 1, 69 | "敏感词3": 1, 70 | }, 71 | }, 72 | } 73 | 74 | for _, tt := range tests { 75 | t.Run(tt.name, func(t *testing.T) { 76 | filter := NewAcModel() 77 | 78 | filter.AddWords(tt.args.words...) 79 | 80 | matchAll := filter.FindAllCount(tt.args.text) 81 | if !reflect.DeepEqual(matchAll, tt.result) { 82 | t.Errorf("FindAllCount() = %v, want %v", matchAll, tt.result) 83 | } 84 | }) 85 | } 86 | } 87 | 88 | func Test_FindOne(t *testing.T) { 89 | type args struct { 90 | words []string 91 | text string 92 | } 93 | 94 | tests := []struct { 95 | name string 96 | args args 97 | result string 98 | }{ 99 | { 100 | name: "1", 101 | args: args{ 102 | words: words1, 103 | text: text1, 104 | }, 105 | result: "敏感词1", 106 | }, 107 | } 108 | 109 | for _, tt := range tests { 110 | t.Run(tt.name, func(t *testing.T) { 111 | filter := NewAcModel() 112 | 113 | filter.AddWords(tt.args.words...) 114 | 115 | matchAll := filter.FindOne(tt.args.text) 116 | if !reflect.DeepEqual(matchAll, tt.result) { 117 | t.Errorf("FindOne() = %v, want %v", matchAll, tt.result) 118 | } 119 | }) 120 | } 121 | } 122 | 123 | func Test_IsSensitive(t *testing.T) { 124 | type args struct { 125 | words []string 126 | text string 127 | } 128 | 129 | tests := []struct { 130 | name string 131 | args args 132 | result bool 133 | }{ 134 | { 135 | name: "1", 136 | args: args{ 137 | words: words1, 138 | text: text1, 139 | }, 140 | result: true, 141 | }, 142 | } 143 | 144 | for _, tt := range tests { 145 | t.Run(tt.name, func(t *testing.T) { 146 | filter := NewAcModel() 147 | 148 | filter.AddWords(tt.args.words...) 149 | 150 | matchAll := filter.IsSensitive(tt.args.text) 151 | if !reflect.DeepEqual(matchAll, tt.result) { 152 | t.Errorf("IsSensitive() = %v, want %v", matchAll, tt.result) 153 | } 154 | }) 155 | } 156 | } 157 | 158 | func Test_Replace(t *testing.T) { 159 | type args struct { 160 | words []string 161 | text string 162 | } 163 | 164 | tests := []struct { 165 | name string 166 | args args 167 | result string 168 | }{ 169 | { 170 | name: "1", 171 | args: args{ 172 | words: words1, 173 | text: text1, 174 | }, 175 | result: "****,这是****,这是****,这是****,这里没有敏感词", 176 | }, 177 | } 178 | 179 | for _, tt := range tests { 180 | t.Run(tt.name, func(t *testing.T) { 181 | filter := NewAcModel() 182 | 183 | filter.AddWords(tt.args.words...) 184 | 185 | result := filter.Replace(tt.args.text, rune('*')) 186 | if !reflect.DeepEqual(result, tt.result) { 187 | t.Errorf("Remove() = %v, want %v", result, tt.result) 188 | } 189 | }) 190 | } 191 | } 192 | 193 | func Test_Remove(t *testing.T) { 194 | type args struct { 195 | words []string 196 | text string 197 | } 198 | 199 | tests := []struct { 200 | name string 201 | args args 202 | result string 203 | }{ 204 | { 205 | name: "1", 206 | args: args{ 207 | words: words1, 208 | text: text1, 209 | }, 210 | result: ",这是,这是,这是,这里没有敏感词", 211 | }, 212 | } 213 | 214 | for _, tt := range tests { 215 | t.Run(tt.name, func(t *testing.T) { 216 | filter := NewAcModel() 217 | 218 | filter.AddWords(tt.args.words...) 219 | 220 | result := filter.Remove(tt.args.text) 221 | if !reflect.DeepEqual(result, tt.result) { 222 | t.Errorf("Remove() = %v, want %v", result, tt.result) 223 | } 224 | }) 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /filter/dfa.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | type dfaNode struct { 4 | children map[rune]*dfaNode 5 | isLeaf bool 6 | } 7 | 8 | func newDfaNode() *dfaNode { 9 | return &dfaNode{ 10 | children: make(map[rune]*dfaNode), 11 | isLeaf: false, 12 | } 13 | } 14 | 15 | type DfaModel struct { 16 | root *dfaNode 17 | } 18 | 19 | func NewDfaModel() *DfaModel { 20 | return &DfaModel{ 21 | root: newDfaNode(), 22 | } 23 | } 24 | 25 | func (m *DfaModel) AddWords(words ...string) { 26 | for _, word := range words { 27 | m.AddWord(word) 28 | } 29 | } 30 | 31 | func (m *DfaModel) AddWord(word string) { 32 | now := m.root 33 | runes := []rune(word) 34 | 35 | for _, r := range runes { 36 | if next, ok := now.children[r]; ok { 37 | now = next 38 | } else { 39 | next = newDfaNode() 40 | now.children[r] = next 41 | now = next 42 | } 43 | } 44 | 45 | now.isLeaf = true 46 | } 47 | 48 | func (m *DfaModel) DelWords(words ...string) { 49 | for _, word := range words { 50 | m.DelWord(word) 51 | } 52 | } 53 | 54 | func (m *DfaModel) DelWord(word string) { 55 | var lastLeaf *dfaNode 56 | var lastLeafNextRune rune 57 | now := m.root 58 | runes := []rune(word) 59 | 60 | for _, r := range runes { 61 | if next, ok := now.children[r]; !ok { 62 | return 63 | } else { 64 | if now.isLeaf { 65 | lastLeaf = now 66 | lastLeafNextRune = r 67 | } 68 | now = next 69 | } 70 | } 71 | 72 | delete(lastLeaf.children, lastLeafNextRune) 73 | } 74 | 75 | func (m *DfaModel) Listen(addChan, delChan <-chan string) { 76 | go func() { 77 | for word := range addChan { 78 | m.AddWord(word) 79 | } 80 | }() 81 | 82 | go func() { 83 | for word := range delChan { 84 | m.DelWord(word) 85 | } 86 | }() 87 | } 88 | 89 | func (m *DfaModel) FindAll(text string) []string { 90 | var matches []string // stores words that match in dict 91 | var found bool // if current rune in node's map 92 | var now *dfaNode // current node 93 | 94 | start := 0 95 | parent := m.root 96 | runes := []rune(text) 97 | length := len(runes) 98 | 99 | for pos := 0; pos < length; pos++ { 100 | now, found = parent.children[runes[pos]] 101 | 102 | if !found { 103 | parent = m.root 104 | pos = start 105 | start++ 106 | continue 107 | } 108 | 109 | if now.isLeaf && start <= pos { 110 | matches = append(matches, string(runes[start:pos+1])) 111 | } 112 | 113 | if pos == length-1 { 114 | parent = m.root 115 | pos = start 116 | start++ 117 | continue 118 | } 119 | 120 | parent = now 121 | } 122 | 123 | var res []string 124 | set := make(map[string]struct{}) 125 | 126 | for _, word := range matches { 127 | if _, ok := set[word]; !ok { 128 | set[word] = struct{}{} 129 | res = append(res, word) 130 | } 131 | } 132 | 133 | return res 134 | } 135 | 136 | func (m *DfaModel) FindAllCount(text string) map[string]int { 137 | res := make(map[string]int) 138 | var found bool 139 | var now *dfaNode 140 | 141 | start := 0 142 | parent := m.root 143 | runes := []rune(text) 144 | length := len(runes) 145 | 146 | for pos := 0; pos < length; pos++ { 147 | now, found = parent.children[runes[pos]] 148 | 149 | if !found { 150 | parent = m.root 151 | pos = start 152 | start++ 153 | continue 154 | } 155 | 156 | if now.isLeaf && start <= pos { 157 | res[string(runes[start:pos+1])]++ 158 | } 159 | 160 | if pos == length-1 { 161 | parent = m.root 162 | pos = start 163 | start++ 164 | continue 165 | } 166 | 167 | parent = now 168 | } 169 | 170 | return res 171 | } 172 | 173 | func (m *DfaModel) FindOne(text string) string { 174 | var found bool 175 | var now *dfaNode 176 | 177 | start := 0 178 | parent := m.root 179 | runes := []rune(text) 180 | length := len(runes) 181 | 182 | for pos := 0; pos < length; pos++ { 183 | now, found = parent.children[runes[pos]] 184 | 185 | if !found || (!now.isLeaf && pos == length-1) { 186 | parent = m.root 187 | pos = start 188 | start++ 189 | continue 190 | } 191 | 192 | if now.isLeaf && start <= pos { 193 | return string(runes[start : pos+1]) 194 | } 195 | 196 | parent = now 197 | } 198 | 199 | return "" 200 | } 201 | 202 | func (m *DfaModel) IsSensitive(text string) bool { 203 | return m.FindOne(text) != "" 204 | } 205 | 206 | func (m *DfaModel) Replace(text string, repl rune) string { 207 | var found bool 208 | var now *dfaNode 209 | 210 | start := 0 211 | parent := m.root 212 | runes := []rune(text) 213 | length := len(runes) 214 | 215 | for pos := 0; pos < length; pos++ { 216 | now, found = parent.children[runes[pos]] 217 | 218 | if !found || (!now.isLeaf && pos == length-1) { 219 | parent = m.root 220 | pos = start 221 | start++ 222 | continue 223 | } 224 | 225 | if now.isLeaf && start <= pos { 226 | for i := start; i <= pos; i++ { 227 | runes[i] = repl 228 | } 229 | } 230 | 231 | parent = now 232 | } 233 | 234 | return string(runes) 235 | } 236 | 237 | func (m *DfaModel) Remove(text string) string { 238 | var found bool 239 | var now *dfaNode 240 | 241 | start := 0 // 从文本的第几个文字开始匹配 242 | parent := m.root 243 | runes := []rune(text) 244 | length := len(runes) 245 | filtered := make([]rune, 0, length) 246 | 247 | for pos := 0; pos < length; pos++ { 248 | now, found = parent.children[runes[pos]] 249 | 250 | if !found || (!now.isLeaf && pos == length-1) { 251 | filtered = append(filtered, runes[start]) 252 | parent = m.root 253 | pos = start 254 | start++ 255 | continue 256 | } 257 | 258 | if now.isLeaf { 259 | start = pos + 1 260 | parent = m.root 261 | } else { 262 | parent = now 263 | } 264 | } 265 | 266 | filtered = append(filtered, runes[start:]...) 267 | 268 | return string(filtered) 269 | } 270 | -------------------------------------------------------------------------------- /filter/filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | type ( 4 | Filter interface { 5 | // FindAll 找到所有敏感词 6 | FindAll(text string) []string 7 | // FindAllCount 找到所有敏感词及出现次数 8 | FindAllCount(text string) map[string]int 9 | // FindOne 找到一个敏感词 10 | FindOne(text string) string 11 | // IsSensitive 是否有敏感词 12 | IsSensitive(text string) bool 13 | // Replace 和谐敏感词 14 | Replace(text string, repl rune) string 15 | // Remove 过滤铭感词 16 | Remove(text string) string 17 | } 18 | ) 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sgoware/go-sensitive 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/go-sql-driver/mysql v1.7.0 7 | github.com/imroc/req/v3 v3.30.0 8 | github.com/jmoiron/sqlx v1.3.5 9 | github.com/orcaman/concurrent-map/v2 v2.0.1 10 | github.com/sgoware/ds v0.0.0-20230824044855-a6df320656ac 11 | go.mongodb.org/mongo-driver v1.11.1 12 | ) 13 | 14 | require ( 15 | github.com/cheekybits/genny v1.0.0 // indirect 16 | github.com/fsnotify/fsnotify v1.5.4 // indirect 17 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect 18 | github.com/golang/snappy v0.0.1 // indirect 19 | github.com/hashicorp/errwrap v1.1.0 // indirect 20 | github.com/hashicorp/go-multierror v1.1.1 // indirect 21 | github.com/klauspost/compress v1.13.6 // indirect 22 | github.com/marten-seemann/qpack v0.2.1 // indirect 23 | github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect 24 | github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect 25 | github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect 26 | github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect 27 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect 28 | github.com/nxadm/tail v1.4.8 // indirect 29 | github.com/onsi/ginkgo v1.16.5 // indirect 30 | github.com/pkg/errors v0.9.1 // indirect 31 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 32 | github.com/xdg-go/scram v1.1.1 // indirect 33 | github.com/xdg-go/stringprep v1.0.3 // indirect 34 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect 35 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect 36 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect 37 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect 38 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect 39 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 40 | golang.org/x/text v0.3.7 // indirect 41 | golang.org/x/tools v0.1.11 // indirect 42 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 43 | ) 44 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 4 | cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= 5 | dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= 6 | dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= 7 | dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= 8 | dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= 9 | git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= 10 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 11 | github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= 12 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 13 | github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= 14 | github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= 15 | github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= 16 | github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= 17 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 18 | github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 19 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 21 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 23 | github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= 24 | github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= 25 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 26 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 27 | github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= 28 | github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= 29 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 30 | github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= 31 | github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= 32 | github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 33 | github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= 34 | github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 35 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= 36 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 37 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 38 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 39 | github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= 40 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 41 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 42 | github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= 43 | github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= 44 | github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= 45 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 46 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 47 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 48 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 49 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 50 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 51 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 52 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 53 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 54 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 55 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 56 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= 57 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 58 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 59 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 60 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 61 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 62 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 63 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 64 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 65 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 66 | github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= 67 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 68 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 69 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 70 | github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= 71 | github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= 72 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 73 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 74 | github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= 75 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 76 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 77 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 78 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 79 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 80 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 81 | github.com/imroc/req/v3 v3.30.0 h1:4iSXgIQfh/3N7JK9Lt7S0q3n/ZvuGICYwV3iv/MWY1M= 82 | github.com/imroc/req/v3 v3.30.0/go.mod h1:DKtNwSxMdpqZKJ6neBw8VwRioq78uwmQB4ynQEXNNUk= 83 | github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= 84 | github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= 85 | github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= 86 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 87 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 88 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 89 | github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= 90 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 91 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 92 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 93 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 94 | github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 95 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 96 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 97 | github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= 98 | github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 99 | github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= 100 | github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 101 | github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= 102 | github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= 103 | github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ= 104 | github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= 105 | github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ= 106 | github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= 107 | github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= 108 | github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= 109 | github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= 110 | github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= 111 | github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= 112 | github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 113 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 114 | github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= 115 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 116 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 117 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= 118 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= 119 | github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= 120 | github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= 121 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 122 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 123 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 124 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 125 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 126 | github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= 127 | github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= 128 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 129 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 130 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 131 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 132 | github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= 133 | github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= 134 | github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= 135 | github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= 136 | github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= 137 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 138 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 139 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 140 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 141 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 142 | github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 143 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 144 | github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 145 | github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 146 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 147 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 148 | github.com/sgoware/ds v0.0.0-20230824044855-a6df320656ac h1:p59i/xcLez9rAlqlBXcy3YTPfuWm+WsH8oLiZrh1lFU= 149 | github.com/sgoware/ds v0.0.0-20230824044855-a6df320656ac/go.mod h1:/dWPV5x7hOW09XyFiXiyLSLN17gY+lXhBSHk2CVdhkU= 150 | github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= 151 | github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= 152 | github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= 153 | github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= 154 | github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= 155 | github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= 156 | github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= 157 | github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= 158 | github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= 159 | github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= 160 | github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= 161 | github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= 162 | github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= 163 | github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= 164 | github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= 165 | github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= 166 | github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= 167 | github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= 168 | github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= 169 | github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 170 | github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= 171 | github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= 172 | github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= 173 | github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= 174 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 175 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 176 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 177 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 178 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 179 | github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= 180 | github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= 181 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 182 | github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= 183 | github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= 184 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 185 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 186 | github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= 187 | github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= 188 | github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= 189 | github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= 190 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= 191 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= 192 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 193 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 194 | github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 195 | go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8= 196 | go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= 197 | go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= 198 | go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= 199 | golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= 200 | golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 201 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 202 | golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 203 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 204 | golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 205 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 206 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 207 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= 208 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 209 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 210 | golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 211 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 212 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 213 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 214 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 215 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= 216 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 217 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 218 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 219 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 220 | golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 221 | golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 222 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 223 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 224 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 225 | golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 226 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 227 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 228 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 229 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 230 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 231 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 232 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 233 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= 234 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 235 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= 236 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 237 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 238 | golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 239 | golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 240 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 241 | golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= 242 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 243 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 244 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 245 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 246 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 247 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 248 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 249 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 250 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 251 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 252 | golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 253 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 254 | golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 255 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 256 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 257 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 258 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 259 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 260 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 261 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 262 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 263 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 264 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 265 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 266 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 267 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 268 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 269 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 270 | golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 271 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= 272 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 273 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 274 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 275 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 276 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 277 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 278 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 279 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 280 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 281 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 282 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 283 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 284 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 285 | golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 286 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 287 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 288 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 289 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 290 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 291 | golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 292 | golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY= 293 | golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= 294 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 295 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 296 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 297 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 298 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 299 | google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= 300 | google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= 301 | google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= 302 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 303 | google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 304 | google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 305 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 306 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 307 | google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 308 | google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 309 | google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= 310 | google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 311 | google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= 312 | google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= 313 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 314 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 315 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 316 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 317 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 318 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 319 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 320 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 321 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 322 | google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= 323 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 324 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 325 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 326 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 327 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 328 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 329 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 330 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 331 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 332 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 333 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 334 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 335 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 336 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 337 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 338 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 339 | grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= 340 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 341 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 342 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 343 | sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= 344 | sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= 345 | -------------------------------------------------------------------------------- /manager.go: -------------------------------------------------------------------------------- 1 | package sensitive 2 | 3 | import ( 4 | "github.com/sgoware/go-sensitive/filter" 5 | "github.com/sgoware/go-sensitive/store" 6 | ) 7 | 8 | type Manager struct { 9 | store.Store 10 | filter.Filter 11 | } 12 | 13 | func NewFilter(storeOption StoreOption, filterOption FilterOption) *Manager { 14 | var filterStore store.Store 15 | var myFilter filter.Filter 16 | 17 | switch storeOption.Type { 18 | case StoreMemory: 19 | filterStore = store.NewMemoryModel() 20 | default: 21 | panic("invalid store type") 22 | } 23 | 24 | switch filterOption.Type { 25 | case FilterDfa: 26 | dfaModel := filter.NewDfaModel() 27 | 28 | go dfaModel.Listen(filterStore.GetAddChan(), filterStore.GetDelChan()) 29 | 30 | myFilter = dfaModel 31 | case FilterAc: 32 | acModel := filter.NewAcModel() 33 | 34 | go acModel.Listen(filterStore.GetAddChan(), filterStore.GetDelChan()) 35 | 36 | myFilter = acModel 37 | default: 38 | panic("invalid filter type") 39 | } 40 | 41 | return &Manager{ 42 | Store: filterStore, 43 | Filter: myFilter, 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /manager_test.go: -------------------------------------------------------------------------------- 1 | package sensitive 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/sgoware/go-sensitive/store" 10 | ) 11 | 12 | func Test_NewFilter(t *testing.T) { 13 | type args struct { 14 | storeOption StoreOption 15 | filterOption FilterOption 16 | } 17 | tests := []struct { 18 | name string 19 | args args 20 | }{ 21 | { 22 | name: "memory+dfa", 23 | args: args{ 24 | storeOption: StoreOption{ 25 | Type: StoreMemory, 26 | }, 27 | filterOption: FilterOption{ 28 | Type: FilterDfa, 29 | }, 30 | }, 31 | }, 32 | { 33 | name: "mongo+dfa", 34 | args: args{ 35 | storeOption: StoreOption{ 36 | Type: StoreMongo, 37 | MongoConfig: &store.MongoConfig{ 38 | Address: "", 39 | Port: "", 40 | Username: "", 41 | Password: "", 42 | Database: "", 43 | Collection: "", 44 | }, 45 | }, 46 | filterOption: FilterOption{ 47 | Type: FilterDfa, 48 | }, 49 | }, 50 | }, 51 | { 52 | name: "mysql+dfa", 53 | args: args{ 54 | storeOption: StoreOption{ 55 | Type: StoreMysql, 56 | MysqlConfig: &store.MysqlConfig{ 57 | Dsn: "", 58 | Database: "", 59 | TableName: "", 60 | }, 61 | }, 62 | filterOption: FilterOption{ 63 | Type: FilterDfa, 64 | }, 65 | }, 66 | }, 67 | } 68 | for _, tt := range tests { 69 | t.Run(tt.name, func(t *testing.T) { 70 | filterManager := NewFilter(tt.args.storeOption, tt.args.filterOption) 71 | 72 | err := filterManager.LoadDictPath("./dict/default_dict.txt") 73 | if err != nil { 74 | fmt.Println(err) 75 | } 76 | 77 | err = filterManager.AddWord("敏感词1", "敏感词2", "敏感词3") 78 | if err != nil { 79 | t.Errorf("add sensitive word failed, err: %v", err) 80 | } 81 | 82 | time.Sleep(1 * time.Second) 83 | 84 | isSensitive := filterManager.IsSensitive("敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词") 85 | if !reflect.DeepEqual(isSensitive, true) { 86 | t.Errorf("IsSensitive() = %v, want %v", isSensitive, true) 87 | } 88 | 89 | filtered := filterManager.Remove("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词") 90 | if !reflect.DeepEqual(filtered, "这是,这是,这是,这是,这里没有敏感词") { 91 | t.Errorf("Remove() = %v, want %v", filtered, "这是,这是,这是,这是,这里没有敏感词") 92 | } 93 | 94 | replaced := filterManager.Replace("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词", '*') 95 | if !reflect.DeepEqual(replaced, "这是****,这是****,这是****,这是****,这里没有敏感词") { 96 | t.Errorf("Replace() = %v, want %v", replaced, "这是****,这是****,这是****,这是****,这里没有敏感词") 97 | } 98 | 99 | matchedOne := filterManager.FindOne("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词") 100 | if !reflect.DeepEqual(matchedOne, "敏感词1") { 101 | t.Errorf("FindOne() = %v, want %v", matchedOne, "敏感词1") 102 | } 103 | 104 | matchedAll := filterManager.FindAll("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词") 105 | if !reflect.DeepEqual(matchedAll, []string{"敏感词1", "敏感词2", "敏感词3"}) { 106 | t.Errorf("FindAll() = %v, want %v", matchedAll, []string{"敏感词1", "敏感词2", "敏感词3"}) 107 | } 108 | 109 | matchedMap := filterManager.FindAllCount("这是敏感词1,这是敏感词2,这是敏感词3,这是敏感词1,这里没有敏感词") 110 | if !reflect.DeepEqual(matchedMap, map[string]int{"敏感词1": 2, "敏感词2": 1, "敏感词3": 1}) { 111 | t.Errorf("FindAllCount() = %v, want %v", matchedMap, map[string]int{"敏感词1": 2, "敏感词2": 1, "敏感词3": 1}) 112 | } 113 | }) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package sensitive 2 | 3 | import ( 4 | "github.com/sgoware/go-sensitive/store" 5 | ) 6 | 7 | const ( 8 | StoreMemory = iota 9 | StoreMysql 10 | StoreMongo 11 | ) 12 | 13 | const ( 14 | FilterDfa = iota 15 | FilterAc 16 | ) 17 | 18 | type StoreOption struct { 19 | Type uint32 20 | MysqlConfig *store.MysqlConfig 21 | MongoConfig *store.MongoConfig 22 | } 23 | 24 | type FilterOption struct { 25 | Type uint32 26 | } 27 | -------------------------------------------------------------------------------- /store/memory.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "github.com/imroc/req/v3" 7 | cmap "github.com/orcaman/concurrent-map/v2" 8 | "io" 9 | "net/http" 10 | "os" 11 | ) 12 | 13 | type MemoryModel struct { 14 | store cmap.ConcurrentMap[string, struct{}] 15 | addChan chan string 16 | delChan chan string 17 | } 18 | 19 | func NewMemoryModel() *MemoryModel { 20 | return &MemoryModel{ 21 | store: cmap.New[struct{}](), 22 | addChan: make(chan string), 23 | delChan: make(chan string), 24 | } 25 | } 26 | 27 | func (m *MemoryModel) LoadDictPath(paths ...string) error { 28 | for _, path := range paths { 29 | err := func(path string) error { 30 | f, err := os.Open(path) 31 | defer func(f *os.File) { 32 | _ = f.Close() 33 | }(f) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | return m.LoadDict(f) 39 | }(path) 40 | if err != nil { 41 | return err 42 | } 43 | } 44 | 45 | return nil 46 | } 47 | 48 | func (m *MemoryModel) LoadDictHttp(urls ...string) error { 49 | for _, url := range urls { 50 | err := func(url string) error { 51 | httpRes, err := req.Get(url) 52 | if err != nil { 53 | return err 54 | } 55 | if httpRes == nil { 56 | return errors.New("nil http response") 57 | } 58 | if httpRes.StatusCode != http.StatusOK { 59 | return errors.New(httpRes.GetStatus()) 60 | } 61 | 62 | defer func(Body io.ReadCloser) { 63 | _ = Body.Close() 64 | }(httpRes.Body) 65 | 66 | return m.LoadDict(httpRes.Body) 67 | }(url) 68 | if err != nil { 69 | return err 70 | } 71 | } 72 | 73 | return nil 74 | } 75 | 76 | func (m *MemoryModel) LoadDict(reader io.Reader) error { 77 | buf := bufio.NewReader(reader) 78 | for { 79 | line, _, err := buf.ReadLine() 80 | if err != nil { 81 | if err != io.EOF { 82 | return err 83 | } 84 | break 85 | } 86 | 87 | m.store.Set(string(line), struct{}{}) 88 | m.addChan <- string(line) 89 | } 90 | 91 | return nil 92 | } 93 | 94 | func (m *MemoryModel) ReadChan() <-chan string { 95 | ch := make(chan string) 96 | 97 | go func() { 98 | for key := range m.store.Items() { 99 | ch <- key 100 | } 101 | close(ch) 102 | }() 103 | 104 | return ch 105 | } 106 | 107 | func (m *MemoryModel) ReadString() []string { 108 | res := make([]string, 0, m.store.Count()) 109 | 110 | for key := range m.store.Items() { 111 | res = append(res, key) 112 | } 113 | 114 | return res 115 | } 116 | 117 | func (m *MemoryModel) GetAddChan() <-chan string { 118 | return m.addChan 119 | } 120 | 121 | func (m *MemoryModel) GetDelChan() <-chan string { 122 | return m.delChan 123 | } 124 | 125 | func (m *MemoryModel) AddWord(words ...string) error { 126 | for _, word := range words { 127 | m.store.Set(word, struct{}{}) 128 | m.addChan <- word 129 | } 130 | 131 | return nil 132 | } 133 | 134 | func (m *MemoryModel) DelWord(words ...string) error { 135 | for _, word := range words { 136 | m.store.Remove(word) 137 | m.delChan <- word 138 | } 139 | 140 | return nil 141 | } 142 | -------------------------------------------------------------------------------- /store/mongo.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "os" 11 | 12 | "github.com/imroc/req/v3" 13 | "go.mongodb.org/mongo-driver/bson" 14 | "go.mongodb.org/mongo-driver/mongo" 15 | "go.mongodb.org/mongo-driver/mongo/options" 16 | ) 17 | 18 | const ( 19 | defaultCollection = "dirties" 20 | ) 21 | 22 | type MongoConfig struct { 23 | Address string 24 | Port string 25 | Username string 26 | Password string 27 | Database string 28 | Collection string 29 | FieldName string 30 | } 31 | 32 | type doc struct { 33 | Id string `bson:"_id"` 34 | Word string `bson:"word"` 35 | } 36 | 37 | type MongoModel struct { 38 | store *mongo.Collection 39 | fieldName string 40 | 41 | addChan chan string 42 | delChan chan string 43 | } 44 | 45 | func NewMongoModel(config *MongoConfig) *MongoModel { 46 | clientOptions := options.Client().ApplyURI( 47 | fmt.Sprintf("mongodb://%s:%s", 48 | config.Address, 49 | config.Port, 50 | ), 51 | ) 52 | 53 | if config.Username != "" { 54 | clientOptions.SetAuth(options.Credential{ 55 | Username: config.Username, 56 | Password: config.Password, 57 | }) 58 | } 59 | 60 | mdb, err := mongo.Connect(context.TODO(), clientOptions) 61 | if err != nil { 62 | return nil 63 | } 64 | 65 | err = mdb.Ping(context.TODO(), nil) 66 | if err != nil { 67 | return nil 68 | } 69 | 70 | if config.Database == "" { 71 | return nil 72 | } 73 | 74 | if config.Collection == "" { 75 | config.Collection = defaultCollection 76 | } 77 | 78 | collection := mdb.Database(config.Database).Collection(config.Collection) 79 | 80 | _, err = collection.Indexes().CreateOne(context.Background(), 81 | mongo.IndexModel{ 82 | Keys: bson.D{{"word", 1}}, 83 | Options: options.Index().SetUnique(true), 84 | }, 85 | ) 86 | if err != nil { 87 | return nil 88 | } 89 | 90 | return &MongoModel{ 91 | store: collection, 92 | fieldName: config.FieldName, 93 | 94 | addChan: make(chan string), 95 | delChan: make(chan string), 96 | } 97 | } 98 | 99 | func (m *MongoModel) LoadDictPath(paths ...string) error { 100 | for _, path := range paths { 101 | err := func(path string) error { 102 | f, err := os.Open(path) 103 | defer func(f *os.File) { 104 | _ = f.Close() 105 | }(f) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | return m.LoadDict(f) 111 | }(path) 112 | if err != nil { 113 | return err 114 | } 115 | } 116 | 117 | return nil 118 | } 119 | 120 | func (m *MongoModel) LoadDictHttp(urls ...string) error { 121 | for _, url := range urls { 122 | err := func(url string) error { 123 | httpRes, err := req.Get(url) 124 | if err != nil { 125 | return err 126 | } 127 | if httpRes == nil { 128 | return errors.New("nil http response") 129 | } 130 | if httpRes.StatusCode != http.StatusOK { 131 | return errors.New(httpRes.GetStatus()) 132 | } 133 | 134 | defer func(Body io.ReadCloser) { 135 | _ = Body.Close() 136 | }(httpRes.Body) 137 | 138 | return m.LoadDict(httpRes.Body) 139 | }(url) 140 | if err != nil { 141 | return err 142 | } 143 | } 144 | 145 | return nil 146 | } 147 | 148 | func (m *MongoModel) LoadDict(reader io.Reader) error { 149 | buf := bufio.NewReader(reader) 150 | var words []interface{} 151 | 152 | for { 153 | line, _, err := buf.ReadLine() 154 | if err != nil { 155 | if err != io.EOF { 156 | return err 157 | } 158 | break 159 | } 160 | 161 | word := string(line) 162 | 163 | words = append(words, bson.D{ 164 | {m.fieldName, word}, 165 | }) 166 | 167 | m.addChan <- word 168 | } 169 | 170 | ctx := context.Background() 171 | 172 | _, _ = m.store.InsertMany(ctx, words, options.InsertMany().SetOrdered(false)) 173 | 174 | return nil 175 | } 176 | 177 | func (m *MongoModel) ReadChan() <-chan string { 178 | ch := make(chan string) 179 | 180 | go func() { 181 | ctx := context.Background() 182 | cur, _ := m.store.Find(ctx, 183 | bson.D{}, 184 | options.Find().SetProjection( 185 | bson.D{ 186 | {"_id", 0}, 187 | {"word", 1}, 188 | }, 189 | ), 190 | ) 191 | 192 | for cur.Next(ctx) { 193 | var word doc 194 | 195 | _ = cur.Decode(&word) 196 | 197 | ch <- word.Word 198 | } 199 | 200 | close(ch) 201 | }() 202 | 203 | return ch 204 | } 205 | 206 | func (m *MongoModel) ReadString() []string { 207 | ctx := context.Background() 208 | cur, err := m.store.Find(ctx, 209 | bson.D{}, 210 | options.Find().SetProjection( 211 | bson.D{ 212 | {"_id", 0}, 213 | {"word", 1}, 214 | }, 215 | ), 216 | ) 217 | if err != nil { 218 | 219 | } 220 | 221 | var words []*doc 222 | 223 | err = cur.All(ctx, &words) 224 | 225 | res := make([]string, 0, len(words)) 226 | 227 | for _, word := range words { 228 | res = append(res, word.Word) 229 | } 230 | 231 | return res 232 | } 233 | 234 | func (m *MongoModel) GetAddChan() <-chan string { 235 | return m.addChan 236 | } 237 | 238 | func (m *MongoModel) GetDelChan() <-chan string { 239 | return m.delChan 240 | } 241 | 242 | func (m *MongoModel) AddWord(words ...string) error { 243 | for _, word := range words { 244 | _, err := m.store.UpdateOne(context.Background(), 245 | bson.D{ 246 | {"word", word}, 247 | }, 248 | bson.D{ 249 | {"$set", bson.D{ 250 | {"word", word}, 251 | }}, 252 | }, 253 | options.Update().SetUpsert(true), 254 | ) 255 | if err != nil { 256 | return err 257 | } 258 | 259 | m.addChan <- word 260 | } 261 | 262 | return nil 263 | } 264 | 265 | func (m *MongoModel) DelWord(words ...string) error { 266 | for _, word := range words { 267 | _, err := m.store.DeleteOne(context.Background(), 268 | bson.D{ 269 | {"word", word}, 270 | }, 271 | ) 272 | if err != nil { 273 | return err 274 | } 275 | 276 | m.delChan <- word 277 | } 278 | 279 | return nil 280 | } 281 | -------------------------------------------------------------------------------- /store/mysql.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bufio" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | _ "github.com/go-sql-driver/mysql" 9 | "github.com/imroc/req/v3" 10 | "github.com/jmoiron/sqlx" 11 | "io" 12 | "net/http" 13 | "os" 14 | ) 15 | 16 | const ( 17 | defaultTable = "dirties" 18 | ) 19 | 20 | type MysqlConfig struct { 21 | Dsn string 22 | Database string 23 | TableName string 24 | } 25 | 26 | type Subject struct { 27 | Id int64 `db:"id"` 28 | Word string `db:"word"` 29 | } 30 | 31 | type MysqlModel struct { 32 | store *sqlx.DB 33 | TableName string 34 | addChan chan string 35 | delChan chan string 36 | } 37 | 38 | func NewMysqlModel(config *MysqlConfig) *MysqlModel { 39 | db, err := sqlx.Connect("mysql", config.Dsn) 40 | if err != nil { 41 | return nil 42 | } 43 | 44 | if config.TableName == "" { 45 | config.TableName = defaultTable 46 | } 47 | 48 | var tableName string 49 | 50 | err = db.Get( 51 | &tableName, 52 | fmt.Sprintf( 53 | "SELECT `TABLE_NAME` FROM information_schema.tables WHERE table_schema = '%s' AND table_name = '%s' LIMIT 1", 54 | config.Database, 55 | config.TableName), 56 | ) 57 | if err != nil { 58 | if err != sql.ErrNoRows { 59 | return nil 60 | } 61 | } 62 | 63 | if tableName == "" { 64 | _, err = db.Exec(fmt.Sprintf("CREATE TABLE `%s` "+ 65 | "(`id` bigint(20) NOT NULL AUTO_INCREMENT, "+ 66 | "`word` varchar(255) NOT NULL, "+ 67 | "PRIMARY KEY (`id`) USING BTREE)", 68 | config.TableName), 69 | ) 70 | if err != nil { 71 | return nil 72 | } 73 | } 74 | 75 | return &MysqlModel{ 76 | store: db, 77 | TableName: config.TableName, 78 | addChan: make(chan string), 79 | delChan: make(chan string), 80 | } 81 | } 82 | 83 | func (m *MysqlModel) LoadDictPath(paths ...string) error { 84 | for _, path := range paths { 85 | err := func(path string) error { 86 | f, err := os.Open(path) 87 | defer func(f *os.File) { 88 | _ = f.Close() 89 | }(f) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | return m.LoadDict(f) 95 | }(path) 96 | if err != nil { 97 | return err 98 | } 99 | } 100 | 101 | return nil 102 | } 103 | 104 | func (m *MysqlModel) LoadDictHttp(urls ...string) error { 105 | for _, url := range urls { 106 | err := func(url string) error { 107 | httpRes, err := req.Get(url) 108 | if err != nil { 109 | return err 110 | } 111 | if httpRes == nil { 112 | return errors.New("nil http response") 113 | } 114 | if httpRes.StatusCode != http.StatusOK { 115 | return errors.New(httpRes.GetStatus()) 116 | } 117 | 118 | defer func(Body io.ReadCloser) { 119 | _ = Body.Close() 120 | }(httpRes.Body) 121 | 122 | return m.LoadDict(httpRes.Body) 123 | }(url) 124 | if err != nil { 125 | return err 126 | } 127 | } 128 | 129 | return nil 130 | } 131 | 132 | func (m *MysqlModel) LoadDict(reader io.Reader) error { 133 | buf := bufio.NewReader(reader) 134 | var words []*Subject 135 | set := make(map[string]struct{}) 136 | 137 | for { 138 | line, _, err := buf.ReadLine() 139 | if err != nil { 140 | if err != io.EOF { 141 | return err 142 | } 143 | break 144 | } 145 | 146 | word := string(line) 147 | 148 | if _, ok := set[word]; !ok { 149 | words = append(words, &Subject{ 150 | Id: 0, 151 | Word: word, 152 | }) 153 | set[word] = struct{}{} 154 | } 155 | 156 | m.addChan <- word 157 | } 158 | 159 | _, err := m.store.NamedExec(fmt.Sprintf("INSERT INTO `%s` (`word`) VALUES (:word)", m.TableName), words) 160 | if err != nil { 161 | return err 162 | } 163 | 164 | _, err = m.store.Exec(fmt.Sprintf("DELETE FROM `%s` AS t1 "+ 165 | "WHERE t1.`id` <> "+ 166 | "(SELECT t.minid FROM "+ 167 | "(SELECT MIN(t2.`id`) AS minid FROM `%s` AS t2 WHERE t1.`word` = t2.`word`) t )", 168 | m.TableName, 169 | m.TableName), 170 | ) 171 | if err != nil { 172 | return err 173 | } 174 | 175 | return nil 176 | } 177 | 178 | func (m *MysqlModel) ReadChan() <-chan string { 179 | ch := make(chan string) 180 | var words []string 181 | 182 | go func() { 183 | defer close(ch) 184 | 185 | err := m.store.Select(&words, fmt.Sprintf("SELECT `word` FROM `%s`", m.TableName)) 186 | if err != nil { 187 | return 188 | } 189 | 190 | for _, word := range words { 191 | ch <- word 192 | } 193 | }() 194 | 195 | return ch 196 | } 197 | 198 | func (m *MysqlModel) ReadString() []string { 199 | var words []string 200 | 201 | err := m.store.Select(&words, fmt.Sprintf("SELECT `word` FROM `%s`", m.TableName)) 202 | if err != nil { 203 | return nil 204 | } 205 | 206 | return words 207 | } 208 | 209 | func (m *MysqlModel) GetAddChan() <-chan string { 210 | return m.addChan 211 | } 212 | 213 | func (m *MysqlModel) GetDelChan() <-chan string { 214 | return m.delChan 215 | } 216 | 217 | func (m *MysqlModel) AddWord(words ...string) error { 218 | insertedWords := make([]*Subject, 0, len(words)) 219 | set := make(map[string]struct{}) 220 | 221 | for _, word := range words { 222 | if _, ok := set[word]; !ok { 223 | insertedWords = append(insertedWords, &Subject{ 224 | Id: 0, 225 | Word: word, 226 | }) 227 | set[word] = struct{}{} 228 | } 229 | } 230 | 231 | _, err := m.store.NamedExec(fmt.Sprintf("INSERT INTO `%s` (`word`) VALUES (:word)", m.TableName), insertedWords) 232 | if err != nil { 233 | return err 234 | } 235 | _, err = m.store.Exec(fmt.Sprintf("DELETE FROM `%s` AS t1 "+ 236 | "WHERE t1.`id` <> "+ 237 | "(SELECT t.minid FROM "+ 238 | "(SELECT MIN(t2.`id`) AS minid FROM `%s` AS t2 WHERE t1.`word` = t2.`word`) t )", 239 | m.TableName, 240 | m.TableName), 241 | ) 242 | if err != nil { 243 | return err 244 | } 245 | 246 | return nil 247 | } 248 | 249 | func (m *MysqlModel) DelWord(words ...string) error { 250 | query, args, _ := sqlx.In(fmt.Sprintf("DELETE FROM `%s` WHERE `word` IN (?)", m.TableName), words) 251 | _, err := m.store.Exec(query, args) 252 | if err != nil { 253 | return err 254 | } 255 | 256 | return nil 257 | } 258 | -------------------------------------------------------------------------------- /store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "io" 4 | 5 | type ( 6 | Store interface { 7 | LoadDictPath(path ...string) error 8 | LoadDictHttp(url ...string) error 9 | LoadDict(reader io.Reader) error 10 | ReadChan() <-chan string 11 | ReadString() []string 12 | GetAddChan() <-chan string 13 | GetDelChan() <-chan string 14 | AddWord(words ...string) error 15 | DelWord(words ...string) error 16 | } 17 | ) 18 | --------------------------------------------------------------------------------