├── .gitignore ├── LICENSE ├── README.md ├── api.go ├── api_tagger.go ├── cmd ├── gen_lemmatize_data │ ├── lang.go │ └── main.go ├── gen_normalize_data │ └── main.go └── ling │ └── main.go ├── common_regex.go ├── dict_tagger.go ├── document.go ├── lemmatize ├── eng.go └── init.go ├── lemmatizer.go ├── normalize ├── cmn.go └── init.go ├── normalizer.go ├── regex_tagger.go ├── tokenizer.go ├── tokentype_jsonenums.go ├── tokentype_string.go ├── types.go ├── unidecoder.go ├── unittests ├── all_test.go ├── api_tagger_test.go └── dict_tagger_test.go └── util ├── io.go ├── types.go └── utils.go /.gitignore: -------------------------------------------------------------------------------- 1 | dict 2 | cmd/ling/ling 3 | *.swp 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Zhanliang Liu. All rights reserved. 2 | 3 | Apache License 4 | Version 2.0, January 2004 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, 12 | and distribution as defined by Sections 1 through 9 of this document. 13 | 14 | "Licensor" shall mean the copyright owner or entity authorized by 15 | the copyright owner that is granting the License. 16 | 17 | "Legal Entity" shall mean the union of the acting entity and all 18 | other entities that control, are controlled by, or are under common 19 | control with that entity. For the purposes of this definition, 20 | "control" means (i) the power, direct or indirect, to cause the 21 | direction or management of such entity, whether by contract or 22 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 23 | outstanding shares, or (iii) beneficial ownership of such entity. 24 | 25 | "You" (or "Your") shall mean an individual or Legal Entity 26 | exercising permissions granted by this License. 27 | 28 | "Source" form shall mean the preferred form for making modifications, 29 | including but not limited to software source code, documentation 30 | source, and configuration files. 31 | 32 | "Object" form shall mean any form resulting from mechanical 33 | transformation or translation of a Source form, including but 34 | not limited to compiled object code, generated documentation, 35 | and conversions to other media types. 36 | 37 | "Work" shall mean the work of authorship, whether in Source or 38 | Object form, made available under the License, as indicated by a 39 | copyright notice that is included in or attached to the work 40 | (an example is provided in the Appendix below). 41 | 42 | "Derivative Works" shall mean any work, whether in Source or Object 43 | form, that is based on (or derived from) the Work and for which the 44 | editorial revisions, annotations, elaborations, or other modifications 45 | represent, as a whole, an original work of authorship. For the purposes 46 | of this License, Derivative Works shall not include works that remain 47 | separable from, or merely link (or bind by name) to the interfaces of, 48 | the Work and Derivative Works thereof. 49 | 50 | "Contribution" shall mean any work of authorship, including 51 | the original version of the Work and any modifications or additions 52 | to that Work or Derivative Works thereof, that is intentionally 53 | submitted to Licensor for inclusion in the Work by the copyright owner 54 | or by an individual or Legal Entity authorized to submit on behalf of 55 | the copyright owner. For the purposes of this definition, "submitted" 56 | means any form of electronic, verbal, or written communication sent 57 | to the Licensor or its representatives, including but not limited to 58 | communication on electronic mailing lists, source code control systems, 59 | and issue tracking systems that are managed by, or on behalf of, the 60 | Licensor for the purpose of discussing and improving the Work, but 61 | excluding communication that is conspicuously marked or otherwise 62 | designated in writing by the copyright owner as "Not a Contribution." 63 | 64 | "Contributor" shall mean Licensor and any individual or Legal Entity 65 | on behalf of whom a Contribution has been received by Licensor and 66 | subsequently incorporated within the Work. 67 | 68 | 2. Grant of Copyright License. Subject to the terms and conditions of 69 | this License, each Contributor hereby grants to You a perpetual, 70 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 71 | copyright license to reproduce, prepare Derivative Works of, 72 | publicly display, publicly perform, sublicense, and distribute the 73 | Work and such Derivative Works in Source or Object form. 74 | 75 | 3. Grant of Patent License. Subject to the terms and conditions of 76 | this License, each Contributor hereby grants to You a perpetual, 77 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 78 | (except as stated in this section) patent license to make, have made, 79 | use, offer to sell, sell, import, and otherwise transfer the Work, 80 | where such license applies only to those patent claims licensable 81 | by such Contributor that are necessarily infringed by their 82 | Contribution(s) alone or by combination of their Contribution(s) 83 | with the Work to which such Contribution(s) was submitted. If You 84 | institute patent litigation against any entity (including a 85 | cross-claim or counterclaim in a lawsuit) alleging that the Work 86 | or a Contribution incorporated within the Work constitutes direct 87 | or contributory patent infringement, then any patent licenses 88 | granted to You under this License for that Work shall terminate 89 | as of the date such litigation is filed. 90 | 91 | 4. Redistribution. You may reproduce and distribute copies of the 92 | Work or Derivative Works thereof in any medium, with or without 93 | modifications, and in Source or Object form, provided that You 94 | meet the following conditions: 95 | 96 | (a) You must give any other recipients of the Work or 97 | Derivative Works a copy of this License; and 98 | 99 | (b) You must cause any modified files to carry prominent notices 100 | stating that You changed the files; and 101 | 102 | (c) You must retain, in the Source form of any Derivative Works 103 | that You distribute, all copyright, patent, trademark, and 104 | attribution notices from the Source form of the Work, 105 | excluding those notices that do not pertain to any part of 106 | the Derivative Works; and 107 | 108 | (d) If the Work includes a "NOTICE" text file as part of its 109 | distribution, then any Derivative Works that You distribute must 110 | include a readable copy of the attribution notices contained 111 | within such NOTICE file, excluding those notices that do not 112 | pertain to any part of the Derivative Works, in at least one 113 | of the following places: within a NOTICE text file distributed 114 | as part of the Derivative Works; within the Source form or 115 | documentation, if provided along with the Derivative Works; or, 116 | within a display generated by the Derivative Works, if and 117 | wherever such third-party notices normally appear. The contents 118 | of the NOTICE file are for informational purposes only and 119 | do not modify the License. You may add Your own attribution 120 | notices within Derivative Works that You distribute, alongside 121 | or as an addendum to the NOTICE text from the Work, provided 122 | that such additional attribution notices cannot be construed 123 | as modifying the License. 124 | 125 | You may add Your own copyright statement to Your modifications and 126 | may provide additional or different license terms and conditions 127 | for use, reproduction, or distribution of Your modifications, or 128 | for any such Derivative Works as a whole, provided Your use, 129 | reproduction, and distribution of the Work otherwise complies with 130 | the conditions stated in this License. 131 | 132 | 5. Submission of Contributions. Unless You explicitly state otherwise, 133 | any Contribution intentionally submitted for inclusion in the Work 134 | by You to the Licensor shall be under the terms and conditions of 135 | this License, without any additional terms or conditions. 136 | Notwithstanding the above, nothing herein shall supersede or modify 137 | the terms of any separate license agreement you may have executed 138 | with Licensor regarding such Contributions. 139 | 140 | 6. Trademarks. This License does not grant permission to use the trade 141 | names, trademarks, service marks, or product names of the Licensor, 142 | except as required for reasonable and customary use in describing the 143 | origin of the Work and reproducing the content of the NOTICE file. 144 | 145 | 7. Disclaimer of Warranty. Unless required by applicable law or 146 | agreed to in writing, Licensor provides the Work (and each 147 | Contributor provides its Contributions) on an "AS IS" BASIS, 148 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 149 | implied, including, without limitation, any warranties or conditions 150 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 151 | PARTICULAR PURPOSE. You are solely responsible for determining the 152 | appropriateness of using or redistributing the Work and assume any 153 | risks associated with Your exercise of permissions under this License. 154 | 155 | 8. Limitation of Liability. In no event and under no legal theory, 156 | whether in tort (including negligence), contract, or otherwise, 157 | unless required by applicable law (such as deliberate and grossly 158 | negligent acts) or agreed to in writing, shall any Contributor be 159 | liable to You for damages, including any direct, indirect, special, 160 | incidental, or consequential damages of any character arising as a 161 | result of this License or out of the use or inability to use the 162 | Work (including but not limited to damages for loss of goodwill, 163 | work stoppage, computer failure or malfunction, or any and all 164 | other commercial damages or losses), even if such Contributor 165 | has been advised of the possibility of such damages. 166 | 167 | 9. Accepting Warranty or Additional Liability. While redistributing 168 | the Work or Derivative Works thereof, You may choose to offer, 169 | and charge a fee for, acceptance of support, warranty, indemnity, 170 | or other liability obligations and/or rights consistent with this 171 | License. However, in accepting such obligations, You may act only 172 | on Your own behalf and on Your sole responsibility, not on behalf 173 | of any other Contributor, and only if You agree to indemnify, 174 | defend, and hold each Contributor harmless for any liability 175 | incurred by, or claims asserted against, such Contributor by reason 176 | of your accepting any such warranty or additional liability. 177 | 178 | END OF TERMS AND CONDITIONS 179 | 180 | APPENDIX: How to apply the Apache License to your work. 181 | 182 | To apply the Apache License to your work, attach the following 183 | boilerplate notice, with the fields enclosed by brackets "[]" 184 | replaced with your own identifying information. (Don't include 185 | the brackets!) The text should be enclosed in the appropriate 186 | comment syntax for the file format. We also recommend that a 187 | file or class name and description of purpose be included on the 188 | same "printed page" as the copyright notice for easier 189 | identification within third-party archives. 190 | 191 | Copyright 2018, Zhanliang Liu. 192 | 193 | Licensed under the Apache License, Version 2.0 (the "License"); 194 | you may not use this file except in compliance with the License. 195 | You may obtain a copy of the License at 196 | 197 | http://www.apache.org/licenses/LICENSE-2.0 198 | 199 | Unless required by applicable law or agreed to in writing, software 200 | distributed under the License is distributed on an "AS IS" BASIS, 201 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 202 | See the License for the specific language governing permissions and 203 | limitations under the License. 204 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ling is a golang toolkit for natural language processing 2 | [![GoDoc](https://godoc.org/github.com/liuzl/ling?status.svg)](https://godoc.org/github.com/liuzl/ling)[![Go Report Card](https://goreportcard.com/badge/github.com/liuzl/ling)](https://goreportcard.com/report/github.com/liuzl/ling) 3 | 4 | # Implementation references 5 | ## Similar NLP tools 6 | * [Stanford CoreNLP](https://stanfordnlp.github.io/CoreNLP/index.html) Java 7 | * [spaCy](https://spacy.io/) Python 8 | * [lingo](https://github.com/chewxy/lingo) Golang 9 | ## Multilingual text toknization 10 | * [Unicode Standard Annex #29](http://www.unicode.org/reports/tr29/) 11 | * [blevesearch segment](https://github.com/liuzl/segment) 12 | ## Text normalization 13 | * [Text normalization in Go](https://blog.golang.org/normalization) 14 | ## Lemmatization 15 | > 词干提取（stemming）和词形还原（lemmatization） 16 | 17 | * [Stemming and lemmatization](https://nlp.stanford.edu/IR-book/html/htmledition/stemming-and-lemmatization-1.html) 18 | * [Lemmatization Lists](http://www.lexiconista.com/datasets/lemmatization/)*Datasets by MBM * 19 | * [The UniMorph Project](https://unimorph.github.io/) 20 | * 中文繁简转换 21 | * [gocc](https://github.com/liuzl/gocc) Golang version OpenCC 22 | * [OpenCC](https://github.com/BYVoid/OpenCC) 23 | * [Chinese-Character Jian<=>Fan converting library in Go](https://github.com/go-cc/cc-jianfan) 24 | * [Traditional and Simplified Chinese Conversion in Go](https://github.com/siongui/gojianfan) 25 | * [Han unification](https://en.wikipedia.org/wiki/Han_unification) 26 | ## Tagging 27 | * Regex tagger 28 | * [commonregex](https://github.com/mingrammer/commonregex), a collection of common regular expressions for Go. 29 | * [xurls](https://github.com/mvdan/xurls), a Go package of regex for urls. 30 | ## Natural language Detection 31 | 32 | `getlang` is much slower than `franco` 33 | 34 | * [getlang](https://github.com/rylans/getlang) 35 | * [franco](https://github.com/liuzl/franco) 36 | * [test scripts](https://github.com/liuzl/org_name_parser/blob/master/parse/pprof.sh) 37 | * franco: Duration: 5.12s, 26.93% 38 | * getlang: Duration: 11.58s, 59.54% 39 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // A Pipeline contains configured annotators and taggers for nl processing 8 | type Pipeline struct { 9 | Annotators []string 10 | 11 | taggers []Processor 12 | } 13 | 14 | // AddTagger adds a new processor t to Pipeline p 15 | func (p *Pipeline) AddTagger(t Processor) error { 16 | if t == nil { 17 | return fmt.Errorf("cannot add nil tagger") 18 | } 19 | p.taggers = append(p.taggers, t) 20 | return nil 21 | } 22 | 23 | // Annotate tags the Document by each configured processors and taggers 24 | func (p *Pipeline) Annotate(d *Document) error { 25 | err := Processors["_"].Process(d) 26 | if err != nil { 27 | return err 28 | } 29 | for _, anno := range p.Annotators { 30 | if err = Processors[anno].Process(d); err != nil { 31 | return err 32 | } 33 | } 34 | 35 | for _, tagger := range p.taggers { 36 | if err = tagger.Process(d); err != nil { 37 | return err 38 | } 39 | } 40 | return nil 41 | } 42 | 43 | // AnnotatePro tags the Document by each configured processors and taggers 44 | func (p *Pipeline) AnnotatePro(d *Document, taggers ...Processor) error { 45 | err := Processors["_"].Process(d) 46 | if err != nil { 47 | return err 48 | } 49 | for _, anno := range p.Annotators { 50 | if err = Processors[anno].Process(d); err != nil { 51 | return err 52 | } 53 | } 54 | for _, tagger := range p.taggers { 55 | if err = tagger.Process(d); err != nil { 56 | return err 57 | } 58 | } 59 | for _, tagger := range taggers { 60 | if err = tagger.Process(d); err != nil { 61 | return err 62 | } 63 | } 64 | return nil 65 | } 66 | 67 | // NLP returns ling handler with the annotators 68 | func NLP(annotators ...string) (*Pipeline, error) { 69 | for _, anno := range annotators { 70 | if Processors[anno] == nil { 71 | return nil, fmt.Errorf("Processor %s doesn't exists", anno) 72 | } 73 | } 74 | return &Pipeline{Annotators: annotators}, nil 75 | } 76 | 77 | // DefaultNLP returns ling handler with norm, lemma, unidecode and regex 78 | func DefaultNLP() (*Pipeline, error) { 79 | return NLP("norm", "lemma", "unidecode", "regex") 80 | } 81 | 82 | // MustNLP is like NLP but panics if the annotators are not correct. It 83 | // simplifies safe initialization of global variables holding ling handler 84 | func MustNLP(annotators ...string) *Pipeline { 85 | p, err := NLP(annotators...) 86 | if err != nil { 87 | panic(err) 88 | } 89 | return p 90 | } 91 | -------------------------------------------------------------------------------- /api_tagger.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/url" 7 | 8 | "crawler.club/dl" 9 | "github.com/juju/errors" 10 | strutils "github.com/torden/go-strutil" 11 | ) 12 | 13 | var strvalidator = strutils.NewStringValidator() 14 | 15 | // APITagger via http interface 16 | type APITagger struct { 17 | apiAddr string 18 | } 19 | 20 | // Entity stores the NER entity 21 | type Entity struct { 22 | Text string `json:"text"` 23 | Type string `json:"type"` 24 | Value interface{} `json:"value"` 25 | Start int `json:"start"` 26 | End int `json:"end"` 27 | } 28 | 29 | // NewAPITagger returns a new tagger 30 | func NewAPITagger(addr string) (*APITagger, error) { 31 | if !strvalidator.IsValidURL(addr) { 32 | return nil, fmt.Errorf("invalid url: %s", addr) 33 | } 34 | return &APITagger{apiAddr: addr}, nil 35 | } 36 | 37 | // Process the input document 38 | func (t *APITagger) Process(d *Document) error { 39 | if d == nil || len(d.Text) == 0 { 40 | return nil 41 | } 42 | if len(d.Tokens) == 0 { 43 | return fmt.Errorf("tokenization required") 44 | } 45 | v := url.Values{} 46 | v.Add("text", d.Text) 47 | req := &dl.HttpRequest{ 48 | Url: t.apiAddr, 49 | Method: "POST", 50 | PostData: v.Encode(), 51 | } 52 | res := dl.Download(req) 53 | if res.Error != nil { 54 | return errors.Annotate(res.Error, "APITagger dl.Download") 55 | } 56 | var entities []Entity 57 | if err := json.Unmarshal(res.Content, &entities); err != nil { 58 | return errors.Annotate(err, "APITagger json.Unmarshal") 59 | } 60 | for _, entity := range entities { 61 | if len(entity.Text) < 1 { 62 | continue 63 | } 64 | start := -1 65 | end := -1 66 | for _, token := range d.Tokens { 67 | if token.StartByte == entity.Start { 68 | start = token.I 69 | } 70 | if token.EndByte == entity.End { 71 | end = token.I + 1 72 | } 73 | } 74 | if start == -1 || end == -1 { 75 | continue 76 | } 77 | span := &Span{Doc: d, Start: start, End: end, 78 | Annotations: map[string]interface{}{ 79 | "from": "api", 80 | "value": map[string]interface{}{entity.Type: entity.Value}, 81 | }, 82 | } 83 | d.Spans = append(d.Spans, span) 84 | } 85 | return nil 86 | } 87 | -------------------------------------------------------------------------------- /cmd/gen_lemmatize_data/lang.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | var langs []string = []string{ 4 | /*"sqi", 5 | "ara", 6 | "hye", 7 | "eus", 8 | "ben", 9 | "bul", 10 | "cat", 11 | "ckb", 12 | "ces", 13 | "dan", 14 | "nld",*/ 15 | "eng", 16 | /*"est", 17 | "fao", 18 | //"fin", 19 | "fra", 20 | "kat", 21 | "deu", 22 | "hai", 23 | "heb", 24 | "hin", 25 | "hun", 26 | "isl", 27 | "gle", 28 | "ita", 29 | "klr", 30 | "lat", 31 | "lav", 32 | "lit", 33 | "dsb", 34 | "mkd", 35 | "nav", 36 | "kmr", 37 | "sme", 38 | "nob", 39 | "nno", 40 | "fas", 41 | "pol", 42 | "por", 43 | "que", 44 | "ron", 45 | "rus", 46 | "gla", 47 | //"slk", 48 | "slv", 49 | "spa", 50 | "swe", 51 | "tur", 52 | "ukr", 53 | "urd", 54 | "cym",*/ 55 | } 56 | -------------------------------------------------------------------------------- /cmd/gen_lemmatize_data/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/liuzl/ling/util" 7 | "log" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | func getInfo(lang string) (srcUrl, dstFile, varName string) { 13 | srcUrl = fmt.Sprintf(`https://raw.githubusercontent.com/unimorph/%s/master/%s`, 14 | lang, lang) 15 | dstFile = fmt.Sprintf(`src/github.com/liuzl/ling/lemmatize/%s.go`, lang) 16 | varName = strings.ToUpper(lang) 17 | return 18 | } 19 | 20 | func gen(lang string) { 21 | srcUrl, dstFile, varName := getInfo(lang) 22 | log.Println("Fetching " + srcUrl) 23 | c := util.UrlToZipContent(srcUrl) 24 | output := bytes.Buffer{} 25 | output.WriteString("package lemmatize\n\n") 26 | output.WriteString(fmt.Sprintf("var %s = %s\n", varName, strconv.Quote(c))) 27 | output.WriteString(fmt.Sprintf("\nfunc init() {\n\tgenDict(%s, %s)\n}\n", 28 | strconv.Quote(lang), varName)) 29 | log.Println("Writing new " + dstFile) 30 | util.WriteFile(dstFile, output.Bytes()) 31 | } 32 | 33 | func main() { 34 | for _, lang := range langs { 35 | gen(lang) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /cmd/gen_normalize_data/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/liuzl/ling/util" 7 | "log" 8 | "strconv" 9 | ) 10 | 11 | func chinese() { 12 | lang := `cmn` 13 | srcUrl1 := `https://raw.githubusercontent.com/liuzl/gocc/master/dictionary/TSPhrases.txt` 14 | srcUrl2 := `https://raw.githubusercontent.com/liuzl/gocc/master/dictionary/TSCharacters.txt` 15 | dstFile := fmt.Sprintf(`src/github.com/liuzl/ling/normalize/%s.go`, lang) 16 | log.Println("Fetching " + srcUrl1) 17 | src1 := util.UrlToZipContent(srcUrl1) 18 | log.Println("Fetching " + srcUrl2) 19 | src2 := util.UrlToZipContent(srcUrl2) 20 | 21 | output := bytes.Buffer{} 22 | output.WriteString("package normalize\n\n") 23 | output.WriteString(fmt.Sprintf("var cmnPhrases = %s\n", strconv.Quote(src1))) 24 | output.WriteString(fmt.Sprintf("var cmnCharacters = %s\n", strconv.Quote(src2))) 25 | output.WriteString(fmt.Sprintf("\nfunc init() {\n\tgenFuncs(%s, &cmnPhrases, &cmnCharacters)\n}\n", strconv.Quote(lang))) 26 | log.Println("Writing new " + dstFile) 27 | util.WriteFile(dstFile, output.Bytes()) 28 | } 29 | 30 | func main() { 31 | chinese() 32 | } 33 | -------------------------------------------------------------------------------- /cmd/ling/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | "strings" 11 | 12 | "github.com/liuzl/ling" 13 | ) 14 | 15 | var ( 16 | input = flag.String("input", "", "file of original text to read") 17 | output = flag.String("output", "", "file of tokenized text to write") 18 | typ = flag.String("type", "token", "type: token, span, all") 19 | full = flag.Bool("full", false, "full output") 20 | ) 21 | 22 | func main() { 23 | flag.Parse() 24 | var err error 25 | var in, out *os.File //io.Reader 26 | var nlp *ling.Pipeline 27 | if *input == "" { 28 | in = os.Stdin 29 | } else { 30 | in, err = os.Open(*input) 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | defer in.Close() 35 | } 36 | br := bufio.NewReader(in) 37 | 38 | if *output == "" { 39 | out = os.Stdout 40 | } else { 41 | out, err = os.OpenFile(*output, os.O_RDWR|os.O_CREATE, 0666) 42 | if err != nil { 43 | log.Fatal(err) 44 | } 45 | defer out.Close() 46 | } 47 | if nlp, err = ling.DefaultNLP(); err != nil { 48 | log.Fatal(err) 49 | } 50 | for { 51 | line, c := br.ReadString('\n') 52 | if c == io.EOF { 53 | break 54 | } 55 | if c != nil { 56 | log.Fatal(c) 57 | } 58 | d := ling.NewDocument(strings.TrimSpace(line)) 59 | if err = nlp.Annotate(d); err != nil { 60 | log.Fatal(err) 61 | } 62 | if *typ == "token" || *typ == "all" { 63 | var ret []string 64 | for _, t := range d.Tokens { 65 | if t.Type == ling.Space { 66 | continue 67 | } 68 | if *full { 69 | ret = append(ret, t.String()+"\t"+t.Type.String()) 70 | } else { 71 | ret = append(ret, t.String()) 72 | } 73 | } 74 | fmt.Fprintf(out, "tokens:\n==========\n%s\n", strings.Join(ret, "\n")) 75 | } 76 | if *typ == "span" || *typ == "all" { 77 | var ret []string 78 | for _, s := range d.Spans { 79 | if *full { 80 | ret = append(ret, 81 | s.String()+"\t"+fmt.Sprintf("%+v", s.Annotations)) 82 | } else { 83 | ret = append(ret, s.String()) 84 | } 85 | } 86 | fmt.Fprintf(out, "spans:\n==========\n%s\n", strings.Join(ret, "\n")) 87 | } 88 | fmt.Fprintf(out, "\n") 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /common_regex.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | // https://github.com/mingrammer/commonregex 8 | // Regular expression patterns 9 | const ( 10 | DatePattern = `(?i)(?:(?:tgl)?\d{1,2}[^0-9^:]\d{1,2}[^0-9^:](?:19|20)?\d{2})|(?:(?:19|20)?\d{2}[^0-9^:]\d{1,2}[^0-9^:]\d{1,2})` 11 | TimePattern = `(?:(?:0?|[12])\d\s*:+\s*[0-5]\d(?:\s*:+\s*[0-5]\d(?:\.\d+)?(?:\s*(?:\+|-)(?:0?\d|1[0-2]):?(?:0|3)0)?)?)` 12 | //TimePattern = `(?is)((?:0?|[12])\d\s*:+\s*[0-5]\d(?:\s*:+\s*[0-5]\d)?(?:\s*[,:.]*\s*(?:am|pm))?|(?:0?|[12])\d\s*[.\s]+\s*[0-5]\d(?:\s*[,:.]*\s*(?:am|pm))+)` 13 | PhonePattern = `(?:(?:\+?\d{1,3}[-.\s*]?)?(?:\(?\d{3}\)?[-.\s*]?)?\d{3}[-.\s*]?\d{4,6})|(?:(?:(?:\(\+?\d{2}\))|(?:\+?\d{2}))\s*\d{2}\s*\d{3}\s*\d{4})` 14 | PhonesWithExtsPattern = `(?i)(?:(?:\+?1\s*(?:[.-]\s*)?)?(?:\(\s*(?:[2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9])\s*\)|(?:[2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9]))\s*(?:[.-]\s*)?)?(?:[2-9]1[02-9]|[2-9][02-9]1|[2-9][02-9]{2})\s*(?:[.-]\s*)?(?:[0-9]{4})(?:\s*(?:#|x\.?|ext\.?|extension)\s*(?:\d+)?)` 15 | LinkPattern = `(?i)(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w\.-]*)*\/?` 16 | EmailPattern = `(?i)([A-Za-z0-9!#$%&'*+\/=?^_{|.}~-]+@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)` 17 | IPv4Pattern = `(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)` 18 | CreditCardPattern = `(?i)(?:(?:(?:[\d\*x]{4}[- ]?){3}[\d\*x]{4}|[\d\*x]{15,16}))` 19 | VISACreditCardPattern = `4\d{3}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}` 20 | MCCreditCardPattern = `5[1-5]\d{2}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}` 21 | BtcAddressPattern = `[13][a-km-zA-HJ-NP-Z1-9]{25,34}` 22 | SSNPattern = `(?:\d{3}-\d{2}-\d{4})` 23 | MD5HexPattern = `[0-9a-fA-F]{32}` 24 | SHA1HexPattern = `[0-9a-fA-F]{40}` 25 | SHA256HexPattern = `[0-9a-fA-F]{64}` 26 | GUIDPattern = `[0-9a-fA-F]{8}-?[a-fA-F0-9]{4}-?[a-fA-F0-9]{4}-?[a-fA-F0-9]{4}-?[a-fA-F0-9]{12}` 27 | ISBN13Pattern = `(?:[\d]-?){12}[\dxX]` 28 | ISBN10Pattern = `(?:[\d]-?){9}[\dxX]` 29 | MACAddressPattern = `(([a-fA-F0-9]{2}[:-]){5}([a-fA-F0-9]{2}))` 30 | IBANPattern = `[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}([A-Z\d]?){0,16}` 31 | NumericPattern = `([+\-]?((\d{1,3}(,\d{3})+))|((?:0|[1-9]\d*)(?:\.\d*)?(?:[eE][+\-]?\d+)?))` 32 | DigitsPattern = `\d+` 33 | ) 34 | 35 | // Regexes is the compiled regular expressions 36 | var Regexes = map[string]*regexp.Regexp{ 37 | "date": regexp.MustCompile(DatePattern), 38 | "time": regexp.MustCompile(TimePattern), 39 | "phone": regexp.MustCompile(PhonePattern), 40 | "phones_with_exts": regexp.MustCompile(PhonesWithExtsPattern), 41 | "link": regexp.MustCompile(LinkPattern), 42 | "email": regexp.MustCompile(EmailPattern), 43 | "ipv4": regexp.MustCompile(IPv4Pattern), 44 | "credit_card": regexp.MustCompile(CreditCardPattern), 45 | "btc_address": regexp.MustCompile(BtcAddressPattern), 46 | "ssn": regexp.MustCompile(SSNPattern), 47 | "md5_hex": regexp.MustCompile(MD5HexPattern), 48 | "sha1_hex": regexp.MustCompile(SHA1HexPattern), 49 | "sha256_hex": regexp.MustCompile(SHA256HexPattern), 50 | "guid": regexp.MustCompile(GUIDPattern), 51 | "isbn13": regexp.MustCompile(ISBN13Pattern), 52 | "isbn10": regexp.MustCompile(ISBN10Pattern), 53 | "visa_credit_card": regexp.MustCompile(VISACreditCardPattern), 54 | "mc_credit_card": regexp.MustCompile(MCCreditCardPattern), 55 | "mac_address": regexp.MustCompile(MACAddressPattern), 56 | "iban": regexp.MustCompile(IBANPattern), 57 | "numeric": regexp.MustCompile(NumericPattern), 58 | "digits": regexp.MustCompile(DigitsPattern), 59 | } 60 | -------------------------------------------------------------------------------- /dict_tagger.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | 7 | "github.com/liuzl/d" 8 | ) 9 | 10 | var ( 11 | dictName = flag.String("dict_name", "dict", "dictionary name") 12 | dictWeb = flag.Bool("dict_web", false, "dictionary web api flag") 13 | ) 14 | 15 | type DictTagger struct { 16 | *d.Dictionary 17 | } 18 | 19 | func NewDictTagger() (*DictTagger, error) { 20 | dict, err := d.Load(*dictName) 21 | if err != nil { 22 | return nil, err 23 | } 24 | if *dictWeb { 25 | dict.RegisterWeb() 26 | } 27 | return &DictTagger{dict}, nil 28 | } 29 | 30 | func (t *DictTagger) Process(d *Document) error { 31 | if d == nil || len(d.Text) == 0 { 32 | return nil 33 | } 34 | if len(d.Tokens) == 0 { 35 | return fmt.Errorf("tokenization required") 36 | } 37 | r := []rune(d.Text) 38 | for i := 0; i < len(r); i++ { 39 | startByte := len(string(r[:i])) 40 | ret, err := t.PrefixMatch(string(r[i:])) 41 | if err != nil { 42 | return err 43 | } 44 | for k, v := range ret { 45 | if len(v) < 1 { 46 | continue 47 | } 48 | start := -1 49 | end := -1 50 | for _, token := range d.Tokens { 51 | if token.StartByte == startByte { 52 | start = token.I 53 | } 54 | if token.EndByte == startByte+len(k) { 55 | end = token.I + 1 56 | } 57 | } 58 | if start == -1 || end == -1 { 59 | continue 60 | } 61 | span := &Span{Doc: d, Start: start, End: end, 62 | Annotations: map[string]interface{}{"from": "dict", "value": v}} 63 | d.Spans = append(d.Spans, span) 64 | } 65 | } 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /document.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | func NewDocument(text string) *Document { 4 | return &Document{Text: text} 5 | } 6 | 7 | func (d *Document) String() string { 8 | return d.Text 9 | } 10 | 11 | func (d *Document) NewSpan(start, end int) *Span { 12 | tokenCnt := len(d.Tokens) 13 | if tokenCnt == 0 { 14 | return nil 15 | } 16 | if start < 0 { 17 | start = 0 18 | } 19 | if end > tokenCnt { 20 | end = tokenCnt 21 | } 22 | return &Span{Doc: d, Start: start, End: end} 23 | } 24 | 25 | func (d *Document) XTokens(anno string) []string { 26 | var ret []string 27 | for _, token := range d.Tokens { 28 | t, has := token.Annotations[anno] 29 | if !has { 30 | t = token.Annotations["norm"] 31 | } 32 | ret = append(ret, t) 33 | } 34 | return ret 35 | } 36 | 37 | func (d *Document) XRealTokens(anno string) []string { 38 | var ret []string 39 | for _, token := range d.Tokens { 40 | if token.Type == Space { 41 | continue 42 | } 43 | t, has := token.Annotations[anno] 44 | if !has { 45 | t = token.Annotations["norm"] 46 | } 47 | ret = append(ret, t) 48 | } 49 | return ret 50 | } 51 | -------------------------------------------------------------------------------- /lemmatize/init.go: -------------------------------------------------------------------------------- 1 | package lemmatize 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/base64" 7 | "github.com/golang/glog" 8 | "github.com/liuzl/ling/util" 9 | "io/ioutil" 10 | "strings" 11 | ) 12 | 13 | var Funcs = make(map[string]util.ConvertFunc) 14 | 15 | func genDict(lang, body string) error { 16 | data, err := base64.StdEncoding.DecodeString(body) 17 | if err != nil { 18 | glog.Error(err) 19 | return err 20 | } 21 | reader, err := gzip.NewReader(bytes.NewReader(data)) 22 | if err != nil { 23 | glog.Error(err) 24 | return err 25 | } 26 | rawBytes, err := ioutil.ReadAll(reader) 27 | if err != nil { 28 | glog.Error(err) 29 | return err 30 | } 31 | m := make(map[string]string) 32 | for _, line := range strings.Split(string(rawBytes), "\n") { 33 | if strings.TrimSpace(line) == "" { 34 | continue 35 | } 36 | items := strings.Fields(line) 37 | if len(items) != 3 || items[0] == items[1] || len(items[1]) < 3 { 38 | // some short words have problems 39 | continue 40 | } 41 | m[items[1]] = items[0] 42 | } 43 | Funcs[lang] = func(in []string) ([]string, error) { 44 | return util.DictConvert(in, m) 45 | } 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /lemmatizer.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/liuzl/ling/lemmatize" 7 | ) 8 | 9 | // Lemma processor name 10 | const Lemma = "lemma" 11 | 12 | func init() { 13 | Processors[Lemma] = &Lemmatizer{} 14 | } 15 | 16 | // Lemmatizer is the processor for lemmatization 17 | type Lemmatizer struct { 18 | } 19 | 20 | // Process is the function to annotate documents 21 | func (l *Lemmatizer) Process(d *Document) error { 22 | if d == nil || len(d.Text) == 0 { 23 | return nil 24 | } 25 | if len(d.Tokens) == 0 { 26 | return fmt.Errorf("tokenization required") 27 | } 28 | 29 | for _, lang := range d.Langs { 30 | if f, has := lemmatize.Funcs[lang]; has { 31 | ret, err := f(d.XTokens(Norm)) 32 | if err != nil { 33 | return err 34 | } 35 | if len(ret) != len(d.Tokens) { 36 | continue 37 | } 38 | for i, str := range ret { 39 | d.Tokens[i].Annotations[Lemma] = str 40 | } 41 | } 42 | } 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /normalize/cmn.go: -------------------------------------------------------------------------------- 1 | package normalize 2 | 3 | var cmnPhrases = "H4sIAAAAAAAA/2yX+XKCyhLG/555ivuoGCOIiLtiAsY1ahbXxAio+DLM9ha3umeiOadOVaq6f983DgOzdfLYEuFavKzE4w/RkKe2ePyheVxRfoVAaGc0j33xspKLKcljP09tuZjSPCmr9ExYkqr0oulypyxPMn5c8uOSsGQPf47FHMs48tGWj7ZxNIDDSkWiA5I/Jzog9V2iA1LUQwozpMGM6GAojy3e3pM71I7GYbavrBH5C7/ObMGr72JxIv9i9NclogMQ7x0JhhVSgOPkAY6ThxHRASkusfWa/GYfoIkwzGMcvs5Q+8b+RTpB+tlp2iPFU3x27Qokd2OiA5B5F/MWqtAmOiA9HYgOSM820cEQS75Vr03uEHwbRzw7LB6Qv0Dz0ysr+TJwWX1KNDBryOpTdCpDObdVoUY0sJkvrw7Nsw7LjnJuy7NLNIBzuVD2MMmvO8LWP6x0pOzhmKdj/tKVgUvy80wDs4aUlT74NWS7HblllDlVaMecKrZwUmVHwjtrLc3jpvDO2rnIuU2Yc2EznzK/qPxM7IcEsk4q9kPK6j6r+zA+aIfAZj62RplowRC3BuJnZTSA0/fdUU797iinfnMWN3lBWQM05TRFp0pYA2SA9oay9jtrtIgOlIUv+PTwBZ8+TeW0blY5m/4op2rWOpstYMPC9E+Z/ap6bd5t8o0v1xXec8jNltcXsINvbbN6n/ccytYfrGvLRYtlB5FdyL+Ysu0jjmL7iKPYnnEtWeSWUdzstxNAE9vW87hpNA13R9nRv5z0mCcZ0YGy84F/dVE4j3RG2eXILleSn675JUTKlkQHyrIDO8+JDkDiUPxDsrBHmm6R3q9AclsEUsMvogPlxRKPXOEvlZ8RA96nag3/d4dOSrnd1EtsBYtLkxhYRhADi3LXgyMTQjuj3F3Ji2umDeHTTBt3M7OPuJuZTcQ9F3/rufhbJHZ8FUvPaBoo9yKYFO5FMCm8/WyOZe6vzbHMe46qzmGd9XcEoDOHddbfUd4753GPlxvklqGW1IgOSKc9W1fJLQONWR8onOdIpZ88tsgtQ811UCg/IXkO72OnOkOteWZti7Vd8hfQ6a1R87tIgznRAenbgZV8y1C7jsTrK7lloPHi0IxJZ6iV9yiU60jdHtEBabKU9Q3RGfNxhDwuyaXuAzPQROOL6ID0ZhMdkD4StvPILQNN1odEByD10CQ6ID0d2Aq/vc4oDzzYnBcXV4iBT1whgfdn7QDc1s4fJ088Vgn/4WuJ8n5NvKwI79fy1AbCNdOv4ZoJ4zxeSeedR+79UMyT+T9UfUDyaAD9zrq82CV/gfJoCQeWDpQPamBlR3LLQOPZTLkl0PjXRbUi0JRfEfM6aKqdiXlda5kqFlDrpKpYoHxZgWmIhuSWUb4v832L8L3L92NDvNzg0afRNIAj3nzQxGIDJMsJkCwnQOrUB1KnPuWHAssORAfKD++y+/b7zQ/v0h//fvMT7nx+asLOF+63cDPCKwfhZkByEQPJxZwKryEDVz7V2LJAhNdg1lADFXVf1EdqVIJdbmCcwF4XnWf4SBBaERWdL5gr0fmCuRLdCe87hG+6vO9Q8XTWl1ITvfDMg4mYpdj+F9B5WbH5A8lTm80fDPGqw2tVo2kAh0ch0YH+FoG6/NOEe6xotDytsWsRHOm93vu7Af2tEPU5pImVt0Zg5S39Ha0Z5+uLeHWIDlQsh+jNLfRWHaRVB2kzg4tGfBbhohHbMeutzRoXa4v5XbPSxa4quh9ErBJWblHxPYJ5E98jmDdpFXjJU5kPxzyAVVDZExzzd+ikVD709Tkuf76IgXYmf76orB6gO1k9YHeNT+gojx38VSuSvYToQGW3jVSJNb3Bq0h/DK8iuyu44GRjAbebDNz8NCXMGuanqaZXTa9ArPIGxCpvhnjvbIQ8raHmuyj4LjV3irlQgHonpN4JaeOw0gEFzO6autREGImvxd2UB1+EgdivoRV3PXC46yG1n5F87XVLSN2SIV5bymxlNIQXdHoJavA9ApdPBkiDMdKigLQoIL0NkaIUaTVDWs0M8bchj4pG41Gan5/BER0clegY2mna0d9bWl/RQB8bpI+NIdEZ5qlnNA3orOP8NBW7KjoaVjh66btQno4r4BgYBOgEejZgfSDxYMQaPvkL6Ly94G83mtaWJpwn+eUgHR6pHCcsCwirv7MsAJJzjy8THi1Bk4sCHzV5tPyHk8c+5NM1fwn+0QqMRUEbVE5695pSTnr3mlLO7TyuEDbz87iCdKkjXepI1zHSdQzE7BYQs1tIzTJSs6zpoumCNGkjTdpI6wBpHSDFX0hxAyn51LQDgmoFqucyerzqIlVdJKxxfSgLgAYbpMEGaYTj5CMcp2hfgUT7ihQ+IIUPmmyigyG4pG+Z1mZa6APJhz38Wj7skbou23oqwf4BNieV4FOUZYOmLOxX2RMke6Lp0wxJlXZmVCo84ag6B6QIn6IifIpKrmzryS6+PcDmJLuucVQhvTuqEPw65xpq5xqVcw9PoEUBTyCs8IkO9F7/3yt/uSzDMSrXYzhG5boC7Vm9j+2R8sTD6ho1KDqm27uDtfWvAxW23D/AgMY+kfsHGNNkQOX+KJ0pzNT2m+TJVjpTmK/tN5VxSfUsuJXKPwQgWOapzctHKtN3PIHTOuwwVVyxVQMO01tGVamlrIkYvrLjThc3bNsk/1Y7Kds2/7utfNvlsc8rXf42ZH6F1V/Z7Pm/O5CbgWkapbemVNkvMEZVGuAYHfw/Ujn4f6RymlA06QAExQCEdkZVuaucORHphbtTqsr4H6BqLfB3bgkWpWpFsCKVW5KNT6A8doBUfw+k+nuq3Bj7bG+xz4ql6QepOlF+xsOYqNZOddI8mVP4L2MSEfj/YjBCgue0M3yOX2GXCdJlAgSntmpncGorv6KqfSDVmSHVepoKVPkh3Iyqc4KbEerLxCGqNRSJ8z+oKxMHRNn45JMSgXHEDp+UQFOOL57OoMEHqzdR80MUOiekeYA0D6iq9fDlOgV8ucZBv6qm5kUMv0y1pVonAF1tqbDAB19q9cz3ZWJgd+Z7l6qwB/u5eGHZG5FPFwPw+cOTrskOSNEepybaI5lnmN5HAXoTR3t6MiZl9MYhHLFWC45cNRndABxmtcxhrCajG1A1XuBiGme4mNJznoY8qBOVXnRG1ekRFm7pk0DWSVnpk5oax1Q3pq75fwAAAP//W7hYquoTAAA=" 4 | var cmnCharacters = "H4sIAAAAAAAA/yxcR357rQ4dn7OK/1Ldux2XuPfee++LyRXim3kJ70fyRghQQwiQuNg/uRU+82aMP7k1fnLf/Mnt8ZOL8id3wk/2yp98AZ9OPMCfQgifWafNn8IEEnvx5zuJz6yd4U9p5XqO/Cnv8OkcX/wpv/GZdev8qcTx6Wy2/KkU8Jn1UvypfOOnfOZPpYafSoY/9aZTYMSfRgg/tTV/Gl/4qff508i6jjF/GmV8uq0kfxo9B2z50zjj0212+NNq4TObJPjTvuEzm0b50/nCTzvCn14Zn9ksx59eE59eMsaf4Qo/gwp/Rhl8evU1f0Z1fHrlAn9Gjsv8zJ/JHZ/ZKsifqRvaKsmfaQM/kzl/pnv8jLP8mR5hCg3+zDL4zNZJ/qwH+PQrWf5sqvj0y1X+HMv49PcR/hxf+MzOaf6cIvjMW3X+nNP4DCI9/pzf+AxiZ/5covgMMjf+XJ8Op8+f2xc+g06SP7cGPrPbnT/3Ez6zR5I/7xI+s+eKP+8yPoN7gj/vMT6D55qeL47P0N+j57viMwwf6Pln+AyjPXr+OzzfkF4gg88w06QXdob1TeiF9w5Y0IvE8Zn7A/TiOQfs6MWr+AwXPnrxBT7zQIxefIfPcNWgFz85nDO9xBKf4S1ML3GGlzrRS9zwmYdq9JJBBzToJb8d0KKXrMJL3egla/BSd3rJkeuY0kte4aXe9JJ3eKkXveTDdazoJd/4jGIFeqmYA0r0UkkHVOilSg5o0Es1HdCml2o7oEUvNXRAh15q44AqvfQAn3k4TC89gpfu0Es/8Rk5IFN1PUt6mS28zIJe0aG2i/SKe3xGmwO9Uti15OiVHZCY0XOePY636FXd0JI7etUuvGqKXnWLz7ixpFeb4jNPrenVQ/jMnaS6M3R6Tq/hg9cY02uUHN8eveYSn3lmTK95x2d8LdJrPuC10vRaMdeQo9dKOpQZvVbBAUt6bWfezI1e2zlWZk+v7aY0s6XXcS1fQXqdLT6TcJ1e13W1B/S6Tq32kF736IARvd4En0nuSm/wO/AbvaHTuOOjN4zCGwXoDcv4zL+T9IZTfCbbEr3h2rVE6A238IYveqMKPpNLiN6oDm90ojdqwxud6Y1+MY/0Rnt8Juc3vXHPobbpzRL4zEtherMvfKZfD3qzMj7TrI/erOJa3vRmB4czpzd7OWBBb+5zwJbePOCADb152wE7evMDPtNSmd7ceWfpRm9+x2dejtKbO/JOjN7C2a+coLdwIqoreoumA9b0Fi0HjOktxg6nTW/pw2faidJbBh0Qo7d0a6O8pbd0M1/e0Fu+8ZmOl/RWv2ZL0Fs5huMNvZVjOPHTW3Vcy5veau5aovRWG3zmlRW91d61fNFbvRzOjN7aB28dpuc2lek0Rm/zwn+7IL2tG3ktS2/rd8AXvW0Vn+krRm/rNK516G0XriVBb7txQJLe9gxv16e3vTj1cvS2T3i7Jr2dH//ta/R2zqtrDXq7kiMJ0NtV8N9+QG+3wWcWD9DbN/CZ1xf09j0HbOjtnbPUV/T2TnL9Rm8/d8CD3iGO/64regfnso0wvcMCn1kmRu+wdS15eoeDa0nSO7gNr/FN77jHZ5YL0zu6CW9U6Z2KDpjQO+0csKd3icK7ROldcvAuOXqXNrxLj95lBLme6F028C4betcYvGuM3vUFue7+edcXvVsA3nVL7xaBd4vQu2Xg3TL0bjl4txy9WxXerULv1oJ3q9G7R+DdI/TuUXj3KL37Gd79TO/+hnd/03tU4T2q9J5ReI85vWcMcgnSe35D3GJ/VuE9XX8d3rNO79mH9+zTex7/dH1FYJpDeq8SvHuO3usE79Wn9zpDtj56bz+8d5/eOwDd3+m9E/DeCXpvdxBHr/Te3/AeDqED79Wh9x7Ce3TpvRfQsZ/eewXvvaL4fPAeQ4ovAu89pfhi8N5zii8O716g+JLwLjOKLwfvvqD4SpDrheJrQ3xtim8O7zGi+Hb4ycUo/ji8R5viz+En+6T4D/BeY4r/CO81ovif8B4zSiCFn+yZEsjBNDeUQAneY0wJVOA9WpRAFTKIUgI9yOpECSx/DSSBNcS/oAT28B4dSuAA71GnBI4Q340SOMN7DCiBF8T3ogQDEP+dEgzDu4cpwTi855ASzMF7TSjB4u9ESrD1p1dwCO/l+icQv6Pf4r96lBI8QoJHSvD6O9ES8sG7zygh/6/hJRSA91pQQiF4tzclFIP3mlJCcXjvJSWUgQQGlFAWEhhTQt/w3iNKqAjv5fhUIJvnP+/lcNsQv+PRx6cT2VFCM3iPCiW0xKeTjFBCO0hgQgmd4b0d9QU/OSftBgm48g4JDCmhB7z3hBKOQqInSjgJCScp4RQknKOE85DQmxIuQ8IlSrgGCdco4TEkEqaEp/AuA0okAYmkKJEcbDBEicwg1wAlsoBEFpRoBBKNUKIxSNRHiaYg0RQlmoZE7pRoFZ9Zq0mJNiDROiXagUQ7lOgWEl1Toqc/BaM3SPRGieUhsTwlNoTE5pTYDBLrUGIrSGxFiZ0gsRMlHoLElpR4FBKrUuL/H2A8A4ldKfEyJLagxOuQWI0Sb0LiQ0p8BYlnKfEDJO6jxI+QWJ0SP+O/wo4Sf+HTqYcpiaBT5N9PYUNJRCHxMSURh8QqlEQCEntSEilI7E1JpCHxHCXxhZ/CkJLIQeIBSqKGn+84JemHJHaUZA6fTrdDSRYhiQklWYUkupRkHZJMUpJN2GCDkmxBEntKsgdJvCnJvgumm5TkFJKsUZJbSKJHSe7/T3iEJLaU5AWScHh3SMpHSS0hqREltYbZRimpLSS1oqR9kNSNkg5C0kFKOgpJRylpt019UdINSLpBSXd+ty1JnyHpMyV9hfEPKOk7JH2gpJ+Q9J6SCUAyAUqmAslUKJkqJJOmZDqQTJGSGUAyEUpmCcmkKJkdJBOnZA6QTJ/yFYR8BShfIciXn/LVg3epU75mkK8Z5WsD+dpQvk6QrxMl64dk/ZRsDJKNUbIJSDZByaYg2RQlm4FkM5RsDpLNUbJ5SH5Aye4h2QMle4LkspRcAJItUnITSG5CyU8geVdeIPkLJf+A5KqU/BOSf1IKIUghRCl0IbkKpbDGp7NuU75DkFyZ8u0ymc6e8l2DZPuU7wvkfaQUv/BfeUUpFiDZNaXYghRylGL7//Ue5LtLKQUgpQCllMFPqUUp1SDfA0ppBu8yppTmkGyQUlrAu35TSitIukgp7SCFNqUcgeTqlHIUUlxQygnId5NSdhYpU8o1SK5BKU8h3z1KeYNPx2205ROk8E0pX1129kWpRCBfV0olDSk9KZUCpLimVL4huSKlUoIUx5RKGbJpUioNyHedUplCCmVKZeYYBSmVFeS7Q6msIcULpeIm0zF0Z3u3RakcIYUCpXKB5PKUisvXuj5K5QkpHihVP4z/RKmGnbk7lGoBUqlQqjX8lBKUahOSb1KqLlnrJinVDqSQp1SHkILrGEEKRUp1BqnOKNU1pLynVJ3ZqpTqAVI6U6oXSHZCqV4hz8K/z6zzpNR8kHydUotCsgVKLQUpNCi17N/Ia0VI1dWrkOKKUqs7J7hQasM/t6mNILkSpTZ2Ce+ZUnN+VqXUVn/+WdvBuzmCPaT0otQO8C5DSu0EqQwotadLkIuUug9SXFLqfkh5TqkHILU+pR7Ep7NtUephSMRPqccguQKlHocUN5R6Cjb5RannIJUtpZ53HDuUegtSd4QzSN11bCD1DaURhdSPlEYSUn9SGmlI/UBp5CH1JaVRgDQilEYJUn9RGhVIvUdpuCnorSjNLKSZpbSOkPaa0u5CWktK24V9/QilvcF/0SilfYZx22snDGnWKJ0EpDWldEqQ1oHSqTmCPKXThLQdwhqycQRbmMSQ0jlAGjdKNwFplindJGzqm9L9grR2lG4e0q5QumWYU4DSrUEiPUq3B2m2KN0BpF2ldI+QjcO7Qjo5SvftBN8pPafRnNJrQJodSm8F6RQpvQ2kWaf0dm7MF0pvD2m2Kb0TpLWm9O6QXpXSe0FaI0o/+kfQj+HTzYco/SSktaX0M38q9AuQTJLSL+PTTQ4o/Qqklaf0q5DGldKvQVoRSr/xf04tSPOL0m9DWo5hB9IKU/pubA1K3x0dgxqlP4U0m5T+DNJ1DNaQ/orSv0H6J0r/AelfKP0nZP2mDDqQQZsy6MH0RpTB8DeylcEFMrhQBi/I4EoZZiHDAGU4hgyHlOEUMvRThjPI4EYZLiDDEmW4ggyTlOEFMrxQhg/I+EoZNSGjCGWchozLlHEDMm5QxqM/vEkFMqlQJnXIZEuZ+iGTEGXaxWfenFOmI8goSpkuIZMwZRZzHRPKbAyZ7CizNWQ0p8w2+Kn7KLMHZDajzJ6QUYwyT8L2wpR5GTKaUeZHyGhKmd8gkwNlfv9/+YZM95SF74/fIgiZLSmLqBO4oCxikEmEskhCpkfKIgWZRCmLKmS+pSy6kHmesnBzMtxQFgPI4kxZjByHKWUxg8y/KYsNZHqgLC6Q6YmyuDuKN2UZ/LPG0iWVzRVl6XL65pKyjDqUOWWZxE99QFk66THKMoNPt9OmLOeQZZWyvECWI8ryjk93OKYsX/jMxj7K8g1ZziirFWQ1p6xOf0H42ve3Utcdh1ihrHuQdZOyHkFWLcp6DFn5KespZNWlrOeQSJWyXkBWT8r6CFl1KGvHsEnZRCGbCGUTg3cpUDZJyPpO2aQh6ytlk4GsH5TNGbI5UDYXyOZC2fp+0yDZRiDbBGWb/n97AbINUrYNyDZO2fYg2y5lO4BsA5TtBLIdUbZTfGaTMmW7gLjtcteFRGKU3RSym1L2a8h+Q9kfILsKZX+C7OqU/ROy71IOacghSDnkYCoRyqEM2ZUph6rL1/59ZpMx5dCD7IeUwwKye1KOWciuRjk6ZD/l+MCnezpQji/8tIeUkx9ybFJOIchhSTlFIbsS5eS2iVuQcvqGHGuUUxlyCFBOFdcRopw6kP2AcnIi+5TTGLJ37RPI3hGuIAdXd+OpUk4XyLFCOV2d2xwppxtk50Q+INs35fSE7PaUcwKf2TRDOSchuzflnIfsR5TzN+QcppxLfxY7l51Jz5RzxVFUKOcJ5DyhnDeQ85pyvkLSN8ol+JsLy6UCuVQolyHkEqZc5pBLjHJZQi5LymUPuWQplxPkMqJcfRAX+lwd/RflWsBPN0O5liGXB+Xq+N0p16ZT4Ey5tiGXBuXah1z8lKub9emFcnWSVpTrAnLZUq47yHVHuV7+ktvr9f/1G+R6o1xff9nK9Q25vim3MeQ2ptzmkFuBcjtCbkfK3f8X5t5dGNyj3EOQzJRyj0EyI8o9A7k9KPcibHpOudcgmQnl3oRkVpR7G3KrUu6dvzD63oXccpR7D3JrU+79P/n3MT6zWYtyn0JcFnpfQG7Zf5/Z7Ity30MyYcojD3nkKI/vv6D4MYK4VP2xh7hU93GEuBT6cYY8bpRn0HF9UJ6Rv6TrGYM8rpRnCvIYUp4ZyGNNeX5DnkXKs/qXTTxbkGeT8uxAnm3Kcwh5jijPKeQ5pTzn+K/u8HeQ54TyPEOe53/e/U15vn8vQOTlh7z8lFfqL2J/5SCvEOXVgnd39Q7kVaS8upBXl/KaQgZpyuvo6P7J60h5uWDtTnm583qeorzLkHeZ8j7+Bt/GH4Hx52n8Q1jfnSYQhgmMaAItmMCKJnCE8YVpAif8Vz7TBJ6QaIgm2P2NO01wC+N/0AR3MP4nTfAO47/ThGIwoRhNqA6NbWlCDZjggiY0gQnMaUIHGF+EJnSG8d9oQnd8etUvmtAL8n7RhN4woTdNOATTKdOEkzA+P004DRPq0IQrMMEJTbgGE1zShNsw/gtNuA8TXNOEB3+CwjMYX4gmPIfxBWjCK9hqjia8hwk4uhPkFqEJ3/AznNGEHzCxGk34BbPY0UQCkPeYJpKACc5pIlkYX4MmkoNElzSRPExoRBMpwYRd3emzoon03IDyNJEBTGhAE5nDBNY0kRWML0gT2UFWM5rICfKO0EQubopuNJGro8zSRH0OKNBEozD+PU00Drl900RTML43TTQNE3X1DD69jp8m2oLNV2mibRjfgybah4nmaKID/AyeNNEJjP9KE93BBKc00RNMNE8TPcP4fDTRC0xgQRO9Qd5fNNEHTHBME33BhHM0MR+MP0kTi8LEIjSxb5iYq9f+jBYbw8TKNLEpTGxBE9vAxCr/Pr1xlia2hYmtaWI7mNgXTewEEzvTxNMw8TRNPAMTz9DEKzDxCk28DxPv0yQqMIkKTTIJk6jTJLMwySxNsgaTrNGkAjCpAE1qApOa0KSmMOkmTWoLk9rRpJ4wqSdN6gWTatOkJ7CxNE16DhOf0aRXMOkVTXr7R5/e/Wb0JhOEic9pMiGYRJcmE8HP+IsmU4ZJNWgyVZhUnybjgvDlkSZT/1Mk0/2NCcyXDybZp/mqwcQXNF91mHSP5msFkxrSfK1hsiGabBom3abJlmByUZpsGSbdp8k2YLINmuwR5utFkz3BJHo0uSQ+vfmCJpd1kl1LDiZzoMk1YDIPmlwbJhugya1hUjWa3B4m0aHJOQ5tmtztz6S5O0zmRpOPwaRbNHl3tC1fNPkvmNSIJp+Fyedo8gWYVIcm38TP2NXbMKkeTb4Pk7nQ5IcwyQlN3m3DqxJNfg6TqNHkdzBxR7CH+XrT5O8wKSfJTUaTJv+GSXdpCn6Y5JCmEILJ7GkKUZjklKaQgETvNIUkTMrhFfAzbtMUvmGSYZpC7W/yCm2n+Zim0IFJDWgKXZjEgqbQd8b89+ktv2gKc3x6izRNYQGTdGzWMLkYTWEDk3TUW5isn6awg8mcaQoHmLhT4wyTd/UbTC5CU7jDFFo0hQdM/kJTeOJnvKEpvGDiG5rvEEwuTPMdgckfaX6/+a5ONN9fMMkezbfTf0TzXYLJh2i+qzBZH813HSZ/o/l2c+/KJkzO9fdgvvM0332YVJ3mewCTS9B8z2CyTtAC5utJU8zAFKs0xQJMsUZTLMMehjTFCuTLtR9hiima4gWmuKEpBWCKdZpSEGZdpSmF8ek9XzSlCExxSlMqwpQKNKUyTKlMU1rAlFz7Gaa0pCld8OkH/TSlB7xb5p8pPWjKAZhymabchyn3aCqR35jQVP4u00zV9Z9oqnGYapKm2oSpPGhqMZhqkabm2ks0tTJMrUxTq8CULzS1HrQYpqnNYWoBmnoIpvpNU4/8XqKZehSmUqWpO/oCTd358+ZEU//6vU0w9RJMbUxT78C446w+xKcfydLUJzCVJ019AeN27foOpr6jqV/gXUc0jSC8R42mMcRnVJ/QNMawbmtsrGAaa5rGFqaxpWkcYBoHmmYEphmhaXZgmh2a5hqmuaZpbn5v901zC+/SoGke8N/3hKb5gmm+aJpvmOabpuWHaflpWoe/3a7dgGk3aNpHmE6Ypn37f/sdpj2k6ZT/zsnOBaZzoelu8elnizTd958iPT9Mz0/T+4bp+Gh6JXxm2zNNrwLTGtP0mjDNLk2vDdNa0vR2MH0/TT8EeQRp+gmYfpKmX4JpTWj6ZZhWn6bfhum49i5+lhma/himE6Ppb2H6W5r+DqZToumfYHovmsEXTLdDM9jhxy3KYRSffj5FM0zANCc0wwJMa04znMB0ezTDJbxLjWZ4/DPJqAJtT2lGo9+Y1IxmME1XX8G6fXO0gxmFaUZvmO6AZpyEaYVoxmmY1oxmXPibkvEApvemGY9hBq4+gRn4acZzfPpfPZrxCmawohnvYboTmvEJZuDaHzCDJs3ED9Od0kwC8K5ZmkkYptWimeT+TD0pwAwfNJMqTCdKM2nBtHo0kw4+/XKCZtLDz3JAM+nDdM40kwE+/e8UzWQMM3IS5vhZHmgmToMZzeTy512TK0wnRzO5wXRHNJM3zGBOM43BdLs00yRMd0gzbcM0bjTTHsxgSTOdw0xmNNM1Pv3Cm2bmh5lNaWZxmPaTZvYN0+3TzBow3THNrA0zrtHMejB9H82sDzNZ08yW+PQrB5rZCqb3oJmtf++hzWwP4zbo2RGf2a5OM7vAtDs0szvM2NVf+IweJZp58M+j52GHmaGZF2CmV5p5DWYUo5nXYTpxmnkTZtCmmbdgpg+auXM4h+9y7t2KZj6EmW5p5mOYcYdmPoHppGjmM5jWkGY+h5m6cgEzitLMt7AVp8EeppOgmV9gRk7QDaYfoJnfYSZbmoWLPb9oFuG//kUUn9k+TbNwnhqkWaTdIhvQLL5gBq50J/Lum2aRw2d2qNIs8jCdF82i4OY8SbMowYxCNIsKPv1ajWbR+pO62P1FnIsn/suPaJaZ31tms8zCLMY0yzbMYkCz7MIsejTL3W/ObJbnv81veYE88zTLN8zKT7OqwKySNKsWzCpPs+rh0588aFYDmFWUZjXBz3ZFs5rDrGo0q+XvBz2zWsGsCjSrNcyqS7Pa4Gc7pVntYJaO/gbT9NGs7pD+nmb1+CvXPpi1n2YdgVkdaNZu9z3maNZvmPWRZhOAxkc0mxjMukuzScFsojSbPszmm2Yzh9n0aDYLmM2UZuOcbNuk2exhNjuazQtmX6fZZv5funSrS7O9QSJ7ml0MZhf7Z3Zxml0eZtel2ZUgkR3N3g+JHGn29T/afRPmtqbZT2D2E5rDFGYfojmcYLZXmuMV5hikOQV+r0bNKQqzf9GcUjCnFM2pDHOq0ZwekGic5uyDOftozvm/DOlcg9nPaM5DmEuKxuX+Udc//f0QaM4zmN2I5nyEuSRozieYU4vmfIM5hmkuVZhTn+ZSh0S/aC49mN2E5jKCOXVpLguYY5LmcoG5XGguLxhn7KuLmFyZxKe/K9NcWzDHGM11/Kf4dQ2zHdDcnLGmNLfS75dKc2vC7No0ty4+s1OI5jaHuTiEFcwxTnPbw5x6NLcHzPVNcw/B7MY09zDUv6S5O8sMaO4Z2PSQ5v6XDJv7AmY3o7mvYe4tmvsO5lKmuZ8haYd/gTnmae53mGOW5v6Cuddo7m+Ye5fm4Ye5fNM8EjC3KM2jAPMo0DxqMLsBzaMHs43TPPowpzbNYwFzSdM8djCnCc3jCONSvIezVJbm8YY5RmiefvzXS9M8czD7B83zG+bgp3kWneUqNC6bdxZ61vBzSNI86zDPBs2zj8+8VaZ5DmBOI5rnGub4TfPcwNyCNM8zzN31v2FOHZqXD+Y0pXmF8en/AhGYY5TmFYU5NWleWZhdiOblnGpM8+rAXHI0rwHM/knzGsHcZzQv56QnmtfCaVCjebm5SdO8tn/e93rhZ9+hefthjgmadxDmWKB5h/Cz79K8I/j0L32ad/FvEt8VmH2L5l2Duflp3vW/FPLdgDkWad5tmGOG5u2GWqd5j/Gz/6Z5z2HuQ5r3BuaZpnlvYe4TmvcN5hameT9g9jeat5vMAc377TaDLdUXgHnVqb4gPv3nmeoLw9x7VF8E5pKh+qL42b2pvjjM/k71Jf8W9aVE9X3BHL+oviLMO0/1Oe0vVJ/z3SHV14Z5Rqm+DswzQvWNYJ4Hqm8Ms3eyJjDvHNW3gTyG/372T6pvB/NMUn0vmFeD6g/C7PpUfwjm3qH6087WTao/B3PIU/15fGbnJ9Vf/HVE9Vdg7lOqv4pP//ag+ptQ/xfV38XPvkj192GuL6p/AHMfUf1jmHub6p/+v3xC/S9q4AbvcqMGv+BdU9TgARoMUkNdmHKHGnYH0GVKDcehgQc1nIcGGtRwFRpsU8M92FSRGh5CQ66cQoMjangBDfio4S1+Tk9qJOAYFamRMDR8oEbi+AxiJ2okhc8gGqJGvqCRL2okD5seUyMFfAaRLjVShgaa1EjNYV6pkS4+g6RD3UKDS2pkh/96GWrkBQ08qVE/NDihRmNQ/5oaddqXqNE8NDimRp32c2q0AQ259jZsdkiNjqD+NzW6+D1YNLqEBuvUqDPLlBo94ed0okbvf6pGH9Dgghp9QUMVaizoZq5LjYWcqkFqLIrP6NSmxtLQoJ8ay0IDcWqs5DCW1FgdGgxQY0N8Ztc9NTZ2TDrU2BLezaHu/iYpdoPGztTYC7L5psZ9kFuCGo/8ftnSeAUaT1PjB2j88M8kjtT4Exo/UhMl2PSImqj/noiaaOEzuy2piR40kaAmxtD4jZr0QRMnavIKTS6pySc0GaOmUo7gSU1VockNNXWCJq7U1B2a3FHTfmjiQk0HIbkINR2CJrfUdBiaXFHT7nC+nanpzO9XKE1PoMkFNT3FZ9AKUtMzaHJNTc+hKVdfQZNNanoDTbao6S1+bmlqegebcYyO0FSGmj5DE65+gYSf1PQNml5S03doykl+QJ2PZnzQVJea8TuBA+rXw1l6SP16QTMbanb7e0uq2Rv0K0vNvqFfO2ouCc2sqLk8NFOj5vrQbJaaG0BzJ2puCpu5UnMzaKZOze2gmS01d8dndncMng7IUvNxaPaLmk840VNqvul6btR8F5rNUPN9aCZHzY+guTM1P4d+baj5NX7uYWp+A82sqfkjNFOl5i/Q3IWa/x3MhJp/QjOO4cs1jKi/D8LvE2ohAc0+qIUv6FeOWviG5jPUQhGfwchHLdSghTy1UMdnMH1SCytoYUX9TkELa+p38f/1HrzbmPrd/385+v2krd/r349PWmzCu02pxR7MukgtzqHf938Sy1OLbisMUItH10YtniDPArX4wGf2jFFLUWjxQC0lIF8dajkKLUep5Sq0HKeWO/gvfKGWV45gRi2/oKUDtRKAlkvUShJaWlMraWipS60UoOUZtdKCVsrUShdaWlErY2ipTa2soJUItbJ1DM/Uyg5aWlIrN2ilQq3c/8pqAFoqU6sRaHlCrTrN5tRqHFoJU6tZfAanBLWa/72N1moFWupQq21oZUatdqHlA7Xag5bz1OoAWipRq8O/IVadRgtqdQqttqjVBbTap1aX0MqbWnWavqjVDbTs+G+hFafADlp15QPiZrbmh9YC1FoNWnP1Nj6Dl49a20BLW2rtAq3tqLUbtLal1oO/twVab0DrZWq9Ba1/UetdyOZJrfeh9Ry1PoDWK9T6GJLuUesTfGavb2p9BpvuUhvOn199aqMFbbSpjSG0MaI2XvAeZWoz7RB61ObX7+sGbTagTT+12YI2s9TmDNp8/dPmjNqqQBtpaqsBz1+itlrQVp7aGuAzjIyprSW0tfzn3SLU1gnacuUD2sxR29HfyELbcSfxQm27HOk1orYL0MaC2q5B3SJsT38TfG2voe05tRv8vVvRbgPa9VO7A2i3RO1OoJ0ytbuCdlfU7hraqVF7RUj7Qe1V8Rlm79TeFNotUntLaMdH7V2gnQa1d/sj7D2gHT+194QXOFL7OWi3Su3XoJ0Std+Bdl17H9prUPtjaK9O7TsFT9T+E9otU/sveMEwdRCCdmvUQQTaPVMHGWjXRx3knUJt6qAK7aWpgxG0/aYOZtDOjTqYQzsv6mABbbvSran3nDrYQjt36jAKHUb/ydeMOixCxxXqsAIduvINHYWoozR0eKGOMtDhmjoqQocZ6qgLHQapo9HvByMdzaCjMHW0gg4e1NEGOuxRR1vo8EwdvaHjIHXs/30gq+MYdPugjpPQcZI6TkPHaeokDJ1kqJMYdByhTr7ghePUSeP3Akgnbajbziar/+NvoeMNdeqD3Zyp0yi8cJs6TUEnferU8V1Tpxn81ytRp2XotEyddqDjF3XahU5z1GkPOn5Sp1PoZE+dzv8OoukSOm1RZ1PorEmdzWAOD+psBZ2eqbM1dJajzo7QWYY6O0FnS+rsBp0NqfMQdNagzsPQWZo6j0JnI+rchS8n6jwBnQWp8xo+w2mbOp9B5yXqfIvP3O8aTtB5i7qMQBff1GUaukhSlznoMkddjqGLInV5gBfdUVfR3yfFukpCFzfq6gu6bFLXPthEgLoOwbojeB3/o18noasxdf0NXT6o6wo+w0WHuu5Al2/qegBdOIIpdFmnrvfQZY26vuAzXL2oGx90NaJuAvjMAxHqJg1dF6ibHGwhR920oKsZddOFrnzUTc8hBqmbPnTtEObQ1Zy6uUKXDermCV06zq/fb9i6DULXK+o2jM9w3qduk07JFnWbgi6v1O03vPiAuq1CVwXqtg5dr6nbBnQ1oW7b0K2fuu1AFw/qdvAnYTuFLlw5g25K1O0Cut5Styvoakrd7n6fQOruCN0NqPs4dPek7pvQvZ+6b0F3M+p+DN2tqPvN7/ss3e+guyZ1f4BuD9T9CbprUPdX6K1N3f/+TqdK3d9/n5jr/gW9dagHH/Q2ox4C0FufeghBbyPqIQy99aiHCPTWpR6i0NuQeohBbxPqIQ69zamHJPR2pR7S0NueeshCXWx3KEBvL+rhG3pbUw9F6G1DPZSgLs04lKG3LfVQgd5W1EMVertQDzWoCwMOdagL8Q4NqAsuD03o7UA9tN1A6tRDF3q7UQ8LeCmHuPr9BKeHNfTgBG6g9wj1sIXeA9TDDnr3Uw97qAt+Dkfo7U49XKH3NPVwg9591MMDek9SD2+oC7OOPug9RT36neA29RiA3mPUY/D3A7weQ9B7iHoMw0tdqccI9J6gHl3u1Q5SjxnovUw9ZqH3AvVYhN6b1GPVcexQj3WoS62OTej9m3psQe8t6rELvTeoxx70XqMeh46gRz2OoPcq9eji+1CXepxA73nq0Q26RD26QTtdXbzvRLlB16nH0/9Fu0E7Rr9uMaGefPiMYinqyQ+956inIPTepZ7caDrUk8s+YknqKQZ1EeApDr33qaekG2aIekpB7wPq6Qt6H1JPWXipB/WUg0lHqKcC9P5FPZWczDH11IDeHUIL+khTT79TO6eeOtD7m3rqOtZh6skZ4EI99aH3O/U0h94cwgJ6P1FPS+j9QD2tofcr9eTM8KKettDbjnraQZ9uUM4MZ+rpAH34qKcj5Dn956We1NMFenO8nGlu1JPzhxX1dIfeXf8Tet9QTy/o3Snp/ONIPbvp31HPUejDTz0noIcx9ZyE3tfUc9oZ7ot6zv2N6pyH3mfUs7PHgnr+ht631HMZ+ghSzxXoI0Q9V6EuuD27FfH+p48s9dyEPuLUszNZhnp2JmtHqOcO9PFNPXehLjQ696GPCvU8cDZ1UkbQR4p6nkAfJep5CnX5/3kBfYSp55VDXFLPa+jNEfz+mq5IPW+hjxz1vIM+YtTzAXp3ep6gDzeus0N0Sl0c4DCv0EeUer7DRg/U8wP6SFAvMdjsiHqJQx9V6iUBffSolyT0kade3IoJbaiXjOP0Tb24JROnXnLQR596Kf6fk7NVi3qpQx816sXZpUG9OLu0qZc29FGnXvqQzJt6Gf+tsMsc+mhSLwsnwBEuoY8u9bKCPqbUy8apsKdenOscqRc36jH1soeXulAvB+htQL0coW71X07Q45l6OUMfRerl4hjsqBdnhCH1cnOistTLE8Z3p16cA02pV3d8hQ7Ua/DPK69h6GNOvUagjxn1GnMIJ+r1y7FoUq/Zv2jimoc+1tRrwXXUqVfnQ1Hq1dlnRb3WoI8l9ep2kiL1+vtjyQz12v1btdce9JGkXidOxoB6nULve+p1Br1XqNf53zlxXTiELfW6hE2NqVdnqQ31uoY+HMEG+thRr3vow43m5AjccM4OcLIu8NJ+6tWtLKfD3SnTpV7dVutYPqEPp8ML+nCi3vDSPurtd8u9UG9R/NfrU28x6GNBvSWgLie8uS1nSb2l//b82xf00aHeCtDHhHr7/n2eobfS36Tdyo7hlXpza+xIvdVcw5B6a0AfEerjif/yR+ozCK8woT5jvw9y9ZmCPgPUZ+b/9Q30WaM+j/hv5trP0GeP+gpDn2XqKwJ9HqivGPTlp76SsIkY9dWHvgbU1xD6alNfE6iLZ19TeNc49XXEZx52DE6wkTv1nYd37VDfC+i7Rn1foe8K9f2Evqu0vgWsb0zrW8H6erT+EqTRp/W38F/5Tutfw/q/af0bWP+M1r+D9De0/j2s70LrP8L6p7T+E6w/QOs/w/pTtP4rrD9N67/jM4980fqfkOyC1v+C9SdpA2HYQJA2cIANHGiDjd9fudhgFzbYpQ2FYYN+2lAMNhSnDdVhg3PaUB/yHaUNDfHppze0oenvb9dsaA4b2tKGrrBBH204ARtO0IZTsOEUbTgDG3jRhkuwwQltuAsbOtGGR7ChHG145hiuacNz2MCbNryHDdVowxfYQIc2EoQNP2kjRdhwhTZSgw0MaCMteF9N2kgHNniijfTwGTUftJEpbPhOG7niM9sOaSNP2GCENvKCDaVpI2/YUIE2GocNXWijadhQljb6BRvJ0UbL+PQzDqj8jSVagw3OaKNt2NAXbbQDG/qmjfZgow3a6BCSHtNGJ/AuB9ro4XcHtdHb7xdKG3PBdoY2FoOED7SxOLzLizaWgCmPaWPp3/dBNlbBf6MIbbwKG9vSxvv4zKN12vgANtanjY9gYxva+Bw2dqeNb2Hjrn6Gje9pE3eIY5QcwyYztMkrbPJKmwr+RvM2tYOEr7TpBGw6QpvOwqaztJkE5BaizZRgUxnaTA82XaPNOD5J2q8ibGJH+7WGpDO0X4fft5b26/J7CWyzUdhkijabgM1sabOL31+B2uwBNt2kzR5hMwfaXPz3J0M2l4dNL2lzZdhcmTbXxGcey9DmBrDpFW1uBJuM0ubWsM45cmfY9IA2786a6JI2n/19qWTzBdjMgzb/DZvZ0+aLsHkfbb76m/rbfAs2E6bNLx3hgTb/gE2maQs+2PSMthD5fehqC0lovURbGOMz2nzRFlawmR1tYQ2b9NMWDn+SCk/YdIP2Owf7naP9/v79eZb9rvz+bMp+t2DzKdrvPmw+Sfs9hM1tab9HsCmHv/x9d2q/127Mbdrv9//HVPTDpie0xQhsPkpbzMCmW7RFZ60FbbEAm7jQFouwGVdWYNN12uKv9Y60xR5sPk5bnP5exdjiDDbRoC0uYbNj2uLeITqEM2yhT1t8OtFP2pIPtrihLUXwmcf9tKUYbLpNW0rAFlyZgk2UaUu5318E2NI37PeZtuR0qNKW2rCZE21p9OdRpSlsdkJb2sCWNv/U7UWlPbxCkbZ0gE2OaUtHeIUCbcl56fc/W7rSlm6w6Q5tOQ5bjv8TX5u2nIYtNGnLzh5T2nITNn6jLQ//nKM8gy2taMtLeIUSbfkAWzzQlk+wyRht2R1p8RBt+Qr7/aAt32C/b7QVH2y2RlsJwRZ3tJUIbGJEW4n+XmbaSuKPUSXpltE/W0nSVmqw5QJtpQX7faWtuDP6XqSt9GBTjmgJCW9oKzfYQoe28obNNmmrEdjvAG21CBmEaKt12GqNttqArX7RVtuQrzNtdQzv5upz2OqctrqGrS5o6zfY+pa2fodtFGkbEdjagrbRhW10aZtF2JqPttmGpha0zRFs9UXbnMBWt7TNC2yjTNsKwdb3tK0WbC1A2+rBZge0rRW8com2dYd122jrDdvy0bZdwOPWYtsRbmjbMdhmjLadgW1NaNsVh5CibTfwmTsHbfdhqyva9gK2Gadtr2HrO9r27m9IbeeHiTtt+wRb/6Jt32FrftqOD5/xV5m244etBWk7YdhmlLYTga2+aTtp2GaGtpOFrR9oOznY5ou28w1ba9J2qo5BibbT/rNKpw9bdwQjJ3JF25nAtr9oO1vYeoK2c4KtFWk7d9i663/hM8590XYjv9eVtpv4W+fdwu+zK9st/v6s0XZrfydmtw5x67nb/P3qb7sT2O6EtnuF7V5/781srwXbXdH2E7+/brH9L/wXTdH2G7DdIW2/Cdt37d3/tz9/f1lrB9HfB9Z2kILtx2gHFdhehHawg+0PaAd72H6PdnCBHdRpB3d4bl8e+vCZp3y0wwjk2qIdRmH7Q9ph/P/lF2xvTTssuLm70g5LsL0Q7e/fSCTPtMPfWb3TDp1ufdphB7YfoR0OYAcH2uEMttegHS5gcpF/XrVAO1zDdhe0ww0+42aTdriD7S1phwfYYZF26AKlVJR2FIEdRf79jm4U+31AZkdJ2LGfdpSBHQdpR1+w4xDtqAA7DtOOSrDjCO2oAjt2POqO2Yx21IUdJ2lHQ9ixYziCHWdoRzN4tx3taAE7TtCO1rDjL9rRDnacpR2dYcc52tHVMVrSjm6w4zjt6OEaJrSjJ+w4Rjt6w45TtGMf7DhAOz7Ajlu04xPseE07vsCOR7QTP+yqQzsJwK56tJMg7KpPO4nBrrq0kwTsak07ScGuxrSTDOxqQjvJwq4cfQ6fudtPJ3nY1ZZ2UoBdLWknRdjVjHZSgl3NaSdlmHjln10taCcV2NWGdlKHXV1pJ24duoNh0oRdnWgnLddQo520YVcP2kkPngs3J33YtY92MoRdnWknLq9ot2gnU9jVnXYyg129aSdL2NWLdrKFXV1oJwfYdZx2coJdB2knN9h1gnZy/33aYicP2LWfduoWtNN76oddB2inEdh1knaacJLatFNngSPtNAu7jtFOfy3QpZ3mYdd52qlzUXfATr9h11+00yLsOkQ7LcGus7TTMuw6SjutwK4ztNM67DpMO23BrtO00w4k7+i6sOsp7bQHu+7RTt3g+7TTEey6SDudwq7LtNM57Hr8P6a+K7t1pWf2GTWKPVTLtqycc7JyIJVF5TiYz/sc1FsP4S7QZ637P6nU7ABUA2i0RDah3kJ0bRIuRdcm0Ep03YN6G9F1HeptRdcdqLcTXY+hXiC6nkC9g+ja5LVlJ2MDGBkNqPcSXVehvkXUzBjqf4quy1A/KrpuQX0jpwT1Y+FjfeqnrKIP9dPhMxTqZ0RXI6hfEF0HUN84OkJ9o+Qb6hsFW6jfFF2voL5RsIT6PdF1E+obBXOob6ovoP5IdL2G+mPR9QbqGxUHqD8TXe+gvi+63kP9uQkyg/qL/wo2opsI1Dfdn1A/PDfHRtzbvPah/lF0E4P6IQs21kV0/YD6VyuYQv276CYJ9Z+iMxPuJbr5gM7fRTef0Pmn6OYLOo+JbuLQeUJ0/YLOU6LrE3SeEV2PoPOc6PoCnRspd+i8KLqJQufmKlfovCa6eYfOW6KbN+i8LbrJQOcd0c03dP4tuulA533RzRg6N/uYQeeWfVu+MR+JbsrQ+dgKAuh8IrppQedT0U0eOp+LbnJ/3Nxy3flSdFOBzlfytzj7o5s6dG65XmYLnW8MbKBzcyTrKhDdNKBz482SmbnxlobOT6KrB3R+Fl2fofOr6KYJnRtrVej8Kbq+QhdvopsUdBER3RShC2OvBF18ivpJ6MJsKw1dWGaXOUMXCQMn6CIpuslCF1nRTRu6yNmFHXSRF91MoYuy6KYAXVRFN33owogcQRcN0c0Eumj+ErawuGPtjNABdNET3Qyhi/5/nzPRTRe6MJp86GIhullAF8bUHLoI2blAF1sDe+giEF3Z59kKDtDFRXQzgy6eohsPuoyEz7Xr0lyp24Muo+F5TRo+gpm5QZeJ3wi0tECTGUCXpuYOusyJrsbQ8JQg21MsTb0VdGnO8wVdmnpr6NLUW0KXZi9b6DKMog/oshs+t65Li5aZCXRpih2gy2V414gu16LrAnS5EV1Zz3vRzR66PMk/SRvR5tQ6eIj6CejStJqb/bygqzfR9Q26iopuEtBVUv7m6tBVStTrQldp+Wl70FVW2OpBV2buNejKVgorr4huNtCVqXWErpqih2/oytSw+mbuAXRzCv8L021c/u7eodts+JeSbvPhCT26Lf/uKbcL+dcW2a0t3dkv6PYoul1BdyX5Nz2H7ixlyKahu4b8dBrQXUd034HuvkX339BdV3Tfg+56ovsudNcX3Q+gu4Hovg/dTUX31tFMdD+F7jzRvQfd+aJ7H7qbi+7X0N1CdD+E7taiexNgI7oPoLud6OENugtE9xvobi+6P0B3R9H9Aro7yc9l88dNL0Xo7iy6P0F3V9H9Ebq7ie4v0N1ddL+H7h6i+xt09xLdn6HBm+j+Cg0iovsdNHgXPbxDgw/RQwQafIruX9AgKnqIQoOY6OETGsRF909okBTd36FBQfQQhwZF0UMSGpTkr62oQUX0EIMGDdFDERrYdCWgQVv0kIcGtrnLetDgW/SQgwZdcdNLFRr0RA9laNAXPZSggdE5ggZD+Zux6yPR/RYaTEQPKWgwFT1UoIHROoMGc9FDBhoYrRNosBQ9pKHBRn6+beS96MH6OYgeatDgKnpoQoOb6KEBDYyxJTR4iB5a0OApemhD9x8mYA26N2oe0L1RU4fujZoudB8XPfSge4tE2Tl0n/w10X1K9NCB7jOihz50nxM9DKD7vOjhC7o3l80uoPuS6OEDui+LHgrQfVX0kIXua/910LSKE+i+JXoYQvft3waHhejBhx6N9SP0aKwfoMeZ6DEFPe5Ej1voqS/6ukDPSdFzEnrOi54O0PNG+PaEnve/5Ze06OkTeimLm+fy0EtF9HSGXrq/HVyGomZ2l5Ho2YdebKHMfUEvV9HTCXq5WUEKen0XPR2h14To8Qy92tBp6DUlerpBrxnR0wf0mpWf7gB6LYie19BrUfT0Db2WRM8L6LUietlAr3Vxs7cI9NoUPV2g1464WdRq9EUvBeh1LHp5Qa8T0WsbevVFbed5exO9rqG3hLhZoga9JUUfE+gtJfqYQm9p0ccMesvI368W9GYrR/4TeiuIPnzorSj68KC3suhjDr1VrEIUequJm4WgJfpYQm+Wq+Zj0FtX9GGDjkQfJ+jNgms+Ab3NRR9n6G1hBUnobSf62EJvllnk49DbWfSxh94uoo8D9HYVfVyht5voYwe93UUfR+jtKfq4Q28va5iG3j9Enx/Q+6dJFYfeo6LPd+g9JvqMQO9x0ecb9J4QfTygdyPiCb2nxPmzI/ReFH1+Qu+l8Ik/vZdFn1HovSL6uEHvVRsqB71bkMznofeW6PMLem+LPrPQe0f0mYDev0WfKei9K/rMQO8Da2AVxqLPJPQ+EX3GoPeQmAL07ok+TWhf9LGC3heiT9NqZRVK0Pta9GkVA9GnyRASVoTez6JPq3ATfZpMd9GnlT9FnzXo/SUa/4I+3n7LH+/yjyXvj0/RZxX6MJoq0MeX6LMOfcSsY7tgPC2gD1t081YjI/psQB9Z0ccL+sjbhRr0YUQ1oY+QF+vD5n8DffSswJr0RZ9l6GMg+rhAw1ug8yaVWUgAfRpTXehzIn+T39DnUvRpnyvR5xz6NPU96HMnf7/a0Nc+PFyFb+/Ct3fwrSL6CsC3huirAb4NRV92fRueP8S3neirBUaS4e3ujKREX3kwkhV9xcCIqfAEI9ZPB4w0hJExGGkL35Jg5Fv4VgQjPdFXG4z0w9v/GfF+x49YtpM/gREbqAtGTqKvCBg5C98SYOQqjNTAiE3PA3yPiL7i4Pun6KsOvsdEXx74nhR9XsD3nPDtBr7nhW8T8CMi/IiAH13hRw78OAjfV+Dnl/CjBH4m5OfSBz+LwvcL+FkWvi/Az6rwYwp+dsXNC3Xwcyp8twZL4UcB/NwI32/g5074cQY/LagVrKe98P0Kfh6FH3nw8yp874DRmDD6Dkbjwo8JGD2HN/Ax/G0/DX5VhdEO+NUOTw3k1/d/37uixRz4NZR/gyr4NZe/6Rv4tRBGt+DXVhidgF9HE+AExt6E0RcYi4j2o2DsQxh9grGoMPoAY7HwQVnGUsJYEoylwx9cGMuHZ2UxVhQWrV1JWPwCY2Vh0epXhEWrVxMWrV1dWEyAsYawmAJjtuB0d2CsIyzGwZhxVxqDsaGwWAJjI2ExD8Ym4TPGjE2FplrM3Lk0AWOesFgEYzvrKgBjgbjZxpQ5CIs1MHYSFjNg7CIsVsDYVVjMgrGn/IyvYPxNWFyD8YiwOAfjH8JiHYx/CotLMB4VFsdg3HLkkg/GzW9LUzAeFxabYDwhLPbAeErcbPsOxjPCYgeMZ4VFD4znhcURGC8Ii0MwbnwVwLjx9A3GjafVn3+vDzAeHp26B+Pm490DGG/ZeDMw/i0s2rg9GycCxvvCYheMG1sNMD4WFhdgfCIstsC4sWUSr4SlNBhfW8NPML4RluJgfGcFH2A8EBaPYHwvLNl3I82EOAuLFzB+EZYyYNzI24Dxh7B4BuNPYekNjL+ExReYMDKfYCJiHUfBRFT+nmZ/WGyDiS9h8QYm4sJSEkwkhKUEmEgJSzEwkRGWUmCiICy9gwlLD0oLMGH8nMCE2VEAJprC0heYaAlLNohle12rYAxdwYSFw9ISTAzCvziZGP3ORMKY2YGJhbB4BxNLYfEBJgJheQ4mjsJSDkxY0O9ewMRZWNqAiauwVAQTN2Eo0sMqnMHE04bag4mX0Mwj+SYsHcFkxC7swGS427qByQ9hKQCTUWGpDSbjVmMNJi3w2yhJY6AJJi016FonRsUQTJaEZgLJqrA0AJN1oemXNCbmYNKYmIDJtmi/DCZ7wlIBTA6FpROYHAtLWzA5EZa6YHImLB3ApCc0f0v6wlIFTM6FoUQLYSkPJrfC0jeYNH7s+9HmNQYmT8LyG5i8CMtxMHkTlm3Au7CcAFMRYakDpt6FlSiY+hCWi2AqKiynwVRMWOyDKcuWtnEwlRaWRmAqJyzVwFRRWE6BKXMD4zPVEJZuYKojLCfBVFdYLoGpcK4PYMp0rYOpkbD8CabGwvIHmJoIy2UwNROWY2DKdLaGpvMVTC1/dUuZl1TB1FpY/gJTG2H5HUwZBw8wdRSWM2DqLCyvwdRdWByA6Yiw/A2m302VFJj+DA8ZYfpLWK6C6divcaXjwrIHpk3FMpjOC8tjMG2qWv2KsNwF03VheQSmG8LyFkzbJK/AdEtYboFpy/6MirTZvYmWNi4WYLonLC/BdF9YboNpM3/rYWQVL2B6bOAJpo2UCZg2Uppg2hOWe2B6Liz3wfRKWDbljIwGmN79p5MZwgZMH8Pf65k2/7iD6buwkgXT5helM5h5E5Y7YCYSPjfNjBmAD2Ysa9wmwEz0N95mLCqUwYyxtAczxlIAZmyfY/xnkvLv+/0PS88/bm42l0kJy1EwkxFW7HvRan6CmbKwfAIz5h19MNMUFqtgxoi7gRmj5wFmzEJMRrOQHZgx5etgZiEsX8DMymTMghkzgSeYOQjLKzBjJnAEM1dhuQZm7r/rVuZpgfgPy1cwaxYfAbOfwvILzMZ+PSBr3t61gqSwUgCzaWFlA2YzJr7VKAsrFTBbFZath7qwkgDDg7zNybLNX8PItoQV+257k20OzH5bjRiY7QsrH2DWJn4GZoei5t/ZibBiFafCUg/Mmg94YNYXlu9gdiGs5MHsUlixflb/fYYrRgbMboSVIpjdCSspMBsIK1Z+EFZKYPZky+4TzJ6FlTSYff0aRO5DWPkGc1/WUxrMGR9dMGerwBPMhRGgCOaMjy8wZ9PaAHN5YaUH5grCSgfM1YSlF5gz9ziAud/DTJhrCys1MNcRVupg7ltYaYG5bnggKnPGSRvMDYSVKpgbCSvvYM58wSJJbmI264G5mTBtIpmy1vNBaAt97iSsjMDcRVjxwdxVfsYPMPf4DX45WwMsquUtwjfAfGjkBTBvuYIFjXwY67dg3ha7PZgPZz0L5rNWtwTmcwbKYN4UH4L5otCcNl8SViZg3uyjD+ZrwnIOzNd/l/98Q1iZg3kjYADme8LKDMyP5WcSBfPm6zbS9L92M2HlE8z7JtMVzM9/5yy/+rXI/Fp+Jh9g3iZ7DeZtspdg3viIgPmjsLIC8+b81rEtji0wfzedrEK4Ot7BvFnBFiy8/TpD4VNo0aFgzt2LgAVzhgVYsMkvg4WCMLEHC8X/Pk3nHViw2DgFCza5Vm4L3CdYGAorAVgYCytjsGAGbPU2v1lbYSusHMCCqXIEC3sT0QayOU2CBZtTDyzYwjUEC4/fJKXwFCZXYOElf2MvsBgxcv/8TL7A4vuvE1S68k/NPq2vF1h9E9ZmYPVdWPPA6oew5oPVL2FtDlbjwtoCrCaEtSVYTcnf+5tFNfuSFtaOYDUjbnZ+B6tZYS0AqzlhbQNW83bhA6wWhLUdWC3apzVfgdWSsLYHq2Vh7QBWw5Pt12C1LqydwaotFj3rNwwW1mFfWLeCobAeAauj8ChaVsfC+htYnQlrV7DqCWsPsDoX1m38hbD+BVaXwtoTrG6F9ShY3f33ebABTJajsJ4EqxdhPQFWb8J6FqzehfUcWH0I6xmw+hTWbeCXsJ4Gax/CehGsfYVnbLAWC48XYc3424K1pLBeBmspYb0C1tLCegGsZYT1GFjLC+tWv2CCHMFaUVivgzUjyTosC+tVsFYR1htgrSqsrcFaW/5GA7D2Law3wVpPWC+BNWMqD9YG4eaZtZGw3gJrY2HtDtZ9Yd3/87fzDjbqwkYCbHSEjQrY+A7PUmHDWD6AjY2wvgcbgbARBxtnYeMLbBgZAdhMCBtTsGlKdsFmVlg/gs2KsDEDmy1hIwo2Z8JmDmz6woYHNtfCxgRsboXNPNjcCxsxsHkWNq39LTyImK1YeCw4Wylh8wG2KuHDe2xVRRv2vSF/c2mw1RK2cmCrLf+YXq1v4bkPtgby7+QGtobC5hNs7cJz5tm6yb8WltsDYbsPtofi5uaS7amw9QTbd3H+YQi2H8J2BOy8C1sB2PkUtpdg50v+5mJgJy7/Rg5gJyF/c19gpynslMBOW9hpgJ2OsFMGOwNhZwB2JqLR+58f7xPsTMXNqx9gZybsjMHvN+G9Bn7n5O/JA7874R3I/J6ED8fw+yH8LoLf4VsYYmA3Et7zye6H8LsAdr/CA6bZTQi/12A3JexOwG5a2J2C3YywOwO7JWHXA7sNYdcHu7a57UXBru3fqmWw2xZ2F2C3I/w2v61WwO63sDsHu5bGVotg9ya0bV/3JX9ty9WLCPsHsPcu7B/B3oewfwJ7UWH/Ava+hP0r2IsJ+2ewFxf2b2AvKezfwV5aOHgHe7bg9L7AXlY4eAN7OeHgA+zlhYNPsFcQ9h9grygcRMBeSdh/gb2acBADe3XhwNq3hYM02OsLBwmwNxIO8mBvIhzkwN7cNFmDvaX8He3B3ko4KIK9jXCQBHs74SAF9vbCQQXsWbyoBmDvJByUwd5ZOIiDvYtwUAJ7N+GgAPYev9/77+I82yn2P8R51SrYj1oXR7D/JRzUwL5pXQf7GeGgA/ZN6SbYN6XbYN+UbID9knDQAvvl/8otbFZXYL8u/3y+bJLOYL8hHPTBfks46IH9b/l3ewP7A+GgC/bN1qsXsD8WDgZgfyrsP8H+XDgYgf2FcDAG+2v5p27lRsUE7G+FgyjYD4SDb7BvlEzBwUw4zIKDpXCYBwcr4bAADtbCYREcbMR5jTE42IY3S3IQhKfLc3AUDkvg4GQVfHBwFg7L4OAqHObAwU04rIKDu3BYAQdPqzgHh2/CYQ0cfgqHdXD4JRw2wGFcOGyCw5S4ua12w5Zw+A0Oe8LxEBwOhOMxOBwKxyNwOBaOJ+DQ0qqaVfSE4xk49IVjHxzOhWMbcSkcL8DhWjhegsOtcDgFhzvheAUO98LxHhyafdQG4PBowIa8C8c7cPgQjrfg8Ckcb8DhSzgOwNGHcHwFR5/C8RkcRYXjOzhKCMcncJSUv9EbOEoLxwdwlBWOL+AoJxxbeUFoCd6oKJxEwFFJOH6Bo/DVKBNwVBFO3sFRVTh+giMzm9oUHNXl59H4w8kbOGqGdwtz1BJOPsFRW2hp1ehbOF6Do77Q8rNRSFQPHIVvZ/HB0Sy8pZUjy9AsYxiZT9XG4Ggh/wbJP25u+cRoaaUrcLSVf4OYlVq3gXBsohzkb/XdCpfg6CicJMDRSTiJgaOzOK+XBkdX4SQLjoxSDxw9hZMcOHoJJ0lwHBFO0uD4QzjJgOOocFIEx3HhJA+OE8JJARwnhZMSOE4JJ3FwnBZOyuC48Gsd46JwfATHNeFwBoZ3SlnCMm4LR9bxt3CSAsddu3AAxxYRLY0aD4STCjhZCL0FOFkKvSU4WcnfrzUYvojHluPJVuitwEkg9DbgZC/0rMJF6O3A8AalehucRoReAE4/rKADTpMGuuA0K86bzsBpXujdwGlB6D3AaVHoXcBpWegdwGlF6D3BaV3ov4HThtA7g9OmdTAHpy2hdwSnHaF3B6eWdtW/wWlX6J3AaT+8nZ7TgdCPg1PbpNd74HRqXezAqWdgD05t+ut9cLoU+jFwuhH6H+B0K/S/wOnRal7B6VnoZ8HpzVoMwelL6OfAWUTop8DZu9DPgLNQ9RE4+zQwBWdRoV8CZ5ab18fgLCRlAs7SQr8MzjJCvwjOskK/AM5yQj8PzsrivNkbOKsK/RY4awj9GjgzMmZJMLy5qu6Bs46VfICzroE4OLNprs/A2UjoN8DZVH7Wn+DMtxoG5kL/G5yZ8nVwthL6HXC2EY0+wdnWeliAM0vu63NwZhGj7oOzo9C7grOT0G+CM6OnC85CepbgzEyiZ7w8hX4f9N5ESxPQM77aoGd8VUHvQ+j3QO9T6A/+OG/2DnpRobcFvaTQj4JeyqZhDXrGWAX0jLEI6BlTQzC8LatuFSpC/xP0akI/DXq2x69vQK8p/3wkQc/M5wV6XaE/Ar2B0NuDnrE0Bb2x0B+D3kTom7wz68Bk8YW+dWCsJUB/Lfq+Auc54TwKzovh7ficl+XnUQLnFeE8Dc6rwnkKnK/CZ+G5+AoPuuMiE96xyUVVnLe8gIvmb4KxCCfuAS4GwkUJXIzDowe5mAprVnEWvmqCC09YT4GLlbBu37e//7AsLsLFEVxmhcsEuKwLlzFw2RRu7+CyJdw+wKVFhKZVGAi3T3A5Fuc9LuDSImczDS5nwt0buFwLdxFwuRfu3sGlGUIzAy5Pwu0LXF6Fu09waTvUXh5cRYS7KLj6EO6+wPBdV80iuApfHlUAV18GquAqKdylwVVKuEuAq4xwlwNXWeEuBq7ywp11WRDuMuCqKNylwFVJuEuCq4o47/kGrprCnQ3RFu7q4Koj3H2Dq2/hrgKuzDceT3A1lJ9dF1xZaGiWwNVEuDNRfOGuDK7mwl0LXC2FO5N1Jdy1wZWFRNsyhW/jalrNQLizHkJGrMlRuPPBla0Ezw9wZdR0wNXLajTB9ZtwNwXXRlEPXH9azQi4DhlpgOuYcOeB64RwNwLXxtAMXJvmC3BdFu5u4Loi3K3BdVW4W4Lr2q/C65ZwdwLXRsAeXFuMbPbBdV+424LrgXB3ANcj4W4HrqfCXQCubb7P4Np4WIFr42EDrhfWgfW8Ee6O4HprMsfAtVnEFVxbpHzcwfVJGLyB64dwNwfXtgFoDsH1SxgkwU1EGMTBzbtwVwM3H8LgHdx8yY/RsYkLdw9wkxAGMXBjCVJzAG7S8rPrg5uMMIiCm5xw9wQ3eWHwCW5sW9q0EYrC4AvclITBB7ip2IUxuGkJgwi4aQuDIrjpCoMCuOkJgxS4GQp3Vi+0iwq4GQuDHLgJHWIEbqbyszNZZsIgC248YZAHN7aSNCfgZikMMuBm9d9QW2HYYCcM0uAmEAYlcHMQBnVwcxLurN1FuBuCG7OWC7i5CQNj5y4MWuDGHKs5BTdPYVAGt+/CIAFuQ8dKgduoMGiAW7OiXhHcxmyO0uA2Lgwq4NYYrYJbW4CaPrg1orrgtiwMOuC2Kgza4LYmDL7BbV1ofrI1O7qD265wlwW3PeHOhhpYR3NwOxQGQ3A7EgZ9cDsVBgNwa/b0ArfmR01wu7IGcXC7FgY9cLuzgiW4Pf1O4vYs3FkF46EPBmPhuQsGU/kbm4OBJzz3wGDx3/ed8DwCg0D+xuxzLzwPweAoPA/A4CRu3mqBgW2TEx9gYEy2mmDwlJ99Cdy/WUEb3Nva3foG9xajWh1w/yU8e+A+JjzPwH1CeJ6A+7xVGIH7ovC8AfeWwbb64L4iPB/AvWWuEStoCc8BuG/b+HFw3xGed+DeNv4rcN8Vnrfgfig8++B+bF0Nwf1EeF6D+5nwvAT3Rl7LRl8Lz2dwvxGeL+B+Z11/gfvAapi8xsIN3J/kZ18B9zfh+QWGb89r2SAv4fkBHiLysy+Dh3fh+QoePoTnO3gwE2mNwUNeeHkDDwXh5QM8FIWXL/BQEl4i4KFpY2fAQ1t4iYOHjvASBQ+2urSW4KEnvHyCh77wkgDDB85bM/DgGQjAgy+8pMFD+ELALXhYCC858LASXvLgYS28ZMHDRtiq/HHz1g487ISXJHg4CS8l8PAUXjLg8VN+9lXwaHM2B48J4aUAHpPCSxE8pm2UA3gsGjiCR9MlBR6rwksDPNrctc7gsS68VMBjU3ipgUdTwboeCy8t8GhhoXUCj57w0gaPvvzsreJCeJ6CR9tJtNbgcS2avYJHWzdaJpVJ3wWPe+FlBB4vwksPPF6Flz54vAkvdfB4F16G4PElvDTB09t/nxHhZQCe3oWXb/D0+fsz0ik03Qd4isvPvg6eEv99mvod8GSm234DT6b1FDyVbQYL4MnUH4OnmvBsnw3h+QSeTPsZeLLdePsDDB8Ib+3BkznnETxNhBcfPM2sqyR4sqltR8DTXHgpgyebQQ88mdpz8BQILzHwtP/v09y0/QmezsLLAjwZHxvwZHwswdNDeLYhX/Kzb4Dnd/nfMQGeY8LLGjwn/yu31aI1AM9p4WULnnPCyw48m2MuwHM4uV3wbJO7B8+m3gQ8t4SXADybAb+DZ/PPJ3g5hMc08nIJ3/zFi0lk5Xf5d7AFLw/5t5UHrzX5MZe6joXXCXg1MrJJ8OoJr2fw6gs7gz+8XsHr/Pc3tutC3Oi1A6/r8MBlXs2XswnwGhhIg9e9XfnD6wW8Hv5PFyFlBfB6l59L/A+vNvjj/+Pbh/D2Cd7SwlsCvH3Lv+8X8DYLX+nH205488HbXti260fhrQ/eTsKbB94ewtsTvL2E9xR4fxfe0+A9IbxnwHslfNUf73vh/QI+YsLHJ/hICh958JGSf6op8JENb+3kIyd8FMBHUfh4gY+y0NK2R0Vo6c+jJny+g4+G8BkFH22hpRiPrtASqUdP+PwCH33h08YZhm8G5GMifMbBx1T+VlZ/3LztgY/QAn3w4QufSfCxED4T4MNcsr0EH5bL9drgw3yyvQAfO+HTBA7fKLoCH0fRqFU9CW0hfVgu096Az4zw2QCfWfmbzYPPvPy9foPPgvwcV+DT9iPtHfg0HZvg03RsgU/TqQs+l7YJy4LP8J/AL/AVkZ/2CK4X+RbXe3vC9dJjcb2vF1wvNxDXS9TgenkLzJ0UXK82lP+VvuB631/iei1Dvay4XjOA6y0a4vxeH64XvsOjd4XrvSri/HYWrv/eF+cPinD9j09xfv9qqCKut28ZGtnVPFz/s2xlA0NDcb3DGa4fzYnr7caG1lavCtcvLMX1UzW4fikizh99Gnpa2Rtcv1wV188M4frfB/lfqwvX71qk7/hw/d5F/td6wfX72d+3dLv+4EP+1+rB9VcJcf1l29DQ0AKuHwTi+sEHXP8cvl7Gg+tfFoYucP1bJTwT2PUfNXH9s1V7buV/XesjfA2214IbZDbifH8PN+jdxQ0aBbjBMC/OXzQNNcUN6h1DS0PfcIORJ86f9w1traxh6GUt1nCD9Umcv7QWtqT4Sys7zsUN5tb2tJDw9H03OK2sLAk3uLzEDRYTuEHo5qsAbhg/yj/NKNwwlRI3jA/hhpm8uF4yATdsZOV/Sytqm65BDW7YqYgb1k9ww++boRLcsOeJG35nDe3EDQtHuOHaytbWdr0XN1yODB3ln7VdfOTE+ccV3PB5kv8FA7jhKyFueJzDjSJDccPrA270aRoei3Cj6FHc8HSCGxUCcaPszdBFnH95gxsVs+JGyRDdrOwCNyqlDfUMeXZ1Azeqv4nzr0O4UWslbtRYG9qKm38P4EbtL7vqw416K0NnuNGgK27UsauDm7iBmefIe7cWU7iRf5X/3epwo7kn/7vl4EaLqDj/vocbWQboPzqGzuJGwSfc6GRO8VjCjc5zcaOj9XG1dfCxhxsnO+LG0RfcOP1hqA43znXE+a8k3LgyFjfOb+DG1aihtSGL2q9vuHEtZugANw5fRP9WMmQB5i0NN/4+iBt/T+DGg4H8RH0DR1PiZMgC+tsKbjzLixtP3+HG/ru4sXeFG69t0Y0EcOP9TNz8vWrobmV29WCCbsuGkuLGywTc+GjJXmQLNz73xI1XKbjxzTKWd1Pt/hA3vozhxo+quPE1Bje2YDT/7MCNX0lxk3gcbvIWHrUzgZu8R62sCDexqDGJ1wxZmP1cw02iMUO+ofDMkQPc5Cu8ezAGNylajh0dw00sGsyjd7hJfWqubP11TY+vDtykZ+nLdxRu0u9Y2RxuMoiLG2ZmcJNhVtyka+NOr+LmsTrcxG/IT+Fh4CFuYvFgsrUML2aiPLviJscb3DQbFzdNj+GmRVMyeYGblsfipsUl3LTSFTdN5OGm7Zr8mMtP2wf5aSThpp1v+am34KbfFrTSKbhp903ctDOFm/ZtqUjP4aYD24elF3DTiUmesT4mlupmunBTz7agGRvKa8hP2+TwTOnMEW46Dxtc4aYLm+TMHW66NBIzT0MXQyZb+LBFUIGbHgL5+U7DTY9lcdNLydBUfszzp6esBfw3uOm5Lm6encFNL3Fx/rgHNw1XvewZbvZu9plLw80suM/etnCzaEbcLLKHm335drUCN4tFxE1fcUPf4mbRuSGTMzczZLad+zZk5pYrwM3iSXGz96uhtLjZ5yfcLLmRn8GbgbP8DCIGbuJm+S+4WWogP4N3uFnalM5n4GYWouf5Mtwsa5OZTxlaWYME3CzXMxQzNDaUhJt9m0V0Tbhu+DTnAm42eImb9Rtws2HckIk0NCW6gaGpuFnPlBiZKKMN3GzcsuXY5Ax/Ly9Zf5O5/EysaNqWn4mJPjXpug9DW6tv48/C20nvhmxXXHrBzbyw3hNu5tv8l1Zws3nRWuThZgvzhHIObrYMkVG3PBuyq6vwtjebgJXt6spFuNk6Kj9j63htwpVLcLON7VnLRkr4O0nJFNuEd+BY0405YPkDbrYLO6nCzYKCuNnZ1Agqhkwz263OzlFDZm3lHdzMtmezs83P4WzI6D6+GbLRjjlDpvjRpuVsdnTcGbKpOoUoCze7h+MaVY+wRR3Oe7MAVmnCedGGoRmclzUKqgVDA3Fe7gnn5cL39ZcMhf/iVg35dvVhyMLlRw3OK9jmu+rDebb+zKtLOK80FOdVK4aMjeoBzisXDJ0MmVTVPZxXtSnqxeC82ps4rzExFL7xP27I3LuWgPPq4VXPUPj2fxujfjFk0tevhhZwXiNuaAPnNVvivO7L0FicZx7pNS1c1vqGDlYWgfNaX4beDZmp10aG+oas55ZnVz/gvHbMkMnSnhj6MhT2koTzOmVDKUPmh7UAzvv+tLKsobBFztBafpZGVTdvRVFD4Z9sNh29vZX14Ly+0dc3wvum2tRoGfQNmZIDqzc1SgdnQzZY+IfndAXnDXOGjoYK8rM22YdzKzobMomnN0Nhx3c4b5Q0ZNyOGpYWm2ajnSHTNvxLbWaCjk2fmek4Ni5mGUMJa7s1VLCyNJw3MWvpGZoat7M8nLe0nhdDOG8dETdvGN/rQJy3LsJ5G5vdhTG/sTnd2PwFI3Hezq7uTea99Xew5TPk0bzDe1wNWYuHaXS0npvG97EuP7tvAxu7aEZw3Boywz1Z2fMTzjt/GDLVzuGveB1DNvXNriGbtKeRcbMZapo5hM8dNY2qW/hTmrnT3TKIps3LI/zFzwR42Tras/l7hZt1m9OXb7vzTzg/XNNbU0MTK4saehpKwfkRWywSCUOWMifScP67pWSJLJz/kTGUN2S5YyJnKPwl5WYo/HHnAud/hi3KhmzfkSjC+VFz6PY7nP8V/iz3hPNjlgAnKnB+PPyFrgTnJywKtpNwfsqS7GwGzk+H+za7mrHpbVuLUrhz68L55YM4v7yE88PUsmlS1W1/0jRJGzmrt4bzu1XbK9m4p5KluUu4edEYL53g5uZY8/Yc/y8AAP//H8dHDd+IAAA=" 5 | 6 | func init() { 7 | genFuncs("cmn", &cmnPhrases, &cmnCharacters) 8 | } 9 | -------------------------------------------------------------------------------- /normalize/init.go: -------------------------------------------------------------------------------- 1 | package normalize 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/base64" 7 | "fmt" 8 | "github.com/golang/glog" 9 | "github.com/liuzl/da" 10 | "github.com/liuzl/ling/util" 11 | "github.com/liuzl/tokenizer" 12 | "strings" 13 | ) 14 | 15 | var Funcs = make(map[string]util.ConvertFunc) 16 | 17 | func genFuncs(lang string, contents ...*string) error { 18 | var dicts []*da.Dict 19 | for _, c := range contents { 20 | data, err := base64.StdEncoding.DecodeString(*c) 21 | if err != nil { 22 | glog.Error(err) 23 | return err 24 | } 25 | reader, err := gzip.NewReader(bytes.NewReader(data)) 26 | if err != nil { 27 | glog.Error(err) 28 | return err 29 | } 30 | dict, err := da.Build(reader) 31 | if err != nil { 32 | glog.Error(err) 33 | return err 34 | } 35 | dicts = append(dicts, dict) 36 | } 37 | Funcs[lang] = func(in []string) ([]string, error) { 38 | out, err := util.Convert(strings.Join(in, ""), dicts) 39 | if err != nil { 40 | return nil, err 41 | } 42 | ret := tokenizer.Tokenize(out) 43 | if len(ret) == len(in) { 44 | return ret, nil 45 | } 46 | return nil, fmt.Errorf("len(ret):%d!=len(in):%d of %s norm function", 47 | len(ret), len(in), lang) 48 | } 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /normalizer.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/liuzl/ling/normalize" 7 | ) 8 | 9 | const Norm = "norm" 10 | 11 | func init() { 12 | Processors[Norm] = &Normalizer{} 13 | } 14 | 15 | // Normalizer is the processor for token normalization 16 | type Normalizer struct { 17 | } 18 | 19 | // Process normalizes the tokens of Document d 20 | func (n *Normalizer) Process(d *Document) error { 21 | if d == nil || len(d.Text) == 0 { 22 | return nil 23 | } 24 | if len(d.Tokens) == 0 { 25 | return fmt.Errorf("tokenization required") 26 | } 27 | if f, has := normalize.Funcs[d.Lang]; has { 28 | ret, err := f(d.XTokens(Norm)) 29 | if err != nil { 30 | return err 31 | } 32 | if len(ret) == len(d.Tokens) { 33 | for i, str := range ret { 34 | d.Tokens[i].Annotations[Norm] = str 35 | } 36 | } 37 | } 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /regex_tagger.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func init() { 8 | Processors["regex"] = &RegexTagger{} 9 | } 10 | 11 | // RegexTagger is the processor that uses regex expression 12 | type RegexTagger struct { 13 | } 14 | 15 | // Process is the function to annotate documents 16 | func (t *RegexTagger) Process(d *Document) error { 17 | if d == nil || len(d.Text) == 0 { 18 | return nil 19 | } 20 | if len(d.Tokens) == 0 { 21 | return fmt.Errorf("tokenization required") 22 | } 23 | for typ, re := range Regexes { 24 | matches := re.FindAllStringIndex(d.Text, -1) 25 | for _, match := range matches { 26 | start := -1 27 | end := -1 28 | for _, token := range d.Tokens { 29 | if token.StartByte == match[0] { 30 | start = token.I 31 | } 32 | if token.EndByte == match[1] { 33 | end = token.I + 1 34 | } 35 | } 36 | if start == -1 || end == -1 { 37 | continue 38 | } 39 | span := &Span{Doc: d, Start: start, End: end, 40 | Annotations: map[string]interface{}{ 41 | "from": "regex", "value": map[string]interface{}{typ: ""}}} 42 | d.Spans = append(d.Spans, span) 43 | } 44 | } 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /tokenizer.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "strings" 5 | "unicode" 6 | 7 | "github.com/liuzl/ling/util" 8 | "github.com/liuzl/tokenizer" 9 | ) 10 | 11 | const Lower = "lower" 12 | 13 | func init() { 14 | Processors["_"] = &Tokenizer{} 15 | } 16 | 17 | func Type(text string) TokenType { 18 | switch { 19 | case util.StringIs(text, unicode.IsSpace): 20 | return Space 21 | case util.StringIs(text, unicode.IsSymbol): 22 | return Symbol 23 | case util.StringIs(text, unicode.IsNumber): 24 | return Number 25 | case util.StringIs(text, unicode.IsPunct): 26 | return Punct 27 | //case util.StringIs(text, unicode.IsLetter): 28 | case util.StringIs(text, func(r rune) bool { 29 | if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') { 30 | return false 31 | } 32 | return true 33 | }): 34 | return Letters 35 | } 36 | return Word 37 | } 38 | 39 | func Script(text string) string { 40 | for k, v := range unicode.Scripts { 41 | if util.StringIs(text, func(r rune) bool { return unicode.Is(v, r) }) { 42 | return k 43 | } 44 | } 45 | return "Unknown" 46 | } 47 | 48 | type Tokenizer struct { 49 | } 50 | 51 | func (t *Tokenizer) Process(d *Document) error { 52 | if d == nil || len(d.Text) == 0 { 53 | return nil 54 | } 55 | var tokens []*Token 56 | var pos int 57 | for i, item := range tokenizer.TokenizePro(d.Text) { 58 | word := item.Text 59 | l := len([]byte(word)) 60 | token := &Token{Doc: d, Text: word, Type: Type(word), Script: Script(word), 61 | I: i, StartByte: pos, EndByte: pos + l, 62 | Annotations: map[string]string{Lower: strings.ToLower(word)}} 63 | if item.Norm != "" { 64 | token.Annotations[Norm] = item.Norm 65 | } 66 | pos += l 67 | tokens = append(tokens, token) 68 | } 69 | d.Tokens = tokens 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /tokentype_jsonenums.go: -------------------------------------------------------------------------------- 1 | // generated by jsonenums -type=TokenType; DO NOT EDIT 2 | 3 | package ling 4 | 5 | import ( 6 | "encoding/json" 7 | "fmt" 8 | ) 9 | 10 | var ( 11 | _TokenTypeNameToValue = map[string]TokenType{ 12 | "EOF": EOF, 13 | "Space": Space, 14 | "Symbol": Symbol, 15 | "Number": Number, 16 | "Letters": Letters, 17 | "Punct": Punct, 18 | "Word": Word, 19 | } 20 | 21 | _TokenTypeValueToName = map[TokenType]string{ 22 | EOF: "EOF", 23 | Space: "Space", 24 | Symbol: "Symbol", 25 | Number: "Number", 26 | Letters: "Letters", 27 | Punct: "Punct", 28 | Word: "Word", 29 | } 30 | ) 31 | 32 | func init() { 33 | var v TokenType 34 | if _, ok := interface{}(v).(fmt.Stringer); ok { 35 | _TokenTypeNameToValue = map[string]TokenType{ 36 | interface{}(EOF).(fmt.Stringer).String(): EOF, 37 | interface{}(Space).(fmt.Stringer).String(): Space, 38 | interface{}(Symbol).(fmt.Stringer).String(): Symbol, 39 | interface{}(Number).(fmt.Stringer).String(): Number, 40 | interface{}(Letters).(fmt.Stringer).String(): Letters, 41 | interface{}(Punct).(fmt.Stringer).String(): Punct, 42 | interface{}(Word).(fmt.Stringer).String(): Word, 43 | } 44 | } 45 | } 46 | 47 | // MarshalJSON is generated so TokenType satisfies json.Marshaler. 48 | func (r TokenType) MarshalJSON() ([]byte, error) { 49 | if s, ok := interface{}(r).(fmt.Stringer); ok { 50 | return json.Marshal(s.String()) 51 | } 52 | s, ok := _TokenTypeValueToName[r] 53 | if !ok { 54 | return nil, fmt.Errorf("invalid TokenType: %d", r) 55 | } 56 | return json.Marshal(s) 57 | } 58 | 59 | // UnmarshalJSON is generated so TokenType satisfies json.Unmarshaler. 60 | func (r *TokenType) UnmarshalJSON(data []byte) error { 61 | var s string 62 | if err := json.Unmarshal(data, &s); err != nil { 63 | return fmt.Errorf("TokenType should be a string, got %s", data) 64 | } 65 | v, ok := _TokenTypeNameToValue[s] 66 | if !ok { 67 | return fmt.Errorf("invalid TokenType %q", s) 68 | } 69 | *r = v 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /tokentype_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=TokenType"; DO NOT EDIT. 2 | 3 | package ling 4 | 5 | import "strconv" 6 | 7 | const _TokenType_name = "EOFSpaceSymbolNumberLettersPunctWord" 8 | 9 | var _TokenType_index = [...]uint8{0, 3, 8, 14, 20, 27, 32, 36} 10 | 11 | func (i TokenType) String() string { 12 | if i >= TokenType(len(_TokenType_index)-1) { 13 | return "TokenType(" + strconv.FormatInt(int64(i), 10) + ")" 14 | } 15 | return _TokenType_name[_TokenType_index[i]:_TokenType_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "bytes" 5 | ) 6 | 7 | type Document struct { 8 | Text string `json:"text"` 9 | Tokens []*Token `json:"tokens"` 10 | Spans []*Span `json:"spans"` 11 | Lang string `json:"lang"` 12 | Langs []string `json:"langs"` 13 | } 14 | 15 | type TokenType byte 16 | 17 | //go:generate jsonenums -type=TokenType 18 | //go:generate stringer -type=TokenType 19 | const ( 20 | EOF TokenType = iota 21 | Space 22 | Symbol 23 | Number 24 | Letters 25 | Punct 26 | Word 27 | ) 28 | 29 | type Token struct { 30 | Doc *Document `json:"-"` 31 | Text string `json:"text"` 32 | Type TokenType `json:"type"` 33 | Script string `json:"script"` 34 | I int `json:"i"` 35 | StartByte int `json:"start_byte"` 36 | EndByte int `json:"end_byte"` 37 | Annotations map[string]string `json:"annotations"` 38 | } 39 | 40 | func (t *Token) String() string { 41 | return t.Text 42 | } 43 | 44 | type Span struct { 45 | Doc *Document `json:"-"` 46 | Start int `json:"start"` 47 | End int `json:"end"` 48 | Annotations map[string]interface{} `json:"annotations"` 49 | } 50 | 51 | func (s *Span) String() string { 52 | output := bytes.Buffer{} 53 | for i := s.Start; i < s.End; i++ { 54 | output.WriteString(s.Doc.Tokens[i].String()) 55 | } 56 | return output.String() 57 | } 58 | 59 | type Processor interface { 60 | Process(d *Document) error 61 | } 62 | 63 | var Processors = make(map[string]Processor) 64 | -------------------------------------------------------------------------------- /unidecoder.go: -------------------------------------------------------------------------------- 1 | package ling 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/liuzl/unidecode" 7 | ) 8 | 9 | const Unidecode = "unidecode" 10 | 11 | func init() { 12 | Processors[Unidecode] = &Unidecoder{} 13 | } 14 | 15 | type Unidecoder struct { 16 | } 17 | 18 | func (u *Unidecoder) Process(d *Document) error { 19 | if d == nil || len(d.Text) == 0 { 20 | return nil 21 | } 22 | if len(d.Tokens) == 0 { 23 | return fmt.Errorf("tokenization is required for unidecode") 24 | } 25 | for _, token := range d.Tokens { 26 | var s string 27 | if s = token.Annotations[Norm]; s == "" { 28 | s = token.Text 29 | } 30 | token.Annotations[Unidecode] = unidecode.Unidecode(s) 31 | } 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /unittests/all_test.go: -------------------------------------------------------------------------------- 1 | package unittests 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/liuzl/ling" 8 | ) 9 | 10 | func TestAll(t *testing.T) { 11 | cases := []string{ 12 | `乾隆爷的乾儿子是谁？`, 13 | `支持中國大陸、臺灣、香港異體字和地區習慣用詞轉換，如「裏」「裡」、「鼠標」「滑鼠」。`, 14 | `自然語言處理是人工智能領域中的一個重要方向。`, 15 | //Chinese 16 | "目前新车的轴距还没有公布，但是长度达到了4915mm，未来定位可能是一款中级轿车，而新车重量为1890kg。作为一款新能源车型，氢燃料的使用无疑让人惊喜，而本田公布它的最大续航为750km，可以说是非常能跑。", 17 | //English 18 | "Emerging after two hours of talks with Chinese President Xi Jinping, Trump said he doesn't fault China for taking advantage of differences between the way the two countries do business.", 19 | //Janpanese 20 | "「東京動画」東京都公式動画チャンネル。都政の仕組みや街の魅力を伝える、いつでも・どこでも・誰でも楽しめるコンテンツがここに集まる！", 21 | //Korean 22 | "트럼프 대통령의 방한은 대통령 취임 후 첫 번째 방한이자 문재인 대통령과의 세 번째 정상회담이고 미국 대통령으로서는 25년 만에 국빈 방한이라는 의미가 있었다. ", 23 | //Russian 24 | "Официальный сайт Московского государственного университета имени М.В.Ломоносова (МГУ)", 25 | //Thai 26 | "ทันทุกเหตุการณ์ข่าววันนี้ข่าวล่าสุดตรวจหวยดวงข่าวบันเทิงคอลัมน์ดังเรื่องย่อ", 27 | //Swedish 28 | "Swedish photograper Per-Anders Jörgensen and Art Director Lotta Jörgensen are the duo behind one of the most interesting Food Magazines in the world.", 29 | //English text 30 | "zhanliangliu@gmail.com, \t\t\t .@#$!@ zliu.org 123 is one two three", 31 | ",，.。有意思quanjian１２３", 32 | "，。！？【】（）％＃＠＆１２３４５６７８９０“”‘’''\"\"『』「」﹃﹄〔〕—-《》：、", 33 | "123hj is goo. goog1e brightliang137 liang@zliu.org", 34 | "1238 3.1415 -1.618 6.023e23 1e-13 1,234,234", 35 | `自建房2樓3室2廳1衛1廚92.00㎡戶型圖，92km到北京`, 36 | `one-hundred and twenty-five`, 37 | `山西省煤炭地质１４８勘查院煤层气工程处`, 38 | } 39 | 40 | pipeline, err := ling.DefaultNLP() 41 | if err != nil { 42 | t.Error(err) 43 | } 44 | for _, c := range cases { 45 | d := ling.NewDocument(c) 46 | err = pipeline.Annotate(d) 47 | if err != nil { 48 | t.Error(err) 49 | } 50 | t.Logf("lang :%s", d.Lang) 51 | t.Logf("langs :%s", d.Langs) 52 | t.Logf("tokens:%s", d.Tokens) 53 | t.Logf("lower :%s", d.XTokens(ling.Lower)) 54 | t.Logf("norm :%s", d.XTokens(ling.Norm)) 55 | t.Logf("lemma :%s", d.XTokens(ling.Lemma)) 56 | t.Logf("unidecode :%s", d.XTokens(ling.Unidecode)) 57 | b, err := json.Marshal(d) 58 | if err != nil { 59 | t.Error(err) 60 | } 61 | t.Log(string(b)) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /unittests/api_tagger_test.go: -------------------------------------------------------------------------------- 1 | package unittests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/juju/errors" 7 | "github.com/liuzl/ling" 8 | ) 9 | 10 | func TestAPITagger(t *testing.T) { 11 | cases := []string{ 12 | `天津大学离北京也不太远，电话是02227890949，主页是https://www.tju.edu.cn，计算机`, 13 | `https://crawler.club是爬虫主页`, 14 | `刘晓明是个好同志，他和周晓华是一对儿好朋友`, 15 | } 16 | 17 | nlp, err := ling.DefaultNLP() 18 | if err != nil { 19 | t.Error(err) 20 | } 21 | tagger, err := ling.NewAPITagger("http://127.0.0.1:5002/api") 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | if err = nlp.AddTagger(tagger); err != nil { 26 | t.Error(err) 27 | } 28 | for _, c := range cases { 29 | d := ling.NewDocument(c) 30 | if err = nlp.Annotate(d); err != nil { 31 | t.Error(errors.ErrorStack(err)) 32 | } 33 | t.Logf("spans:%s", d.Spans) 34 | for i, s := range d.Spans { 35 | t.Logf("span %d %s: %+v", i, s, s.Annotations) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /unittests/dict_tagger_test.go: -------------------------------------------------------------------------------- 1 | package unittests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/liuzl/ling" 7 | ) 8 | 9 | func TestDictTagger(t *testing.T) { 10 | cases := []string{ 11 | `天津大学离北京也不太远，电话是02227890949，主页是https://www.tju.edu.cn，计算机`, 12 | `https://crawler.club是爬虫主页`, 13 | } 14 | 15 | nlp, err := ling.DefaultNLP() 16 | if err != nil { 17 | t.Error(err) 18 | } 19 | tagger, err := ling.NewDictTagger() 20 | if err != nil { 21 | t.Error(err) 22 | } 23 | if err = nlp.AddTagger(tagger); err != nil { 24 | t.Error(err) 25 | } 26 | for _, c := range cases { 27 | d := ling.NewDocument(c) 28 | if err = nlp.Annotate(d); err != nil { 29 | t.Error(err) 30 | } 31 | t.Logf("spans:%s", d.Spans) 32 | for i, s := range d.Spans { 33 | t.Logf("span %d %s: %+v", i, s, s.Annotations) 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /util/io.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/base64" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | // FetchURL gets the content of url 15 | func FetchURL(url string) []byte { 16 | resp, err := http.Get(url) 17 | if err != nil || resp.StatusCode != 200 { 18 | log.Fatalf("Error fetching URL '%s': %s", url, err) 19 | } 20 | defer resp.Body.Close() 21 | body, err := ioutil.ReadAll(resp.Body) 22 | if err != nil { 23 | log.Fatalf("Error reading body: %s", err) 24 | } 25 | return body 26 | } 27 | 28 | // UrlToZipConent gets the content of srcUrl and returns the compressed content 29 | func UrlToZipContent(srcUrl string) string { 30 | body := FetchURL(srcUrl) 31 | var compressed bytes.Buffer 32 | w := gzip.NewWriter(&compressed) 33 | w.Write(body) 34 | w.Close() 35 | return base64.StdEncoding.EncodeToString(compressed.Bytes()) 36 | } 37 | 38 | // WriteFile writes data to file at filePath 39 | func WriteFile(filePath string, data []byte) { 40 | gopath, found := os.LookupEnv("GOPATH") 41 | if !found { 42 | log.Fatal("Missing $GOPATH environment variable") 43 | } 44 | path := filepath.Join(gopath, filePath) 45 | err := ioutil.WriteFile(path, data, os.FileMode(0664)) 46 | if err != nil { 47 | log.Fatalf("Error writing '%s': %s", path, err) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /util/types.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // ConvertFunc is a class of function that convert a string slice to another 4 | type ConvertFunc func([]string) ([]string, error) 5 | -------------------------------------------------------------------------------- /util/utils.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "github.com/liuzl/da" 6 | "strings" 7 | ) 8 | 9 | type is func(rune) bool 10 | 11 | func StringIs(s string, f is) bool { 12 | for _, c := range s { 13 | if !f(c) { 14 | return false 15 | } 16 | } 17 | return true 18 | } 19 | 20 | func Convert(in string, dicts []*da.Dict) (string, error) { 21 | r := []rune(in) 22 | var tokens []string 23 | for i := 0; i < len(r); { 24 | s := r[i:] 25 | var token string 26 | max := 0 27 | for _, dict := range dicts { 28 | ret, err := dict.PrefixMatch(string(s)) 29 | if err != nil { 30 | return "", err 31 | } 32 | if len(ret) > 0 { 33 | o := "" 34 | for k, v := range ret { 35 | if len(k) > max { 36 | max = len(k) 37 | token = v[0] 38 | o = k 39 | } 40 | } 41 | i += len([]rune(o)) 42 | break 43 | } 44 | } 45 | if max == 0 { //no match 46 | token = string(r[i]) 47 | i++ 48 | } 49 | tokens = append(tokens, token) 50 | } 51 | return strings.Join(tokens, ""), nil 52 | } 53 | 54 | func DictConvert(in []string, m map[string]string) ([]string, error) { 55 | if m == nil { 56 | return nil, fmt.Errorf("lemma dict is nil") 57 | } 58 | var ret []string 59 | for _, token := range in { 60 | if str, has := m[token]; has { 61 | ret = append(ret, str) 62 | } else { 63 | ret = append(ret, token) 64 | } 65 | } 66 | return ret, nil 67 | } 68 | --------------------------------------------------------------------------------