├── LICENSE
├── README.md
├── one-grams.txt
└── segment.py


/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2022 Jeremy Kun
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | Word Segmentation, or Makingsenseofthis
2 | =======
3 | 
4 | Python code and data for the post ["Word Segmentation, or Makingsenseofthis"](http://jeremykun.wordpress.com/2012/01/15/word-segmentation/)
5 | 


--------------------------------------------------------------------------------
/segment.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | 
 3 | import functools, math
 4 | 
 5 | class OneGramDist(dict):
 6 |    def __init__(self, filename):
 7 |       self.gramCount = 0
 8 | 
 9 |       for line in open(filename):
10 |          (word, count) = line[:-1].split('\t')
11 |          self[word] = int(count)
12 |          self.gramCount += self[word]
13 | 
14 |    def __call__(self, key):
15 |       if key in self:
16 |          return float(self[key]) / self.gramCount
17 |       else:
18 |          return 1.0 / (self.gramCount * 10**(len(key)-2))
19 | 
20 | singleWordProb = OneGramDist('one-grams.txt')
21 | def wordSeqFitness(words):
22 |    return sum(math.log10(singleWordProb(w)) for w in words)
23 | 
24 | def memoize(f):
25 |    cache = {}
26 | 
27 |    def memoizedFunction(*args):
28 |       if args not in cache:
29 |          cache[args] = f(*args)
30 |       return cache[args]
31 | 
32 |    memoizedFunction.cache = cache
33 |    return memoizedFunction
34 | 
35 | @memoize
36 | def segment(word):
37 |    if not word: return []
38 |    word = word.lower() # change to lower case
39 |    allSegmentations = [[first] + segment(rest) for (first,rest) in splitPairs(word)]
40 |    return max(allSegmentations, key = wordSeqFitness)
41 | 
42 | def splitPairs(word, maxLen=20):
43 |    return [(word[:i+1], word[i+1:]) for i in range(max(len(word), maxLen))]
44 | 
45 | @memoize
46 | def segmentWithProb(word):
47 |    segmented = segment(word)
48 |    return (wordSeqFitness(segmented), segmented)
49 | 


--------------------------------------------------------------------------------