├── .gitignore ├── README.md ├── requirements.txt ├── segment ├── __init__.py ├── data │ ├── __init__.py │ ├── anchor │ │ ├── frequencies.tsv │ │ └── total.tsv │ ├── en │ │ ├── __init__.py │ │ ├── frequencies.tsv │ │ └── total.tsv │ └── twitter │ │ ├── __init__.py │ │ ├── frequencies.tsv │ │ └── total.tsv ├── segmenter.py └── test │ ├── __init__.py │ └── segment_tests.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/README.md -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /segment/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/segment/__init__.py -------------------------------------------------------------------------------- /segment/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/segment/data/__init__.py -------------------------------------------------------------------------------- /segment/data/anchor/frequencies.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/segment/data/anchor/frequencies.tsv -------------------------------------------------------------------------------- /segment/data/anchor/total.tsv: -------------------------------------------------------------------------------- 1 | 879911520048 2 | -------------------------------------------------------------------------------- /segment/data/en/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /segment/data/en/frequencies.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/segment/data/en/frequencies.tsv -------------------------------------------------------------------------------- /segment/data/en/total.tsv: -------------------------------------------------------------------------------- 1 | 1024908267229 2 | -------------------------------------------------------------------------------- /segment/data/twitter/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /segment/data/twitter/frequencies.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/segment/data/twitter/frequencies.tsv -------------------------------------------------------------------------------- /segment/data/twitter/total.tsv: -------------------------------------------------------------------------------- 1 | 1174595007 2 | -------------------------------------------------------------------------------- /segment/segmenter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/segment/segmenter.py -------------------------------------------------------------------------------- /segment/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /segment/test/segment_tests.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/segment/test/segment_tests.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/willf/segment/HEAD/setup.py --------------------------------------------------------------------------------