├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── appveyor.yml ├── data.go ├── doc.go ├── document.go ├── document_test.go ├── extract.go ├── extract_test.go ├── go.mod ├── go.sum ├── model.go ├── model ├── AveragedPerceptron │ ├── classes.gob │ ├── tags.gob │ └── weights.gob └── Maxent │ ├── labels.gob │ ├── mapping.gob │ ├── weights.gob │ └── words.gob ├── model_test.go ├── scripts ├── cover.sh ├── fuzzit.sh ├── get_article.py ├── pipe.py ├── requirements.txt ├── test_model.py ├── treebank_words.py ├── update_model.py └── word_punct.py ├── segment.go ├── segment_test.go ├── tag.go ├── tag_test.go ├── testdata ├── PRODUCT │ └── Maxent │ │ ├── labels.gob │ │ ├── mapping.gob │ │ └── weights.gob ├── article.txt ├── golden_rules_en.json ├── reddit_product.jsonl ├── sherlock.txt ├── tokenize.json ├── treebank_sents.json ├── treebank_tags.json ├── treebank_tokens.json └── treebank_words.json ├── tokenize.go ├── tokenize_test.go ├── types.go ├── utilities.go └── words.go /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/README.md -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/appveyor.yml -------------------------------------------------------------------------------- /data.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/data.go -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/doc.go -------------------------------------------------------------------------------- /document.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/document.go -------------------------------------------------------------------------------- /document_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/document_test.go -------------------------------------------------------------------------------- /extract.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/extract.go -------------------------------------------------------------------------------- /extract_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/extract_test.go -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/go.mod -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/go.sum -------------------------------------------------------------------------------- /model.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/model.go -------------------------------------------------------------------------------- /model/AveragedPerceptron/classes.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/model/AveragedPerceptron/classes.gob -------------------------------------------------------------------------------- /model/AveragedPerceptron/tags.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/model/AveragedPerceptron/tags.gob -------------------------------------------------------------------------------- /model/AveragedPerceptron/weights.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/model/AveragedPerceptron/weights.gob -------------------------------------------------------------------------------- /model/Maxent/labels.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/model/Maxent/labels.gob -------------------------------------------------------------------------------- /model/Maxent/mapping.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/model/Maxent/mapping.gob -------------------------------------------------------------------------------- /model/Maxent/weights.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/model/Maxent/weights.gob -------------------------------------------------------------------------------- /model/Maxent/words.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/model/Maxent/words.gob -------------------------------------------------------------------------------- /model_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/model_test.go -------------------------------------------------------------------------------- /scripts/cover.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/scripts/cover.sh -------------------------------------------------------------------------------- /scripts/fuzzit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/scripts/fuzzit.sh -------------------------------------------------------------------------------- /scripts/get_article.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/scripts/get_article.py -------------------------------------------------------------------------------- /scripts/pipe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/scripts/pipe.py -------------------------------------------------------------------------------- /scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | nltk==3.4.5 2 | tabulate==0.7.7 3 | newspaper3k>=0.2.2 4 | -------------------------------------------------------------------------------- /scripts/test_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/scripts/test_model.py -------------------------------------------------------------------------------- /scripts/treebank_words.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/scripts/treebank_words.py -------------------------------------------------------------------------------- /scripts/update_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/scripts/update_model.py -------------------------------------------------------------------------------- /scripts/word_punct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/scripts/word_punct.py -------------------------------------------------------------------------------- /segment.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/segment.go -------------------------------------------------------------------------------- /segment_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/segment_test.go -------------------------------------------------------------------------------- /tag.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/tag.go -------------------------------------------------------------------------------- /tag_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/tag_test.go -------------------------------------------------------------------------------- /testdata/PRODUCT/Maxent/labels.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/PRODUCT/Maxent/labels.gob -------------------------------------------------------------------------------- /testdata/PRODUCT/Maxent/mapping.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/PRODUCT/Maxent/mapping.gob -------------------------------------------------------------------------------- /testdata/PRODUCT/Maxent/weights.gob: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/PRODUCT/Maxent/weights.gob -------------------------------------------------------------------------------- /testdata/article.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/article.txt -------------------------------------------------------------------------------- /testdata/golden_rules_en.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/golden_rules_en.json -------------------------------------------------------------------------------- /testdata/reddit_product.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/reddit_product.jsonl -------------------------------------------------------------------------------- /testdata/sherlock.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/sherlock.txt -------------------------------------------------------------------------------- /testdata/tokenize.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/tokenize.json -------------------------------------------------------------------------------- /testdata/treebank_sents.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/treebank_sents.json -------------------------------------------------------------------------------- /testdata/treebank_tags.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/treebank_tags.json -------------------------------------------------------------------------------- /testdata/treebank_tokens.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/treebank_tokens.json -------------------------------------------------------------------------------- /testdata/treebank_words.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/testdata/treebank_words.json -------------------------------------------------------------------------------- /tokenize.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/tokenize.go -------------------------------------------------------------------------------- /tokenize_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/tokenize_test.go -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/types.go -------------------------------------------------------------------------------- /utilities.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/utilities.go -------------------------------------------------------------------------------- /words.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdkato/prose/HEAD/words.go --------------------------------------------------------------------------------