├── .gitignore ├── LineBreak.txt ├── Makefile ├── README.md ├── bin └── generate-tokens ├── examples ├── linebreak.js └── package.json ├── exclude-classes.txt ├── include-ranges.txt ├── lib ├── BreakType.js ├── LineBreak.js ├── TokenType.js ├── index.js ├── parse-data.js ├── tokenizer.js └── tokens.js ├── package.json └── test ├── LineBreak-test.js └── Tokenizer-test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /LineBreak.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/LineBreak.txt -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/README.md -------------------------------------------------------------------------------- /bin/generate-tokens: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/bin/generate-tokens -------------------------------------------------------------------------------- /examples/linebreak.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/examples/linebreak.js -------------------------------------------------------------------------------- /examples/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/examples/package.json -------------------------------------------------------------------------------- /exclude-classes.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/exclude-classes.txt -------------------------------------------------------------------------------- /include-ranges.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/include-ranges.txt -------------------------------------------------------------------------------- /lib/BreakType.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/lib/BreakType.js -------------------------------------------------------------------------------- /lib/LineBreak.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/lib/LineBreak.js -------------------------------------------------------------------------------- /lib/TokenType.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/lib/TokenType.js -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/lib/index.js -------------------------------------------------------------------------------- /lib/parse-data.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/lib/parse-data.js -------------------------------------------------------------------------------- /lib/tokenizer.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/lib/tokenizer.js -------------------------------------------------------------------------------- /lib/tokens.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/lib/tokens.js -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/package.json -------------------------------------------------------------------------------- /test/LineBreak-test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/test/LineBreak-test.js -------------------------------------------------------------------------------- /test/Tokenizer-test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bramstein/unicode-tokenizer/HEAD/test/Tokenizer-test.js --------------------------------------------------------------------------------