├── .gitignore ├── README.md ├── ctok.c ├── demo.py ├── setup.cfg ├── setup.py ├── test_ctok.py ├── v35tokenizer.h ├── v36tokenizer.h ├── v37tokenizer.h └── v38tokenizer.h /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/README.md -------------------------------------------------------------------------------- /ctok.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/ctok.c -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/demo.py -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/setup.py -------------------------------------------------------------------------------- /test_ctok.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/test_ctok.py -------------------------------------------------------------------------------- /v35tokenizer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/v35tokenizer.h -------------------------------------------------------------------------------- /v36tokenizer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/v36tokenizer.h -------------------------------------------------------------------------------- /v37tokenizer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/v37tokenizer.h -------------------------------------------------------------------------------- /v38tokenizer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gvanrossum/ctok/HEAD/v38tokenizer.h --------------------------------------------------------------------------------