├── .gitignore ├── README.md ├── figure1.png ├── lib ├── datasets.py ├── ddp.py ├── decay_to_init.py ├── ema.py ├── models.py ├── ops.py ├── rotary.py ├── scalinglaw_utils.py └── utils.py ├── misc ├── bw_tokenizer │ ├── nonbreaking_prefixes │ │ ├── nonbreaking_prefix.de │ │ ├── nonbreaking_prefix.el │ │ └── nonbreaking_prefix.en │ ├── normalize-punctuation.perl │ └── tokenizer.perl ├── gpt2_nll.py ├── owt2_preprocess.py ├── owt2_tokenizer.json ├── owt2_train_tokenizer.py ├── scalinglaw_plots.py └── scalinglaw_sweep.py ├── requirements.txt ├── sample.py ├── train.py ├── train_ar.py └── train_cdcd.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | __pycache__ 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/README.md -------------------------------------------------------------------------------- /figure1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/figure1.png -------------------------------------------------------------------------------- /lib/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/lib/datasets.py -------------------------------------------------------------------------------- /lib/ddp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/lib/ddp.py -------------------------------------------------------------------------------- /lib/decay_to_init.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/lib/decay_to_init.py -------------------------------------------------------------------------------- /lib/ema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/lib/ema.py -------------------------------------------------------------------------------- /lib/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/lib/models.py -------------------------------------------------------------------------------- /lib/ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/lib/ops.py -------------------------------------------------------------------------------- /lib/rotary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/lib/rotary.py -------------------------------------------------------------------------------- /lib/scalinglaw_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/lib/scalinglaw_utils.py -------------------------------------------------------------------------------- /lib/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/lib/utils.py -------------------------------------------------------------------------------- /misc/bw_tokenizer/nonbreaking_prefixes/nonbreaking_prefix.de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/bw_tokenizer/nonbreaking_prefixes/nonbreaking_prefix.de -------------------------------------------------------------------------------- /misc/bw_tokenizer/nonbreaking_prefixes/nonbreaking_prefix.el: -------------------------------------------------------------------------------- 1 | # for now, just include the Greek equivalent of "Mr." 2 | κ 3 | -------------------------------------------------------------------------------- /misc/bw_tokenizer/nonbreaking_prefixes/nonbreaking_prefix.en: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/bw_tokenizer/nonbreaking_prefixes/nonbreaking_prefix.en -------------------------------------------------------------------------------- /misc/bw_tokenizer/normalize-punctuation.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/bw_tokenizer/normalize-punctuation.perl -------------------------------------------------------------------------------- /misc/bw_tokenizer/tokenizer.perl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/bw_tokenizer/tokenizer.perl -------------------------------------------------------------------------------- /misc/gpt2_nll.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/gpt2_nll.py -------------------------------------------------------------------------------- /misc/owt2_preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/owt2_preprocess.py -------------------------------------------------------------------------------- /misc/owt2_tokenizer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/owt2_tokenizer.json -------------------------------------------------------------------------------- /misc/owt2_train_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/owt2_train_tokenizer.py -------------------------------------------------------------------------------- /misc/scalinglaw_plots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/scalinglaw_plots.py -------------------------------------------------------------------------------- /misc/scalinglaw_sweep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/misc/scalinglaw_sweep.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/requirements.txt -------------------------------------------------------------------------------- /sample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/sample.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/train.py -------------------------------------------------------------------------------- /train_ar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/train_ar.py -------------------------------------------------------------------------------- /train_cdcd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/igul222/plaid/HEAD/train_cdcd.py --------------------------------------------------------------------------------