├── README.md ├── continue_training.ipynb ├── data ├── 2Mix │ ├── 160M.txt │ └── 70M.txt ├── 3Mix │ ├── 160M.txt │ ├── 410M.txt │ └── 70M.txt ├── 5Mix │ ├── 160M.txt │ ├── 1B.txt │ ├── 305M.txt │ ├── 410M.txt │ ├── 70M.txt │ ├── mix5mlp.pt │ └── proportions.txt ├── RPJ.predictions.json ├── RedPajama │ ├── 160M.txt │ ├── 1B.txt │ ├── 305M.txt │ ├── 410M.txt │ ├── 70M.txt │ └── proportions.txt └── ct │ └── 70M.txt ├── law.py ├── mix_2_domains.ipynb ├── mix_3_domains.ipynb ├── mix_5_domains.ipynb ├── pipeline ├── find_opt.py ├── get_loss.py ├── mixlaw.py ├── ratios.txt ├── run.sh ├── sizelaw.py ├── steplaw.py ├── utils.py └── valid_weight.json └── predict_vs_observe.ipynb /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/README.md -------------------------------------------------------------------------------- /continue_training.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/continue_training.ipynb -------------------------------------------------------------------------------- /data/2Mix/160M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/2Mix/160M.txt -------------------------------------------------------------------------------- /data/2Mix/70M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/2Mix/70M.txt -------------------------------------------------------------------------------- /data/3Mix/160M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/3Mix/160M.txt -------------------------------------------------------------------------------- /data/3Mix/410M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/3Mix/410M.txt -------------------------------------------------------------------------------- /data/3Mix/70M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/3Mix/70M.txt -------------------------------------------------------------------------------- /data/5Mix/160M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/5Mix/160M.txt -------------------------------------------------------------------------------- /data/5Mix/1B.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/5Mix/1B.txt -------------------------------------------------------------------------------- /data/5Mix/305M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/5Mix/305M.txt -------------------------------------------------------------------------------- /data/5Mix/410M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/5Mix/410M.txt -------------------------------------------------------------------------------- /data/5Mix/70M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/5Mix/70M.txt -------------------------------------------------------------------------------- /data/5Mix/mix5mlp.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/5Mix/mix5mlp.pt -------------------------------------------------------------------------------- /data/5Mix/proportions.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/5Mix/proportions.txt -------------------------------------------------------------------------------- /data/RPJ.predictions.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/RPJ.predictions.json -------------------------------------------------------------------------------- /data/RedPajama/160M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/RedPajama/160M.txt -------------------------------------------------------------------------------- /data/RedPajama/1B.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/RedPajama/1B.txt -------------------------------------------------------------------------------- /data/RedPajama/305M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/RedPajama/305M.txt -------------------------------------------------------------------------------- /data/RedPajama/410M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/RedPajama/410M.txt -------------------------------------------------------------------------------- /data/RedPajama/70M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/RedPajama/70M.txt -------------------------------------------------------------------------------- /data/RedPajama/proportions.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/RedPajama/proportions.txt -------------------------------------------------------------------------------- /data/ct/70M.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/data/ct/70M.txt -------------------------------------------------------------------------------- /law.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/law.py -------------------------------------------------------------------------------- /mix_2_domains.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/mix_2_domains.ipynb -------------------------------------------------------------------------------- /mix_3_domains.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/mix_3_domains.ipynb -------------------------------------------------------------------------------- /mix_5_domains.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/mix_5_domains.ipynb -------------------------------------------------------------------------------- /pipeline/find_opt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/pipeline/find_opt.py -------------------------------------------------------------------------------- /pipeline/get_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/pipeline/get_loss.py -------------------------------------------------------------------------------- /pipeline/mixlaw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/pipeline/mixlaw.py -------------------------------------------------------------------------------- /pipeline/ratios.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipeline/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/pipeline/run.sh -------------------------------------------------------------------------------- /pipeline/sizelaw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/pipeline/sizelaw.py -------------------------------------------------------------------------------- /pipeline/steplaw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/pipeline/steplaw.py -------------------------------------------------------------------------------- /pipeline/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/pipeline/utils.py -------------------------------------------------------------------------------- /pipeline/valid_weight.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/pipeline/valid_weight.json -------------------------------------------------------------------------------- /predict_vs_observe.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yegcjs/mixinglaws/HEAD/predict_vs_observe.ipynb --------------------------------------------------------------------------------