├── .gitignore ├── .python-version ├── README.md ├── generated_text.txt ├── play.ipynb ├── pytest.ini ├── requirements.txt ├── src ├── __pycache__ │ ├── attention.cpython-312.pyc │ ├── config.cpython-312.pyc │ ├── data.cpython-312.pyc │ ├── layer.cpython-312.pyc │ ├── model.cpython-312.pyc │ ├── moe.cpython-312.pyc │ ├── utils.cpython-311.pyc │ └── utils.cpython-312.pyc ├── data.py ├── generate_text.py ├── main.py ├── model │ ├── __pycache__ │ │ ├── attention.cpython-312.pyc │ │ ├── config.cpython-312.pyc │ │ ├── config_.cpython-312.pyc │ │ ├── layer.cpython-312.pyc │ │ ├── model.cpython-312.pyc │ │ ├── moe.cpython-312.pyc │ │ └── utils.cpython-312.pyc │ ├── attention.py │ ├── config.py │ ├── layer.py │ ├── model.py │ ├── moe.py │ └── utils.py └── train.py ├── static └── first_2_epochs.png └── tests ├── __pycache__ ├── test_attention.cpython-312-pytest-8.3.5.pyc ├── test_layer.cpython-312-pytest-8.3.5.pyc ├── test_model.cpython-312-pytest-8.3.5.pyc ├── test_moe.cpython-312-pytest-8.3.5.pyc ├── test_utils.cpython-311-pytest-7.4.0.pyc └── test_utils.cpython-312-pytest-8.3.5.pyc ├── test_attention.py ├── test_layer.py ├── test_model.py ├── test_moe.py └── test_utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/.gitignore -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.12 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/README.md -------------------------------------------------------------------------------- /generated_text.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/generated_text.txt -------------------------------------------------------------------------------- /play.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/play.ipynb -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/pytest.ini -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/requirements.txt -------------------------------------------------------------------------------- /src/__pycache__/attention.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/__pycache__/attention.cpython-312.pyc -------------------------------------------------------------------------------- /src/__pycache__/config.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/__pycache__/config.cpython-312.pyc -------------------------------------------------------------------------------- /src/__pycache__/data.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/__pycache__/data.cpython-312.pyc -------------------------------------------------------------------------------- /src/__pycache__/layer.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/__pycache__/layer.cpython-312.pyc -------------------------------------------------------------------------------- /src/__pycache__/model.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/__pycache__/model.cpython-312.pyc -------------------------------------------------------------------------------- /src/__pycache__/moe.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/__pycache__/moe.cpython-312.pyc -------------------------------------------------------------------------------- /src/__pycache__/utils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/__pycache__/utils.cpython-311.pyc -------------------------------------------------------------------------------- /src/__pycache__/utils.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/__pycache__/utils.cpython-312.pyc -------------------------------------------------------------------------------- /src/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/data.py -------------------------------------------------------------------------------- /src/generate_text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/generate_text.py -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/model/__pycache__/attention.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/__pycache__/attention.cpython-312.pyc -------------------------------------------------------------------------------- /src/model/__pycache__/config.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/__pycache__/config.cpython-312.pyc -------------------------------------------------------------------------------- /src/model/__pycache__/config_.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/__pycache__/config_.cpython-312.pyc -------------------------------------------------------------------------------- /src/model/__pycache__/layer.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/__pycache__/layer.cpython-312.pyc -------------------------------------------------------------------------------- /src/model/__pycache__/model.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/__pycache__/model.cpython-312.pyc -------------------------------------------------------------------------------- /src/model/__pycache__/moe.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/__pycache__/moe.cpython-312.pyc -------------------------------------------------------------------------------- /src/model/__pycache__/utils.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/__pycache__/utils.cpython-312.pyc -------------------------------------------------------------------------------- /src/model/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/attention.py -------------------------------------------------------------------------------- /src/model/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/config.py -------------------------------------------------------------------------------- /src/model/layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/layer.py -------------------------------------------------------------------------------- /src/model/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/model.py -------------------------------------------------------------------------------- /src/model/moe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/moe.py -------------------------------------------------------------------------------- /src/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/model/utils.py -------------------------------------------------------------------------------- /src/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/src/train.py -------------------------------------------------------------------------------- /static/first_2_epochs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/static/first_2_epochs.png -------------------------------------------------------------------------------- /tests/__pycache__/test_attention.cpython-312-pytest-8.3.5.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/__pycache__/test_attention.cpython-312-pytest-8.3.5.pyc -------------------------------------------------------------------------------- /tests/__pycache__/test_layer.cpython-312-pytest-8.3.5.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/__pycache__/test_layer.cpython-312-pytest-8.3.5.pyc -------------------------------------------------------------------------------- /tests/__pycache__/test_model.cpython-312-pytest-8.3.5.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/__pycache__/test_model.cpython-312-pytest-8.3.5.pyc -------------------------------------------------------------------------------- /tests/__pycache__/test_moe.cpython-312-pytest-8.3.5.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/__pycache__/test_moe.cpython-312-pytest-8.3.5.pyc -------------------------------------------------------------------------------- /tests/__pycache__/test_utils.cpython-311-pytest-7.4.0.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/__pycache__/test_utils.cpython-311-pytest-7.4.0.pyc -------------------------------------------------------------------------------- /tests/__pycache__/test_utils.cpython-312-pytest-8.3.5.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/__pycache__/test_utils.cpython-312-pytest-8.3.5.pyc -------------------------------------------------------------------------------- /tests/test_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/test_attention.py -------------------------------------------------------------------------------- /tests/test_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/test_layer.py -------------------------------------------------------------------------------- /tests/test_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/test_model.py -------------------------------------------------------------------------------- /tests/test_moe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/test_moe.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabir2505/tiny-mixtral/HEAD/tests/test_utils.py --------------------------------------------------------------------------------