├── assets
    ├── nanogpt.jpg
    └── gpt2_124M_loss.png
├── .gitignore
├── data
    ├── shakespeare
    │   ├── readme.md
    │   └── prepare.py
    ├── shakespeare_char
    │   ├── readme.md
    │   └── prepare.py
    └── openwebtext
    │   ├── readme.md
    │   └── prepare.py
├── .gitattributes
├── config
    ├── eval_gpt2.py
    ├── eval_gpt2_xl.py
    ├── eval_gpt2_large.py
    ├── eval_gpt2_medium.py
    ├── finetune_shakespeare.py
    ├── train_gpt2.py
    └── train_shakespeare_char.py
├── LICENSE
├── configurator.py
├── sample.py
├── bench.py
├── README.md
├── train.py
├── model.py
└── data.json


/assets/nanogpt.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/phkhanhtrinh23/milliGPT/HEAD/assets/nanogpt.jpg


--------------------------------------------------------------------------------
/assets/gpt2_124M_loss.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/phkhanhtrinh23/milliGPT/HEAD/assets/gpt2_124M_loss.png


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | .DS_Store
 2 | .idea
 3 | .ipynb_checkpoints/
 4 | .vscode
 5 | __pycache__/
 6 | *.bin
 7 | *.pkl
 8 | *.pt
 9 | *.pyc
10 | input.txt
11 | 


--------------------------------------------------------------------------------
/data/shakespeare/readme.md:
--------------------------------------------------------------------------------
 1 | 
 2 | # tiny shakespeare
 3 | 
 4 | Tiny shakespeare, of the good old char-rnn fame :)
 5 | 
 6 | After running `prepare.py`:
 7 | 
 8 | - train.bin has 301,966 tokens
 9 | - val.bin has 36,059 tokens
10 | 


--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Override jupyter in Github language stats for more accurate estimate of repo code languages
2 | # reference: https://github.com/github/linguist/blob/master/docs/overrides.md#generated-code
3 | *.ipynb linguist-generated
4 | 


--------------------------------------------------------------------------------
/config/eval_gpt2.py:
--------------------------------------------------------------------------------
1 | # evaluate the base gpt2
2 | # n_layer=12, n_head=12, n_embd=768
3 | # 124M parameters
4 | batch_size = 8
5 | eval_iters = 500 # use more iterations to get good estimate
6 | eval_only = True
7 | wandb_log = False
8 | init_from = 'gpt2'
9 | 


--------------------------------------------------------------------------------
/config/eval_gpt2_xl.py:
--------------------------------------------------------------------------------
1 | # evaluate the base gpt2
2 | # n_layer=48, n_head=25, n_embd=1600
3 | # 1558M parameters
4 | batch_size = 8
5 | eval_iters = 500 # use more iterations to get good estimate
6 | eval_only = True
7 | wandb_log = False
8 | init_from = 'gpt2-xl'
9 | 


--------------------------------------------------------------------------------
/config/eval_gpt2_large.py:
--------------------------------------------------------------------------------
1 | # evaluate the base gpt2
2 | # n_layer=36, n_head=20, n_embd=1280
3 | # 774M parameters
4 | batch_size = 8
5 | eval_iters = 500 # use more iterations to get good estimate
6 | eval_only = True
7 | wandb_log = False
8 | init_from = 'gpt2-large'
9 | 


--------------------------------------------------------------------------------
/config/eval_gpt2_medium.py:
--------------------------------------------------------------------------------
1 | # evaluate the base gpt2
2 | # n_layer=24, n_head=16, n_embd=1024
3 | # 350M parameters
4 | batch_size = 8
5 | eval_iters = 500 # use more iterations to get good estimate
6 | eval_only = True
7 | wandb_log = False
8 | init_from = 'gpt2-medium'
9 | 


--------------------------------------------------------------------------------
/data/shakespeare_char/readme.md:
--------------------------------------------------------------------------------
 1 | 
 2 | # tiny shakespeare, character-level
 3 | 
 4 | Tiny shakespeare, of the good old char-rnn fame :) Treated on character-level.
 5 | 
 6 | After running `prepare.py`:
 7 | 
 8 | - train.bin has 1,003,854 tokens
 9 | - val.bin has 111,540 tokens
10 | 


--------------------------------------------------------------------------------
/data/openwebtext/readme.md:
--------------------------------------------------------------------------------
 1 | 
 2 | ## openwebtext dataset
 3 | 
 4 | after running `prepare.py` (preprocess) we get:
 5 | 
 6 | - train.bin is ~17GB, val.bin ~8.5MB
 7 | - train has ~9B tokens (9,035,582,198)
 8 | - val has ~4M tokens (4,434,897)
 9 | 
10 | this came from 8,013,769 documents in total.
11 | 
12 | references:
13 | 
14 | - OpenAI's WebText dataset is discussed in [GPT-2 paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf)
15 | - [OpenWebText](https://skylion007.github.io/OpenWebTextCorpus/) dataset
16 | 


--------------------------------------------------------------------------------
/config/finetune_shakespeare.py:
--------------------------------------------------------------------------------
 1 | import time
 2 | 
 3 | out_dir = 'out-shakespeare'
 4 | eval_interval = 5
 5 | eval_iters = 40
 6 | wandb_log = False # feel free to turn on
 7 | wandb_project = 'shakespeare'
 8 | wandb_run_name = 'ft-' + str(time.time())
 9 | 
10 | dataset = 'shakespeare'
11 | init_from = 'gpt2-xl' # this is the largest GPT-2 model
12 | 
13 | # only save checkpoints if the validation loss improves
14 | always_save_checkpoint = False
15 | 
16 | # the number of examples per iter:
17 | # 1 batch_size * 32 grad_accum * 1024 tokens = 32,768 tokens/iter
18 | # shakespeare has 301,966 tokens, so 1 epoch ~= 9.2 iters
19 | batch_size = 1
20 | gradient_accumulation_steps = 32
21 | max_iters = 20
22 | 
23 | # finetune at constant LR
24 | learning_rate = 3e-5
25 | decay_lr = False
26 | 


--------------------------------------------------------------------------------
/config/train_gpt2.py:
--------------------------------------------------------------------------------
 1 | # config for training GPT-2 (124M) down to very nice loss of ~2.85 on 1 node of 8X A100 40GB
 2 | # launch as the following (e.g. in a screen session) and wait ~5 days:
 3 | # $ torchrun --standalone --nproc_per_node=8 train.py config/train_gpt2.py
 4 | 
 5 | wandb_log = True
 6 | wandb_project = 'owt'
 7 | wandb_run_name='gpt2-124M'
 8 | 
 9 | # these make the total batch size be ~0.5M
10 | # 12 batch size * 1024 block size * 5 gradaccum * 8 GPUs = 491,520
11 | batch_size = 12
12 | block_size = 1024
13 | gradient_accumulation_steps = 5 * 8
14 | 
15 | # this makes total number of tokens be 300B
16 | max_iters = 600000
17 | lr_decay_iters = 600000
18 | 
19 | # eval stuff
20 | eval_interval = 1000
21 | eval_iters = 200
22 | log_interval = 10
23 | 
24 | # weight decay
25 | weight_decay = 1e-1
26 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2022 Andrej Karpathy
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/data/shakespeare/prepare.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import requests
 3 | import tiktoken
 4 | import numpy as np
 5 | 
 6 | # download the tiny shakespeare dataset
 7 | input_file_path = os.path.join(os.path.dirname(__file__), 'input.txt')
 8 | if not os.path.exists(input_file_path):
 9 |     data_url = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt'
10 |     with open(input_file_path, 'w') as f:
11 |         f.write(requests.get(data_url).text)
12 | 
13 | with open(input_file_path, 'r') as f:
14 |     data = f.read()
15 | n = len(data)
16 | train_data = data[:int(n*0.9)]
17 | val_data = data[int(n*0.9):]
18 | 
19 | # encode with tiktoken gpt2 bpe
20 | enc = tiktoken.get_encoding("gpt2")
21 | train_ids = enc.encode_ordinary(train_data)
22 | val_ids = enc.encode_ordinary(val_data)
23 | print(f"train has {len(train_ids):,} tokens")
24 | print(f"val has {len(val_ids):,} tokens")
25 | 
26 | # export to bin files
27 | train_ids = np.array(train_ids, dtype=np.uint16)
28 | val_ids = np.array(val_ids, dtype=np.uint16)
29 | train_ids.tofile(os.path.join(os.path.dirname(__file__), 'train.bin'))
30 | val_ids.tofile(os.path.join(os.path.dirname(__file__), 'val.bin'))
31 | 
32 | # train.bin has 301,966 tokens
33 | # val.bin has 36,059 tokens
34 | 


--------------------------------------------------------------------------------
/config/train_shakespeare_char.py:
--------------------------------------------------------------------------------
 1 | # train a miniature character-level shakespeare model
 2 | # good for debugging and playing on macbooks and such
 3 | 
 4 | out_dir = 'out-shakespeare-char'
 5 | eval_interval = 250 # keep frequent because we'll overfit
 6 | eval_iters = 200
 7 | log_interval = 10 # don't print too too often
 8 | 
 9 | # we expect to overfit on this small dataset, so only save when val improves
10 | always_save_checkpoint = False
11 | 
12 | wandb_log = False # override via command line if you like
13 | wandb_project = 'shakespeare-char'
14 | wandb_run_name = 'mini-gpt'
15 | 
16 | dataset = 'shakespeare_char'
17 | gradient_accumulation_steps = 1
18 | batch_size = 64
19 | block_size = 256 # context of up to 256 previous characters
20 | 
21 | # baby GPT model :)
22 | n_layer = 6
23 | n_head = 6
24 | n_embd = 384
25 | dropout = 0.2
26 | 
27 | learning_rate = 1e-3 # with baby networks can afford to go a bit higher
28 | max_iters = 5000
29 | lr_decay_iters = 5000 # make equal to max_iters usually
30 | min_lr = 1e-4 # learning_rate / 10 usually
31 | beta2 = 0.99 # make a bit bigger because number of tokens per iter is small
32 | 
33 | warmup_iters = 100 # not super necessary potentially
34 | 
35 | # on macbook also add
36 | # device = 'cpu'  # run on cpu only
37 | # compile = False # do not torch compile the model
38 | 


--------------------------------------------------------------------------------
/configurator.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Poor Man's Configurator. Probably a terrible idea. Example usage:
 3 | $ python train.py config/override_file.py --batch_size=32
 4 | this will first run config/override_file.py, then override batch_size to 32
 5 | 
 6 | The code in this file will be run as follows from e.g. train.py:
 7 | >>> exec(open('configurator.py').read())
 8 | 
 9 | So it's not a Python module, it's just shuttling this code away from train.py
10 | The code in this script then overrides the globals()
11 | 
12 | I know people are not going to love this, I just really dislike configuration
13 | complexity and having to prepend config. to every single variable. If someone
14 | comes up with a better simple Python solution I am all ears.
15 | """
16 | 
17 | import sys
18 | from ast import literal_eval
19 | 
20 | for arg in sys.argv[1:]:
21 |     if '=' not in arg:
22 |         # assume it's the name of a config file
23 |         assert not arg.startswith('--')
24 |         config_file = arg
25 |         print(f"Overriding config with {config_file}:")
26 |         with open(config_file) as f:
27 |             print(f.read())
28 |         exec(open(config_file).read())
29 |     else:
30 |         # assume it's a --key=value argument
31 |         assert arg.startswith('--')
32 |         key, val = arg.split('=')
33 |         key = key[2:]
34 |         if key in globals():
35 |             try:
36 |                 # attempt to eval it it (e.g. if bool, number, or etc)
37 |                 attempt = literal_eval(val)
38 |             except (SyntaxError, ValueError):
39 |                 # if that goes wrong, just use the string
40 |                 attempt = val
41 |             # ensure the types match ok
42 |             assert type(attempt) == type(globals()[key])
43 |             # cross fingers
44 |             print(f"Overriding: {key} = {attempt}")
45 |             globals()[key] = attempt
46 |         else:
47 |             raise ValueError(f"Unknown config key: {key}")
48 | 


--------------------------------------------------------------------------------
/data/shakespeare_char/prepare.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Prepare the Shakespeare dataset for character-level language modeling.
 3 | So instead of encoding with GPT-2 BPE tokens, we just map characters to ints.
 4 | Will save train.bin, val.bin containing the ids, and meta.pkl containing the
 5 | encoder and decoder and some other related info.
 6 | """
 7 | import os
 8 | import pickle
 9 | import requests
10 | import numpy as np
11 | 
12 | # download the tiny shakespeare dataset
13 | input_file_path = os.path.join(os.path.dirname(__file__), 'input.txt')
14 | if not os.path.exists(input_file_path):
15 |     data_url = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt'
16 |     with open(input_file_path, 'w') as f:
17 |         f.write(requests.get(data_url).text)
18 | 
19 | with open(input_file_path, 'r') as f:
20 |     data = f.read()
21 | print(f"length of dataset in characters: {len(data):,}")
22 | 
23 | # get all the unique characters that occur in this text
24 | chars = sorted(list(set(data)))
25 | vocab_size = len(chars)
26 | print("all the unique characters:", ''.join(chars))
27 | print(f"vocab size: {vocab_size:,}")
28 | 
29 | # create a mapping from characters to integers
30 | stoi = { ch:i for i,ch in enumerate(chars) }
31 | itos = { i:ch for i,ch in enumerate(chars) }
32 | def encode(s):
33 |     return [stoi[c] for c in s] # encoder: take a string, output a list of integers
34 | def decode(l):
35 |     return ''.join([itos[i] for i in l]) # decoder: take a list of integers, output a string
36 | 
37 | # create the train and test splits
38 | n = len(data)
39 | train_data = data[:int(n*0.9)]
40 | val_data = data[int(n*0.9):]
41 | 
42 | # encode both to integers
43 | train_ids = encode(train_data)
44 | val_ids = encode(val_data)
45 | print(f"train has {len(train_ids):,} tokens")
46 | print(f"val has {len(val_ids):,} tokens")
47 | 
48 | # export to bin files
49 | train_ids = np.array(train_ids, dtype=np.uint16)
50 | val_ids = np.array(val_ids, dtype=np.uint16)
51 | train_ids.tofile(os.path.join(os.path.dirname(__file__), 'train.bin'))
52 | val_ids.tofile(os.path.join(os.path.dirname(__file__), 'val.bin'))
53 | 
54 | # save the meta information as well, to help us encode/decode later
55 | meta = {
56 |     'vocab_size': vocab_size,
57 |     'itos': itos,
58 |     'stoi': stoi,
59 | }
60 | with open(os.path.join(os.path.dirname(__file__), 'meta.pkl'), 'wb') as f:
61 |     pickle.dump(meta, f)
62 | 
63 | # length of dataset in characters:  1115394
64 | # all the unique characters:
65 | #  !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
66 | # vocab size: 65
67 | # train has 1003854 tokens
68 | # val has 111540 tokens
69 | 


--------------------------------------------------------------------------------
/data/openwebtext/prepare.py:
--------------------------------------------------------------------------------
 1 | # saves the openwebtext dataset to a binary file for training. following was helpful:
 2 | # https://github.com/HazyResearch/flash-attention/blob/main/training/src/datamodules/language_modeling_hf.py
 3 | 
 4 | import os
 5 | from tqdm import tqdm
 6 | import numpy as np
 7 | import tiktoken
 8 | from datasets import load_dataset # huggingface datasets
 9 | 
10 | # number of workers in .map() call
11 | # good number to use is ~order number of cpu cores // 2
12 | num_proc = 8
13 | 
14 | # number of workers in load_dataset() call
15 | # best number might be different from num_proc above as it also depends on NW speed.
16 | # it is better than 1 usually though
17 | num_proc_load_dataset = num_proc
18 | 
19 | if __name__ == '__main__':
20 |     # takes 54GB in huggingface .cache dir, about 8M documents (8,013,769)
21 |     dataset = load_dataset("openwebtext", num_proc=num_proc_load_dataset)
22 | 
23 |     # owt by default only contains the 'train' split, so create a test split
24 |     split_dataset = dataset["train"].train_test_split(test_size=0.0005, seed=2357, shuffle=True)
25 |     split_dataset['val'] = split_dataset.pop('test') # rename the test split to val
26 | 
27 |     # this results in:
28 |     # >>> split_dataset
29 |     # DatasetDict({
30 |     #     train: Dataset({
31 |     #         features: ['text'],
32 |     #         num_rows: 8009762
33 |     #     })
34 |     #     val: Dataset({
35 |     #         features: ['text'],
36 |     #         num_rows: 4007
37 |     #     })
38 |     # })
39 | 
40 |     # we now want to tokenize the dataset. first define the encoding function (gpt2 bpe)
41 |     enc = tiktoken.get_encoding("gpt2")
42 |     def process(example):
43 |         ids = enc.encode_ordinary(example['text']) # encode_ordinary ignores any special tokens
44 |         ids.append(enc.eot_token) # add the end of text token, e.g. 50256 for gpt2 bpe
45 |         # note: I think eot should be prepended not appended... hmm. it's called "eot" though...
46 |         out = {'ids': ids, 'len': len(ids)}
47 |         return out
48 | 
49 |     # tokenize the dataset
50 |     tokenized = split_dataset.map(
51 |         process,
52 |         remove_columns=['text'],
53 |         desc="tokenizing the splits",
54 |         num_proc=num_proc,
55 |     )
56 | 
57 |     # concatenate all the ids in each dataset into one large file we can use for training
58 |     for split, dset in tokenized.items():
59 |         arr_len = np.sum(dset['len'], dtype=np.uint64)
60 |         filename = os.path.join(os.path.dirname(__file__), f'{split}.bin')
61 |         dtype = np.uint16 # (can do since enc.max_token_value == 50256 is < 2**16)
62 |         arr = np.memmap(filename, dtype=dtype, mode='w+', shape=(arr_len,))
63 |         total_batches = 1024
64 | 
65 |         idx = 0
66 |         for batch_idx in tqdm(range(total_batches), desc=f'writing {filename}'):
67 |             # Batch together samples for faster write
68 |             batch = dset.shard(num_shards=total_batches, index=batch_idx, contiguous=True).with_format('numpy')
69 |             arr_batch = np.concatenate(batch['ids'])
70 |             # Write into mmap
71 |             arr[idx : idx + len(arr_batch)] = arr_batch
72 |             idx += len(arr_batch)
73 |         arr.flush()
74 | 
75 |     # train.bin is ~17GB, val.bin ~8.5MB
76 |     # train has ~9B tokens (9,035,582,198)
77 |     # val has ~4M tokens (4,434,897)
78 | 
79 |     # to read the bin files later, e.g. with numpy:
80 |     # m = np.memmap('train.bin', dtype=np.uint16, mode='r')
81 | 


--------------------------------------------------------------------------------
/sample.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Sample from a trained model
 3 | """
 4 | import os
 5 | import pickle
 6 | from contextlib import nullcontext
 7 | import torch
 8 | import tiktoken
 9 | from model import GPTConfig, GPT
10 | 
11 | # -----------------------------------------------------------------------------
12 | init_from = 'resume' # either 'resume' (from an out_dir) or a gpt2 variant (e.g. 'gpt2-xl')
13 | out_dir = 'out' # ignored if init_from is not 'resume'
14 | start = "\n" # or "<|endoftext|>" or etc. Can also specify a file, use as: "FILE:prompt.txt"
15 | num_samples = 5 # number of samples to draw
16 | max_new_tokens = 256 # number of tokens generated in each sample
17 | temperature = 0.6 # 1.0 = no change, < 1.0 = less random, > 1.0 = more random, in predictions
18 | top_k = 200 # retain only the top_k most likely tokens, clamp others to have 0 probability
19 | top_p = 0.96
20 | seed = 1337
21 | device = 'cuda' # examples: 'cpu', 'cuda', 'cuda:0', 'cuda:1', etc.
22 | dtype = 'bfloat16' if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else 'float16' # 'float32' or 'bfloat16' or 'float16'
23 | compile = False # use PyTorch 2.0 to compile the model to be faster
24 | exec(open('configurator.py').read()) # overrides from command line or config file
25 | # -----------------------------------------------------------------------------
26 | 
27 | torch.manual_seed(seed)
28 | torch.cuda.manual_seed(seed)
29 | torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul
30 | torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
31 | device_type = 'cuda' if 'cuda' in device else 'cpu' # for later use in torch.autocast
32 | ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]
33 | ctx = nullcontext() if device_type == 'cpu' else torch.amp.autocast(device_type=device_type, dtype=ptdtype)
34 | 
35 | # model
36 | if init_from == 'resume':
37 |     # init from a model saved in a specific directory
38 |     ckpt_path = os.path.join(out_dir, 'ckpt.pt')
39 |     checkpoint = torch.load(ckpt_path, map_location=device)
40 |     gptconf = GPTConfig(**checkpoint['model_args'])
41 |     model = GPT(gptconf)
42 |     state_dict = checkpoint['model']
43 |     unwanted_prefix = '_orig_mod.'
44 |     for k,v in list(state_dict.items()):
45 |         if k.startswith(unwanted_prefix):
46 |             state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k)
47 |     model.load_state_dict(state_dict)
48 | elif init_from.startswith('gpt2'):
49 |     # init from a given GPT-2 model
50 |     model = GPT.from_pretrained(init_from, dict(dropout=0.0))
51 | 
52 | model.eval()
53 | model.to(device)
54 | if compile:
55 |     model = torch.compile(model) # requires PyTorch 2.0 (optional)
56 | 
57 | # look for the meta pickle in case it is available in the dataset folder
58 | load_meta = False
59 | if init_from == 'resume' and 'config' in checkpoint and 'dataset' in checkpoint['config']: # older checkpoints might not have these...
60 |     meta_path = os.path.join('data', checkpoint['config']['dataset'], 'meta.pkl')
61 |     load_meta = os.path.exists(meta_path)
62 | if load_meta:
63 |     print(f"Loading meta from {meta_path}...")
64 |     with open(meta_path, 'rb') as f:
65 |         meta = pickle.load(f)
66 |     # TODO want to make this more general to arbitrary encoder/decoder schemes
67 |     stoi, itos = meta['stoi'], meta['itos']
68 |     encode = lambda s: [stoi[c] for c in s]
69 |     decode = lambda l: ''.join([itos[i] for i in l])
70 | else:
71 |     # ok let's assume gpt-2 encodings by default
72 |     print("No meta.pkl found, assuming GPT-2 encodings...")
73 |     enc = tiktoken.get_encoding("gpt2")
74 |     encode = lambda s: enc.encode(s, allowed_special={"<|endoftext|>"})
75 |     decode = lambda l: enc.decode(l)
76 | 
77 | # encode the beginning of the prompt
78 | if start.startswith('FILE:'):
79 |     with open(start[5:], 'r', encoding='utf-8') as f:
80 |         start = f.read()
81 | print("start:", start)
82 | start_ids = encode(start)
83 | x = (torch.tensor(start_ids, dtype=torch.long, device=device)[None, ...])
84 | 
85 | # run generation
86 | with torch.no_grad():
87 |     with ctx:
88 |         for k in range(num_samples):
89 |             y = model.generate(x, max_new_tokens, temperature=temperature, top_k=top_k, top_p=top_p)
90 |             print(decode(y[0].tolist()))
91 |             print('---------------')
92 | 


--------------------------------------------------------------------------------
/bench.py:
--------------------------------------------------------------------------------
  1 | """
  2 | A much shorter version of train.py for benchmarking
  3 | """
  4 | import os
  5 | from contextlib import nullcontext
  6 | import numpy as np
  7 | import time
  8 | import torch
  9 | from model import GPTConfig, GPT
 10 | 
 11 | # -----------------------------------------------------------------------------
 12 | batch_size = 12
 13 | block_size = 1024
 14 | bias = False
 15 | real_data = True
 16 | seed = 1337
 17 | device = 'cuda' # examples: 'cpu', 'cuda', 'cuda:0', 'cuda:1', etc.
 18 | dtype = 'bfloat16' if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else 'float16' # 'float32' or 'bfloat16' or 'float16'
 19 | compile = True # use PyTorch 2.0 to compile the model to be faster
 20 | profile = False # use pytorch profiler, or just simple benchmarking?
 21 | exec(open('configurator.py').read()) # overrides from command line or config file
 22 | # -----------------------------------------------------------------------------
 23 | 
 24 | torch.manual_seed(seed)
 25 | torch.cuda.manual_seed(seed)
 26 | torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul
 27 | torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
 28 | device_type = 'cuda' if 'cuda' in device else 'cpu' # for later use in torch.autocast
 29 | ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]
 30 | ctx = nullcontext() if device_type == 'cpu' else torch.amp.autocast(device_type=device_type, dtype=ptdtype)
 31 | 
 32 | # data loading init
 33 | if real_data:
 34 |     dataset = 'openwebtext'
 35 |     data_dir = os.path.join('data', dataset)
 36 |     train_data = np.memmap(os.path.join(data_dir, 'train.bin'), dtype=np.uint16, mode='r')
 37 |     def get_batch(split):
 38 |         data = train_data # note ignore split in benchmarking script
 39 |         ix = torch.randint(len(data) - block_size, (batch_size,))
 40 |         x = torch.stack([torch.from_numpy((data[i:i+block_size]).astype(np.int64)) for i in ix])
 41 |         y = torch.stack([torch.from_numpy((data[i+1:i+1+block_size]).astype(np.int64)) for i in ix])
 42 |         x, y = x.pin_memory().to(device, non_blocking=True), y.pin_memory().to(device, non_blocking=True)
 43 |         return x, y
 44 | else:
 45 |     # alternatively, if fixed data is desired to not care about data loading
 46 |     x = torch.randint(50304, (batch_size, block_size), device=device)
 47 |     y = torch.randint(50304, (batch_size, block_size), device=device)
 48 |     get_batch = lambda split: (x, y)
 49 | 
 50 | # model init
 51 | gptconf = GPTConfig(
 52 |     block_size = block_size, # how far back does the model look? i.e. context size
 53 |     n_layer = 12, n_head = 12, n_embd = 768, # size of the model
 54 |     dropout = 0, # for determinism
 55 |     bias = bias,
 56 | )
 57 | model = GPT(gptconf)
 58 | model.to(device)
 59 | 
 60 | optimizer = model.configure_optimizers(weight_decay=1e-2, learning_rate=1e-4, betas=(0.9, 0.95), device_type=device_type)
 61 | 
 62 | if compile:
 63 |     print("Compiling model...")
 64 |     model = torch.compile(model) # pytorch 2.0
 65 | 
 66 | if profile:
 67 |     # useful docs on pytorch profiler:
 68 |     # - tutorial https://pytorch.org/tutorials/intermediate/tensorboard_profiler_tutorial.html
 69 |     # - api https://pytorch.org/docs/stable/profiler.html#torch.profiler.profile
 70 |     wait, warmup, active = 5, 5, 5
 71 |     num_steps = wait + warmup + active
 72 |     with torch.profiler.profile(
 73 |         activities=[torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA],
 74 |         schedule=torch.profiler.schedule(wait=wait, warmup=warmup, active=active, repeat=1),
 75 |         on_trace_ready=torch.profiler.tensorboard_trace_handler('./bench_log'),
 76 |         record_shapes=False,
 77 |         profile_memory=False,
 78 |         with_stack=False, # incurs an additional overhead, disable if not needed
 79 |         with_flops=True,
 80 |         with_modules=False, # only for torchscript models atm
 81 |     ) as prof:
 82 | 
 83 |         X, Y = get_batch('train')
 84 |         for k in range(num_steps):
 85 |             with ctx:
 86 |                 logits, loss = model(X, Y)
 87 |             X, Y = get_batch('train')
 88 |             optimizer.zero_grad(set_to_none=True)
 89 |             loss.backward()
 90 |             optimizer.step()
 91 |             lossf = loss.item()
 92 |             print(f"{k}/{num_steps} loss: {lossf:.4f}")
 93 | 
 94 |             prof.step() # notify the profiler at end of each step
 95 | 
 96 | else:
 97 | 
 98 |     # simple benchmarking
 99 |     torch.cuda.synchronize()
100 |     for stage, num_steps in enumerate([10, 20]): # burnin, then benchmark
101 |         t0 = time.time()
102 |         X, Y = get_batch('train')
103 |         for k in range(num_steps):
104 |             with ctx:
105 |                 logits, loss = model(X, Y)
106 |             X, Y = get_batch('train')
107 |             optimizer.zero_grad(set_to_none=True)
108 |             loss.backward()
109 |             optimizer.step()
110 |             lossf = loss.item()
111 |             print(f"{k}/{num_steps} loss: {lossf:.4f}")
112 |         torch.cuda.synchronize()
113 |         t1 = time.time()
114 |         dt = t1-t0
115 |         mfu = model.estimate_mfu(batch_size * 1 * num_steps, dt)
116 |         if stage == 1:
117 |             print(f"time per iteration: {dt/num_steps*1000:.4f}ms, MFU: {mfu*100:.2f}%")
118 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | *Note: This repository orignates from Karpathy's famous [nanoGPT](https://github.com/karpathy/nanoGPT).*
  2 | 
  3 | # milliGPT
  4 | 
  5 | ## Install requirements
  6 | 
  7 | ```
  8 | pip install torch numpy transformers datasets tiktoken wandb tqdm
  9 | ```
 10 | 
 11 | Dependencies:
 12 | 
 13 | - [pytorch](https://pytorch.org) (should be PyTorch >= 2.0)
 14 | - [numpy](https://numpy.org/install/)
 15 | -  `transformers` for huggingface transformers (to load GPT-2 checkpoints)
 16 | -  `datasets` for huggingface datasets (if you want to download + preprocess OpenWebText)
 17 | -  `tiktoken` for OpenAI's fast BPE code
 18 | -  `wandb` for optional logging
 19 | -  `tqdm` for progress bars
 20 | 
 21 | ## Training a minimal example GPT from start
 22 | 
 23 | First step is to select a text corpus to train your model. This repo provides the default set-up script to download a public text dataset (Shakespeare books). You need to run the tokenizer script to conver the raw text data to token ids which can be interpreted by the model:
 24 | 
 25 | 
 26 | ```
 27 | $ python data/shakespeare_char/prepare.py
 28 | ```
 29 | 
 30 | Want to use another dataset? Train a milliGPT model on your favorite book? Check out this open [online e-book resources](https://www.gutenberg.org/ebooks/1513). Select one and pick the option `Plain Text UTF-8` to download in text format (`.txt`). Or just use any text file that you can find. Rename it into `input.txt` and copy it into `data/shakespeare_char`, then re-run the above command. 
 31 | 
 32 | This creates a `train.bin` and `val.bin` in that data directory. Now it is time to train your GPT. The size of it very much depends on the computational resources of your system:
 33 | 
 34 | **I have a GPU**. Great, we can quickly train a milliGPT with the settings provided in the [config/train_shakespeare_char.py](config/train_shakespeare_char.py) config file:
 35 | 
 36 | ```
 37 | $ python train.py config/train_shakespeare_char.py
 38 | ```
 39 | 
 40 | If you peek inside it, you'll see that we're training a GPT with a context size of up to 256 characters, 384 feature channels, and it is a 6-layer Transformer with 6 heads in each layer. On one A100 GPU this training run takes about 3 minutes and the best validation loss is *1.4697*. Based on the configuration, the model checkpoints are being written into the `--out_dir` directory `out-shakespeare-char`. So once the training finishes we can sample from the best model by pointing the sampling script at this directory:
 41 | 
 42 | ```
 43 | $ python sample.py --out_dir=out-shakespeare-char
 44 | ```
 45 | 
 46 | This generates a few samples, for example:
 47 | 
 48 | ```
 49 | ANGELO:
 50 | And cowards it be strawn to my bed,
 51 | And thrust the gates of my threats,
 52 | Because he that ale away, and hang'd
 53 | An one with him.
 54 | 
 55 | DUKE VINCENTIO:
 56 | I thank your eyes against it.
 57 | 
 58 | DUKE VINCENTIO:
 59 | Then will answer him to save the malm:
 60 | And what have you tyrannous shall do this?
 61 | 
 62 | DUKE VINCENTIO:
 63 | If you have done evils of all disposition
 64 | To end his power, the day of thrust for a common men
 65 | That I leave, to fight with over-liking
 66 | Hasting in a roseman.
 67 | ```
 68 | 
 69 | Not bad for a character-level model after 3 minutes of training on a GPU. Better results are quite likely obtainable by instead finetuning a pretrained GPT-2 model on this dataset (see finetuning section later).
 70 | 
 71 | ## Training on a CPU machine
 72 | If we are running on CPU instead of GPU we must set both `--device=cpu` and also turn off PyTorch 2.0 compile with `--compile=False`. Then when we evaluate we get a bit more noisy but faster estimate (`--eval_iters=20`, down from 200), our context size is only 64 characters instead of 256, and the batch size only 12 examples per iteration, not 64. We'll also use a much smaller Transformer (4 layers, 4 heads, 128 embedding size), and decrease the number of iterations to 2000 (and correspondingly usually decay the learning rate to around max_iters with `--lr_decay_iters`). Because our network is so small we also ease down on regularization (`--dropout=0.0`). This still runs in almost 10 minutes on a CPU, but gets us a loss of around *1.48* and therefore also worse samples, but it's still good fun:
 73 | 
 74 | ```
 75 | $ python sample.py --out_dir=out-shakespeare-char --device=cpu
 76 | ```
 77 | Generates samples like this:
 78 | 
 79 | ```
 80 | GLEORKEN VINGHARD III:
 81 | Whell's the couse, the came light gacks,
 82 | And the for mought you in Aut fries the not high shee
 83 | bot thou the sought bechive in that to doth groan you,
 84 | No relving thee post mose the wear
 85 | ```
 86 | 
 87 | If you're willing to wait longer, feel free to tune the hyperparameters, increase the size of the network, the context length (`--block_size`), the length of training, etc.
 88 | 
 89 | Finally, on Apple Silicon Macbooks and with a recent PyTorch version make sure to add `--device=mps` (short for "Metal Performance Shaders"); PyTorch then uses the on-chip GPU that can *significantly* accelerate training (2-3X) and allow you to use larger networks. See [Issue 28](https://github.com/karpathy/nanoGPT/issues/28) for more.
 90 | 
 91 | 
 92 | ## sampling / inference
 93 | 
 94 | Use the script `sample.py` to sample either from pre-trained GPT-2 models released by OpenAI, or from a model you trained yourself. For example, here is a way to sample from the largest available `gpt2-xl` model:
 95 | 
 96 | ```
 97 | $ python sample.py \
 98 |     --init_from=gpt2-xl \
 99 |     --start="What is the answer to life, the universe, and everything?" \
100 |     --num_samples=5 --max_new_tokens=100
101 | ```
102 | 
103 | If you'd like to sample from a model you trained, use the `--out_dir` to point the code appropriately. You can also prompt the model with some text from a file, e.g. `$ python sample.py --start=FILE:prompt.txt`.
104 | 


--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
  1 | """
  2 | This training script can be run both on a single gpu in debug mode,
  3 | and also in a larger training run with distributed data parallel (ddp).
  4 | 
  5 | To run on a single GPU, example:
  6 | $ python train.py --batch_size=32 --compile=False
  7 | 
  8 | To run with DDP on 4 gpus on 1 node, example:
  9 | $ torchrun --standalone --nproc_per_node=4 train.py
 10 | 
 11 | To run with DDP on 4 gpus across 2 nodes, example:
 12 | - Run on the first (master) node with example IP 123.456.123.456:
 13 | $ torchrun --nproc_per_node=8 --nnodes=2 --node_rank=0 --master_addr=123.456.123.456 --master_port=1234 train.py
 14 | - Run on the worker node:
 15 | $ torchrun --nproc_per_node=8 --nnodes=2 --node_rank=1 --master_addr=123.456.123.456 --master_port=1234 train.py
 16 | (If your cluster does not have Infiniband interconnect prepend NCCL_IB_DISABLE=1)
 17 | """
 18 | 
 19 | import os
 20 | import time
 21 | import math
 22 | import pickle
 23 | from contextlib import nullcontext
 24 | 
 25 | import numpy as np
 26 | import torch
 27 | from torch.nn.parallel import DistributedDataParallel as DDP
 28 | from torch.distributed import init_process_group, destroy_process_group
 29 | 
 30 | from model import GPTConfig, GPT
 31 | 
 32 | # -----------------------------------------------------------------------------
 33 | # default config values designed to train a gpt2 (124M) on OpenWebText
 34 | # I/O
 35 | out_dir = 'out'
 36 | eval_interval = 2000
 37 | log_interval = 1
 38 | eval_iters = 200
 39 | eval_only = False # if True, script exits right after the first eval
 40 | always_save_checkpoint = True # if True, always save a checkpoint after each eval
 41 | init_from = 'scratch' # 'scratch' or 'resume' or 'gpt2*'
 42 | # wandb logging
 43 | wandb_log = False # disabled by default
 44 | wandb_project = 'owt'
 45 | wandb_run_name = 'gpt2' # 'run' + str(time.time())
 46 | # data
 47 | dataset = 'openwebtext'
 48 | gradient_accumulation_steps = 5 * 8 # used to simulate larger batch sizes
 49 | batch_size = 12 # if gradient_accumulation_steps > 1, this is the micro-batch size
 50 | block_size = 1024
 51 | # model
 52 | n_layer = 12
 53 | n_head = 12
 54 | n_embd = 768
 55 | dropout = 0.0 # for pretraining 0 is good, for finetuning try 0.1+
 56 | bias = False # do we use bias inside LayerNorm and Linear layers?
 57 | # adamw optimizer
 58 | learning_rate = 6e-4 # max learning rate
 59 | max_iters = 600000 # total number of training iterations
 60 | weight_decay = 1e-1
 61 | beta1 = 0.9
 62 | beta2 = 0.95
 63 | grad_clip = 1.0 # clip gradients at this value, or disable if == 0.0
 64 | # learning rate decay settings
 65 | decay_lr = True # whether to decay the learning rate
 66 | warmup_iters = 2000 # how many steps to warm up for
 67 | lr_decay_iters = 600000 # should be ~= max_iters per Chinchilla
 68 | min_lr = 6e-5 # minimum learning rate, should be ~= learning_rate/10 per Chinchilla
 69 | # DDP settings
 70 | backend = 'nccl' # 'nccl', 'gloo', etc.
 71 | # system
 72 | device = 'cuda' # examples: 'cpu', 'cuda', 'cuda:0', 'cuda:1' etc., or try 'mps' on macbooks
 73 | dtype = 'bfloat16' if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else 'float16' # 'float32', 'bfloat16', or 'float16', the latter will auto implement a GradScaler
 74 | compile = False # use PyTorch 2.0 to compile the model to be faster
 75 | # -----------------------------------------------------------------------------
 76 | config_keys = [k for k,v in globals().items() if not k.startswith('_') and isinstance(v, (int, float, bool, str))]
 77 | exec(open('configurator.py').read()) # overrides from command line or config file
 78 | config = {k: globals()[k] for k in config_keys} # will be useful for logging
 79 | # -----------------------------------------------------------------------------
 80 | 
 81 | # various inits, derived attributes, I/O setup
 82 | ddp = int(os.environ.get('RANK', -1)) != -1 # is this a ddp run?
 83 | if ddp:
 84 |     init_process_group(backend=backend)
 85 |     ddp_rank = int(os.environ['RANK'])
 86 |     ddp_local_rank = int(os.environ['LOCAL_RANK'])
 87 |     ddp_world_size = int(os.environ['WORLD_SIZE'])
 88 |     device = f'cuda:{ddp_local_rank}'
 89 |     torch.cuda.set_device(device)
 90 |     master_process = ddp_rank == 0 # this process will do logging, checkpointing etc.
 91 |     seed_offset = ddp_rank # each process gets a different seed
 92 |     # world_size number of processes will be training simultaneously, so we can scale
 93 |     # down the desired gradient accumulation iterations per process proportionally
 94 |     assert gradient_accumulation_steps % ddp_world_size == 0
 95 |     gradient_accumulation_steps //= ddp_world_size
 96 | else:
 97 |     # if not ddp, we are running on a single gpu, and one process
 98 |     master_process = True
 99 |     seed_offset = 0
100 |     ddp_world_size = 1
101 | tokens_per_iter = gradient_accumulation_steps * ddp_world_size * batch_size * block_size
102 | print(f"tokens per iteration will be: {tokens_per_iter:,}")
103 | 
104 | if master_process:
105 |     os.makedirs(out_dir, exist_ok=True)
106 | torch.manual_seed(1337 + seed_offset)
107 | torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul
108 | torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
109 | device_type = 'cuda' if 'cuda' in device else 'cpu' # for later use in torch.autocast
110 | # note: float16 data type will automatically use a GradScaler
111 | ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]
112 | ctx = nullcontext() if device_type == 'cpu' else torch.amp.autocast(device_type=device_type, dtype=ptdtype)
113 | 
114 | # poor man's data loader
115 | data_dir = os.path.join('data', dataset)
116 | train_data = np.memmap(os.path.join(data_dir, 'train.bin'), dtype=np.uint16, mode='r')
117 | val_data = np.memmap(os.path.join(data_dir, 'val.bin'), dtype=np.uint16, mode='r')
118 | def get_batch(split):
119 |     data = train_data if split == 'train' else val_data
120 |     ix = torch.randint(len(data) - block_size, (batch_size,))
121 |     x = torch.stack([torch.from_numpy((data[i:i+block_size]).astype(np.int64)) for i in ix])
122 |     y = torch.stack([torch.from_numpy((data[i+1:i+1+block_size]).astype(np.int64)) for i in ix])
123 |     if device_type == 'cuda':
124 |         # pin arrays x,y, which allows us to move them to GPU asynchronously (non_blocking=True)
125 |         x, y = x.pin_memory().to(device, non_blocking=True), y.pin_memory().to(device, non_blocking=True)
126 |     else:
127 |         x, y = x.to(device), y.to(device)
128 |     return x, y
129 | 
130 | # init these up here, can override if init_from='resume' (i.e. from a checkpoint)
131 | iter_num = 0
132 | best_val_loss = 1e9
133 | 
134 | # attempt to derive vocab_size from the dataset
135 | meta_path = os.path.join(data_dir, 'meta.pkl')
136 | meta_vocab_size = None
137 | if os.path.exists(meta_path):
138 |     with open(meta_path, 'rb') as f:
139 |         meta = pickle.load(f)
140 |     meta_vocab_size = meta['vocab_size']
141 |     print(f"found vocab_size = {meta_vocab_size} (inside {meta_path})")
142 | 
143 | # model init
144 | model_args = dict(n_layer=n_layer, n_head=n_head, n_embd=n_embd, block_size=block_size,
145 |                   bias=bias, vocab_size=None, dropout=dropout) # start with model_args from command line
146 | if init_from == 'scratch':
147 |     # init a new model from scratch
148 |     print("Initializing a new model from scratch")
149 |     # determine the vocab size we'll use for from-scratch training
150 |     if meta_vocab_size is None:
151 |         print("defaulting to vocab_size of GPT-2 to 50304 (50257 rounded up for efficiency)")
152 |     model_args['vocab_size'] = meta_vocab_size if meta_vocab_size is not None else 50304
153 |     gptconf = GPTConfig(**model_args)
154 |     model = GPT(gptconf)
155 | elif init_from == 'resume':
156 |     print(f"Resuming training from {out_dir}")
157 |     # resume training from a checkpoint.
158 |     ckpt_path = os.path.join(out_dir, 'ckpt.pt')
159 |     checkpoint = torch.load(ckpt_path, map_location=device)
160 |     checkpoint_model_args = checkpoint['model_args']
161 |     # force these config attributes to be equal otherwise we can't even resume training
162 |     # the rest of the attributes (e.g. dropout) can stay as desired from command line
163 |     for k in ['n_layer', 'n_head', 'n_embd', 'block_size', 'bias', 'vocab_size']:
164 |         model_args[k] = checkpoint_model_args[k]
165 |     # create the model
166 |     gptconf = GPTConfig(**model_args)
167 |     model = GPT(gptconf)
168 |     state_dict = checkpoint['model']
169 |     # fix the keys of the state dictionary :(
170 |     # honestly no idea how checkpoints sometimes get this prefix, have to debug more
171 |     unwanted_prefix = '_orig_mod.'
172 |     for k,v in list(state_dict.items()):
173 |         if k.startswith(unwanted_prefix):
174 |             state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k)
175 |     model.load_state_dict(state_dict)
176 |     iter_num = checkpoint['iter_num']
177 |     best_val_loss = checkpoint['best_val_loss']
178 | elif init_from.startswith('gpt2'):
179 |     print(f"Initializing from OpenAI GPT-2 weights: {init_from}")
180 |     # initialize from OpenAI GPT-2 weights
181 |     override_args = dict(dropout=dropout)
182 |     model = GPT.from_pretrained(init_from, override_args)
183 |     # read off the created config params, so we can store them into checkpoint correctly
184 |     for k in ['n_layer', 'n_head', 'n_embd', 'block_size', 'bias', 'vocab_size']:
185 |         model_args[k] = getattr(model.config, k)
186 | # crop down the model block size if desired, using model surgery
187 | if block_size < model.config.block_size:
188 |     model.crop_block_size(block_size)
189 |     model_args['block_size'] = block_size # so that the checkpoint will have the right value
190 | model.to(device)
191 | 
192 | # initialize a GradScaler. If enabled=False scaler is a no-op
193 | scaler = torch.cuda.amp.GradScaler(enabled=(dtype == 'float16'))
194 | 
195 | # optimizer
196 | optimizer = model.configure_optimizers(weight_decay, learning_rate, (beta1, beta2), device_type)
197 | if init_from == 'resume':
198 |     optimizer.load_state_dict(checkpoint['optimizer'])
199 | checkpoint = None # free up memory
200 | 
201 | # compile the model
202 | if compile:
203 |     print("compiling the model... (takes a ~minute)")
204 |     unoptimized_model = model
205 |     model = torch.compile(model) # requires PyTorch 2.0
206 | 
207 | # wrap model into DDP container
208 | if ddp:
209 |     model = DDP(model, device_ids=[ddp_local_rank])
210 | 
211 | # helps estimate an arbitrarily accurate loss over either split using many batches
212 | @torch.no_grad()
213 | def estimate_loss():
214 |     out = {}
215 |     model.eval()
216 |     for split in ['train', 'val']:
217 |         losses = torch.zeros(eval_iters)
218 |         for k in range(eval_iters):
219 |             X, Y = get_batch(split)
220 |             with ctx:
221 |                 logits, loss = model(X, Y)
222 |             losses[k] = loss.item()
223 |         out[split] = losses.mean()
224 |     model.train()
225 |     return out
226 | 
227 | # learning rate decay scheduler (cosine with warmup)
228 | def get_lr(it):
229 |     # 1) linear warmup for warmup_iters steps
230 |     if it < warmup_iters:
231 |         return learning_rate * it / warmup_iters
232 |     # 2) if it > lr_decay_iters, return min learning rate
233 |     if it > lr_decay_iters:
234 |         return min_lr
235 |     # 3) in between, use cosine decay down to min learning rate
236 |     decay_ratio = (it - warmup_iters) / (lr_decay_iters - warmup_iters)
237 |     assert 0 <= decay_ratio <= 1
238 |     coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio)) # coeff ranges 0..1
239 |     return min_lr + coeff * (learning_rate - min_lr)
240 | 
241 | # logging
242 | if wandb_log and master_process:
243 |     import wandb
244 |     wandb.init(project=wandb_project, name=wandb_run_name, config=config)
245 | 
246 | # training loop
247 | X, Y = get_batch('train') # fetch the very first batch
248 | t0 = time.time()
249 | local_iter_num = 0 # number of iterations in the lifetime of this process
250 | raw_model = model.module if ddp else model # unwrap DDP container if needed
251 | running_mfu = -1.0
252 | while True:
253 | 
254 |     # determine and set the learning rate for this iteration
255 |     lr = get_lr(iter_num) if decay_lr else learning_rate
256 |     for param_group in optimizer.param_groups:
257 |         param_group['lr'] = lr
258 | 
259 |     # evaluate the loss on train/val sets and write checkpoints
260 |     if iter_num % eval_interval == 0 and master_process:
261 |         losses = estimate_loss()
262 |         print(f"step {iter_num}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}")
263 |         if wandb_log:
264 |             wandb.log({
265 |                 "iter": iter_num,
266 |                 "train/loss": losses['train'],
267 |                 "val/loss": losses['val'],
268 |                 "lr": lr,
269 |                 "mfu": running_mfu*100, # convert to percentage
270 |             })
271 |         if losses['val'] < best_val_loss or always_save_checkpoint:
272 |             best_val_loss = losses['val']
273 |             if iter_num > 0:
274 |                 checkpoint = {
275 |                     'model': raw_model.state_dict(),
276 |                     'optimizer': optimizer.state_dict(),
277 |                     'model_args': model_args,
278 |                     'iter_num': iter_num,
279 |                     'best_val_loss': best_val_loss,
280 |                     'config': config,
281 |                 }
282 |                 print(f"saving checkpoint to {out_dir}")
283 |                 torch.save(checkpoint, os.path.join(out_dir, 'ckpt.pt'))
284 |     if iter_num == 0 and eval_only:
285 |         break
286 | 
287 |     # forward backward update, with optional gradient accumulation to simulate larger batch size
288 |     # and using the GradScaler if data type is float16
289 |     for micro_step in range(gradient_accumulation_steps):
290 |         if ddp:
291 |             # in DDP training we only need to sync gradients at the last micro step.
292 |             # the official way to do this is with model.no_sync() context manager, but
293 |             # I really dislike that this bloats the code and forces us to repeat code
294 |             # looking at the source of that context manager, it just toggles this variable
295 |             model.require_backward_grad_sync = (micro_step == gradient_accumulation_steps - 1)
296 |         with ctx:
297 |             logits, loss = model(X, Y)
298 |             loss = loss / gradient_accumulation_steps # scale the loss to account for gradient accumulation
299 |         # immediately async prefetch next batch while model is doing the forward pass on the GPU
300 |         X, Y = get_batch('train')
301 |         # backward pass, with gradient scaling if training in fp16
302 |         scaler.scale(loss).backward()
303 |     # clip the gradient
304 |     if grad_clip != 0.0:
305 |         scaler.unscale_(optimizer)
306 |         torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip)
307 |     # step the optimizer and scaler if training in fp16
308 |     scaler.step(optimizer)
309 |     scaler.update()
310 |     # flush the gradients as soon as we can, no need for this memory anymore
311 |     optimizer.zero_grad(set_to_none=True)
312 | 
313 |     # timing and logging
314 |     t1 = time.time()
315 |     dt = t1 - t0
316 |     t0 = t1
317 |     if iter_num % log_interval == 0 and master_process:
318 |         # get loss as float. note: this is a CPU-GPU sync point
319 |         # scale up to undo the division above, approximating the true total loss (exact would have been a sum)
320 |         lossf = loss.item() * gradient_accumulation_steps
321 |         if local_iter_num >= 5: # let the training loop settle a bit
322 |             mfu = raw_model.estimate_mfu(batch_size * gradient_accumulation_steps, dt)
323 |             running_mfu = mfu if running_mfu == -1.0 else 0.9*running_mfu + 0.1*mfu
324 |         print(f"iter {iter_num}: loss {lossf:.4f}, time {dt*1000:.2f}ms, mfu {running_mfu*100:.2f}%")
325 |     iter_num += 1
326 |     local_iter_num += 1
327 | 
328 |     # termination conditions
329 |     if iter_num > max_iters:
330 |         break
331 | 
332 | if ddp:
333 |     destroy_process_group()
334 | 


--------------------------------------------------------------------------------
/model.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Full definition of a GPT Language Model, all of it in this single file.
  3 | References:
  4 | 1) the official GPT-2 TensorFlow implementation released by OpenAI:
  5 | https://github.com/openai/gpt-2/blob/master/src/model.py
  6 | 2) huggingface/transformers PyTorch implementation:
  7 | https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt2/modeling_gpt2.py
  8 | """
  9 | 
 10 | import math
 11 | import inspect
 12 | from dataclasses import dataclass
 13 | 
 14 | import torch
 15 | import torch.nn as nn
 16 | from torch.nn import functional as F
 17 | 
 18 | class LayerNorm(nn.Module):
 19 |     """ LayerNorm but with an optional bias. PyTorch doesn't support simply bias=False """
 20 | 
 21 |     def __init__(self, ndim, bias):
 22 |         super().__init__()
 23 |         self.weight = nn.Parameter(torch.ones(ndim))
 24 |         self.bias = nn.Parameter(torch.zeros(ndim)) if bias else None
 25 | 
 26 |     def forward(self, input):
 27 |         return F.layer_norm(input, self.weight.shape, self.weight, self.bias, 1e-5)
 28 | 
 29 | class CausalSelfAttention(nn.Module):
 30 | 
 31 |     def __init__(self, config):
 32 |         super().__init__()
 33 |         assert config.n_embd % config.n_head == 0
 34 |         # key, query, value projections for all heads, but in a batch
 35 |         self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd, bias=config.bias)
 36 |         # output projection
 37 |         self.c_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
 38 |         # regularization
 39 |         self.attn_dropout = nn.Dropout(config.dropout)
 40 |         self.resid_dropout = nn.Dropout(config.dropout)
 41 |         self.n_head = config.n_head
 42 |         self.n_embd = config.n_embd
 43 |         self.dropout = config.dropout
 44 |         self.block_size = config.block_size
 45 |         self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size))
 46 |                                         .view(1, 1, config.block_size, config.block_size))
 47 | 
 48 |     def forward(self, x, kv_cache=None):
 49 |         B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
 50 | 
 51 |         # calculate query, key, values for all heads in batch and move head forward to be the batch dim
 52 |         q, k, v  = self.c_attn(x).split(self.n_embd, dim=2)
 53 | 
 54 |         # TODO: get previous k and v from cache (kv_cache) and concat to the current k and v
 55 |         if kv_cache:
 56 |             prev_k, prev_v = kv_cache
 57 |             k = torch.cat([prev_k, k], dim=1)
 58 |             v = torch.cat([prev_v, v], dim=1)
 59 | 
 60 |         # TODO: update new k and v and return
 61 |         kv_cache = [k, v]
 62 | 
 63 |         q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) with T = 1
 64 |         k = k.view(B, -1, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
 65 |         v = v.view(B, -1, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
 66 | 
 67 |         current_T = k.size(2)
 68 |         assert current_T <= self.block_size, f"Cannot forward sequence of length {current_T}, block size is only {self.block_size}"
 69 | 
 70 |         # manual implementation of attention
 71 |         att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
 72 |         att = att.masked_fill(self.bias[:, :, current_T - T:current_T, :current_T] == 0, float('-inf'))
 73 |         att = F.softmax(att, dim=-1)
 74 |         att = self.attn_dropout(att)
 75 |         y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)
 76 | 
 77 |         y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side
 78 |         
 79 |         # output projection
 80 |         y = self.resid_dropout(self.c_proj(y))
 81 |         
 82 |         return y, kv_cache
 83 | 
 84 | class MLP(nn.Module):
 85 | 
 86 |     def __init__(self, config):
 87 |         super().__init__()
 88 |         self.c_fc    = nn.Linear(config.n_embd, 4 * config.n_embd, bias=config.bias)
 89 |         self.gelu    = nn.GELU()
 90 |         self.c_proj  = nn.Linear(4 * config.n_embd, config.n_embd, bias=config.bias)
 91 |         self.dropout = nn.Dropout(config.dropout)
 92 | 
 93 |     def forward(self, x):
 94 |         x = self.c_fc(x)
 95 |         x = self.gelu(x)
 96 |         x = self.c_proj(x)
 97 |         x = self.dropout(x)
 98 |         return x
 99 | 
100 | class Block(nn.Module):
101 | 
102 |     def __init__(self, config):
103 |         super().__init__()
104 |         self.ln_1 = LayerNorm(config.n_embd, bias=config.bias)
105 |         self.attn = CausalSelfAttention(config)
106 |         self.ln_2 = LayerNorm(config.n_embd, bias=config.bias)
107 |         self.mlp = MLP(config)
108 | 
109 |     def forward(self, x, kv_cache=None):
110 |         # TODO: return kv_cache at each block
111 |         attn_logits, kv_cache = self.attn(self.ln_1(x), kv_cache)
112 | 
113 |         x = x + attn_logits
114 |         x = x + self.mlp(self.ln_2(x))
115 |         return x, kv_cache
116 | 
117 | @dataclass
118 | class GPTConfig:
119 |     block_size: int = 1024
120 |     vocab_size: int = 50304 # GPT-2 vocab_size of 50257, padded up to nearest multiple of 64 for efficiency
121 |     n_layer: int = 12
122 |     n_head: int = 12
123 |     n_embd: int = 768
124 |     dropout: float = 0.0
125 |     bias: bool = True # True: bias in Linears and LayerNorms, like GPT-2. False: a bit better and faster
126 | 
127 | class GPT(nn.Module):
128 | 
129 |     def __init__(self, config):
130 |         super().__init__()
131 |         assert config.vocab_size is not None
132 |         assert config.block_size is not None
133 |         self.config = config
134 | 
135 |         self.transformer = nn.ModuleDict(dict(
136 |             wte = nn.Embedding(config.vocab_size, config.n_embd),
137 |             wpe = nn.Embedding(config.block_size, config.n_embd),
138 |             drop = nn.Dropout(config.dropout),
139 |             h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
140 |             ln_f = LayerNorm(config.n_embd, bias=config.bias),
141 |         ))
142 |         self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
143 |         # with weight tying when using torch.compile() some warnings get generated:
144 |         # "UserWarning: functional_call was passed multiple values for tied weights.
145 |         # This behavior is deprecated and will be an error in future versions"
146 |         # not 100% sure what this is, so far seems to be harmless. TODO investigate
147 |         self.transformer.wte.weight = self.lm_head.weight # https://paperswithcode.com/method/weight-tying
148 | 
149 |         # init all weights
150 |         self.apply(self._init_weights)
151 |         # apply special scaled init to the residual projections, per GPT-2 paper
152 |         for pn, p in self.named_parameters():
153 |             if pn.endswith('c_proj.weight'):
154 |                 torch.nn.init.normal_(p, mean=0.0, std=0.02/math.sqrt(2 * config.n_layer))
155 | 
156 |         # report number of parameters
157 |         print("number of parameters: %.2fM" % (self.get_num_params()/1e6,))
158 | 
159 |     def get_num_params(self, non_embedding=True):
160 |         """
161 |         Return the number of parameters in the model.
162 |         For non-embedding count (default), the position embeddings get subtracted.
163 |         The token embeddings would too, except due to the parameter sharing these
164 |         params are actually used as weights in the final layer, so we include them.
165 |         """
166 |         n_params = sum(p.numel() for p in self.parameters())
167 |         if non_embedding:
168 |             n_params -= self.transformer.wpe.weight.numel()
169 |         return n_params
170 | 
171 |     def _init_weights(self, module):
172 |         if isinstance(module, nn.Linear):
173 |             torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
174 |             if module.bias is not None:
175 |                 torch.nn.init.zeros_(module.bias)
176 |         elif isinstance(module, nn.Embedding):
177 |             torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
178 | 
179 |     def forward(self, idx, targets=None, kv_cache=None):
180 |         device = idx.device
181 |         b, t = idx.size()
182 |         assert t <= self.config.block_size, f"Cannot forward sequence of length {t}, block size is only {self.config.block_size}"
183 |         pos = torch.arange(0, t, dtype=torch.long, device=device) # shape (t)
184 |         
185 |         # TODO: initialize kv_cache + update pos corresponding timestep t
186 |         if kv_cache:
187 |             start_pos = idx.size(1) - 1
188 |             pos = torch.arange(start_pos, start_pos + 1, dtype=torch.long, device=device)
189 |             idx = idx[:, [-1]]
190 |         
191 |         # forward the GPT model itself
192 |         tok_emb = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)
193 |         pos_emb = self.transformer.wpe(pos) # position embeddings of shape (t, n_embd)
194 |         x = self.transformer.drop(tok_emb + pos_emb)
195 |     
196 |         # TODO: update kv_cache after each transformer block
197 |         new_kv_cache = []
198 |         if kv_cache:
199 |             x_output = None
200 |             for block, kv_cache_block in zip(self.transformer.h, kv_cache):
201 |                 x, kv_cache = block(x, kv_cache_block)
202 |                 new_kv_cache.append(kv_cache)
203 |                 x_output = torch.cat([x_output, x], dim=1) if x_output is not None else x
204 |             x = x_output
205 |         else:
206 |             for block in self.transformer.h:
207 |                 x, _ = block(x)
208 |         x = self.transformer.ln_f(x)
209 |         
210 |         if targets is not None:
211 |             # if we are given some desired targets also calculate the loss
212 |             logits = self.lm_head(x)
213 |             loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
214 |         else:
215 |             # inference-time mini-optimization: only forward the lm_head on the very last position
216 |             logits = self.lm_head(x[:, [-1], :]) # note: using list [-1] to preserve the time dim
217 |             loss = None
218 |         
219 |         if len(new_kv_cache) > 0:
220 |             return logits, loss, new_kv_cache
221 |         else:
222 |             return logits, loss
223 | 
224 |     def crop_block_size(self, block_size):
225 |         # model surgery to decrease the block size if necessary
226 |         # e.g. we may load the GPT2 pretrained model checkpoint (block size 1024)
227 |         # but want to use a smaller block size for some smaller, simpler model
228 |         assert block_size <= self.config.block_size
229 |         self.config.block_size = block_size
230 |         self.transformer.wpe.weight = nn.Parameter(self.transformer.wpe.weight[:block_size])
231 |         for block in self.transformer.h:
232 |             if hasattr(block.attn, 'bias'):
233 |                 block.attn.bias = block.attn.bias[:,:,:block_size,:block_size]
234 | 
235 |     @classmethod
236 |     def from_pretrained(cls, model_type, override_args=None):
237 |         assert model_type in {'gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'}
238 |         override_args = override_args or {} # default to empty dict
239 |         # only dropout can be overridden see more notes below
240 |         assert all(k == 'dropout' for k in override_args)
241 |         from transformers import GPT2LMHeadModel
242 |         print("loading weights from pretrained gpt: %s" % model_type)
243 | 
244 |         # n_layer, n_head and n_embd are determined from model_type
245 |         config_args = {
246 |             'gpt2':         dict(n_layer=12, n_head=12, n_embd=768),  # 124M params
247 |             'gpt2-medium':  dict(n_layer=24, n_head=16, n_embd=1024), # 350M params
248 |             'gpt2-large':   dict(n_layer=36, n_head=20, n_embd=1280), # 774M params
249 |             'gpt2-xl':      dict(n_layer=48, n_head=25, n_embd=1600), # 1558M params
250 |         }[model_type]
251 |         print("forcing vocab_size=50257, block_size=1024, bias=True")
252 |         config_args['vocab_size'] = 50257 # always 50257 for GPT model checkpoints
253 |         config_args['block_size'] = 1024 # always 1024 for GPT model checkpoints
254 |         config_args['bias'] = True # always True for GPT model checkpoints
255 |         # we can override the dropout rate, if desired
256 |         if 'dropout' in override_args:
257 |             print(f"overriding dropout rate to {override_args['dropout']}")
258 |             config_args['dropout'] = override_args['dropout']
259 |         # create a from-scratch initialized minGPT model
260 |         config = GPTConfig(**config_args)
261 |         model = GPT(config)
262 |         sd = model.state_dict()
263 |         sd_keys = sd.keys()
264 |         sd_keys = [k for k in sd_keys if not k.endswith('.attn.bias')] # discard this mask / buffer, not a param
265 | 
266 |         # init a huggingface/transformers model
267 |         model_hf = GPT2LMHeadModel.from_pretrained(model_type)
268 |         sd_hf = model_hf.state_dict()
269 | 
270 |         # copy while ensuring all of the parameters are aligned and match in names and shapes
271 |         sd_keys_hf = sd_hf.keys()
272 |         sd_keys_hf = [k for k in sd_keys_hf if not k.endswith('.attn.masked_bias')] # ignore these, just a buffer
273 |         sd_keys_hf = [k for k in sd_keys_hf if not k.endswith('.attn.bias')] # same, just the mask (buffer)
274 |         transposed = ['attn.c_attn.weight', 'attn.c_proj.weight', 'mlp.c_fc.weight', 'mlp.c_proj.weight']
275 |         # basically the openai checkpoints use a "Conv1D" module, but we only want to use a vanilla Linear
276 |         # this means that we have to transpose these weights when we import them
277 |         assert len(sd_keys_hf) == len(sd_keys), f"mismatched keys: {len(sd_keys_hf)} != {len(sd_keys)}"
278 |         for k in sd_keys_hf:
279 |             if any(k.endswith(w) for w in transposed):
280 |                 # special treatment for the Conv1D weights we need to transpose
281 |                 assert sd_hf[k].shape[::-1] == sd[k].shape
282 |                 with torch.no_grad():
283 |                     sd[k].copy_(sd_hf[k].t())
284 |             else:
285 |                 # vanilla copy over the other parameters
286 |                 assert sd_hf[k].shape == sd[k].shape
287 |                 with torch.no_grad():
288 |                     sd[k].copy_(sd_hf[k])
289 | 
290 |         return model
291 | 
292 |     def configure_optimizers(self, weight_decay, learning_rate, betas, device_type):
293 |         # start with all of the candidate parameters
294 |         param_dict = {pn: p for pn, p in self.named_parameters()}
295 |         # filter out those that do not require grad
296 |         param_dict = {pn: p for pn, p in param_dict.items() if p.requires_grad}
297 |         # create optim groups. Any parameters that is 2D will be weight decayed, otherwise no.
298 |         # i.e. all weight tensors in matmuls + embeddings decay, all biases and layernorms don't.
299 |         decay_params = [p for n, p in param_dict.items() if p.dim() >= 2]
300 |         nodecay_params = [p for n, p in param_dict.items() if p.dim() < 2]
301 |         optim_groups = [
302 |             {'params': decay_params, 'weight_decay': weight_decay},
303 |             {'params': nodecay_params, 'weight_decay': 0.0}
304 |         ]
305 |         num_decay_params = sum(p.numel() for p in decay_params)
306 |         num_nodecay_params = sum(p.numel() for p in nodecay_params)
307 |         print(f"num decayed parameter tensors: {len(decay_params)}, with {num_decay_params:,} parameters")
308 |         print(f"num non-decayed parameter tensors: {len(nodecay_params)}, with {num_nodecay_params:,} parameters")
309 |         # Create AdamW optimizer and use the fused version if it is available
310 |         fused_available = 'fused' in inspect.signature(torch.optim.AdamW).parameters
311 |         use_fused = fused_available and device_type == 'cuda'
312 |         extra_args = dict(fused=True) if use_fused else dict()
313 |         optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=betas, **extra_args)
314 |         print(f"using fused AdamW: {use_fused}")
315 | 
316 |         return optimizer
317 | 
318 |     def estimate_mfu(self, fwdbwd_per_iter, dt):
319 |         """ estimate model flops utilization (MFU) in units of A100 bfloat16 peak FLOPS """
320 |         # first estimate the number of flops we do per iteration.
321 |         # see PaLM paper Appendix B as ref: https://arxiv.org/abs/2204.02311
322 |         N = self.get_num_params()
323 |         cfg = self.config
324 |         L, H, Q, T = cfg.n_layer, cfg.n_head, cfg.n_embd//cfg.n_head, cfg.block_size
325 |         flops_per_token = 6*N + 12*L*H*Q*T
326 |         flops_per_fwdbwd = flops_per_token * T
327 |         flops_per_iter = flops_per_fwdbwd * fwdbwd_per_iter
328 |         # express our flops throughput as ratio of A100 bfloat16 peak flops
329 |         flops_achieved = flops_per_iter * (1.0/dt) # per second
330 |         flops_promised = 312e12 # A100 GPU bfloat16 peak flops is 312 TFLOPS
331 |         mfu = flops_achieved / flops_promised
332 |         return mfu
333 | 
334 |     @torch.no_grad()
335 |     def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None, top_p=None):
336 |         """
337 |         Take a conditioning sequence of indices idx (LongTensor of shape (b,t)) and complete
338 |         the sequence max_new_tokens times, feeding the predictions back into the model each time.
339 |         Most likely you'll want to make sure to be in model.eval() mode of operation for this.
340 |         """
341 |         def sampling_generation(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')):
342 |             """ Filter a distribution of logits using top-k and/or nucleus (top-p) filtering
343 |                 Args:
344 |                     logits: logits distribution shape (vocabulary size)
345 |                     top_k >0: keep only top k tokens with highest probability (top-k filtering).
346 |                     top_p >0.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).
347 |             """
348 |             assert logits.size(0) == 1  # batch size = 1
349 |             top_k = min(top_k, logits.size(-1))  # Safety check
350 |             if top_k > 0:
351 |                 # Remove all tokens with a probability less than the last token of the top-k
352 |                 indices_to_remove = logits < torch.topk(logits, top_k)[0][:, -1].unsqueeze(0)
353 |                 logits[indices_to_remove] = filter_value
354 | 
355 |             if top_p > 0.0:
356 |                 sorted_logits, sorted_indices = torch.sort(logits, descending=True)
357 |                 cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
358 | 
359 |                 # Remove tokens with cumulative probability above the threshold
360 |                 sorted_indices_to_remove = cumulative_probs > top_p
361 |                 # Shift the indices to the right to keep also the first token above the threshold
362 |                 sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
363 |                 sorted_indices_to_remove[..., 0] = 0
364 |                 
365 |                 # indices_to_remove = sorted_indices[sorted_indices_to_remove]
366 |                 indices_to_remove = torch.zeros_like(logits, dtype=sorted_indices_to_remove.dtype).scatter_(
367 |                                                     dim=-1, index=sorted_indices, src=sorted_indices_to_remove)
368 |                 logits[indices_to_remove] = filter_value
369 |             return logits
370 |         
371 |         # TODO: initialize kv_cache for forward pass
372 |         kv_cache = [None] * self.config.n_layer
373 | 
374 |         for _ in range(max_new_tokens):
375 |             # if the sequence context is growing too long we must crop it at block_size
376 |             idx_cond = idx if idx.size(1) <= self.config.block_size else idx[:, -self.config.block_size:]
377 |             # forward the model to get the logits for the index in the sequence
378 |             logits, _, kv_cache = self(idx_cond, kv_cache=kv_cache)
379 |             logits = logits[:, -1, :] # crop to just the final time step
380 |             if temperature == 0.0:
381 |                 # "sample" the single most likely index
382 |                 _, idx_next = torch.topk(logits, k=1, dim=-1)
383 |             else:
384 |                 # implement your top-k sampling here
385 |                 # raise NotImplementedError("temperature > 0 is not implemented")
386 |                 ## TODO: implement temperature, top-k, and top-p
387 |                 logits = logits / temperature
388 |                 filtered_logits = sampling_generation(logits, top_k=top_k, top_p=top_p)
389 |                 # Sample from the filtered distribution
390 |                 probabilities = F.softmax(filtered_logits, dim=-1)
391 |                 idx_next = torch.multinomial(probabilities, 1)
392 |                 
393 |             # append sampled index to the running sequence and continue
394 |             idx = torch.cat((idx, idx_next), dim=1)
395 |         
396 |         return idx
397 | 


--------------------------------------------------------------------------------
/data.json:
--------------------------------------------------------------------------------
1 | [{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}, {"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}, {"instruction": "What are the three primary colors?", "input": "", "output": "The three primary colors are red, blue, and yellow."}]


--------------------------------------------------------------------------------