├── .gitignore ├── README.md ├── data ├── dataloader.py └── readdata.py ├── dataset └── assist2009 │ ├── builder_test.csv │ └── builder_train.csv ├── evaluation ├── eval.py └── run.py └── model ├── DKT └── RNNModel.py └── SAKT ├── __init__.py ├── attention.py ├── embedding.py └── model.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | 134 | # pytype static type analyzer 135 | .pytype/ 136 | 137 | # Cython debug symbols 138 | cython_debug/ 139 | 140 | # others 141 | .vscode/ 142 | .DS_Store 143 | log/* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KnowledgeTracing 2 | 3 | ## Introduction 4 | 5 | Some implementations of knowledge tracing with pytorch 6 | 7 | 1. DKT, paper: , reference: 8 | 9 | ## Usage 10 | 11 | ```bash 12 | # at the root path 13 | python -m DKT.evaluation.run 14 | ``` 15 | -------------------------------------------------------------------------------- /data/dataloader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author: jarvis.zhang 3 | # @Date: 2020-05-08 16:21:21 4 | # @Last Modified by: jarvis.zhang 5 | # @Last Modified time: 2020-05-10 11:47:28 6 | import torch 7 | import torch.utils.data as Data 8 | from data.readdata import DataReader 9 | 10 | 11 | def getDataLoader(batch_size, num_of_questions, max_step): 12 | handle = DataReader('dataset/assist2009/builder_train.csv', 13 | 'dataset/assist2009/builder_test.csv', max_step, 14 | num_of_questions) 15 | dtrain = torch.tensor(handle.getTrainData().astype(float).tolist(), 16 | dtype=torch.float32) 17 | dtest = torch.tensor(handle.getTestData().astype(float).tolist(), 18 | dtype=torch.float32) 19 | trainLoader = Data.DataLoader(dtrain, batch_size=batch_size, shuffle=True) 20 | testLoader = Data.DataLoader(dtest, batch_size=batch_size, shuffle=False) 21 | return trainLoader, testLoader 22 | -------------------------------------------------------------------------------- /data/readdata.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author: jarvis.zhang 3 | # @Date: 2020-05-08 18:46:52 4 | # @Last Modified by: jarvis.zhang 5 | # @Last Modified time: 2020-05-10 00:51:05 6 | import numpy as np 7 | import itertools 8 | 9 | 10 | class DataReader(): 11 | def __init__(self, train_path, test_path, maxstep, numofques): 12 | self.train_path = train_path 13 | self.test_path = test_path 14 | self.maxstep = maxstep 15 | self.numofques = numofques 16 | 17 | def getData(self, file_path): 18 | data = [] 19 | with open(file_path, 'r') as file: 20 | for len, ques, ans in itertools.zip_longest(*[file] * 3): 21 | len = int(len.strip().strip(',')) 22 | ques = [int(q) for q in ques.strip().strip(',').split(',')] 23 | ans = [int(a) for a in ans.strip().strip(',').split(',')] 24 | slices = len//self.maxstep + (1 if len % self.maxstep > 0 else 0) 25 | for i in range(slices): 26 | temp = temp = np.zeros(shape=[self.maxstep, 2 * self.numofques]) 27 | if len > 0: 28 | if len >= self.maxstep: 29 | steps = self.maxstep 30 | else: 31 | steps = len 32 | for j in range(steps): 33 | if ans[i*self.maxstep + j] == 1: 34 | temp[j][ques[i*self.maxstep + j]] = 1 35 | else: 36 | temp[j][ques[i*self.maxstep + j] + self.numofques] = 1 37 | len = len - self.maxstep 38 | data.append(temp.tolist()) 39 | print('done: ' + str(np.array(data).shape)) 40 | return data 41 | 42 | def getTrainData(self): 43 | print('loading train data...') 44 | trainData = self.getData(self.train_path) 45 | return np.array(trainData) 46 | 47 | def getTestData(self): 48 | print('loading test data...') 49 | testData = self.getData(self.test_path) 50 | return np.array(testData) 51 | -------------------------------------------------------------------------------- /evaluation/eval.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author: jarvis.zhang 3 | # @Date: 2020-05-09 13:42:11 4 | # @Last Modified by: jarvis.zhang 5 | # @Last Modified time: 2020-05-10 13:33:06 6 | import tqdm 7 | import torch 8 | import logging 9 | 10 | import torch.nn as nn 11 | from sklearn import metrics 12 | 13 | logger = logging.getLogger('main.eval') 14 | 15 | 16 | def performance(ground_truth, prediction): 17 | fpr, tpr, thresholds = metrics.roc_curve(ground_truth.detach().cpu().numpy(), 18 | prediction.detach().cpu().numpy()) 19 | auc = metrics.auc(fpr, tpr) 20 | 21 | f1 = metrics.f1_score(ground_truth.detach().cpu().numpy(), 22 | torch.round(prediction).detach().cpu().numpy()) 23 | recall = metrics.recall_score(ground_truth.detach().cpu().numpy(), 24 | torch.round(prediction).detach().cpu().numpy()) 25 | precision = metrics.precision_score( 26 | ground_truth.detach().cpu().numpy(), 27 | torch.round(prediction).detach().cpu().numpy()) 28 | logger.info('auc: ' + str(auc) + ' f1: ' + str(f1) + ' recall: ' + 29 | str(recall) + ' precision: ' + str(precision)) 30 | print('auc: ' + str(auc) + ' f1: ' + str(f1) + ' recall: ' + str(recall) + 31 | ' precision: ' + str(precision)) 32 | 33 | 34 | class lossFunc(nn.Module): 35 | def __init__(self, num_of_questions, max_step, device): 36 | super(lossFunc, self).__init__() 37 | self.crossEntropy = nn.BCELoss() 38 | self.num_of_questions = num_of_questions 39 | self.max_step = max_step 40 | self.device = device 41 | 42 | def forward(self, pred, batch): 43 | loss = 0 44 | prediction = torch.tensor([], device=self.device) 45 | ground_truth = torch.tensor([], device=self.device) 46 | for student in range(pred.shape[0]): 47 | delta = batch[student][:, 0:self.num_of_questions] + batch[ 48 | student][:, self.num_of_questions:] # shape: [length, questions] 49 | temp = pred[student][:self.max_step - 1].mm(delta[1:].t()) 50 | index = torch.tensor([[i for i in range(self.max_step - 1)]], 51 | dtype=torch.long, device=self.device) 52 | p = temp.gather(0, index)[0] 53 | a = (((batch[student][:, 0:self.num_of_questions] - 54 | batch[student][:, self.num_of_questions:]).sum(1) + 1) // 55 | 2)[1:] 56 | for i in range(len(p) - 1, -1, -1): 57 | if p[i] > 0: 58 | p = p[:i + 1] 59 | a = a[:i + 1] 60 | break 61 | loss += self.crossEntropy(p, a) 62 | prediction = torch.cat([prediction, p]) 63 | ground_truth = torch.cat([ground_truth, a]) 64 | return loss, prediction, ground_truth 65 | 66 | 67 | def train_epoch(model, trainLoader, optimizer, loss_func, device): 68 | model.to(device) 69 | for batch in tqdm.tqdm(trainLoader, desc='Training: ', mininterval=2): 70 | batch = batch.to(device) 71 | pred = model(batch) 72 | loss, prediction, ground_truth = loss_func(pred, batch) 73 | optimizer.zero_grad() 74 | loss.backward() 75 | optimizer.step() 76 | return model, optimizer 77 | 78 | 79 | def test_epoch(model, testLoader, loss_func, device): 80 | model.to(device) 81 | ground_truth = torch.tensor([], device=device) 82 | prediction = torch.tensor([], device=device) 83 | for batch in tqdm.tqdm(testLoader, desc='Testing: ', mininterval=2): 84 | batch = batch.to(device) 85 | pred = model(batch) 86 | loss, p, a = loss_func(pred, batch) 87 | prediction = torch.cat([prediction, p]) 88 | ground_truth = torch.cat([ground_truth, a]) 89 | performance(ground_truth, prediction) 90 | -------------------------------------------------------------------------------- /evaluation/run.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author: jarvis.zhang 3 | # @Date: 2020-05-09 21:50:46 4 | # @Last Modified by: jarvis.zhang 5 | # @Last Modified time: 2020-05-10 13:20:09 6 | """ 7 | Usage: 8 | run.py (rnn|sakt) --hidden= [options] 9 | 10 | Options: 11 | --length= max length of question sequence [default: 50] 12 | --questions= num of question [default: 124] 13 | --lr= learning rate [default: 0.001] 14 | --bs= batch size [default: 64] 15 | --seed= random seed [default: 59] 16 | --epochs= number of epochs [default: 10] 17 | --cuda= use GPU id [default: 0] 18 | --hidden= dimention of hidden state [default: 128] 19 | --layers= layers of rnn or transformer [default: 1] 20 | --heads= head number of transformer [default: 8] 21 | --dropout= dropout rate [default: 0.1] 22 | --model= model type 23 | """ 24 | 25 | import os 26 | import random 27 | import logging 28 | import torch 29 | 30 | import torch.optim as optim 31 | import numpy as np 32 | 33 | from datetime import datetime 34 | from docopt import docopt 35 | from data.dataloader import getDataLoader 36 | from evaluation import eval 37 | 38 | 39 | def setup_seed(seed=0): 40 | random.seed(seed) 41 | np.random.seed(seed) 42 | torch.random.manual_seed(seed) 43 | torch.cuda.manual_seed_all(seed) 44 | torch.backends.cudnn.deterministic = True 45 | torch.backends.cudnn.benchmark = False 46 | 47 | 48 | def main(): 49 | 50 | args = docopt(__doc__) 51 | length = int(args['--length']) 52 | questions = int(args['--questions']) 53 | lr = float(args['--lr']) 54 | bs = int(args['--bs']) 55 | seed = int(args['--seed']) 56 | epochs = int(args['--epochs']) 57 | cuda = args['--cuda'] 58 | hidden = int(args['--hidden']) 59 | layers = int(args['--layers']) 60 | heads = int(args['--heads']) 61 | dropout = float(args['--dropout']) 62 | if args['rnn']: 63 | model_type = 'RNN' 64 | elif args['sakt']: 65 | model_type = 'SAKT' 66 | 67 | logger = logging.getLogger('main') 68 | logger.setLevel(level=logging.DEBUG) 69 | date = datetime.now() 70 | handler = logging.FileHandler( 71 | f'log/{date.year}_{date.month}_{date.day}_{model_type}_result.log') 72 | handler.setLevel(logging.INFO) 73 | formatter = logging.Formatter( 74 | '%(asctime)s - %(name)s - %(levelname)s - %(message)s') 75 | handler.setFormatter(formatter) 76 | logger.addHandler(handler) 77 | 78 | logger.info(list(args.items())) 79 | 80 | setup_seed(seed) 81 | 82 | if torch.cuda.is_available(): 83 | os.environ["CUDA_VISIBLE_DEVICES"] = cuda 84 | device = torch.device('cuda') 85 | else: 86 | device = torch.device('cpu') 87 | 88 | trainLoader, testLoade = getDataLoader(bs, questions, length) 89 | 90 | if model_type == 'RNN': 91 | from model.DKT.RNNModel import RNNModel 92 | model = RNNModel(questions * 2, hidden, layers, questions, device) 93 | elif model_type == 'SAKT': 94 | from model.SAKT.model import SAKTModel 95 | model = SAKTModel(heads, length, hidden, questions, dropout) 96 | 97 | optimizer = optim.Adam(model.parameters(), lr=lr) 98 | loss_func = eval.lossFunc(questions, length, device) 99 | 100 | for epoch in range(epochs): 101 | print('epoch: ' + str(epoch)) 102 | model, optimizer = eval.train_epoch(model, trainLoader, optimizer, 103 | loss_func, device) 104 | logger.info(f'epoch {epoch}') 105 | eval.test_epoch(model, testLoade, loss_func, device) 106 | 107 | 108 | if __name__ == '__main__': 109 | main() 110 | -------------------------------------------------------------------------------- /model/DKT/RNNModel.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Author: jarvis.zhang 3 | # @Date: 2020-05-10 00:29:34 4 | # @Last Modified by: jarvis.zhang 5 | # @Last Modified time: 2020-05-10 13:14:50 6 | import torch 7 | import torch.nn as nn 8 | 9 | 10 | class RNNModel(nn.Module): 11 | def __init__(self, input_dim, hidden_dim, layer_dim, output_dim, device): 12 | super(RNNModel, self).__init__() 13 | self.hidden_dim = hidden_dim 14 | self.layer_dim = layer_dim 15 | self.output_dim = output_dim 16 | self.rnn = nn.RNN(input_dim, 17 | hidden_dim, 18 | layer_dim, 19 | batch_first=True, 20 | nonlinearity='tanh') 21 | self.fc = nn.Linear(self.hidden_dim, self.output_dim) 22 | self.sig = nn.Sigmoid() 23 | self.device = device 24 | 25 | def forward(self, x): # shape of input: [batch_size, length, questions * 2] 26 | h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim, device=self.device) # shape: [num_layers * num_directions, batch_size, hidden_size] 27 | out, hn = self.rnn(x, h0) # shape of out: [batch_size, length, hidden_size] 28 | res = self.sig(self.fc(out)) # shape of res: [batch_size, length, question] 29 | return res 30 | -------------------------------------------------------------------------------- /model/SAKT/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jarviszhb/KnowledgeTracing/de28cee3cfb22d50c8916eb01946fe085803fb61/model/SAKT/__init__.py -------------------------------------------------------------------------------- /model/SAKT/attention.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import copy 4 | import torch.nn as nn 5 | 6 | class Encoder(nn.Module): 7 | "Encoder is made up of self-attn and feed forward (defined below)" 8 | def __init__(self, h, length, d_model, dropout): 9 | super(Encoder, self).__init__() 10 | self.multi_headed_attention = MultiHeadedAttention(h, d_model) 11 | self.feed_forward = PositionwiseFeedForward(d_model, d_model * 4) 12 | self.sublayer = clones(SublayerConnection(length, d_model, dropout), 2) 13 | 14 | def forward(self, x, y, mask=None): 15 | "Follow Figure 1 (left) for connections." 16 | y = self.sublayer[0](y, lambda y: self.multi_headed_attention(x, y, y, mask)) 17 | return self.sublayer[1](y, self.feed_forward) 18 | 19 | class MultiHeadedAttention(nn.Module): 20 | def __init__(self, h, d_model, dropout=0.1): 21 | "Take in model size and number of heads." 22 | super(MultiHeadedAttention, self).__init__() 23 | assert d_model % h == 0 24 | # We assume d_v always equals d_k 25 | self.d_k = d_model // h 26 | self.h = h 27 | self.linears = clones(nn.Linear(d_model, d_model), 4) # (3 + 1) 28 | self.attn = None 29 | self.dropout = nn.Dropout(p=dropout) 30 | self.softmax = nn.Softmax(dim=-1) 31 | 32 | def forward(self, query, key, value, mask=None): 33 | if mask is not None: 34 | # Same mask applied to all h heads. 35 | mask = mask.unsqueeze(1) 36 | nbatches = query.size(0) 37 | 38 | # 1) Do all the linear projections in batch from d_model => h x d_k 39 | query, key, value = \ 40 | [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2) 41 | for l, x in zip(self.linears, (query, key, value))] 42 | 43 | # 2) Apply attention on all the projected vectors in batch. 44 | x = self.attention(query, key, value,causality=True, mask=mask, 45 | dropout=self.dropout) 46 | 47 | # 3) "Concat" using a view and apply a final linear. 48 | x = x.transpose(1, 2).contiguous() \ 49 | .view(nbatches, -1, self.h * self.d_k) 50 | return self.linears[-1](x) 51 | 52 | def attention(self, query, key, value, causality=True, mask=None, dropout=None): 53 | "Compute 'Scaled Dot Product Attention'" 54 | d_k = query.size(-1) 55 | scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k) 56 | if mask is not None: 57 | scores = scores.masked_fill(mask == 0, -1e9) 58 | if causality: 59 | scores = torch.tril(scores, diagonal=0, out=None) 60 | p_attn = self.softmax(scores) 61 | if dropout is not None: 62 | p_attn = dropout(p_attn) 63 | return torch.matmul(p_attn, value) 64 | 65 | class PositionwiseFeedForward(nn.Module): 66 | "Implements FFN equation." 67 | def __init__(self, d_model, d_ff, dropout=0.1): 68 | super(PositionwiseFeedForward, self).__init__() 69 | self.w_1 = nn.Linear(d_model, d_ff) 70 | self.w_2 = nn.Linear(d_ff, d_model) 71 | self.dropout = nn.Dropout(dropout) 72 | self.relu = nn.ReLU() 73 | 74 | def forward(self, x): 75 | return self.w_2(self.dropout(self.relu(self.w_1(x)))) 76 | 77 | class SublayerConnection(nn.Module): 78 | """ 79 | A residual connection followed by a layer norm. 80 | Note for code simplicity the norm is first as opposed to last. 81 | """ 82 | def __init__(self, length, d_model, dropout): 83 | super(SublayerConnection, self).__init__() 84 | self.norm = nn.LayerNorm(normalized_shape = [length, d_model]) 85 | self.dropout = nn.Dropout(dropout) 86 | 87 | def forward(self, x, sublayer): 88 | "Apply residual connection to any sublayer with the same size." 89 | return x + self.dropout(sublayer(self.norm(x))) 90 | 91 | def clones(module, N): 92 | "Produce N identical layers." 93 | return nn.ModuleList([copy.deepcopy(module) for _ in range(N)]) -------------------------------------------------------------------------------- /model/SAKT/embedding.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | class Embedding(nn.Module): 5 | def __init__(self, n_questions, length, embedding_dim): 6 | super().__init__() 7 | self.n_questions = n_questions 8 | self.x_emb = nn.Linear(n_questions, embedding_dim, bias=False) # 问题的 embedding 9 | self.y_emb = nn.Linear(n_questions * 2, embedding_dim, bias=False) # 问题和结果的 embedding 10 | self.pos_emb = nn.Embedding(length, embedding_dim) # 位置编码 11 | self.length = length 12 | def forward(self, y): # shape of input: [batch_size, length, questions * 2] 13 | n_batch = y.shape[0] 14 | x = y[:, :, 0:self.n_questions] + y[:, :, self.n_questions:] 15 | p = torch.LongTensor([[i for i in range(self.length)] for j in range(n_batch)]) 16 | pos = self.pos_emb(p) 17 | y = self.y_emb(y) # shape: [batch_size, length, embedding_dim] 18 | x = self.x_emb(x) # shape: [batch_size, length, embedding_dim] 19 | return (x+pos, y) -------------------------------------------------------------------------------- /model/SAKT/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from model.SAKT.attention import Encoder 5 | from model.SAKT.embedding import Embedding 6 | 7 | class SAKTModel(nn.Module): 8 | def __init__(self, h, length, d_model, n_question, dropout): 9 | super(SAKTModel, self).__init__() 10 | self.embedding = Embedding(n_question, length, d_model) 11 | self.encoder = Encoder(h, length, d_model, dropout) 12 | self.w = nn.Linear(d_model, n_question) 13 | self.sig = nn.Sigmoid() 14 | 15 | def forward(self, y): # shape of input: [batch_size, length, questions * 2] 16 | x, y = self.embedding(y) # shape: [batch_size, length, d_model] 17 | encode = self.encoder(x, y) # shape: [batch_size, length, d_model] 18 | res = self.sig(self.w(encode)) 19 | return res 20 | 21 | --------------------------------------------------------------------------------